@adobe/spacecat-shared-utils 1.89.0 → 1.90.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-utils-v1.90.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.89.1...@adobe/spacecat-shared-utils-v1.90.0) (2026-01-29)
+
+
+### Features
+
+* LLMO-1534 Pretty format old and new log forwarding credentials ([#1290](https://github.com/adobe/spacecat-shared/issues/1290)) ([df84629](https://github.com/adobe/spacecat-shared/commit/df84629c1a49e81b43e352eb4b8b542b9eb3085e))
+
+# [@adobe/spacecat-shared-utils-v1.89.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.89.0...@adobe/spacecat-shared-utils-v1.89.1) (2026-01-26)
+
+
+### Bug Fixes
+
+* Additional checks and methods on bot protection ([#1250](https://github.com/adobe/spacecat-shared/issues/1250)) ([0c34a8d](https://github.com/adobe/spacecat-shared/commit/0c34a8d850abef6e2a024132bc1c61d10865c1a0))
+
 # [@adobe/spacecat-shared-utils-v1.89.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.88.0...@adobe/spacecat-shared-utils-v1.89.0) (2026-01-22)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-utils",
-  "version": "1.89.0",
+  "version": "1.90.0",
   "description": "Shared modules of the Spacecat Services - utils",
   "type": "module",
   "exports": {

package/src/bot-blocker-detect/bot-blocker-detect.js

@@ -30,7 +30,85 @@ const CONFIDENCE_MEDIUM = 0.95;
 const CONFIDENCE_ABSOLUTE = 1.0;
 const DEFAULT_TIMEOUT = 5000;
 
-function analyzeResponse(response) {
+/**
+ * SpaceCat bot identification constants
+ */
+export const SPACECAT_BOT_USER_AGENT = 'Spacecat/1.0';
+
+/**
+ * Gets SpaceCat bot IPs from environment variable
+ * @param {string} ipsString - Comma-separated IPs (from env/secrets) - REQUIRED
+ * @returns {Array<string>} Array of IP addresses
+ * @throws {Error} If ipsString is not provided
+ */
+export function getSpacecatBotIps(ipsString) {
+  if (!ipsString) {
+    throw new Error('SPACECAT_BOT_IPS environment variable is required but not set');
+  }
+
+  return ipsString.split(',').map((ip) => ip.trim()).filter((ip) => ip);
+}
+
+/**
+ * Formats allowlist message with current bot IPs
+ * @param {string} botIps - Comma-separated IPs from secrets - REQUIRED
+ * @returns {object} Formatted message with IPs and user-agent
+ * @throws {Error} If botIps is not provided
+ */
+export function formatAllowlistMessage(botIps) {
+  const ips = getSpacecatBotIps(botIps);
+
+  return {
+    title: 'To allowlist SpaceCat bot:',
+    ips,
+    userAgent: SPACECAT_BOT_USER_AGENT,
+  };
+}
+
+/**
+ * HTML patterns for detecting challenge pages
+ */
+const CHALLENGE_PATTERNS = {
+  cloudflare: [
+    /Checking your browser/i,
+    /Just a moment\.\.\./i,
+    /Verifying you are human/i,
+    /Please wait.*CloudFlare/i,
+    /cf-turnstile/i,
+    /challenge-platform/i,
+    /cf-chl-widget/i, // Cloudflare challenge widget
+    /ray\s*id.*cloudflare/i, // Cloudflare Ray ID in error pages
+    /__cf_chl_tk/i, // Cloudflare challenge token
+    /cloudflare.*security/i,
+    /attention required.*cloudflare/i,
+  ],
+  imperva: [
+    /_Incapsula_Resource/i,
+    /Incapsula incident ID/i,
+    /incap_ses/i, // Imperva session cookie
+    /visid_incap/i, // Imperva visitor ID
+  ],
+  akamai: [
+    /Access Denied.*Akamai/i,
+    /Reference.*Akamai/i,
+  ],
+  general: [
+    /captcha/i,
+    /human verification/i,
+    /recaptcha/i,
+    /hcaptcha/i,
+    /datadome/i,
+    /dd-request-id/i,
+  ],
+};
+
+/**
+ * Analyzes response for bot protection indicators
+ * @param {Object} response - Response object with status and headers
+ * @param {string} [html] - Optional HTML content for deeper analysis
+ * @returns {Object} Detection result
+ */
+function analyzeResponse(response, html = null) {
   const { status, headers } = response;
 
   // Check for CDN/blocker infrastructure presence (lazy evaluation for performance)
@@ -45,6 +123,12 @@ function analyzeResponse(response) {
     || headers.get('x-amz-cf-pop')
     || headers.get('via')?.includes('CloudFront');
 
+  // Check HTML content for challenge page patterns (if HTML provided)
+  const htmlHasChallenge = (patterns) => {
+    if (!html) return false;
+    return patterns.some((pattern) => pattern.test(html));
+  };
+
   // Active blocking (403 status with known blocker)
   if (status === 403 && hasCloudflare()) {
     return {
@@ -88,6 +172,16 @@ function analyzeResponse(response) {
 
   // Success with known infrastructure present (infrastructure detected but allowing requests)
   if (status === 200 && hasCloudflare()) {
+    // Check if HTML contains challenge page (even though status is 200)
+    if (htmlHasChallenge(CHALLENGE_PATTERNS.cloudflare)) {
+      return {
+        crawlable: false,
+        type: 'cloudflare',
+        confidence: CONFIDENCE_HIGH,
+        reason: 'Challenge page detected despite 200 status',
+      };
+    }
+
     return {
       crawlable: true,
       type: 'cloudflare-allowed',
@@ -96,6 +190,14 @@ function analyzeResponse(response) {
   }
 
   if (status === 200 && hasImperva()) {
+    if (htmlHasChallenge(CHALLENGE_PATTERNS.imperva)) {
+      return {
+        crawlable: false,
+        type: 'imperva',
+        confidence: CONFIDENCE_HIGH,
+        reason: 'Challenge page detected despite 200 status',
+      };
+    }
     return {
       crawlable: true,
       type: 'imperva-allowed',
@@ -104,6 +206,14 @@ function analyzeResponse(response) {
   }
 
   if (status === 200 && hasAkamai()) {
+    if (htmlHasChallenge(CHALLENGE_PATTERNS.akamai)) {
+      return {
+        crawlable: false,
+        type: 'akamai',
+        confidence: CONFIDENCE_HIGH,
+        reason: 'Challenge page detected despite 200 status',
+      };
+    }
     return {
       crawlable: true,
       type: 'akamai-allowed',
@@ -129,6 +239,15 @@ function analyzeResponse(response) {
 
   // Success with no known infrastructure
   if (status === 200) {
+    // Still check for generic challenge patterns
+    if (htmlHasChallenge(CHALLENGE_PATTERNS.general)) {
+      return {
+        crawlable: false,
+        type: 'unknown',
+        confidence: 0.7,
+        reason: 'Generic challenge patterns detected',
+      };
+    }
     return {
       crawlable: true,
       type: 'none',
@@ -136,7 +255,16 @@ function analyzeResponse(response) {
     };
   }
 
-  // Unknown status without known blocker signature
+  // Potential CDN/protection blocked the request
+  if (status === 403) {
+    return {
+      crawlable: false,
+      type: 'unknown',
+      confidence: 0.7,
+      reason: 'HTTP 403 Forbidden - access denied',
+    };
+  }
+
   return {
     crawlable: true,
     type: 'unknown',
@@ -207,3 +335,27 @@ export async function detectBotBlocker({ baseUrl, timeout = DEFAULT_TIMEOUT }) {
     return analyzeError(error);
   }
 }
+
+/**
+ * Analyzes already-fetched response data for bot protection.
+ * Used by content scraper to analyze Puppeteer results without making another request.
+ *
+ * @param {Object} data - Response data to analyze
+ * @param {number} data.status - HTTP status code
+ * @param {Object} data.headers - Response headers (plain object or Headers object)
+ * @param {string} [data.html] - Optional HTML content for challenge page detection
+ * @returns {Object} Detection result (same format as detectBotBlocker)
+ */
+export function analyzeBotProtection({ status, headers, html }) {
+  // Convert headers to Headers object if plain object
+  const headersObj = headers instanceof Headers
+    ? headers
+    : new Headers(Object.entries(headers || {}));
+
+  const response = {
+    status,
+    headers: headersObj,
+  };
+
+  return analyzeResponse(response, html);
+}

package/src/cdn-helpers.js

@@ -11,9 +11,44 @@
  */
 
 /**
- * CDN-specific transformations for log forwarding configuration preparation
+ * Transforms credential fields object with backwards compatibility
+ * @param {Object} payload - The payload containing credential information
+ * @returns {Object} - Object with credential fields
  */
+const transformCredentialFields = (payload) => {
+  const fields = {};
+
+  if (payload.currentAccessKey) {
+    fields['Access Key (current)'] = payload.currentAccessKey;
+  } else if (payload.accessKey) {
+    fields['Access Key'] = payload.accessKey;
+  }
+
+  if (payload.currentSecretKey) {
+    fields['Secret Key (current)'] = payload.currentSecretKey;
+  } else if (payload.secretKey) {
+    fields['Secret Key'] = payload.secretKey;
+  }
+
+  if (payload.oldAccessKey) {
+    fields['Access Key (to be retired)'] = payload.oldAccessKey;
+  }
 
+  if (payload.oldSecretKey) {
+    fields['Secret Key (to be retired)'] = payload.oldSecretKey;
+  }
+
+  fields.currentCredentialsCreatedAt = payload.currentCredentialsCreatedAt;
+  fields.currentCredentialsLastUsed = payload.currentCredentialsLastUsed;
+  fields.oldCredentialsCreatedAt = payload.oldCredentialsCreatedAt;
+  fields.oldCredentialsLastUsed = payload.oldCredentialsLastUsed;
+
+  return fields;
+};
+
+/**
+ * CDN-specific transformations for log forwarding configuration preparation
+ */
 const FASTLY_LOG_FORMAT = `{
     "timestamp": "%{strftime(\\{"%Y-%m-%dT%H:%M:%S%z"\\}, time.start)}V",
     "host": "%{if(req.http.Fastly-Orig-Host, req.http.Fastly-Orig-Host, req.http.Host)}V",
@@ -35,8 +70,7 @@ const CDN_TRANSFORMATIONS = {
     Placement: 'Format Version Default',
     'Log format': FASTLY_LOG_FORMAT,
     'Access method': 'User credentials',
-    'Access key': payload.accessKey,
-    'Secret key': payload.secretKey,
+    ...transformCredentialFields(payload),
     Period: 300,
     'Log line format': 'Blank',
     Compression: 'Gzip',
@@ -67,8 +101,7 @@ const CDN_TRANSFORMATIONS = {
     'Log file prefix': '{%Y}-{%m}-{%d}T{%H}:{%M}:{%S}.000',
     'Log file suffix': '.log',
     'Log interval': '60 seconds',
-    'Access key': payload.accessKey,
-    'Secret key': payload.secretKey,
+    ...transformCredentialFields(payload),
     HelpUrl: 'https://techdocs.akamai.com/datastream2/docs/stream-amazon-s3',
   }),
   'byocdn-cloudflare': (payload) => ({
@@ -156,8 +189,7 @@ const CDN_TRANSFORMATIONS = {
     'Bucket name': payload.bucketName,
     Region: payload.region,
     Path: `${payload.allowedPaths?.[0] || ''}<year>/<month>/<day>`,
-    'Access Key': payload.accessKey,
-    'Secret Key': payload.secretKey,
+    ...transformCredentialFields(payload),
     'Timestamp format': 'RFC3339',
     'Log format': 'JSON lines (one log per line)',
     Compression: 'Optional, but prefered. Please use Gzip compression if you decide to compress the log files.',
@@ -206,11 +238,11 @@ const prettifyLogForwardingConfig = (payload) => {
   }
 
   if (payload.logSource === 'byocdn-fastly' || payload.logSource === 'byocdn-akamai' || payload.logSource === 'byocdn-other') {
-    if (!payload.accessKey) {
-      throw new Error('accessKey is required in payload');
+    if (!payload.accessKey && !payload.currentAccessKey) {
+      throw new Error('accessKey or currentAccessKey is required in payload');
     }
-    if (!payload.secretKey) {
-      throw new Error('secretKey is required in payload');
+    if (!payload.secretKey && !payload.currentSecretKey) {
+      throw new Error('secretKey or currentSecretKey is required in payload');
     }
   }
 

package/src/index.js

@@ -111,7 +111,13 @@ export * as llmoStrategy from './llmo-strategy.js';
 export * as schemas from './schemas.js';
 
 export { detectLocale } from './locale-detect/locale-detect.js';
-export { detectBotBlocker } from './bot-blocker-detect/bot-blocker-detect.js';
+export {
+  detectBotBlocker,
+  analyzeBotProtection,
+  SPACECAT_BOT_USER_AGENT,
+  getSpacecatBotIps,
+  formatAllowlistMessage,
+} from './bot-blocker-detect/bot-blocker-detect.js';
 export { prettifyLogForwardingConfig } from './cdn-helpers.js';
 
 export {