@adobe/spacecat-shared-utils 1.85.2 → 1.87.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-utils-v1.87.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.86.0...@adobe/spacecat-shared-utils-v1.87.0) (2026-01-08)
+
+
+### Features
+
+* propagate traceId across workers for cross-worker tracing ([#1247](https://github.com/adobe/spacecat-shared/issues/1247)) ([f7d194d](https://github.com/adobe/spacecat-shared/commit/f7d194d2a8d23426c9dd66aae33f1118d226e827))
+
+# [@adobe/spacecat-shared-utils-v1.86.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.85.2...@adobe/spacecat-shared-utils-v1.86.0) (2025-12-12)
+
+
+### Features
+
+* add detection for Akamai, Fastly, and CloudFront ([#1238](https://github.com/adobe/spacecat-shared/issues/1238)) ([3f7aad9](https://github.com/adobe/spacecat-shared/commit/3f7aad96fbc823b2e9d59541a71ba3b4e6d315e8))
+
 # [@adobe/spacecat-shared-utils-v1.85.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.85.1...@adobe/spacecat-shared-utils-v1.85.2) (2025-12-11)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-utils",
-  "version": "1.85.2",
+  "version": "1.87.0",
   "description": "Shared modules of the Spacecat Services - utils",
   "type": "module",
   "exports": {

package/src/bot-blocker-detect/bot-blocker-detect.js

@@ -16,7 +16,8 @@ import { isValidUrl } from '../functions.js';
 /**
  * Confidence levels used in bot blocker detection:
  * - 1.0 (ABSOLUTE): Site responds successfully with 200 OK - definitively crawlable
- * - 0.99 (HIGH): Known bot blocker signature detected (Cloudflare cf-ray, Imperva x-iinfo/x-cdn)
+ * - 0.99 (HIGH): Known bot blocker signature detected
+ *   (Cloudflare, Imperva, Akamai, Fastly, CloudFront)
  * - 0.95 (MEDIUM): HTTP/2 protocol errors indicating potential blocking
  * - 0.5: Unknown status code without known blocker signature (e.g., 403 without headers)
  * - 0.3: Unknown error occurred during request
@@ -32,7 +33,20 @@ const DEFAULT_TIMEOUT = 5000;
 function analyzeResponse(response) {
   const { status, headers } = response;
 
-  if (status === 403 && headers.get('cf-ray')) {
+  // Check for CDN/blocker infrastructure presence (lazy evaluation for performance)
+  const hasCloudflare = () => headers.get('cf-ray') || headers.get('server') === 'cloudflare';
+  const hasImperva = () => headers.get('x-iinfo') || headers.get('x-cdn') === 'Incapsula';
+  const hasAkamai = () => headers.get('x-akamai-request-id')
+    || headers.get('x-akamai-session-id')
+    || headers.get('server')?.includes('AkamaiGHost');
+  const hasFastly = () => headers.get('x-served-by')?.startsWith('cache-')
+    || headers.get('fastly-io-info');
+  const hasCloudFront = () => headers.get('x-amz-cf-id')
+    || headers.get('x-amz-cf-pop')
+    || headers.get('via')?.includes('CloudFront');
+
+  // Active blocking (403 status with known blocker)
+  if (status === 403 && hasCloudflare()) {
     return {
       crawlable: false,
       type: 'cloudflare',
@@ -40,7 +54,7 @@ function analyzeResponse(response) {
     };
   }
 
-  if (status === 403 && (headers.get('x-iinfo') || headers.get('x-cdn') === 'Incapsula')) {
+  if (status === 403 && hasImperva()) {
     return {
       crawlable: false,
       type: 'imperva',
@@ -48,6 +62,72 @@ function analyzeResponse(response) {
     };
   }
 
+  if (status === 403 && hasAkamai()) {
+    return {
+      crawlable: false,
+      type: 'akamai',
+      confidence: CONFIDENCE_HIGH,
+    };
+  }
+
+  if (status === 403 && hasFastly()) {
+    return {
+      crawlable: false,
+      type: 'fastly',
+      confidence: CONFIDENCE_HIGH,
+    };
+  }
+
+  if (status === 403 && hasCloudFront()) {
+    return {
+      crawlable: false,
+      type: 'cloudfront',
+      confidence: CONFIDENCE_HIGH,
+    };
+  }
+
+  // Success with known infrastructure present (infrastructure detected but allowing requests)
+  if (status === 200 && hasCloudflare()) {
+    return {
+      crawlable: true,
+      type: 'cloudflare-allowed',
+      confidence: CONFIDENCE_ABSOLUTE,
+    };
+  }
+
+  if (status === 200 && hasImperva()) {
+    return {
+      crawlable: true,
+      type: 'imperva-allowed',
+      confidence: CONFIDENCE_ABSOLUTE,
+    };
+  }
+
+  if (status === 200 && hasAkamai()) {
+    return {
+      crawlable: true,
+      type: 'akamai-allowed',
+      confidence: CONFIDENCE_ABSOLUTE,
+    };
+  }
+
+  if (status === 200 && hasFastly()) {
+    return {
+      crawlable: true,
+      type: 'fastly-allowed',
+      confidence: CONFIDENCE_ABSOLUTE,
+    };
+  }
+
+  if (status === 200 && hasCloudFront()) {
+    return {
+      crawlable: true,
+      type: 'cloudfront-allowed',
+      confidence: CONFIDENCE_ABSOLUTE,
+    };
+  }
+
+  // Success with no known infrastructure
   if (status === 200) {
     return {
       crawlable: true,
@@ -56,6 +136,7 @@ function analyzeResponse(response) {
     };
   }
 
+  // Unknown status without known blocker signature
   return {
     crawlable: true,
     type: 'unknown',
@@ -86,14 +167,24 @@ function analyzeError(error) {
  * Currently detects:
  * - Cloudflare bot blocking (403 + cf-ray header)
  * - Imperva/Incapsula (403 + x-iinfo or x-cdn: Incapsula header)
+ * - Akamai (403 + x-akamai-request-id or related headers)
+ * - Fastly (403 + x-served-by or fastly-io-info headers)
+ * - AWS CloudFront (403 + x-amz-cf-id or via: CloudFront header)
  * - HTTP/2 stream errors (NGHTTP2_INTERNAL_ERROR, ERR_HTTP2_STREAM_ERROR)
  *
+ * Also detects infrastructure presence on successful requests (200 OK):
+ * - Returns 'cloudflare-allowed', 'imperva-allowed', 'akamai-allowed',
+ *   'fastly-allowed', or 'cloudfront-allowed' when infrastructure is present
+ *   but allowing the request through
+ *
  * @param {Object} config - Configuration object
  * @param {string} config.baseUrl - The base URL to check
  * @param {number} [config.timeout=5000] - Request timeout in milliseconds
  * @returns {Promise<Object>} Detection result with:
  *   - crawlable {boolean}: Whether the site can be crawled by bots
- *   - type {string}: Blocker type ('cloudflare', 'imperva', 'http2-block', 'none', 'unknown')
+ *   - type {string}: Blocker type ('cloudflare', 'imperva', 'akamai', 'fastly',
+ *     'cloudfront', 'http2-block', 'cloudflare-allowed', 'imperva-allowed',
+ *     'akamai-allowed', 'fastly-allowed', 'cloudfront-allowed', 'none', 'unknown')
  *   - confidence {number}: Confidence level (0.0-1.0, see confidence level constants)
  * @throws {Error} If baseUrl is invalid
  */

package/src/bot-blocker-detect/index.d.ts

@@ -17,7 +17,10 @@ export interface BotBlockerConfig {
 
 export interface BotBlockerResult {
   crawlable: boolean;
-  type: 'cloudflare' | 'imperva' | 'http2-block' | 'none' | 'unknown';
+  type: 'cloudflare' | 'imperva' | 'akamai' | 'fastly' | 'cloudfront'
+    | 'cloudflare-allowed' | 'imperva-allowed' | 'akamai-allowed'
+    | 'fastly-allowed' | 'cloudfront-allowed'
+    | 'http2-block' | 'none' | 'unknown';
   confidence: number;
 }
 

package/src/cdn-helpers.js

@@ -152,6 +152,17 @@ const CDN_TRANSFORMATIONS = {
     'Compress logs': 'Yes',
     HelpUrl: 'https://docs-cybersec.thalesgroup.com/bundle/cloud-application-security/page/siem-log-configuration.htm',
   }),
+  'byocdn-other': (payload) => ({
+    'Bucket name': payload.bucketName,
+    Region: payload.region,
+    Path: `${payload.allowedPaths?.[0] || ''}<year>/<month>/<day>`,
+    'Access Key': payload.accessKey,
+    'Secret Key': payload.secretKey,
+    'Timestamp format': 'RFC3339',
+    'Log format': 'JSON lines (one log per line)',
+    Compression: 'Optional, but prefered. Please use Gzip compression if you decide to compress the log files.',
+    'Example of valid log line': '{"timestamp":"2025-12-01T13:00:05Z","host":"www.example.com","url":"/docs/getting-started","request_method":"GET","request_user_agent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)","response_status":200,"request_referer":"https://www.chatgpt.com/","response_content_type":"text/html; charset=utf-8","time_to_first_byte":123}',
+  }),
 };
 
 /**
@@ -164,7 +175,8 @@ const CDN_TRANSFORMATIONS = {
  *
  * @param {Object} payload - The result from CDN-Logs-Infrastructure-Provisioning API
  * @param {string} payload.logSource - The CDN type ('byocdn-fastly' | 'byocdn-akamai'
- *   | 'byocdn-cloudflare' | 'byocdn-cloudfront' | 'ams-cloudfront' | 'byocdn-imperva')
+ *   | 'byocdn-cloudflare' | 'byocdn-cloudfront' | 'ams-cloudfront' | 'byocdn-imperva'
+ *   | 'byocdn-other')
  * @returns {Object} - The prepared log forwarding configuration parameters
  * @throws {Error} - If logSource is not supported or missing
  */
@@ -193,7 +205,7 @@ const prettifyLogForwardingConfig = (payload) => {
     throw new Error('allowedPaths is required in payload');
   }
 
-  if (payload.logSource === 'byocdn-fastly' || payload.logSource === 'byocdn-akamai') {
+  if (payload.logSource === 'byocdn-fastly' || payload.logSource === 'byocdn-akamai' || payload.logSource === 'byocdn-other') {
     if (!payload.accessKey) {
       throw new Error('accessKey is required in payload');
     }

package/src/log-wrapper.js

@@ -43,8 +43,9 @@ export function logWrapper(fn) {
         markers.push(`[jobId=${jobId}]`);
       }
 
-      // Extract traceId from AWS X-Ray
-      const traceId = getTraceId();
+      // Extract traceId: prioritize context.traceId (from SQS message propagation)
+      // over X-Ray segment (which is new for each Lambda invocation)
+      const traceId = context.traceId || getTraceId();
       if (traceId) {
         markers.push(`[traceId=${traceId}]`);
       }