@adobe/spacecat-shared-utils 1.85.1 → 1.86.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-utils-v1.86.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.85.2...@adobe/spacecat-shared-utils-v1.86.0) (2025-12-12)
+
+
+### Features
+
+* add detection for Akamai, Fastly, and CloudFront ([#1238](https://github.com/adobe/spacecat-shared/issues/1238)) ([3f7aad9](https://github.com/adobe/spacecat-shared/commit/3f7aad96fbc823b2e9d59541a71ba3b4e6d315e8))
+
+# [@adobe/spacecat-shared-utils-v1.85.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.85.1...@adobe/spacecat-shared-utils-v1.85.2) (2025-12-11)
+
+
+### Bug Fixes
+
+* Implement Structured (JSON) Logging for Spacecat Audits - rollback ([#1239](https://github.com/adobe/spacecat-shared/issues/1239)) ([1f174d7](https://github.com/adobe/spacecat-shared/commit/1f174d7dd188dbdc610b75bf58644992925755b1))
+
 # [@adobe/spacecat-shared-utils-v1.85.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.85.0...@adobe/spacecat-shared-utils-v1.85.1) (2025-12-11)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-utils",
-  "version": "1.85.1",
+  "version": "1.86.0",
   "description": "Shared modules of the Spacecat Services - utils",
   "type": "module",
   "exports": {

package/src/bot-blocker-detect/bot-blocker-detect.js

@@ -16,7 +16,8 @@ import { isValidUrl } from '../functions.js';
 /**
  * Confidence levels used in bot blocker detection:
  * - 1.0 (ABSOLUTE): Site responds successfully with 200 OK - definitively crawlable
- * - 0.99 (HIGH): Known bot blocker signature detected (Cloudflare cf-ray, Imperva x-iinfo/x-cdn)
+ * - 0.99 (HIGH): Known bot blocker signature detected
+ *   (Cloudflare, Imperva, Akamai, Fastly, CloudFront)
  * - 0.95 (MEDIUM): HTTP/2 protocol errors indicating potential blocking
  * - 0.5: Unknown status code without known blocker signature (e.g., 403 without headers)
  * - 0.3: Unknown error occurred during request
@@ -32,7 +33,20 @@ const DEFAULT_TIMEOUT = 5000;
 function analyzeResponse(response) {
   const { status, headers } = response;
 
-  if (status === 403 && headers.get('cf-ray')) {
+  // Check for CDN/blocker infrastructure presence (lazy evaluation for performance)
+  const hasCloudflare = () => headers.get('cf-ray') || headers.get('server') === 'cloudflare';
+  const hasImperva = () => headers.get('x-iinfo') || headers.get('x-cdn') === 'Incapsula';
+  const hasAkamai = () => headers.get('x-akamai-request-id')
+    || headers.get('x-akamai-session-id')
+    || headers.get('server')?.includes('AkamaiGHost');
+  const hasFastly = () => headers.get('x-served-by')?.startsWith('cache-')
+    || headers.get('fastly-io-info');
+  const hasCloudFront = () => headers.get('x-amz-cf-id')
+    || headers.get('x-amz-cf-pop')
+    || headers.get('via')?.includes('CloudFront');
+
+  // Active blocking (403 status with known blocker)
+  if (status === 403 && hasCloudflare()) {
     return {
       crawlable: false,
       type: 'cloudflare',
@@ -40,7 +54,7 @@ function analyzeResponse(response) {
     };
   }
 
-  if (status === 403 && (headers.get('x-iinfo') || headers.get('x-cdn') === 'Incapsula')) {
+  if (status === 403 && hasImperva()) {
     return {
       crawlable: false,
       type: 'imperva',
@@ -48,6 +62,72 @@ function analyzeResponse(response) {
     };
   }
 
+  if (status === 403 && hasAkamai()) {
+    return {
+      crawlable: false,
+      type: 'akamai',
+      confidence: CONFIDENCE_HIGH,
+    };
+  }
+
+  if (status === 403 && hasFastly()) {
+    return {
+      crawlable: false,
+      type: 'fastly',
+      confidence: CONFIDENCE_HIGH,
+    };
+  }
+
+  if (status === 403 && hasCloudFront()) {
+    return {
+      crawlable: false,
+      type: 'cloudfront',
+      confidence: CONFIDENCE_HIGH,
+    };
+  }
+
+  // Success with known infrastructure present (infrastructure detected but allowing requests)
+  if (status === 200 && hasCloudflare()) {
+    return {
+      crawlable: true,
+      type: 'cloudflare-allowed',
+      confidence: CONFIDENCE_ABSOLUTE,
+    };
+  }
+
+  if (status === 200 && hasImperva()) {
+    return {
+      crawlable: true,
+      type: 'imperva-allowed',
+      confidence: CONFIDENCE_ABSOLUTE,
+    };
+  }
+
+  if (status === 200 && hasAkamai()) {
+    return {
+      crawlable: true,
+      type: 'akamai-allowed',
+      confidence: CONFIDENCE_ABSOLUTE,
+    };
+  }
+
+  if (status === 200 && hasFastly()) {
+    return {
+      crawlable: true,
+      type: 'fastly-allowed',
+      confidence: CONFIDENCE_ABSOLUTE,
+    };
+  }
+
+  if (status === 200 && hasCloudFront()) {
+    return {
+      crawlable: true,
+      type: 'cloudfront-allowed',
+      confidence: CONFIDENCE_ABSOLUTE,
+    };
+  }
+
+  // Success with no known infrastructure
   if (status === 200) {
     return {
       crawlable: true,
@@ -56,6 +136,7 @@ function analyzeResponse(response) {
     };
   }
 
+  // Unknown status without known blocker signature
   return {
     crawlable: true,
     type: 'unknown',
@@ -86,14 +167,24 @@ function analyzeError(error) {
  * Currently detects:
  * - Cloudflare bot blocking (403 + cf-ray header)
  * - Imperva/Incapsula (403 + x-iinfo or x-cdn: Incapsula header)
+ * - Akamai (403 + x-akamai-request-id or related headers)
+ * - Fastly (403 + x-served-by or fastly-io-info headers)
+ * - AWS CloudFront (403 + x-amz-cf-id or via: CloudFront header)
  * - HTTP/2 stream errors (NGHTTP2_INTERNAL_ERROR, ERR_HTTP2_STREAM_ERROR)
  *
+ * Also detects infrastructure presence on successful requests (200 OK):
+ * - Returns 'cloudflare-allowed', 'imperva-allowed', 'akamai-allowed',
+ *   'fastly-allowed', or 'cloudfront-allowed' when infrastructure is present
+ *   but allowing the request through
+ *
  * @param {Object} config - Configuration object
  * @param {string} config.baseUrl - The base URL to check
  * @param {number} [config.timeout=5000] - Request timeout in milliseconds
  * @returns {Promise<Object>} Detection result with:
  *   - crawlable {boolean}: Whether the site can be crawled by bots
- *   - type {string}: Blocker type ('cloudflare', 'imperva', 'http2-block', 'none', 'unknown')
+ *   - type {string}: Blocker type ('cloudflare', 'imperva', 'akamai', 'fastly',
+ *     'cloudfront', 'http2-block', 'cloudflare-allowed', 'imperva-allowed',
+ *     'akamai-allowed', 'fastly-allowed', 'cloudfront-allowed', 'none', 'unknown')
  *   - confidence {number}: Confidence level (0.0-1.0, see confidence level constants)
  * @throws {Error} If baseUrl is invalid
  */

package/src/bot-blocker-detect/index.d.ts

@@ -17,7 +17,10 @@ export interface BotBlockerConfig {
 
 export interface BotBlockerResult {
   crawlable: boolean;
-  type: 'cloudflare' | 'imperva' | 'http2-block' | 'none' | 'unknown';
+  type: 'cloudflare' | 'imperva' | 'akamai' | 'fastly' | 'cloudfront'
+    | 'cloudflare-allowed' | 'imperva-allowed' | 'akamai-allowed'
+    | 'fastly-allowed' | 'cloudfront-allowed'
+    | 'http2-block' | 'none' | 'unknown';
   confidence: number;
 }
 

package/src/log-wrapper.js

@@ -13,92 +13,64 @@
 import { getTraceId } from './xray.js';
 
 /**
- * Check if a value is a plain object (not Array, not Error, not null, not other special objects)
- * @param {*} value - The value to check
- * @returns {boolean} - True if the value is a plain object
- */
-function isPlainObject(value) {
-  return typeof value === 'object'
-    && value !== null
-    && !Array.isArray(value)
-    && !(value instanceof Error)
-    && value.constructor === Object;
-}
-
-/**
- * A higher-order function that wraps a given function and enhances logging by converting
- * all logs to JSON format and appending `severity`, `jobId`and `traceId`
- * to log messages when available.
+ * A higher-order function that wraps a given function and enhances logging by appending
+ * a `jobId` and `traceId` to log messages when available. This improves traceability of logs
+ * associated with specific jobs or processes.
+ *
+ * The wrapper checks if a `log` object exists in the `context` and whether the `message`
+ * contains a `jobId`. It also extracts the AWS X-Ray trace ID if available. If found, log
+ * methods (e.g., `info`, `error`, etc.) will prepend the `jobId` and/or `traceId` to all log
+ * statements. All existing code using `context.log` will automatically include these markers.
  *
- * All log messages are automatically converted to structured JSON format:
- * - String messages become: { severity: "info", message: "...", jobId: "...", traceId: "..." }
- * - Object messages are merged with:
- * { severity: "info", ...yourObject, jobId: "...", traceId: "..." }
+ * @param {function} fn - The original function to be wrapped, called with the provided
+ * message and context after logging enhancement.
+ * @returns {function(object, object): Promise<Response>} - A wrapped function that enhances
+ * logging and returns the result of the original function.
  *
- * @param {function} fn - The original function to be wrapped
- * @returns {function(object, object): Promise<Response>} - A wrapped function with JSON logging
+ * `context.log` will be enhanced in place to include `jobId` and/or `traceId` prefixed to all
+ * log messages. No code changes needed - existing `context.log` calls work automatically.
  */
 export function logWrapper(fn) {
   return async (message, context) => {
     const { log } = context;
 
     if (log && !context.contextualLog) {
-      const markers = {};
+      const markers = [];
 
       // Extract jobId from message if available
       if (typeof message === 'object' && message !== null && 'jobId' in message) {
-        markers.jobId = message.jobId;
+        const { jobId } = message;
+        markers.push(`[jobId=${jobId}]`);
       }
 
       // Extract traceId from AWS X-Ray
       const traceId = getTraceId();
       if (traceId) {
-        markers.traceId = traceId;
+        markers.push(`[traceId=${traceId}]`);
       }
 
-      // Define log levels
-      const logLevels = ['info', 'error', 'debug', 'warn', 'trace', 'verbose', 'silly', 'fatal'];
+      // If we have markers, enhance the log object directly
+      if (markers.length > 0) {
+        const markerString = markers.join(' ');
 
-      // Wrap all log methods to output structured JSON
-      context.log = logLevels.reduce((accumulator, level) => {
-        if (typeof log[level] === 'function') {
-          accumulator[level] = (...args) => {
-            // If first argument is a plain object, merge with markers
-            if (args.length > 0 && isPlainObject(args[0])) {
-              return log[level](JSON.stringify({ severity: level, ...markers, ...args[0] }));
-            }
+        // Define log levels
+        const logLevels = ['info', 'error', 'debug', 'warn', 'trace', 'verbose', 'silly', 'fatal'];
 
-            // If first argument is a string, convert to structured format
-            if (args.length > 0 && typeof args[0] === 'string') {
-              const logObject = {
-                severity: level,
-                ...markers,
-                message: args[0],
-              };
-
-              // If second argument is a plain object, merge it into the log object
-              if (args.length > 1 && isPlainObject(args[1])) {
-                Object.assign(logObject, args[1]);
-
-                // If there are more arguments after the object, add them as 'data'
-                if (args.length > 2) {
-                  logObject.data = args.slice(2);
-                }
-              } else if (args.length > 1) {
-                // If there are additional arguments but second is not a plain object,
-                // add all additional args as 'data'
-                logObject.data = args.slice(1);
+        // Enhance context.log directly to include markers in all log statements
+        context.log = logLevels.reduce((accumulator, level) => {
+          if (typeof log[level] === 'function') {
+            accumulator[level] = (...args) => {
+              // If first argument is a string (format string), prepend the marker to it
+              if (args.length > 0 && typeof args[0] === 'string') {
+                const enhancedArgs = [`${markerString} ${args[0]}`, ...args.slice(1)];
+                return log[level](...enhancedArgs);
               }
-
-              return log[level](JSON.stringify(logObject));
-            }
-
-            // For other types (arrays, primitives, Error objects), wrap in object
-            return log[level](JSON.stringify({ severity: level, ...markers, data: args }));
-          };
-        }
-        return accumulator;
-      }, {});
+              return log[level](...args);
+            };
+          }
+          return accumulator;
+        }, {});
+      }
 
       // Mark that we've processed this context
       context.contextualLog = context.log;