@adobe/spacecat-shared-utils 1.82.3 → 1.84.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-utils-v1.84.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.83.0...@adobe/spacecat-shared-utils-v1.84.0) (2025-12-10)
+
+
+### Features
+
+* add bot-blocker-detect ([#1233](https://github.com/adobe/spacecat-shared/issues/1233)) ([5d73f1b](https://github.com/adobe/spacecat-shared/commit/5d73f1b07ba5ea9735577b0bb0519d9d1cfd278c))
+
+# [@adobe/spacecat-shared-utils-v1.83.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.82.3...@adobe/spacecat-shared-utils-v1.83.0) (2025-12-10)
+
+
+### Features
+
+* Implement Structured (JSON) Logging for Spacecat Audits ([#1232](https://github.com/adobe/spacecat-shared/issues/1232)) ([7eae4d6](https://github.com/adobe/spacecat-shared/commit/7eae4d62fe9f0592f8124082fc66e9754803dd2b))
+
 # [@adobe/spacecat-shared-utils-v1.82.3](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.82.2...@adobe/spacecat-shared-utils-v1.82.3) (2025-12-10)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-utils",
-  "version": "1.82.3",
+  "version": "1.84.0",
   "description": "Shared modules of the Spacecat Services - utils",
   "type": "module",
   "exports": {

package/src/bot-blocker-detect/bot-blocker-detect.js

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { tracingFetch, SPACECAT_USER_AGENT } from '../tracing-fetch.js';
+import { isValidUrl } from '../functions.js';
+
+/**
+ * Confidence levels used in bot blocker detection:
+ * - 1.0 (ABSOLUTE): Site responds successfully with 200 OK - definitively crawlable
+ * - 0.99 (HIGH): Known bot blocker signature detected (Cloudflare cf-ray, Imperva x-iinfo/x-cdn)
+ * - 0.95 (MEDIUM): HTTP/2 protocol errors indicating potential blocking
+ * - 0.5: Unknown status code without known blocker signature (e.g., 403 without headers)
+ * - 0.3: Unknown error occurred during request
+ *
+ * Only detections with confidence >= 0.95 should be considered reliable indicators of bot blocking.
+ * Lower confidence values indicate uncertain situations that may require manual investigation.
+ */
+const CONFIDENCE_HIGH = 0.99;
+const CONFIDENCE_MEDIUM = 0.95;
+const CONFIDENCE_ABSOLUTE = 1.0;
+const DEFAULT_TIMEOUT = 5000;
+
+function analyzeResponse(response) {
+  const { status, headers } = response;
+
+  if (status === 403 && headers.get('cf-ray')) {
+    return {
+      crawlable: false,
+      type: 'cloudflare',
+      confidence: CONFIDENCE_HIGH,
+    };
+  }
+
+  if (status === 403 && (headers.get('x-iinfo') || headers.get('x-cdn') === 'Incapsula')) {
+    return {
+      crawlable: false,
+      type: 'imperva',
+      confidence: CONFIDENCE_HIGH,
+    };
+  }
+
+  if (status === 200) {
+    return {
+      crawlable: true,
+      type: 'none',
+      confidence: CONFIDENCE_ABSOLUTE,
+    };
+  }
+
+  return {
+    crawlable: true,
+    type: 'unknown',
+    confidence: 0.5,
+  };
+}
+
+function analyzeError(error) {
+  if (error.code === 'NGHTTP2_INTERNAL_ERROR' || error.code === 'ERR_HTTP2_STREAM_ERROR') {
+    return {
+      crawlable: false,
+      type: 'http2-block',
+      confidence: CONFIDENCE_MEDIUM,
+    };
+  }
+
+  return {
+    crawlable: true,
+    type: 'unknown',
+    confidence: 0.3,
+  };
+}
+
+/**
+ * Detects bot blocker technology on a website.
+ * Makes a single HEAD request and analyzes the response for blocking patterns.
+ *
+ * Currently detects:
+ * - Cloudflare bot blocking (403 + cf-ray header)
+ * - Imperva/Incapsula (403 + x-iinfo or x-cdn: Incapsula header)
+ * - HTTP/2 stream errors (NGHTTP2_INTERNAL_ERROR, ERR_HTTP2_STREAM_ERROR)
+ *
+ * @param {Object} config - Configuration object
+ * @param {string} config.baseUrl - The base URL to check
+ * @param {number} [config.timeout=5000] - Request timeout in milliseconds
+ * @returns {Promise<Object>} Detection result with:
+ *   - crawlable {boolean}: Whether the site can be crawled by bots
+ *   - type {string}: Blocker type ('cloudflare', 'imperva', 'http2-block', 'none', 'unknown')
+ *   - confidence {number}: Confidence level (0.0-1.0, see confidence level constants)
+ * @throws {Error} If baseUrl is invalid
+ */
+export async function detectBotBlocker({ baseUrl, timeout = DEFAULT_TIMEOUT }) {
+  if (!baseUrl || !isValidUrl(baseUrl)) {
+    throw new Error('Invalid baseUrl');
+  }
+
+  try {
+    const response = await tracingFetch(baseUrl, {
+      method: 'HEAD',
+      headers: {
+        'User-Agent': SPACECAT_USER_AGENT,
+      },
+      signal: AbortSignal.timeout(timeout),
+    });
+
+    return analyzeResponse(response);
+  } catch (error) {
+    return analyzeError(error);
+  }
+}

package/src/bot-blocker-detect/index.d.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+export interface BotBlockerConfig {
+  baseUrl: string;
+  timeout?: number;
+}
+
+export interface BotBlockerResult {
+  crawlable: boolean;
+  type: 'cloudflare' | 'imperva' | 'http2-block' | 'none' | 'unknown';
+  confidence: number;
+}
+
+export function detectBotBlocker(config: BotBlockerConfig): Promise<BotBlockerResult>;

package/src/index.d.ts

@@ -330,3 +330,4 @@ export * as llmoConfig from './llmo-config.js';
 export * as schemas from './schemas.js';
 
 export { type detectLocale } from './locale-detect/index.js';
+export { type detectBotBlocker } from './bot-blocker-detect/index.js';

package/src/index.js

@@ -110,6 +110,7 @@ export * as llmoConfig from './llmo-config.js';
 export * as schemas from './schemas.js';
 
 export { detectLocale } from './locale-detect/locale-detect.js';
+export { detectBotBlocker } from './bot-blocker-detect/bot-blocker-detect.js';
 export { prettifyLogForwardingConfig } from './cdn-helpers.js';
 
 export {

package/src/log-wrapper.js

@@ -13,64 +13,89 @@
 import { getTraceId } from './xray.js';
 
 /**
- * A higher-order function that wraps a given function and enhances logging by appending
- * a `jobId` and `traceId` to log messages when available. This improves traceability of logs
- * associated with specific jobs or processes.
- *
- * The wrapper checks if a `log` object exists in the `context` and whether the `message`
- * contains a `jobId`. It also extracts the AWS X-Ray trace ID if available. If found, log
- * methods (e.g., `info`, `error`, etc.) will prepend the `jobId` and/or `traceId` to all log
- * statements. All existing code using `context.log` will automatically include these markers.
+ * Check if a value is a plain object (not Array, not Error, not null, not other special objects)
+ * @param {*} value - The value to check
+ * @returns {boolean} - True if the value is a plain object
+ */
+function isPlainObject(value) {
+  return typeof value === 'object'
+    && value !== null
+    && !Array.isArray(value)
+    && !(value instanceof Error)
+    && value.constructor === Object;
+}
+
+/**
+ * A higher-order function that wraps a given function and enhances logging by converting
+ * all logs to JSON format and appending `jobId` and `traceId` to log messages when available.
  *
- * @param {function} fn - The original function to be wrapped, called with the provided
- * message and context after logging enhancement.
- * @returns {function(object, object): Promise<Response>} - A wrapped function that enhances
- * logging and returns the result of the original function.
+ * All log messages are automatically converted to structured JSON format:
+ * - String messages become: { message: "...", jobId: "...", traceId: "..." }
+ * - Object messages are merged with: { ...yourObject, jobId: "...", traceId: "..." }
  *
- * `context.log` will be enhanced in place to include `jobId` and/or `traceId` prefixed to all
- * log messages. No code changes needed - existing `context.log` calls work automatically.
+ * @param {function} fn - The original function to be wrapped
+ * @returns {function(object, object): Promise<Response>} - A wrapped function with JSON logging
  */
 export function logWrapper(fn) {
   return async (message, context) => {
     const { log } = context;
 
     if (log && !context.contextualLog) {
-      const markers = [];
+      const markers = {};
 
       // Extract jobId from message if available
       if (typeof message === 'object' && message !== null && 'jobId' in message) {
-        const { jobId } = message;
-        markers.push(`[jobId=${jobId}]`);
+        markers.jobId = message.jobId;
       }
 
       // Extract traceId from AWS X-Ray
       const traceId = getTraceId();
       if (traceId) {
-        markers.push(`[traceId=${traceId}]`);
+        markers.traceId = traceId;
       }
 
-      // If we have markers, enhance the log object directly
-      if (markers.length > 0) {
-        const markerString = markers.join(' ');
+      // Define log levels
+      const logLevels = ['info', 'error', 'debug', 'warn', 'trace', 'verbose', 'silly', 'fatal'];
 
-        // Define log levels
-        const logLevels = ['info', 'error', 'debug', 'warn', 'trace', 'verbose', 'silly', 'fatal'];
+      // Wrap all log methods to output structured JSON
+      context.log = logLevels.reduce((accumulator, level) => {
+        if (typeof log[level] === 'function') {
+          accumulator[level] = (...args) => {
+            // If first argument is a plain object, merge with markers
+            if (args.length > 0 && isPlainObject(args[0])) {
+              return log[level]({ ...markers, ...args[0] });
+            }
 
-        // Enhance context.log directly to include markers in all log statements
-        context.log = logLevels.reduce((accumulator, level) => {
-          if (typeof log[level] === 'function') {
-            accumulator[level] = (...args) => {
-              // If first argument is a string (format string), prepend the marker to it
-              if (args.length > 0 && typeof args[0] === 'string') {
-                const enhancedArgs = [`${markerString} ${args[0]}`, ...args.slice(1)];
-                return log[level](...enhancedArgs);
+            // If first argument is a string, convert to structured format
+            if (args.length > 0 && typeof args[0] === 'string') {
+              const logObject = {
+                ...markers,
+                message: args[0],
+              };
+
+              // If second argument is a plain object, merge it into the log object
+              if (args.length > 1 && isPlainObject(args[1])) {
+                Object.assign(logObject, args[1]);
+
+                // If there are more arguments after the object, add them as 'data'
+                if (args.length > 2) {
+                  logObject.data = args.slice(2);
+                }
+              } else if (args.length > 1) {
+                // If there are additional arguments but second is not a plain object,
+                // add all additional args as 'data'
+                logObject.data = args.slice(1);
               }
-              return log[level](...args);
-            };
-          }
-          return accumulator;
-        }, {});
-      }
+
+              return log[level](logObject);
+            }
+
+            // For other types (arrays, primitives, Error objects), wrap in object
+            return log[level]({ ...markers, data: args });
+          };
+        }
+        return accumulator;
+      }, {});
 
       // Mark that we've processed this context
       context.contextualLog = context.log;