@adobe/spacecat-shared-utils 1.88.0 → 1.89.1

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-utils-v1.89.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.89.0...@adobe/spacecat-shared-utils-v1.89.1) (2026-01-26)
+
+
+### Bug Fixes
+
+* Additional checks and methods on bot protection ([#1250](https://github.com/adobe/spacecat-shared/issues/1250)) ([0c34a8d](https://github.com/adobe/spacecat-shared/commit/0c34a8d850abef6e2a024132bc1c61d10865c1a0))
+
+# [@adobe/spacecat-shared-utils-v1.89.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.88.0...@adobe/spacecat-shared-utils-v1.89.0) (2026-01-22)
+
+
+### Features
+
+* moved `calculateCpcValue` function to spacecat-shared-utils ([#1048](https://github.com/adobe/spacecat-shared/issues/1048)) ([5006e5b](https://github.com/adobe/spacecat-shared/commit/5006e5be51c8959f2e6d72664251e1e0264d44a1))
+
 # [@adobe/spacecat-shared-utils-v1.88.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-utils-v1.87.1...@adobe/spacecat-shared-utils-v1.88.0) (2026-01-22)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-utils",
-  "version": "1.88.0",
+  "version": "1.89.1",
   "description": "Shared modules of the Spacecat Services - utils",
   "type": "module",
   "exports": {

package/src/bot-blocker-detect/bot-blocker-detect.js

@@ -30,7 +30,85 @@ const CONFIDENCE_MEDIUM = 0.95;
 const CONFIDENCE_ABSOLUTE = 1.0;
 const DEFAULT_TIMEOUT = 5000;
 
-function analyzeResponse(response) {
+/**
+ * SpaceCat bot identification constants
+ */
+export const SPACECAT_BOT_USER_AGENT = 'Spacecat/1.0';
+
+/**
+ * Gets SpaceCat bot IPs from environment variable
+ * @param {string} ipsString - Comma-separated IPs (from env/secrets) - REQUIRED
+ * @returns {Array<string>} Array of IP addresses
+ * @throws {Error} If ipsString is not provided
+ */
+export function getSpacecatBotIps(ipsString) {
+  if (!ipsString) {
+    throw new Error('SPACECAT_BOT_IPS environment variable is required but not set');
+  }
+
+  return ipsString.split(',').map((ip) => ip.trim()).filter((ip) => ip);
+}
+
+/**
+ * Formats allowlist message with current bot IPs
+ * @param {string} botIps - Comma-separated IPs from secrets - REQUIRED
+ * @returns {object} Formatted message with IPs and user-agent
+ * @throws {Error} If botIps is not provided
+ */
+export function formatAllowlistMessage(botIps) {
+  const ips = getSpacecatBotIps(botIps);
+
+  return {
+    title: 'To allowlist SpaceCat bot:',
+    ips,
+    userAgent: SPACECAT_BOT_USER_AGENT,
+  };
+}
+
+/**
+ * HTML patterns for detecting challenge pages
+ */
+const CHALLENGE_PATTERNS = {
+  cloudflare: [
+    /Checking your browser/i,
+    /Just a moment\.\.\./i,
+    /Verifying you are human/i,
+    /Please wait.*CloudFlare/i,
+    /cf-turnstile/i,
+    /challenge-platform/i,
+    /cf-chl-widget/i, // Cloudflare challenge widget
+    /ray\s*id.*cloudflare/i, // Cloudflare Ray ID in error pages
+    /__cf_chl_tk/i, // Cloudflare challenge token
+    /cloudflare.*security/i,
+    /attention required.*cloudflare/i,
+  ],
+  imperva: [
+    /_Incapsula_Resource/i,
+    /Incapsula incident ID/i,
+    /incap_ses/i, // Imperva session cookie
+    /visid_incap/i, // Imperva visitor ID
+  ],
+  akamai: [
+    /Access Denied.*Akamai/i,
+    /Reference.*Akamai/i,
+  ],
+  general: [
+    /captcha/i,
+    /human verification/i,
+    /recaptcha/i,
+    /hcaptcha/i,
+    /datadome/i,
+    /dd-request-id/i,
+  ],
+};
+
+/**
+ * Analyzes response for bot protection indicators
+ * @param {Object} response - Response object with status and headers
+ * @param {string} [html] - Optional HTML content for deeper analysis
+ * @returns {Object} Detection result
+ */
+function analyzeResponse(response, html = null) {
   const { status, headers } = response;
 
   // Check for CDN/blocker infrastructure presence (lazy evaluation for performance)
@@ -45,6 +123,12 @@ function analyzeResponse(response) {
     || headers.get('x-amz-cf-pop')
     || headers.get('via')?.includes('CloudFront');
 
+  // Check HTML content for challenge page patterns (if HTML provided)
+  const htmlHasChallenge = (patterns) => {
+    if (!html) return false;
+    return patterns.some((pattern) => pattern.test(html));
+  };
+
   // Active blocking (403 status with known blocker)
   if (status === 403 && hasCloudflare()) {
     return {
@@ -88,6 +172,16 @@ function analyzeResponse(response) {
 
   // Success with known infrastructure present (infrastructure detected but allowing requests)
   if (status === 200 && hasCloudflare()) {
+    // Check if HTML contains challenge page (even though status is 200)
+    if (htmlHasChallenge(CHALLENGE_PATTERNS.cloudflare)) {
+      return {
+        crawlable: false,
+        type: 'cloudflare',
+        confidence: CONFIDENCE_HIGH,
+        reason: 'Challenge page detected despite 200 status',
+      };
+    }
+
     return {
       crawlable: true,
       type: 'cloudflare-allowed',
@@ -96,6 +190,14 @@ function analyzeResponse(response) {
   }
 
   if (status === 200 && hasImperva()) {
+    if (htmlHasChallenge(CHALLENGE_PATTERNS.imperva)) {
+      return {
+        crawlable: false,
+        type: 'imperva',
+        confidence: CONFIDENCE_HIGH,
+        reason: 'Challenge page detected despite 200 status',
+      };
+    }
     return {
       crawlable: true,
       type: 'imperva-allowed',
@@ -104,6 +206,14 @@ function analyzeResponse(response) {
   }
 
   if (status === 200 && hasAkamai()) {
+    if (htmlHasChallenge(CHALLENGE_PATTERNS.akamai)) {
+      return {
+        crawlable: false,
+        type: 'akamai',
+        confidence: CONFIDENCE_HIGH,
+        reason: 'Challenge page detected despite 200 status',
+      };
+    }
     return {
       crawlable: true,
       type: 'akamai-allowed',
@@ -129,6 +239,15 @@ function analyzeResponse(response) {
 
   // Success with no known infrastructure
   if (status === 200) {
+    // Still check for generic challenge patterns
+    if (htmlHasChallenge(CHALLENGE_PATTERNS.general)) {
+      return {
+        crawlable: false,
+        type: 'unknown',
+        confidence: 0.7,
+        reason: 'Generic challenge patterns detected',
+      };
+    }
     return {
       crawlable: true,
       type: 'none',
@@ -136,7 +255,16 @@ function analyzeResponse(response) {
     };
   }
 
-  // Unknown status without known blocker signature
+  // Potential CDN/protection blocked the request
+  if (status === 403) {
+    return {
+      crawlable: false,
+      type: 'unknown',
+      confidence: 0.7,
+      reason: 'HTTP 403 Forbidden - access denied',
+    };
+  }
+
   return {
     crawlable: true,
     type: 'unknown',
@@ -207,3 +335,27 @@ export async function detectBotBlocker({ baseUrl, timeout = DEFAULT_TIMEOUT }) {
     return analyzeError(error);
   }
 }
+
+/**
+ * Analyzes already-fetched response data for bot protection.
+ * Used by content scraper to analyze Puppeteer results without making another request.
+ *
+ * @param {Object} data - Response data to analyze
+ * @param {number} data.status - HTTP status code
+ * @param {Object} data.headers - Response headers (plain object or Headers object)
+ * @param {string} [data.html] - Optional HTML content for challenge page detection
+ * @returns {Object} Detection result (same format as detectBotBlocker)
+ */
+export function analyzeBotProtection({ status, headers, html }) {
+  // Convert headers to Headers object if plain object
+  const headersObj = headers instanceof Headers
+    ? headers
+    : new Headers(Object.entries(headers || {}));
+
+  const response = {
+    status,
+    headers: headersObj,
+  };
+
+  return analyzeResponse(response, html);
+}

package/src/constants.js

@@ -68,3 +68,5 @@ export const OPPORTUNITY_TYPES = /** @type {const} */ ({
   // Wikipedia Analysis (LLMO)
   WIKIPEDIA_ANALYSIS: 'wikipedia-analysis',
 });
+
+export const DEFAULT_CPC_VALUE = 1.5;

package/src/index.d.ts

@@ -17,6 +17,8 @@ export { AUTHORING_TYPES, DELIVERY_TYPES } from './aem.js';
 
 export { OPPORTUNITY_TYPES } from './constants.js';
 
+export const DEFAULT_CPC_VALUE: number;
+
 /** UTILITY FUNCTIONS */
 export function arrayEquals<T>(a: T[], b: T[]): boolean;
 
@@ -287,6 +289,35 @@ export function getStoredMetrics(config: object, context: object):
  */
 export function storeMetrics(content: object, config: object, context: object): Promise<string>;
 
+/**
+ * Retrieves an object from S3 by its key and returns its JSON parsed content.
+ * If the object is not JSON, returns the raw body.
+ * If the object is not found, returns null.
+ * @param s3Client - The S3 client
+ * @param bucketName - The name of the S3 bucket
+ * @param key - The key of the S3 object
+ * @param log - A logger instance
+ * @returns The content of the S3 object or null if not found
+ */
+export function getObjectFromKey(
+  s3Client: any,
+  bucketName: string,
+  key: string,
+  log: any
+): Promise<any | null>;
+
+/**
+ * Fetches the organic traffic data for a site from S3 and calculates the CPC value
+ * @param context - Context object
+ * @param context.env - Environment variables
+ * @param context.env.S3_IMPORTER_BUCKET_NAME - S3 importer bucket name
+ * @param context.s3Client - S3 client
+ * @param context.log - Logger
+ * @param siteId - The site ID
+ * @returns CPC value in dollars
+ */
+export function calculateCPCValue(context: object, siteId: string): Promise<number>;
+
 export function s3Wrapper(fn: (request: object, context: object) => Promise<Response>):
   (request: object, context: object) => Promise<Response>;
 

package/src/index.js

@@ -76,11 +76,11 @@ export {
   extractUrlsFromSuggestion,
 } from './url-extractors.js';
 
-export { getStoredMetrics, storeMetrics } from './metrics-store.js';
+export { getStoredMetrics, storeMetrics, calculateCPCValue } from './metrics-store.js';
 
-export { s3Wrapper } from './s3.js';
+export { s3Wrapper, getObjectFromKey } from './s3.js';
 
-export { OPPORTUNITY_TYPES } from './constants.js';
+export { OPPORTUNITY_TYPES, DEFAULT_CPC_VALUE } from './constants.js';
 
 export { fetch } from './adobe-fetch.js';
 export { tracingFetch, SPACECAT_USER_AGENT } from './tracing-fetch.js';
@@ -111,7 +111,13 @@ export * as llmoStrategy from './llmo-strategy.js';
 export * as schemas from './schemas.js';
 
 export { detectLocale } from './locale-detect/locale-detect.js';
-export { detectBotBlocker } from './bot-blocker-detect/bot-blocker-detect.js';
+export {
+  detectBotBlocker,
+  analyzeBotProtection,
+  SPACECAT_BOT_USER_AGENT,
+  getSpacecatBotIps,
+  formatAllowlistMessage,
+} from './bot-blocker-detect/bot-blocker-detect.js';
 export { prettifyLogForwardingConfig } from './cdn-helpers.js';
 
 export {

package/src/metrics-store.js

@@ -10,6 +10,8 @@
  * governing permissions and limitations under the License.
  */
 import { GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3';
+import { getObjectFromKey } from './s3.js';
+import { DEFAULT_CPC_VALUE } from './constants.js';
 
 function createFilePath({ siteId, source, metric }) {
   if (!siteId) {
@@ -80,3 +82,61 @@ export async function storeMetrics(content, config, context) {
     throw new Error(`Failed to upload metrics to ${filePath}, error: ${e.message}`);
   }
 }
+
+/**
+ * Fetches the organic traffic data for a site from S3 and calculate the CPC value as per
+ * https://wiki.corp.adobe.com/pages/viewpage.action?spaceKey=AEMSites&title=Success+Studio+Projected+Business+Impact+Metrics#SuccessStudioProjectedBusinessImpactMetrics-IdentifyingCPCvalueforadomain
+ * @param context
+ * @param siteId
+ * @returns {object} Object containing either { success: true, value: number } on success
+ * or { success: false, reason: string, value: number } on failure
+ */
+export async function calculateCPCValue(context, siteId) {
+  if (!context?.env?.S3_IMPORTER_BUCKET_NAME) {
+    throw new Error('S3 importer bucket name is required');
+  }
+  if (!context.s3Client) {
+    throw new Error('S3 client is required');
+  }
+  if (!context.log) {
+    throw new Error('Logger is required');
+  }
+  if (!siteId) {
+    throw new Error('SiteId is required');
+  }
+  const { s3Client, log } = context;
+  const bucketName = context.env.S3_IMPORTER_BUCKET_NAME;
+  const key = `metrics/${siteId}/ahrefs/organic-traffic.json`;
+  try {
+    const organicTrafficData = await getObjectFromKey(s3Client, bucketName, key, log);
+    if (!Array.isArray(organicTrafficData) || organicTrafficData.length === 0) {
+      log.warn(`Organic traffic data not available for ${siteId}. Using Default CPC value.`);
+      return {
+        success: false,
+        reason: 'Organic traffic data not available',
+        value: DEFAULT_CPC_VALUE,
+      };
+    }
+    const lastTraffic = organicTrafficData.at(-1);
+    if (!lastTraffic.cost || !lastTraffic.value) {
+      log.warn(`Invalid organic traffic data present for ${siteId} - cost:${lastTraffic.cost} value:${lastTraffic.value}, Using Default CPC value.`);
+      return {
+        success: false,
+        reason: 'Invalid organic traffic data',
+        value: DEFAULT_CPC_VALUE,
+      };
+    }
+    // dividing by 100 for cents to dollar conversion
+    return {
+      success: true,
+      value: lastTraffic.cost / lastTraffic.value / 100,
+    };
+  } catch (err) {
+    log.error(`Error fetching organic traffic data for site ${siteId}. Using Default CPC value.`, err);
+    return {
+      success: false,
+      reason: 'Error fetching organic traffic data',
+      value: DEFAULT_CPC_VALUE,
+    };
+  }
+}

package/src/s3.js

@@ -10,9 +10,55 @@
  * governing permissions and limitations under the License.
  */
 
-import { S3Client } from '@aws-sdk/client-s3';
+import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
 import { instrumentAWSClient } from './xray.js';
 
+/**
+ * Retrieves an object from S3 by its key and returns its JSON parsed content.
+ * If the object is not JSON, returns the raw body.
+ * If the object is not found, returns null.
+ * @param {import('@aws-sdk/client-s3').S3Client} s3Client - an S3 client
+ * @param {string} bucketName - the name of the S3 bucket
+ * @param {string} key - the key of the S3 object
+ * @param {import('@azure/logger').Logger} log - a logger instance
+ * @returns {Promise<import('@aws-sdk/client-s3').GetObjectOutput['Body'] | null>}
+ * - the content of the S3 object
+ */
+export async function getObjectFromKey(s3Client, bucketName, key, log) {
+  if (!s3Client || !bucketName || !key) {
+    log.error(
+      'Invalid input parameters in getObjectFromKey: ensure s3Client, bucketName, and key are provided.',
+    );
+    return null;
+  }
+  const command = new GetObjectCommand({
+    Bucket: bucketName,
+    Key: key,
+  });
+  try {
+    const response = await s3Client.send(command);
+    const contentType = response.ContentType;
+    const body = await response.Body.transformToString();
+
+    if (contentType && contentType.includes('application/json')) {
+      try {
+        return JSON.parse(body);
+      } catch (parseError) {
+        log.error(`Unable to parse content for key ${key}`, parseError);
+        return null;
+      }
+    }
+    // Always return body for non-JSON content types
+    return body;
+  } catch (err) {
+    log.error(
+      `Error while fetching S3 object from bucket ${bucketName} using key ${key}`,
+      err,
+    );
+    return null;
+  }
+}
+
 /**
  * Adds an S3Client instance and bucket to the context.
  *