@adobe/spacecat-shared-data-access 3.70.2 → 3.71.0

package/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [@adobe/spacecat-shared-data-access-v3.71.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v3.70.2...@adobe/spacecat-shared-data-access-v3.71.0) (2026-05-25)
+
+### Features
+
+* add scraperConfig to SiteConfig ([#1618](https://github.com/adobe/spacecat-shared/issues/1618)) ([8e3707b](https://github.com/adobe/spacecat-shared/commit/8e3707b505a2834c9351c50a965d838b3bad6123))
+
 ## [@adobe/spacecat-shared-data-access-v3.70.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v3.70.1...@adobe/spacecat-shared-data-access-v3.70.2) (2026-05-23)
 
 ### Bug Fixes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-data-access",
-  "version": "3.70.2",
+  "version": "3.71.0",
   "description": "Shared modules of the Spacecat Services - Data Access",
   "type": "module",
   "engines": {

package/src/models/audit/audit.model.js

@@ -173,14 +173,28 @@ class Audit extends BaseModel {
       getQueueUrl: (context) => context.env?.CONTENT_SCRAPER_QUEUE_URL,
       /**
        * Formats the payload for the content scraper queue.
+       *
+       * Per-site scraper headers (configured via the site's `scraperConfig.headers`)
+       * are auto-injected into the SQS payload's `customHeaders`, reusing the
+       * `site` already loaded by the audit framework on the context — no extra
+       * Site.findById is performed. Steps that need a different value can set
+       * `stepResult.customHeaders` explicitly to forward it verbatim instead.
+       *
        * @param {object} stepResult - The result of the audit step.
        * @param {object[]} stepResult.urls - The list of URLs to scrape.
        * @param {string} stepResult.urls[].url - The URL to scrape.
        * @param {string} stepResult.siteId - The site ID. Will be used as the job ID.
        * @param {string} stepResult.options - The options for the scraper.
        * @param {string} stepResult.processingType - The scraping processing type to trigger.
+       * @param {Object<string,string>} [stepResult.customHeaders] - Explicit
+       *   override for the HTTP headers forwarded to the scraper. When unset,
+       *   the dispatcher auto-loads from `context.site.getConfig().getScraperConfig()?.headers`.
        * @param {object} auditContext - The audit context.
-       * @param {object} context - The context object.
+       * @param {object} context - The context object. `context.site` is the
+       *   loaded site model when available; the auto-load uses optional
+       *   chaining so older code paths that have not yet attached a site to
+       *   context (or sites whose Config predates the scraperConfig getter)
+       *   degrade to "no headers" rather than throwing.
        * @param {object} auditContext.next - The next audit step to run.
        * @param {string} auditContext.auditId - The audit ID.
        * @param {string} auditContext.auditType - The audit type.
@@ -188,16 +202,29 @@ class Audit extends BaseModel {
        *
        * @returns {object} - The formatted payload.
        */
-      formatPayload: (stepResult, auditContext, context) => ({
-        urls: stepResult.urls,
-        jobId: stepResult.siteId,
-        processingType: stepResult.processingType || 'default',
-        skipMessage: false,
-        allowCache: isBoolean(stepResult.allowCache) ? stepResult.allowCache : true,
-        options: stepResult.options || {},
-        completionQueueUrl: stepResult.completionQueueUrl || context.env?.AUDIT_JOBS_QUEUE_URL,
-        auditContext,
-      }),
+      formatPayload: (stepResult, auditContext, context) => {
+        const payload = {
+          urls: stepResult.urls,
+          jobId: stepResult.siteId,
+          processingType: stepResult.processingType || 'default',
+          skipMessage: false,
+          allowCache: isBoolean(stepResult.allowCache) ? stepResult.allowCache : true,
+          options: stepResult.options || {},
+          completionQueueUrl: stepResult.completionQueueUrl || context.env?.AUDIT_JOBS_QUEUE_URL,
+          auditContext,
+        };
+
+        // Prefer step-supplied customHeaders; otherwise auto-load from site config.
+        const customHeaders = stepResult.customHeaders
+          ?? context?.site?.getConfig?.()?.getScraperConfig?.()?.headers;
+
+        // Reject empty object so the scraper does not receive `customHeaders: {}`.
+        if (customHeaders && Object.keys(customHeaders).length > 0) {
+          payload.customHeaders = customHeaders;
+        }
+
+        return payload;
+      },
     },
     [Audit.AUDIT_STEP_DESTINATIONS.SCRAPE_CLIENT]: {
       /**

package/src/models/site/config.js

@@ -46,6 +46,28 @@ export const IMPORT_SOURCES = {
   RUM: 'rum',
 };
 
+// HTTP header names that must not be set via `scraperConfig.headers`
+// (case-insensitive). These are owned by the scraper, its HTTP client, or its
+// auth flow; persisting them as per-site overrides would either be stripped at
+// scrape time or open a security surface outside the per-site config contract.
+const RESERVED_SCRAPER_HEADER_NAMES = new Set([
+  // Credential / authentication headers.
+  'authorization',
+  'cookie',
+  'proxy-authorization',
+  // Routing / fingerprint headers the scraper or its environment owns.
+  'host',
+  'user-agent',
+  // Hop-by-hop / connection-management headers (RFC 7230 section 6.1).
+  'content-length',
+  'transfer-encoding',
+  'connection',
+  'keep-alive',
+  'upgrade',
+  'te',
+  'trailer',
+]);
+
 const LLMO_TAG_PATTERN = /^(market|product|topic):\s?.+/;
 const AWS_REGION_PATTERN = /^[a-z]{2}(?:-[a-z]+)+-\d+$/i;
 const LLMO_TAG = Joi.alternatives()
@@ -379,6 +401,42 @@ export const configSchema = Joi.object({
     ).optional(),
     outputLocation: Joi.string().required(),
   }).optional(),
+  /**
+   * Per-site configuration intended to be forwarded by the content scraper as
+   * HTTP headers on outbound requests. Values are bounded and restricted to
+   * printable ASCII to avoid header-injection (CR/LF) and oversized-payload
+   * hazards. Header names are restricted to RFC 7230 token characters.
+   * Reserved names (see `RESERVED_SCRAPER_HEADER_NAMES`) are rejected.
+   *
+   * Enforcement lives at the schema layer (rather than at each consumer or API
+   * boundary) so any writer using `updateScraperConfig` -- API endpoint,
+   * internal tool, Slack command -- inherits the same checks.
+   */
+  scraperConfig: Joi.object({
+    headers: Joi.object()
+      .pattern(
+        // RFC 7230 token characters for header names, 64 chars max.
+        Joi.string().pattern(/^[A-Za-z0-9!#$%&'*+\-.^_`|~]+$/).max(64),
+        // Visible ASCII + tab, no CR/LF/NUL, 1 to 1024 chars (empty rejected).
+        Joi.string().pattern(/^[\t\x20-\x7E]+$/).max(1024),
+      )
+      .max(32)
+      // Case-insensitive denylist for credential / routing / hop-by-hop names.
+      // Done as an object-level custom validator (rather than on the key
+      // pattern) so the error message names the offending header instead of
+      // surfacing as Joi's generic "is not allowed" pattern-object error.
+      .custom((value, helpers) => {
+        for (const key of Object.keys(value)) {
+          if (RESERVED_SCRAPER_HEADER_NAMES.has(key.toLowerCase())) {
+            return helpers.message(
+              `"${key}" is a reserved scraper header name and cannot be set via scraperConfig.headers`,
+            );
+          }
+        }
+        return value;
+      })
+      .optional(),
+  }).optional(),
   tokowakaConfig: Joi.object({
     apiKey: Joi.string().optional(),
     forwardedHost: Joi.string().optional(),
@@ -529,6 +587,7 @@ export const Config = (data = {}) => {
   self.getBrandConfig = () => state?.brandConfig;
   self.getBrandProfile = () => state?.brandProfile;
   self.getCdnLogsConfig = () => state?.cdnLogsConfig;
+  self.getScraperConfig = () => state?.scraperConfig;
   self.getLlmoConfig = () => state?.llmo;
   self.getLlmoDataFolder = () => state?.llmo?.dataFolder;
   self.getLlmoBrand = () => state?.llmo?.brand;
@@ -949,6 +1008,21 @@ export const Config = (data = {}) => {
     state.cdnLogsConfig = cdnLogsConfig;
   };
 
+  self.updateScraperConfig = (scraperConfig) => {
+    // Validate eagerly (unlike neighboring setters) so bad input is rejected
+    // at the setter rather than silently round-tripping through Dynamo. This
+    // eagerness is the contract every writer relies on for scraperConfig;
+    // do not "harmonize" with cdnLogs/tokowaka setters.
+    //
+    // Use the sanitized value Joi returns so any future `default()` or
+    // `lowercase()` on the schema applies here too — assigning raw input
+    // would silently drift from what validation guarantees.
+    const { scraperConfig: validated } = validateConfiguration(
+      { ...state, scraperConfig },
+    );
+    state.scraperConfig = validated;
+  };
+
   self.updateTokowakaConfig = (tokowakaConfig) => {
     state.tokowakaConfig = tokowakaConfig;
   };
@@ -1003,6 +1077,7 @@ Config.toDynamoItem = (config) => ({
   brandConfig: config.getBrandConfig(),
   brandProfile: config.getBrandProfile(),
   cdnLogsConfig: config.getCdnLogsConfig(),
+  scraperConfig: config.getScraperConfig(),
   llmo: config.getLlmoConfig(),
   tokowakaConfig: config.getTokowakaConfig(),
   edgeOptimizeConfig: config.getEdgeOptimizeConfig(),