@adobe/spacecat-shared-tokowaka-client 1.18.1 → 1.18.2

package/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [@adobe/spacecat-shared-tokowaka-client-v1.18.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.18.1...@adobe/spacecat-shared-tokowaka-client-v1.18.2) (2026-06-02)
+
+### Bug Fixes
+
+* **tokowaka-client:** use scraperConfig headers in edge optimize probe (LLMO-5280) ([#1647](https://github.com/adobe/spacecat-shared/issues/1647)) ([6be14a1](https://github.com/adobe/spacecat-shared/commit/6be14a153ea8d9de010880afaa1c3c9290a32f13))
+
 ## [@adobe/spacecat-shared-tokowaka-client-v1.18.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.18.0...@adobe/spacecat-shared-tokowaka-client-v1.18.1) (2026-06-01)
 
 ### Bug Fixes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-tokowaka-client",
-  "version": "1.18.1",
+  "version": "1.18.2",
   "description": "Tokowaka Client for SpaceCat - Edge optimization config management",
   "type": "module",
   "engines": {

package/src/index.js

@@ -1287,10 +1287,18 @@ class TokowakaClient {
       return { edgeOptimizeEnabled: true };
     }
 
+    const defaultProbeHeaders = {
+      // eslint-disable-next-line max-len
+      'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Tokowaka-AI Tokowaka/1.0 AdobeEdgeOptimize-AI AdobeEdgeOptimize/1.0',
+      'fastly-debug': '1',
+    };
+    const customHeaders = currentConfig?.getScraperConfig?.()?.headers ?? {};
+    const probeHeaders = { ...customHeaders, ...defaultProbeHeaders };
+
     const baseURL = getEffectiveBaseURL(site);
     const targetUrl = new URL(path, baseURL).toString();
 
-    this.log.info(`Checking edge optimize status for ${targetUrl}`);
+    this.log.info(`[edge-optimize-status] Checking edge optimize status for ${targetUrl}`);
 
     const maxRetries = 3;
     let attempt = 0;
@@ -1300,25 +1308,21 @@ class TokowakaClient {
     while (attempt <= maxRetries) {
       try {
         // eslint-disable-next-line max-len
-        this.log.debug(`Attempt ${attempt + 1}/${maxRetries + 1}: Checking edge optimize status for ${targetUrl}`);
+        this.log.debug(`[edge-optimize-status] Attempt ${attempt + 1}/${maxRetries + 1}: Checking edge optimize status for ${targetUrl}`);
 
         // eslint-disable-next-line no-await-in-loop
         const response = await tracingFetch(targetUrl, {
           method: 'GET',
-          headers: {
-            // eslint-disable-next-line max-len
-            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Tokowaka-AI Tokowaka/1.0 AdobeEdgeOptimize-AI AdobeEdgeOptimize/1.0',
-            'fastly-debug': '1',
-          },
+          headers: probeHeaders,
           timeout: REQUEST_TIMEOUT_MS,
         });
 
-        this.log.debug(`Response status: ${response.status}`);
+        this.log.info(`[edge-optimize-status] Response status: ${response.status} for ${targetUrl}`);
 
         const edgeOptimizeEnabled = response.headers.get('x-tokowaka-request-id') !== null
           || response.headers.get('x-edgeoptimize-request-id') !== null;
 
-        this.log.debug(`Edge optimize headers found: ${edgeOptimizeEnabled}`);
+        this.log.info(`[edge-optimize-status] Edge optimize headers found: ${edgeOptimizeEnabled} for ${targetUrl}`);
 
         return {
           edgeOptimizeEnabled,
@@ -1328,7 +1332,7 @@ class TokowakaClient {
 
         if (isTimeout) {
           // eslint-disable-next-line max-len
-          this.log.warn(`Request timed out after ${REQUEST_TIMEOUT_MS}ms for ${targetUrl}, returning edgeOptimizeEnabled: false`);
+          this.log.warn(`[edge-optimize-status] Request timed out after ${REQUEST_TIMEOUT_MS}ms for ${targetUrl}, returning edgeOptimizeEnabled: false`);
           return { edgeOptimizeEnabled: false };
         }
 
@@ -1336,7 +1340,7 @@ class TokowakaClient {
 
         if (attempt > maxRetries) {
           // All retries exhausted
-          this.log.error(`Failed after ${maxRetries + 1} attempts: ${error.message}`);
+          this.log.error(`[edge-optimize-status] Failed after ${maxRetries + 1} attempts: ${error.message} for ${targetUrl}`);
           throw this.#createError(
             `Failed to check edge optimize status: ${error.message}`,
             HTTP_INTERNAL_SERVER_ERROR,
@@ -1346,7 +1350,7 @@ class TokowakaClient {
         // Exponential backoff: 200ms, 400ms, 800ms
         const delay = 100 * (2 ** attempt);
         this.log.warn(
-          `Attempt ${attempt} to fetch failed: ${error.message}. Retrying in ${delay}ms...`,
+          `[edge-optimize-status] Attempt ${attempt} to fetch failed: ${error.message}. Retrying in ${delay}ms... for ${targetUrl}`,
         );
         // eslint-disable-next-line no-await-in-loop
         await new Promise((res) => {

package/test/index.test.js

@@ -5035,6 +5035,165 @@ describe('TokowakaClient', () => {
         expect(userAgent).to.include('Mozilla/5.0');
       });
 
+      it('should ignore custom User-Agent from scraperConfig — default takes priority', async () => {
+        const customSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({
+            getEdgeOptimizeConfig: () => undefined,
+            getScraperConfig: () => ({ headers: { 'User-Agent': 'AllowListedBot/1.0' } }),
+          }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = {
+          status: 200,
+          headers: new Map(),
+        };
+        mockResponse.headers.get = () => null;
+
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(customSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['User-Agent']).to.include('Tokowaka-AI');
+        expect(fetchOptions.headers['User-Agent']).to.not.include('AllowListedBot/1.0');
+      });
+
+      it('should add extra custom headers while defaults always win on collision', async () => {
+        const customSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({
+            getEdgeOptimizeConfig: () => undefined,
+            getScraperConfig: () => ({ headers: { 'x-allowlist-token': 'secret123', 'fastly-debug': 'custom' } }),
+          }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = {
+          status: 200,
+          headers: new Map(),
+        };
+        mockResponse.headers.get = () => null;
+
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(customSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['x-allowlist-token']).to.equal('secret123');
+        expect(fetchOptions.headers['fastly-debug']).to.equal('1');
+        expect(fetchOptions.headers['User-Agent']).to.include('Mozilla/5.0');
+      });
+
+      it('should use default probe headers when scraperConfig has no headers', async () => {
+        const noHeadersSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({
+            getEdgeOptimizeConfig: () => undefined,
+            getScraperConfig: () => ({}),
+          }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = {
+          status: 200,
+          headers: new Map(),
+        };
+        mockResponse.headers.get = () => null;
+
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(noHeadersSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['User-Agent']).to.include('Tokowaka-AI');
+        expect(fetchOptions.headers['fastly-debug']).to.equal('1');
+      });
+
+      it('should use default probe headers when getScraperConfig is not present on config', async () => {
+        const noScraperSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({ getEdgeOptimizeConfig: () => undefined }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = {
+          status: 200,
+          headers: new Map(),
+        };
+        mockResponse.headers.get = () => null;
+
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(noScraperSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['User-Agent']).to.include('AdobeEdgeOptimize-AI');
+        expect(fetchOptions.headers['fastly-debug']).to.equal('1');
+      });
+
+      it('should use default probe headers when getScraperConfig returns null', async () => {
+        const nullScraperSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({
+            getEdgeOptimizeConfig: () => undefined,
+            getScraperConfig: () => null,
+          }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = { status: 200, headers: new Map() };
+        mockResponse.headers.get = () => null;
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(nullScraperSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['User-Agent']).to.include('Tokowaka-AI');
+        expect(fetchOptions.headers['fastly-debug']).to.equal('1');
+      });
+
+      it('should use default probe headers when getConfig returns null', async () => {
+        const nullConfigSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => null,
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = { status: 200, headers: new Map() };
+        mockResponse.headers.get = () => null;
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(nullConfigSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['User-Agent']).to.include('Tokowaka-AI');
+        expect(fetchOptions.headers['fastly-debug']).to.equal('1');
+      });
+
+      it('should use default probe headers when scraperConfig has an empty headers object', async () => {
+        const emptyHeadersSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({
+            getEdgeOptimizeConfig: () => undefined,
+            getScraperConfig: () => ({ headers: {} }),
+          }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = { status: 200, headers: new Map() };
+        mockResponse.headers.get = () => null;
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(emptyHeadersSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['User-Agent']).to.include('Tokowaka-AI');
+        expect(fetchOptions.headers['fastly-debug']).to.equal('1');
+        expect(Object.keys(fetchOptions.headers)).to.have.lengthOf(2);
+      });
+
       it('should pass timeout option to tracingFetch', async () => {
         const mockResponse = {
           status: 200,
@@ -5168,7 +5327,7 @@ describe('TokowakaClient', () => {
         await promise;
 
         expect(log.warn).to.have.been.calledWith(
-          sinon.match(/Attempt 1 to fetch failed.*Connection refused.*Retrying in 200ms/),
+          sinon.match(/\[edge-optimize-status\] Attempt 1 to fetch failed.*Connection refused.*Retrying in 200ms/),
         );
       });
 
@@ -5182,9 +5341,38 @@ describe('TokowakaClient', () => {
         expect(result).to.deep.equal({ edgeOptimizeEnabled: false });
         expect(tracingFetchStub).to.have.been.calledOnce;
         expect(log.warn).to.have.been.calledWith(
-          sinon.match(/Request timed out after 5000ms for https:\/\/example.com\/, returning edgeOptimizeEnabled: false/),
+          sinon.match(/\[edge-optimize-status\] Request timed out after 5000ms for https:\/\/example.com\/, returning edgeOptimizeEnabled: false/),
         );
       });
+
+      it('should send the same custom headers on every retry attempt', async () => {
+        const customSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({
+            getEdgeOptimizeConfig: () => undefined,
+            getScraperConfig: () => ({ headers: { 'x-allowlist-token': 'secret123' } }),
+          }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = { status: 200, headers: new Map() };
+        mockResponse.headers.get = () => null;
+
+        tracingFetchStub.onFirstCall().rejects(new Error('Temporary failure'));
+        tracingFetchStub.onSecondCall().resolves(mockResponse);
+
+        const promise = esmockClient.checkEdgeOptimizeStatus(customSite, '/');
+        await clock.tickAsync(200);
+        await promise;
+
+        expect(tracingFetchStub).to.have.been.calledTwice;
+        const firstCallHeaders = tracingFetchStub.firstCall.args[1].headers;
+        const secondCallHeaders = tracingFetchStub.secondCall.args[1].headers;
+        expect(firstCallHeaders['x-allowlist-token']).to.equal('secret123');
+        expect(secondCallHeaders['x-allowlist-token']).to.equal('secret123');
+        expect(firstCallHeaders['User-Agent']).to.include('Tokowaka-AI');
+        expect(secondCallHeaders['User-Agent']).to.include('Tokowaka-AI');
+      });
     });
 
     describe('URL Construction', () => {