@adobe/spacecat-shared-tokowaka-client 1.18.1 → 1.19.0

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+## [@adobe/spacecat-shared-tokowaka-client-v1.19.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.18.2...@adobe/spacecat-shared-tokowaka-client-v1.19.0) (2026-06-03)
+
+### Features
+
+* **tokowaka-client:** mark path-level suggestions as coveredByDomainWide on domain-wide deploy ([#1648](https://github.com/adobe/spacecat-shared/issues/1648)) ([9cc5992](https://github.com/adobe/spacecat-shared/commit/9cc5992bb98acf9da48a432fedfbec694d30fe23))
+
+## [@adobe/spacecat-shared-tokowaka-client-v1.18.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.18.1...@adobe/spacecat-shared-tokowaka-client-v1.18.2) (2026-06-02)
+
+### Bug Fixes
+
+* **tokowaka-client:** use scraperConfig headers in edge optimize probe (LLMO-5280) ([#1647](https://github.com/adobe/spacecat-shared/issues/1647)) ([6be14a1](https://github.com/adobe/spacecat-shared/commit/6be14a153ea8d9de010880afaa1c3c9290a32f13))
+
 ## [@adobe/spacecat-shared-tokowaka-client-v1.18.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.18.0...@adobe/spacecat-shared-tokowaka-client-v1.18.1) (2026-06-01)
 
 ### Bug Fixes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-tokowaka-client",
-  "version": "1.18.1",
+  "version": "1.19.0",
   "description": "Tokowaka Client for SpaceCat - Edge optimization config management",
   "type": "module",
   "engines": {

package/src/index.js

@@ -1287,10 +1287,18 @@ class TokowakaClient {
       return { edgeOptimizeEnabled: true };
     }
 
+    const defaultProbeHeaders = {
+      // eslint-disable-next-line max-len
+      'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Tokowaka-AI Tokowaka/1.0 AdobeEdgeOptimize-AI AdobeEdgeOptimize/1.0',
+      'fastly-debug': '1',
+    };
+    const customHeaders = currentConfig?.getScraperConfig?.()?.headers ?? {};
+    const probeHeaders = { ...customHeaders, ...defaultProbeHeaders };
+
     const baseURL = getEffectiveBaseURL(site);
     const targetUrl = new URL(path, baseURL).toString();
 
-    this.log.info(`Checking edge optimize status for ${targetUrl}`);
+    this.log.info(`[edge-optimize-status] Checking edge optimize status for ${targetUrl}`);
 
     const maxRetries = 3;
     let attempt = 0;
@@ -1300,25 +1308,21 @@ class TokowakaClient {
     while (attempt <= maxRetries) {
       try {
         // eslint-disable-next-line max-len
-        this.log.debug(`Attempt ${attempt + 1}/${maxRetries + 1}: Checking edge optimize status for ${targetUrl}`);
+        this.log.debug(`[edge-optimize-status] Attempt ${attempt + 1}/${maxRetries + 1}: Checking edge optimize status for ${targetUrl}`);
 
         // eslint-disable-next-line no-await-in-loop
         const response = await tracingFetch(targetUrl, {
           method: 'GET',
-          headers: {
-            // eslint-disable-next-line max-len
-            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Tokowaka-AI Tokowaka/1.0 AdobeEdgeOptimize-AI AdobeEdgeOptimize/1.0',
-            'fastly-debug': '1',
-          },
+          headers: probeHeaders,
           timeout: REQUEST_TIMEOUT_MS,
         });
 
-        this.log.debug(`Response status: ${response.status}`);
+        this.log.info(`[edge-optimize-status] Response status: ${response.status} for ${targetUrl}`);
 
         const edgeOptimizeEnabled = response.headers.get('x-tokowaka-request-id') !== null
           || response.headers.get('x-edgeoptimize-request-id') !== null;
 
-        this.log.debug(`Edge optimize headers found: ${edgeOptimizeEnabled}`);
+        this.log.info(`[edge-optimize-status] Edge optimize headers found: ${edgeOptimizeEnabled} for ${targetUrl}`);
 
         return {
           edgeOptimizeEnabled,
@@ -1328,7 +1332,7 @@ class TokowakaClient {
 
         if (isTimeout) {
           // eslint-disable-next-line max-len
-          this.log.warn(`Request timed out after ${REQUEST_TIMEOUT_MS}ms for ${targetUrl}, returning edgeOptimizeEnabled: false`);
+          this.log.warn(`[edge-optimize-status] Request timed out after ${REQUEST_TIMEOUT_MS}ms for ${targetUrl}, returning edgeOptimizeEnabled: false`);
           return { edgeOptimizeEnabled: false };
         }
 
@@ -1336,7 +1340,7 @@ class TokowakaClient {
 
         if (attempt > maxRetries) {
           // All retries exhausted
-          this.log.error(`Failed after ${maxRetries + 1} attempts: ${error.message}`);
+          this.log.error(`[edge-optimize-status] Failed after ${maxRetries + 1} attempts: ${error.message} for ${targetUrl}`);
           throw this.#createError(
             `Failed to check edge optimize status: ${error.message}`,
             HTTP_INTERNAL_SERVER_ERROR,
@@ -1346,7 +1350,7 @@ class TokowakaClient {
         // Exponential backoff: 200ms, 400ms, 800ms
         const delay = 100 * (2 ** attempt);
         this.log.warn(
-          `Attempt ${attempt} to fetch failed: ${error.message}. Retrying in ${delay}ms...`,
+          `[edge-optimize-status] Attempt ${attempt} to fetch failed: ${error.message}. Retrying in ${delay}ms... for ${targetUrl}`,
         );
         // eslint-disable-next-line no-await-in-loop
         await new Promise((res) => {
@@ -1528,12 +1532,14 @@ class TokowakaClient {
       if (!isEdgeDeployableSuggestionStatus(s.getStatus())) {
         return false;
       }
-      if (isPatternSuggestion(s)) {
-        return false;
-      }
       if (s.getData()?.edgeDeployed) {
         return false;
       }
+      // Path-level pattern suggestions (not domain-wide) are fully covered by a
+      // domain-wide deployment. Other pattern suggestions (including other DW ones) are not.
+      if (isPatternSuggestion(s)) {
+        return coverageField === 'coveredByDomainWide' && !s.getData()?.isDomainWide;
+      }
       const url = s.getData()?.url;
       return url && matchers.some((match) => match(url));
     });

package/test/index.test.js

@@ -3117,6 +3117,55 @@ describe('TokowakaClient', () => {
       expect(coveredData).to.not.have.property('coveredByDomainWide');
     });
 
+    it('cleans up coveredByDomainWide on path-level suggestions when rolling back domain-wide', async () => {
+      const prerenderOpportunity = { getId: () => 'opp-dw', getType: () => 'prerender' };
+      const dwSuggestion = {
+        getId: () => 'dw-1',
+        getData: () => ({
+          isDomainWide: true,
+          allowedRegexPatterns: ['/*'],
+          edgeDeployed: Date.now(),
+        }),
+        setData: sinon.stub(),
+        setUpdatedBy: sinon.stub(),
+        save: sinon.stub().resolves(),
+      };
+      const pathSuggestion = {
+        getId: () => 'path-1',
+        getData: () => ({
+          allowedRegexPatterns: ['/products/*'],
+          coveredByDomainWide: 'dw-1',
+        }),
+        setData: sinon.stub(),
+        setUpdatedBy: sinon.stub(),
+        save: sinon.stub().resolves(),
+      };
+
+      sinon.stub(client, 'fetchMetaconfig').resolves({
+        siteId: 'site-123',
+        prerender: { allowList: ['/*'] },
+      });
+      sinon.stub(client, 'uploadMetaconfig').resolves();
+
+      const result = await client.rollbackSuggestions(
+        mockSite,
+        prerenderOpportunity,
+        [dwSuggestion],
+        { allSuggestions: [dwSuggestion, pathSuggestion], updatedBy: 'test@example.com' },
+      );
+
+      expect(result.succeededSuggestions).to.include(dwSuggestion);
+      expect(pathSuggestion.setData.calledOnce).to.be.true;
+      const pathData = pathSuggestion.setData.firstCall.args[0];
+      expect(pathData).to.not.have.property('coveredByDomainWide');
+      expect(pathData).to.have.property('allowedRegexPatterns');
+      // Verify the cleaned-up path suggestion was actually persisted via saveMany
+      expect(client.dataAccess.Suggestion.saveMany).to.have.been.calledWith(
+        sinon.match((arr) => arr.includes(pathSuggestion)),
+        sinon.match.any,
+      );
+    });
+
     it('cleans up covered suggestions (coveredByPattern) when rolling back a path-level pattern', async () => {
       const prerenderOpportunity = { getId: () => 'opp-p', getType: () => 'prerender' };
       const pathSuggestion = {
@@ -5035,6 +5084,165 @@ describe('TokowakaClient', () => {
         expect(userAgent).to.include('Mozilla/5.0');
       });
 
+      it('should ignore custom User-Agent from scraperConfig — default takes priority', async () => {
+        const customSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({
+            getEdgeOptimizeConfig: () => undefined,
+            getScraperConfig: () => ({ headers: { 'User-Agent': 'AllowListedBot/1.0' } }),
+          }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = {
+          status: 200,
+          headers: new Map(),
+        };
+        mockResponse.headers.get = () => null;
+
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(customSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['User-Agent']).to.include('Tokowaka-AI');
+        expect(fetchOptions.headers['User-Agent']).to.not.include('AllowListedBot/1.0');
+      });
+
+      it('should add extra custom headers while defaults always win on collision', async () => {
+        const customSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({
+            getEdgeOptimizeConfig: () => undefined,
+            getScraperConfig: () => ({ headers: { 'x-allowlist-token': 'secret123', 'fastly-debug': 'custom' } }),
+          }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = {
+          status: 200,
+          headers: new Map(),
+        };
+        mockResponse.headers.get = () => null;
+
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(customSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['x-allowlist-token']).to.equal('secret123');
+        expect(fetchOptions.headers['fastly-debug']).to.equal('1');
+        expect(fetchOptions.headers['User-Agent']).to.include('Mozilla/5.0');
+      });
+
+      it('should use default probe headers when scraperConfig has no headers', async () => {
+        const noHeadersSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({
+            getEdgeOptimizeConfig: () => undefined,
+            getScraperConfig: () => ({}),
+          }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = {
+          status: 200,
+          headers: new Map(),
+        };
+        mockResponse.headers.get = () => null;
+
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(noHeadersSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['User-Agent']).to.include('Tokowaka-AI');
+        expect(fetchOptions.headers['fastly-debug']).to.equal('1');
+      });
+
+      it('should use default probe headers when getScraperConfig is not present on config', async () => {
+        const noScraperSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({ getEdgeOptimizeConfig: () => undefined }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = {
+          status: 200,
+          headers: new Map(),
+        };
+        mockResponse.headers.get = () => null;
+
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(noScraperSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['User-Agent']).to.include('AdobeEdgeOptimize-AI');
+        expect(fetchOptions.headers['fastly-debug']).to.equal('1');
+      });
+
+      it('should use default probe headers when getScraperConfig returns null', async () => {
+        const nullScraperSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({
+            getEdgeOptimizeConfig: () => undefined,
+            getScraperConfig: () => null,
+          }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = { status: 200, headers: new Map() };
+        mockResponse.headers.get = () => null;
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(nullScraperSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['User-Agent']).to.include('Tokowaka-AI');
+        expect(fetchOptions.headers['fastly-debug']).to.equal('1');
+      });
+
+      it('should use default probe headers when getConfig returns null', async () => {
+        const nullConfigSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => null,
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = { status: 200, headers: new Map() };
+        mockResponse.headers.get = () => null;
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(nullConfigSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['User-Agent']).to.include('Tokowaka-AI');
+        expect(fetchOptions.headers['fastly-debug']).to.equal('1');
+      });
+
+      it('should use default probe headers when scraperConfig has an empty headers object', async () => {
+        const emptyHeadersSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({
+            getEdgeOptimizeConfig: () => undefined,
+            getScraperConfig: () => ({ headers: {} }),
+          }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = { status: 200, headers: new Map() };
+        mockResponse.headers.get = () => null;
+        tracingFetchStub.resolves(mockResponse);
+
+        await esmockClient.checkEdgeOptimizeStatus(emptyHeadersSite, '/');
+
+        const fetchOptions = tracingFetchStub.firstCall.args[1];
+        expect(fetchOptions.headers['User-Agent']).to.include('Tokowaka-AI');
+        expect(fetchOptions.headers['fastly-debug']).to.equal('1');
+        expect(Object.keys(fetchOptions.headers)).to.have.lengthOf(2);
+      });
+
       it('should pass timeout option to tracingFetch', async () => {
         const mockResponse = {
           status: 200,
@@ -5168,7 +5376,7 @@ describe('TokowakaClient', () => {
         await promise;
 
         expect(log.warn).to.have.been.calledWith(
-          sinon.match(/Attempt 1 to fetch failed.*Connection refused.*Retrying in 200ms/),
+          sinon.match(/\[edge-optimize-status\] Attempt 1 to fetch failed.*Connection refused.*Retrying in 200ms/),
         );
       });
 
@@ -5182,9 +5390,38 @@ describe('TokowakaClient', () => {
         expect(result).to.deep.equal({ edgeOptimizeEnabled: false });
         expect(tracingFetchStub).to.have.been.calledOnce;
         expect(log.warn).to.have.been.calledWith(
-          sinon.match(/Request timed out after 5000ms for https:\/\/example.com\/, returning edgeOptimizeEnabled: false/),
+          sinon.match(/\[edge-optimize-status\] Request timed out after 5000ms for https:\/\/example.com\/, returning edgeOptimizeEnabled: false/),
         );
       });
+
+      it('should send the same custom headers on every retry attempt', async () => {
+        const customSite = {
+          getId: () => 'site-id',
+          getBaseURL: () => 'https://example.com',
+          getConfig: () => ({
+            getEdgeOptimizeConfig: () => undefined,
+            getScraperConfig: () => ({ headers: { 'x-allowlist-token': 'secret123' } }),
+          }),
+          getDeliveryType: () => 'aem_edge',
+        };
+        const mockResponse = { status: 200, headers: new Map() };
+        mockResponse.headers.get = () => null;
+
+        tracingFetchStub.onFirstCall().rejects(new Error('Temporary failure'));
+        tracingFetchStub.onSecondCall().resolves(mockResponse);
+
+        const promise = esmockClient.checkEdgeOptimizeStatus(customSite, '/');
+        await clock.tickAsync(200);
+        await promise;
+
+        expect(tracingFetchStub).to.have.been.calledTwice;
+        const firstCallHeaders = tracingFetchStub.firstCall.args[1].headers;
+        const secondCallHeaders = tracingFetchStub.secondCall.args[1].headers;
+        expect(firstCallHeaders['x-allowlist-token']).to.equal('secret123');
+        expect(secondCallHeaders['x-allowlist-token']).to.equal('secret123');
+        expect(firstCallHeaders['User-Agent']).to.include('Tokowaka-AI');
+        expect(secondCallHeaders['User-Agent']).to.include('Tokowaka-AI');
+      });
     });
 
     describe('URL Construction', () => {
@@ -5721,6 +5958,51 @@ describe('TokowakaClient', () => {
       expect(result.coveredSuggestions).to.not.include(dw2);
     });
 
+    it('should mark path-level pattern suggestions as coveredByDomainWide when domain-wide is deployed', async () => {
+      const dw = makeSuggestion('dw1', {
+        isDomainWide: true,
+        allowedRegexPatterns: ['^https://example\\.com/.*'],
+      });
+      const pathLevel = makeSuggestion('path1', {
+        allowedRegexPatterns: ['/products/*'],
+      });
+
+      fetchMetaconfigStub.resolves({ siteId: 'site-123' });
+
+      const result = await client.deployToEdge({
+        site: mockSite,
+        opportunity: mockOpportunity,
+        targetSuggestions: [dw],
+        allSuggestions: [dw, pathLevel],
+      });
+
+      expect(result.coveredSuggestions).to.include(pathLevel);
+      expect(pathLevel.getData()).to.have.property('coveredByDomainWide', 'dw1');
+    });
+
+    it('should not mark already-deployed path-level suggestions as coveredByDomainWide', async () => {
+      const dw = makeSuggestion('dw1', {
+        isDomainWide: true,
+        allowedRegexPatterns: ['^https://example\\.com/.*'],
+      });
+      const pathLevel = makeSuggestion('path1', {
+        allowedRegexPatterns: ['/products/*'],
+        edgeDeployed: Date.now(),
+      });
+
+      fetchMetaconfigStub.resolves({ siteId: 'site-123' });
+
+      const result = await client.deployToEdge({
+        site: mockSite,
+        opportunity: mockOpportunity,
+        targetSuggestions: [dw],
+        allSuggestions: [dw, pathLevel],
+      });
+
+      expect(result.coveredSuggestions).to.not.include(pathLevel);
+      expect(pathLevel.getData()).to.not.have.property('coveredByDomainWide');
+    });
+
     it('should not mark non-NEW suggestions as covered', async () => {
       const dw = makeSuggestion('dw1', {
         isDomainWide: true,