@adobe/spacecat-shared-tokowaka-client 1.20.0 → 1.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,3 +1,15 @@
1
+ ## [@adobe/spacecat-shared-tokowaka-client-v1.21.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.21.0...@adobe/spacecat-shared-tokowaka-client-v1.21.1) (2026-07-15)
2
+
3
+ ### Bug Fixes
4
+
5
+ * utility for marking suggestions as covered ([#1810](https://github.com/adobe/spacecat-shared/issues/1810)) ([2709d47](https://github.com/adobe/spacecat-shared/commit/2709d4741bfe2968dfb481ceedf3e283525f5e71))
6
+
7
+ ## [@adobe/spacecat-shared-tokowaka-client-v1.21.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.20.0...@adobe/spacecat-shared-tokowaka-client-v1.21.0) (2026-07-14)
8
+
9
+ ### Features
10
+
11
+ * **tokowaka-client:** offload covered-marking saves to import worker queue above threshold ([#1802](https://github.com/adobe/spacecat-shared/issues/1802)) ([ad1fe43](https://github.com/adobe/spacecat-shared/commit/ad1fe433b4628c38ef0d2d4f4008c2932b5981e0))
12
+
1
13
  ## [@adobe/spacecat-shared-tokowaka-client-v1.20.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.19.1...@adobe/spacecat-shared-tokowaka-client-v1.20.0) (2026-06-27)
2
14
 
3
15
  ### Features
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@adobe/spacecat-shared-tokowaka-client",
3
- "version": "1.20.0",
3
+ "version": "1.21.1",
4
4
  "description": "Tokowaka Client for SpaceCat - Edge optimization config management",
5
5
  "type": "module",
6
6
  "engines": {
package/src/index.js CHANGED
@@ -27,7 +27,6 @@ import {
27
27
  } from './utils/s3-utils.js';
28
28
  import {
29
29
  omitKeys,
30
- isEdgeDeployableSuggestionStatus,
31
30
  isPatternSuggestion,
32
31
  groupSuggestionsByUrlPath,
33
32
  filterEligibleSuggestions,
@@ -36,8 +35,8 @@ import {
36
35
  cleanupCoveredSuggestions,
37
36
  classifySuggestions,
38
37
  filterBatchCoveredSuggestions,
38
+ findCoveredSuggestions,
39
39
  } from './utils/suggestion-utils.js';
40
- import { buildUrlMatcher } from './utils/pattern-utils.js';
41
40
  import { getEffectiveBaseURL } from './utils/site-utils.js';
42
41
  import { removePatternFromMetaconfig, addPatternsToMetaconfig } from './utils/metaconfig-utils.js';
43
42
  import { fetchHtmlWithWarmup, calculateForwardedHost } from './utils/custom-html-utils.js';
@@ -50,6 +49,7 @@ import {
50
49
 
51
50
  export { FastlyKVClient } from './fastly-kv-client.js';
52
51
  export { calculateForwardedHost } from './utils/custom-html-utils.js';
52
+ export { SUGGESTION_BULK_UPDATE_TYPE } from './utils/suggestion-utils.js';
53
53
 
54
54
  // CloudFront control-plane (free functions + constants).
55
55
  export {
@@ -88,6 +88,7 @@ class TokowakaClient {
88
88
  const {
89
89
  TOKOWAKA_SITE_CONFIG_BUCKET: bucketName,
90
90
  TOKOWAKA_PREVIEW_BUCKET: previewBucketName,
91
+ IMPORT_WORKER_QUEUE_URL: importWorkerQueueUrl,
91
92
  } = env;
92
93
 
93
94
  if (context.tokowakaClient) {
@@ -100,6 +101,8 @@ class TokowakaClient {
100
101
  s3Client: s3?.s3Client ?? context.s3Client,
101
102
  env,
102
103
  dataAccess: context.dataAccess,
104
+ sqs: context.sqs,
105
+ importWorkerQueueUrl,
103
106
  }, log);
104
107
  context.tokowakaClient = client;
105
108
  return client;
@@ -114,10 +117,13 @@ class TokowakaClient {
114
117
  * @param {Object} config.env - Environment variables (for CDN credentials)
115
118
  * @param {Object} [config.dataAccess] - Data access layer
116
119
  * (provides Suggestion.saveMany for batch saves)
120
+ * @param {Object} [config.sqs] - SQS client (sendMessage(queueUrl, payload)). Used to offload
121
+ * very large covered-suggestion saves to the import worker instead of saving them directly.
122
+ * @param {string} [config.importWorkerQueueUrl] - Import worker SQS queue URL.
117
123
  * @param {Object} log - Logger instance
118
124
  */
119
125
  constructor({
120
- bucketName, previewBucketName, s3Client, env = {}, dataAccess,
126
+ bucketName, previewBucketName, s3Client, env = {}, dataAccess, sqs, importWorkerQueueUrl,
121
127
  }, log) {
122
128
  this.log = log;
123
129
 
@@ -134,6 +140,8 @@ class TokowakaClient {
134
140
  this.s3Client = s3Client;
135
141
  this.env = env;
136
142
  this.dataAccess = dataAccess;
143
+ this.sqs = sqs;
144
+ this.importWorkerQueueUrl = importWorkerQueueUrl;
137
145
 
138
146
  this.mapperRegistry = new MapperRegistry(log);
139
147
  this.cdnClientRegistry = new CdnClientRegistry(env, log);
@@ -1077,7 +1085,14 @@ class TokowakaClient {
1077
1085
  this.log.info(`[edge-rollback] Cleaning ${covered.length} covered suggestion(s) for pattern ${suggestion.getId()} (isDomainWide=${isDomainWide}, fallback=${coveredFallback})`);
1078
1086
  }
1079
1087
  // eslint-disable-next-line no-await-in-loop, max-len
1080
- await cleanupCoveredSuggestions(this.dataAccess, covered, coveredFallback, updatedBy, fieldsToStrip, this.log);
1088
+ await cleanupCoveredSuggestions(this.dataAccess, covered, coveredFallback, updatedBy, fieldsToStrip, this.log, {
1089
+ sqs: this.sqs,
1090
+ queueUrl: this.importWorkerQueueUrl,
1091
+ siteId: site.getId(),
1092
+ unset: fieldsToStrip,
1093
+ updatedBy: updatedBy ?? coveredFallback,
1094
+ log: this.log,
1095
+ });
1081
1096
  }
1082
1097
  }
1083
1098
  }
@@ -1504,6 +1519,7 @@ class TokowakaClient {
1504
1519
  * @param {Array} allSuggestions - Full opportunity suggestion list
1505
1520
  * @param {string} updatedBy
1506
1521
  * @param {Array} coveredSuggestions - Accumulator (mutated)
1522
+ * @param {string} siteId
1507
1523
  * @returns {Promise<void>} Throws on metaconfig upload failure
1508
1524
  * @private
1509
1525
  */
@@ -1516,6 +1532,7 @@ class TokowakaClient {
1516
1532
  allSuggestions,
1517
1533
  updatedBy,
1518
1534
  coveredSuggestions,
1535
+ siteId,
1519
1536
  ) {
1520
1537
  const data = suggestion.getData();
1521
1538
  const coverageField = data?.isDomainWide ? 'coveredByDomainWide' : 'coveredByPattern';
@@ -1541,41 +1558,13 @@ class TokowakaClient {
1541
1558
  suggestion.setUpdatedBy(updatedBy);
1542
1559
  // suggestion.save() is deferred — caller batches saves via saveSuggestions.
1543
1560
 
1544
- // Mark per-URL suggestions covered by this pattern.
1545
- const matchers = allowedRegexPatterns.flatMap((p) => {
1546
- const m = buildUrlMatcher(p);
1547
- if (!m) {
1548
- // eslint-disable-next-line max-len
1549
- this.log.warn(`[edge-deploy] Pattern '${p}' for suggestion ${suggestion.getId()} is invalid, skipping`);
1550
- }
1551
- return m ? [m] : [];
1552
- });
1553
-
1554
- if (matchers.length === 0) {
1555
- return;
1556
- }
1557
-
1558
- const covered = allSuggestions.filter((s) => {
1559
- if (s.getId() === suggestion.getId()) {
1560
- return false;
1561
- }
1562
- if (skippedInBatchIds.has(s.getId())) {
1563
- return false;
1564
- }
1565
- if (!isEdgeDeployableSuggestionStatus(s.getStatus())) {
1566
- return false;
1567
- }
1568
- if (s.getData()?.edgeDeployed) {
1569
- return false;
1570
- }
1571
- // Path-level pattern suggestions (not domain-wide) are fully covered by a
1572
- // domain-wide deployment. Other pattern suggestions (including other DW ones) are not.
1573
- if (isPatternSuggestion(s)) {
1574
- return coverageField === 'coveredByDomainWide' && !s.getData()?.isDomainWide;
1575
- }
1576
- const url = s.getData()?.url;
1577
- return url && matchers.some((match) => match(url));
1578
- });
1561
+ const covered = findCoveredSuggestions(
1562
+ suggestion,
1563
+ allowedRegexPatterns,
1564
+ allSuggestions,
1565
+ skippedInBatchIds,
1566
+ this.log,
1567
+ );
1579
1568
 
1580
1569
  // eslint-disable-next-line max-len
1581
1570
  this.log.info(`[edge-deploy] Pattern ${suggestion.getId()}: found ${covered.length} coverable per-URL suggestions (field=${coverageField})`);
@@ -1586,7 +1575,14 @@ class TokowakaClient {
1586
1575
  cs.setUpdatedBy(updatedBy);
1587
1576
  });
1588
1577
  try {
1589
- await saveSuggestions(this.dataAccess, covered);
1578
+ await saveSuggestions(this.dataAccess, covered, {
1579
+ sqs: this.sqs,
1580
+ queueUrl: this.importWorkerQueueUrl,
1581
+ siteId,
1582
+ set: { [coverageField]: suggestion.getId() },
1583
+ updatedBy,
1584
+ log: this.log,
1585
+ });
1590
1586
  coveredSuggestions.push(...covered);
1591
1587
  // eslint-disable-next-line max-len
1592
1588
  this.log.info(`[edge-deploy] Marked ${covered.length} suggestions as ${coverageField}=${suggestion.getId()}`);
@@ -1679,6 +1675,7 @@ class TokowakaClient {
1679
1675
  allSuggestions,
1680
1676
  updatedBy,
1681
1677
  coveredSuggestions,
1678
+ site.getId(),
1682
1679
  );
1683
1680
  deployedPatternSuggestions.push(suggestion);
1684
1681
  } catch (error) {
@@ -1733,6 +1730,64 @@ class TokowakaClient {
1733
1730
  return { succeededSuggestions, failedSuggestions, coveredSuggestions };
1734
1731
  }
1735
1732
 
1733
+ /**
1734
+ * Finds and marks the suggestions that fall within a pattern suggestion's
1735
+ * scope (domain-wide or segment/path) as covered and saves them.
1736
+ *
1737
+ * @param {Object} patternSuggestion - The domain-wide / segment pattern suggestion
1738
+ * @param {Array} allSuggestions - Full opportunity suggestion list to search for matches
1739
+ * @param {string} siteId - Site ID
1740
+ * @param {string} [updatedBy]
1741
+ * @returns {Promise<Array>} the marked suggestions
1742
+ */
1743
+ async markPatternCoveredSuggestions(
1744
+ patternSuggestion,
1745
+ allSuggestions,
1746
+ siteId,
1747
+ updatedBy = 'edge-deploy',
1748
+ ) {
1749
+ const allowedRegexPatterns = patternSuggestion.getData()?.allowedRegexPatterns;
1750
+ if (!Array.isArray(allowedRegexPatterns) || allowedRegexPatterns.length === 0) {
1751
+ this.log.warn(`[edge-deploy] Pattern suggestion ${patternSuggestion.getId()} has `
1752
+ + 'no allowedRegexPatterns, skipping cover-marking');
1753
+ return [];
1754
+ }
1755
+ const covered = findCoveredSuggestions(
1756
+ patternSuggestion,
1757
+ allowedRegexPatterns,
1758
+ allSuggestions,
1759
+ new Set(),
1760
+ this.log,
1761
+ );
1762
+ if (covered.length === 0) {
1763
+ return [];
1764
+ }
1765
+ const coverageField = patternSuggestion.getData()?.isDomainWide
1766
+ ? 'coveredByDomainWide' : 'coveredByPattern';
1767
+ covered.forEach((s) => {
1768
+ s.setData({ ...s.getData(), [coverageField]: patternSuggestion.getId() });
1769
+ s.setUpdatedBy(updatedBy);
1770
+ });
1771
+ try {
1772
+ await saveSuggestions(this.dataAccess, covered, {
1773
+ sqs: this.sqs,
1774
+ queueUrl: this.importWorkerQueueUrl,
1775
+ siteId,
1776
+ set: { [coverageField]: patternSuggestion.getId() },
1777
+ updatedBy,
1778
+ log: this.log,
1779
+ });
1780
+ this.log.info(`[edge-deploy] Marked ${covered.length} suggestions as ${coverageField}=${patternSuggestion.getId()}`);
1781
+ return covered;
1782
+ } catch (saveError) {
1783
+ this.log.warn(
1784
+ '[edge-deploy] Failed to mark covered suggestions for pattern suggestion '
1785
+ + `${patternSuggestion.getId()}: ${saveError.message}`,
1786
+ );
1787
+ return [];
1788
+ }
1789
+ }
1790
+
1736
1791
  /**
1737
1792
  * Strips the `applyStale` flag from patches in S3 for the given per-URL suggestions.
1738
1793
  * Called when an experiment completes so edge reverts to normal (non-stale) behaviour.
@@ -92,26 +92,68 @@ export function filterEligibleSuggestions(suggestions, mapper) {
92
92
  return { eligible, ineligible };
93
93
  }
94
94
 
95
+ // Import worker job type for a bulk suggestion field update (agnostic set/unset by id).
96
+ export const SUGGESTION_BULK_UPDATE_TYPE = 'suggestion-bulk-update';
97
+
95
98
  /**
96
99
  * Batch-saves suggestions using the optimal strategy based on count.
97
100
  *
98
101
  * - <= 1700: sequential chunked upsert via saveMany (chunkSize 25).
99
102
  * ~4s for 1700 suggestions from the DB layer, but ~11s observed end-to-end
100
103
  * from the API due to serialization, network, and Lambda overhead.
101
- * - > 1700: parallel individual .save() via Promise.allSettled to avoid
102
- * sequential chunk bottleneck at scale.
104
+ * - > 1700 with no queueContext: parallel individual .save() via Promise.allSettled to
105
+ * avoid sequential chunk bottleneck at scale.
106
+ * - > 1700 with queueContext: enqueues a SUGGESTION_BULK_UPDATE_TYPE job to the import
107
+ * worker instead of saving here (the worker fetches by id and applies
108
+ * queueContext.set/unset itself). Falls back to the Promise.allSettled path above
109
+ * if sqs/queueUrl aren't configured or sendMessage fails, so a queue outage
110
+ * degrades rather than silently drops the save.
103
111
  *
104
112
  * @param {Object} dataAccess - Data access layer
105
113
  * @param {Array} suggestions - Suggestion entities to save
114
+ * @param {Object} [queueContext] - sqs, queueUrl, siteId, set/unset,
115
+ * updatedBy (already resolved), log
106
116
  * @returns {Promise<void>}
107
117
  */
108
118
  const PARALLEL_SAVE_THRESHOLD = 1700;
109
119
 
110
- export async function saveSuggestions(dataAccess, suggestions) {
120
+ export async function saveSuggestions(dataAccess, suggestions, queueContext) {
111
121
  if (suggestions.length === 0) {
112
122
  return;
113
123
  }
114
124
  if (suggestions.length > PARALLEL_SAVE_THRESHOLD) {
125
+ if (queueContext) {
126
+ const {
127
+ sqs, queueUrl, siteId, set, unset, updatedBy, log,
128
+ } = queueContext;
129
+
130
+ if (queueUrl && sqs) {
131
+ try {
132
+ await sqs.sendMessage(queueUrl, {
133
+ type: SUGGESTION_BULK_UPDATE_TYPE,
134
+ siteId,
135
+ suggestionIds: suggestions.map((s) => s.getId()),
136
+ ...(set && { set }),
137
+ ...(unset && { unset }),
138
+ updatedBy,
139
+ });
140
+ log.info(
141
+ `[suggestion-bulk-update] Queued bulk update for ${suggestions.length} suggestion(s)`,
142
+ );
143
+ return;
144
+ } catch (error) {
145
+ log.error(
146
+ `[suggestion-bulk-update] Failed to queue bulk update, falling back to direct save: ${error.message}`,
147
+ );
148
+ }
149
+ } else {
150
+ log.warn(
151
+ '[suggestion-bulk-update] SQS/IMPORT_WORKER_QUEUE_URL not configured; '
152
+ + `falling back to direct save for ${suggestions.length} suggestion(s)`,
153
+ );
154
+ }
155
+ }
156
+
115
157
  const results = await Promise.allSettled(suggestions.map((s) => s.save()));
116
158
  const failed = results.filter((r) => r.status === 'rejected');
117
159
  if (failed.length > 0) {
@@ -150,10 +192,11 @@ export function stripSuggestion(suggestion, actorFallback, updatedBy) {
150
192
  * @param {string|undefined} updatedBy - Explicit actor
151
193
  * @param {string[]} fieldsToStrip - Specific fields to remove
152
194
  * @param {Object} log - Logger instance
195
+ * @param {Object} [queueContext] - Passed through to saveSuggestions unchanged
153
196
  * @returns {Promise<void>}
154
197
  */
155
198
  // eslint-disable-next-line max-len
156
- export async function cleanupCoveredSuggestions(dataAccess, covered, actorFallback, updatedBy, fieldsToStrip, log) {
199
+ export async function cleanupCoveredSuggestions(dataAccess, covered, actorFallback, updatedBy, fieldsToStrip, log, queueContext) {
157
200
  if (covered.length === 0) {
158
201
  return;
159
202
  }
@@ -162,7 +205,7 @@ export async function cleanupCoveredSuggestions(dataAccess, covered, actorFallba
162
205
  cs.setUpdatedBy(updatedBy ?? actorFallback);
163
206
  });
164
207
  try {
165
- await saveSuggestions(dataAccess, covered);
208
+ await saveSuggestions(dataAccess, covered, queueContext);
166
209
  } catch (error) {
167
210
  // eslint-disable-next-line max-len
168
211
  log.error(`[edge-rollback-failed] Failed to clean ${covered.length} covered suggestion(s): ${error.message}`);
@@ -241,3 +284,58 @@ export function filterBatchCoveredSuggestions(validSuggestions, patternSuggestio
241
284
 
242
285
  return { remaining, skippedInBatch };
243
286
  }
287
+
288
+ /**
289
+ * Finds the suggestions in `allSuggestions` that are covered by a pattern suggestion's
290
+ * allowedRegexPatterns
291
+ *
292
+ * @param {Object} patternSuggestion - The domain-wide / segment pattern suggestion
293
+ * @param {Array<string>} allowedRegexPatterns
294
+ * @param {Array} allSuggestions - Full opportunity suggestion list to search
295
+ * @param {Set<string>} [excludeIds] - Suggestion IDs to skip (e.g. already handled in-batch)
296
+ * @param {Object} log - Logger instance
297
+ * @returns {Array} Suggestions covered by the pattern
298
+ */
299
+ export function findCoveredSuggestions(
300
+ patternSuggestion,
301
+ allowedRegexPatterns,
302
+ allSuggestions,
303
+ excludeIds,
304
+ log,
305
+ ) {
306
+ const isDomainWide = patternSuggestion.getData()?.isDomainWide === true;
307
+ const matchers = allowedRegexPatterns.flatMap((p) => {
308
+ const m = buildUrlMatcher(p);
309
+ if (!m) {
310
+ // eslint-disable-next-line max-len
311
+ log.warn(`[edge-deploy] Pattern '${p}' for suggestion ${patternSuggestion.getId()} is invalid, skipping`);
312
+ }
313
+ return m ? [m] : [];
314
+ });
315
+
316
+ if (matchers.length === 0) {
317
+ return [];
318
+ }
319
+
320
+ return allSuggestions.filter((s) => {
321
+ if (s.getId() === patternSuggestion.getId()) {
322
+ return false;
323
+ }
324
+ if (excludeIds?.has(s.getId())) {
325
+ return false;
326
+ }
327
+ if (!isEdgeDeployableSuggestionStatus(s.getStatus())) {
328
+ return false;
329
+ }
330
+ if (s.getData()?.edgeDeployed) {
331
+ return false;
332
+ }
333
+ // Path-level pattern suggestions (not domain-wide) are fully covered by a
334
+ // domain-wide deployment. Other pattern suggestions (including other DW ones) are not.
335
+ if (isPatternSuggestion(s)) {
336
+ return isDomainWide && !s.getData()?.isDomainWide;
337
+ }
338
+ const url = s.getData()?.url;
339
+ return url && matchers.some((match) => match(url));
340
+ });
341
+ }
@@ -147,6 +147,98 @@ describe('TokowakaClient', () => {
147
147
  });
148
148
  });
149
149
 
150
+ describe('markPatternCoveredSuggestions', () => {
151
+ const mkPattern = (isDomainWide) => ({
152
+ getId: () => (isDomainWide ? 'dw-1' : 'path-1'),
153
+ getData: () => (isDomainWide
154
+ ? { isDomainWide: true, allowedRegexPatterns: ['/*'] }
155
+ : { allowedRegexPatterns: ['/foo/*'] }),
156
+ });
157
+ const mkCovered = (id, url = `https://example.com/foo/${id}`) => ({
158
+ getId: () => id,
159
+ getStatus: () => 'NEW',
160
+ getData: () => ({ url }),
161
+ setData: sinon.stub(),
162
+ setUpdatedBy: sinon.stub(),
163
+ save: sinon.stub().resolves(),
164
+ });
165
+
166
+ it('marks all matching suggestions in the opportunity with coveredByDomainWide', async () => {
167
+ const covered = [mkCovered('a'), mkCovered('b')];
168
+
169
+ const result = await client.markPatternCoveredSuggestions(
170
+ mkPattern(true),
171
+ covered,
172
+ 'site-1',
173
+ 'tester',
174
+ );
175
+
176
+ expect(result).to.deep.equal(covered);
177
+ covered.forEach((s) => {
178
+ expect(s.setData.calledWithMatch({ coveredByDomainWide: 'dw-1' })).to.be.true;
179
+ expect(s.setUpdatedBy.calledOnceWith('tester')).to.be.true;
180
+ });
181
+ expect(client.dataAccess.Suggestion.saveMany.calledOnceWith(covered)).to.be.true;
182
+ });
183
+
184
+ it('marks only URLs matching the segment/path pattern with coveredByPattern', async () => {
185
+ const matching = mkCovered('a', 'https://example.com/foo/a');
186
+ const nonMatching = mkCovered('b', 'https://example.com/bar/b');
187
+
188
+ const result = await client.markPatternCoveredSuggestions(
189
+ mkPattern(false),
190
+ [matching, nonMatching],
191
+ 'site-1',
192
+ );
193
+
194
+ expect(result).to.deep.equal([matching]);
195
+ expect(matching.setData.calledWithMatch({ coveredByPattern: 'path-1' })).to.be.true;
196
+ expect(matching.setUpdatedBy.calledOnceWith('edge-deploy')).to.be.true;
197
+ expect(nonMatching.setData.called).to.be.false;
198
+ });
199
+
200
+ it('is a no-op when the pattern suggestion has no allowedRegexPatterns', async () => {
201
+ const noPattern = { getId: () => 'dw-1', getData: () => ({ isDomainWide: true }) };
202
+
203
+ expect(await client.markPatternCoveredSuggestions(noPattern, [mkCovered('a')], 'site-1'))
204
+ .to.deep.equal([]);
205
+ expect(client.dataAccess.Suggestion.saveMany.called).to.be.false;
206
+ });
207
+
208
+ it('is a no-op when nothing in allSuggestions matches', async () => {
209
+ const result = await client.markPatternCoveredSuggestions(mkPattern(true), [], 'site-1');
210
+
211
+ expect(result).to.deep.equal([]);
212
+ expect(client.dataAccess.Suggestion.saveMany.called).to.be.false;
213
+ });
214
+
215
+ it('enqueues a bulk-update job with the passed-in siteId when covered count is large', async () => {
216
+ const covered = Array.from({ length: 1701 }, (_, i) => mkCovered(`c${i}`));
217
+ client.sqs = { sendMessage: sinon.stub().resolves() };
218
+ client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
219
+
220
+ await client.markPatternCoveredSuggestions(mkPattern(true), covered, 'site-1', 'tester');
221
+
222
+ expect(client.sqs.sendMessage).to.have.been.calledOnce;
223
+ const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
224
+ expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
225
+ expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-1' });
226
+ expect(payload.set).to.deep.equal({ coveredByDomainWide: 'dw-1' });
227
+ expect(payload.updatedBy).to.equal('tester');
228
+ expect(client.dataAccess.Suggestion.saveMany.called).to.be.false;
229
+ });
230
+
231
+ it('warns and returns [] instead of throwing when saveSuggestions fails', async () => {
232
+ client.dataAccess.Suggestion.saveMany.rejects(new Error('DB down'));
233
+ const covered = [mkCovered('a')];
234
+
235
+ const result = await client.markPatternCoveredSuggestions(mkPattern(true), covered, 'site-1');
236
+
237
+ expect(result).to.deep.equal([]);
238
+ expect(client.log.warn.calledWithMatch(/Failed to mark covered suggestions/)).to.be.true;
239
+ });
240
+ });
241
+
150
242
  describe('createFrom', () => {
151
243
  it('should create client from context', () => {
152
244
  const context = {
@@ -3197,6 +3289,57 @@ describe('TokowakaClient', () => {
3197
3289
  );
3198
3290
  });
3199
3291
 
3292
+ it('enqueues a suggestion-bulk-update job instead of saveMany when cleaning up more than the threshold (domain-wide rollback)', async () => {
3293
+ const prerenderOpportunity = { getId: () => 'opp-dw', getType: () => 'prerender' };
3294
+ const dwSuggestion = {
3295
+ getId: () => 'dw-1',
3296
+ getData: () => ({
3297
+ isDomainWide: true,
3298
+ allowedRegexPatterns: ['/*'],
3299
+ edgeDeployed: Date.now(),
3300
+ }),
3301
+ setData: sinon.stub(),
3302
+ setUpdatedBy: sinon.stub(),
3303
+ save: sinon.stub().resolves(),
3304
+ };
3305
+ const covered = Array.from({ length: 1701 }, (_, i) => ({
3306
+ getId: () => `covered-${i}`,
3307
+ getData: () => ({ url: `https://example.com/page${i}`, coveredByDomainWide: 'dw-1' }),
3308
+ setData: sinon.stub(),
3309
+ setUpdatedBy: sinon.stub(),
3310
+ save: sinon.stub().resolves(),
3311
+ }));
3312
+
3313
+ sinon.stub(client, 'fetchMetaconfig').resolves({
3314
+ siteId: 'site-123',
3315
+ prerender: { allowList: ['/*'] },
3316
+ });
3317
+ sinon.stub(client, 'uploadMetaconfig').resolves();
3318
+ client.sqs = { sendMessage: sinon.stub().resolves() };
3319
+ client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
3320
+
3321
+ await client.rollbackSuggestions(
3322
+ mockSite,
3323
+ prerenderOpportunity,
3324
+ [dwSuggestion],
3325
+ { allSuggestions: [dwSuggestion, ...covered], updatedBy: 'test@example.com' },
3326
+ );
3327
+
3328
+ expect(client.sqs.sendMessage).to.have.been.calledOnce;
3329
+ const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
3330
+ expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
3331
+ expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-123' });
3332
+ expect(payload).to.not.have.property('opportunityId');
3333
+ expect(payload.suggestionIds).to.have.length(1701);
3334
+ expect(payload.unset).to.deep.equal(['coveredByDomainWide']);
3335
+ expect(payload.updatedBy).to.equal('test@example.com');
3336
+ // Not saved directly — the enqueue replaces the saveMany call for this batch.
3337
+ expect(client.dataAccess.Suggestion.saveMany).to.not.have.been.calledWith(
3338
+ sinon.match((arr) => arr.length === 1701),
3339
+ sinon.match.any,
3340
+ );
3341
+ });
3342
+
3200
3343
  it('cleans up covered suggestions (coveredByPattern) when rolling back a path-level pattern', async () => {
3201
3344
  const prerenderOpportunity = { getId: () => 'opp-p', getType: () => 'prerender' };
3202
3345
  const pathSuggestion = {
@@ -3240,6 +3383,54 @@ describe('TokowakaClient', () => {
3240
3383
  expect(coveredData).to.not.have.property('coveredByPattern');
3241
3384
  });
3242
3385
 
3386
+ it('enqueues a suggestion-bulk-update job instead of saveMany when cleaning up more than the threshold (path-level rollback)', async () => {
3387
+ const prerenderOpportunity = { getId: () => 'opp-p', getType: () => 'prerender' };
3388
+ const pathSuggestion = {
3389
+ getId: () => 'path-1',
3390
+ getData: () => ({
3391
+ allowedRegexPatterns: ['/products/*'],
3392
+ edgeDeployed: Date.now(),
3393
+ }),
3394
+ setData: sinon.stub(),
3395
+ setUpdatedBy: sinon.stub(),
3396
+ save: sinon.stub().resolves(),
3397
+ };
3398
+ const covered = Array.from({ length: 1701 }, (_, i) => ({
3399
+ getId: () => `covered-${i}`,
3400
+ getData: () => ({
3401
+ url: `https://example.com/products/item-${i}`,
3402
+ edgeDeployed: Date.now(),
3403
+ coveredByPattern: 'path-1',
3404
+ }),
3405
+ setData: sinon.stub(),
3406
+ setUpdatedBy: sinon.stub(),
3407
+ save: sinon.stub().resolves(),
3408
+ }));
3409
+
3410
+ sinon.stub(client, 'fetchMetaconfig').resolves({
3411
+ siteId: 'site-123',
3412
+ prerender: { allowList: ['/products/*'] },
3413
+ });
3414
+ sinon.stub(client, 'uploadMetaconfig').resolves();
3415
+ client.sqs = { sendMessage: sinon.stub().resolves() };
3416
+ client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
3417
+
3418
+ await client.rollbackSuggestions(
3419
+ mockSite,
3420
+ prerenderOpportunity,
3421
+ [pathSuggestion],
3422
+ { allSuggestions: [pathSuggestion, ...covered] },
3423
+ );
3424
+
3425
+ expect(client.sqs.sendMessage).to.have.been.calledOnce;
3426
+ const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
3427
+ expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
3428
+ expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-123' });
3429
+ expect(payload).to.not.have.property('opportunityId');
3430
+ expect(payload.suggestionIds).to.have.length(1701);
3431
+ expect(payload.unset).to.deep.equal(['edgeDeployed', 'tokowakaDeployed', 'coveredByPattern']);
3432
+ });
3433
+
3243
3434
  it('uses domain-wide-rollback fallback for covered suggestions when updatedBy is not provided (domain-wide parent)', async () => {
3244
3435
  const prerenderOpportunity = { getId: () => 'opp-dw', getType: () => 'prerender' };
3245
3436
  const dwSuggestion = {
@@ -5965,6 +6156,37 @@ describe('TokowakaClient', () => {
5965
6156
  expect(covered.getData()).to.have.property('coveredByDomainWide', 'dw1');
5966
6157
  });
5967
6158
 
6159
+ it('enqueues a suggestion-bulk-update job instead of saveMany when covered count exceeds the threshold (domain-wide)', async () => {
6160
+ const dw = makeSuggestion('dw1', {
6161
+ isDomainWide: true,
6162
+ allowedRegexPatterns: ['^https://example\\.com/.*'],
6163
+ });
6164
+ const covered = Array.from({ length: 1701 }, (_, i) => makeSuggestion(
6165
+ `covered-${i}`,
6166
+ { url: `https://example.com/page${i}` },
6167
+ ));
6168
+
6169
+ fetchMetaconfigStub.resolves({ siteId: 'site-123' });
6170
+ client.sqs = { sendMessage: sinon.stub().resolves() };
6171
+ client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
6172
+
6173
+ const result = await client.deployToEdge({
6174
+ site: mockSite,
6175
+ opportunity: mockOpportunity,
6176
+ targetSuggestions: [dw],
6177
+ allSuggestions: [dw, ...covered],
6178
+ });
6179
+
6180
+ expect(result.coveredSuggestions).to.have.length(1701);
6181
+ expect(client.sqs.sendMessage).to.have.been.calledOnce;
6182
+ const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
6183
+ expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
6184
+ expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-123' });
6185
+ expect(payload).to.not.have.property('opportunityId');
6186
+ expect(payload.suggestionIds).to.have.length(1701);
6187
+ expect(payload.set).to.deep.equal({ coveredByDomainWide: 'dw1' });
6188
+ });
6189
+
5968
6190
  it('should not mark already-domain-wide suggestions as covered', async () => {
5969
6191
  const dw1 = makeSuggestion('dw1', {
5970
6192
  isDomainWide: true,
@@ -6467,6 +6689,36 @@ describe('TokowakaClient', () => {
6467
6689
  expect(result.coveredSuggestions).to.not.include(urlElsewhere);
6468
6690
  });
6469
6691
 
6692
+ it('enqueues a suggestion-bulk-update job instead of saveMany when covered count exceeds the threshold (path-level)', async () => {
6693
+ const path = makeSuggestion('p1', {
6694
+ allowedRegexPatterns: ['/products/*'],
6695
+ });
6696
+ const covered = Array.from({ length: 1701 }, (_, i) => makeSuggestion(
6697
+ `covered-${i}`,
6698
+ { url: `https://example.com/products/item-${i}` },
6699
+ ));
6700
+
6701
+ fetchMetaconfigStub.resolves({ siteId: 'site-123' });
6702
+ client.sqs = { sendMessage: sinon.stub().resolves() };
6703
+ client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
6704
+
6705
+ const result = await client.deployToEdge({
6706
+ site: mockSite,
6707
+ opportunity: mockOpportunity,
6708
+ targetSuggestions: [path],
6709
+ allSuggestions: [path, ...covered],
6710
+ });
6711
+
6712
+ expect(result.coveredSuggestions).to.have.length(1701);
6713
+ expect(client.sqs.sendMessage).to.have.been.calledOnce;
6714
+ const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
6715
+ expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
6716
+ expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-123' });
6717
+ expect(payload).to.not.have.property('opportunityId');
6718
+ expect(payload.suggestionIds).to.have.length(1701);
6719
+ expect(payload.set).to.deep.equal({ coveredByPattern: 'p1' });
6720
+ });
6721
+
6470
6722
  it('path deploy marks as failed with statusCode 500 when metaconfig upload fails', async () => {
6471
6723
  const path = makeSuggestion('p1', {
6472
6724
  allowedRegexPatterns: ['/products/*'],
@@ -12,7 +12,12 @@
12
12
 
13
13
  import { expect } from 'chai';
14
14
  import sinon from 'sinon';
15
- import { groupSuggestionsByUrlPath, filterEligibleSuggestions, saveSuggestions } from '../../src/utils/suggestion-utils.js';
15
+ import {
16
+ groupSuggestionsByUrlPath,
17
+ filterEligibleSuggestions,
18
+ saveSuggestions,
19
+ SUGGESTION_BULK_UPDATE_TYPE,
20
+ } from '../../src/utils/suggestion-utils.js';
16
21
 
17
22
  describe('Suggestion Utils', () => {
18
23
  describe('groupSuggestionsByUrlPath', () => {
@@ -223,5 +228,153 @@ describe('Suggestion Utils', () => {
223
228
  expect(error.message).to.include('DB error');
224
229
  }
225
230
  });
231
+
232
+ it('should enqueue a bulk update job instead of saving when queueContext is provided', async () => {
233
+ const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
234
+ const items = Array.from({ length: 1701 }, (_, i) => ({
235
+ getId: () => `sugg-${i}`,
236
+ save: sinon.stub().resolves(),
237
+ }));
238
+ const queueContext = {
239
+ sqs: { sendMessage: sinon.stub().resolves() },
240
+ queueUrl: 'https://sqs.example.com/queue',
241
+ siteId: 'site-1',
242
+ set: { coveredByDomainWide: 'pattern-sugg-id' },
243
+ updatedBy: 'system',
244
+ log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
245
+ };
246
+
247
+ await saveSuggestions(dataAccess, items, queueContext);
248
+
249
+ expect(dataAccess.Suggestion.saveMany.called).to.be.false;
250
+ expect(items[0].save.called).to.be.false;
251
+ expect(queueContext.sqs.sendMessage.calledOnce).to.be.true;
252
+ const [queueUrl, payload] = queueContext.sqs.sendMessage.firstCall.args;
253
+ expect(queueUrl).to.equal('https://sqs.example.com/queue');
254
+ expect(payload).to.deep.equal({
255
+ type: SUGGESTION_BULK_UPDATE_TYPE,
256
+ siteId: 'site-1',
257
+ suggestionIds: items.map((s) => s.getId()),
258
+ set: { coveredByDomainWide: 'pattern-sugg-id' },
259
+ updatedBy: 'system',
260
+ });
261
+ expect(queueContext.log.info.calledOnce).to.be.true;
262
+ });
263
+
264
+ it('should include unset instead of set when queueContext.unset is provided', async () => {
265
+ const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
266
+ const items = Array.from({ length: 1701 }, (_, i) => ({
267
+ getId: () => `sugg-${i}`,
268
+ save: sinon.stub().resolves(),
269
+ }));
270
+ const queueContext = {
271
+ sqs: { sendMessage: sinon.stub().resolves() },
272
+ queueUrl: 'https://sqs.example.com/queue',
273
+ siteId: 'site-1',
274
+ unset: ['coveredByDomainWide'],
275
+ updatedBy: 'system',
276
+ log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
277
+ };
278
+
279
+ await saveSuggestions(dataAccess, items, queueContext);
280
+
281
+ const [, payload] = queueContext.sqs.sendMessage.firstCall.args;
282
+ expect(payload.set).to.be.undefined;
283
+ expect(payload.unset).to.deep.equal(['coveredByDomainWide']);
284
+ });
285
+
286
+ it('falls back to direct save when queueUrl is not configured', async () => {
287
+ const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
288
+ const items = Array.from({ length: 1701 }, (_, i) => ({
289
+ getId: () => `sugg-${i}`,
290
+ save: sinon.stub().resolves(),
291
+ }));
292
+ const queueContext = {
293
+ sqs: { sendMessage: sinon.stub().resolves() },
294
+ queueUrl: undefined,
295
+ siteId: 'site-1',
296
+ updatedBy: 'system',
297
+ log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
298
+ };
299
+
300
+ await saveSuggestions(dataAccess, items, queueContext);
301
+
302
+ expect(queueContext.sqs.sendMessage.called).to.be.false;
303
+ expect(queueContext.log.warn.calledOnce).to.be.true;
304
+ expect(queueContext.log.warn.firstCall.args[0]).to.include('SQS/IMPORT_WORKER_QUEUE_URL not configured');
305
+ expect(items[0].save.calledOnce).to.be.true;
306
+ expect(items[1700].save.calledOnce).to.be.true;
307
+ expect(dataAccess.Suggestion.saveMany.called).to.be.false;
308
+ });
309
+
310
+ it('falls back to direct save when sqs is not configured', async () => {
311
+ const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
312
+ const items = Array.from({ length: 1701 }, (_, i) => ({
313
+ getId: () => `sugg-${i}`,
314
+ save: sinon.stub().resolves(),
315
+ }));
316
+ const queueContext = {
317
+ sqs: undefined,
318
+ queueUrl: 'https://sqs.example.com/queue',
319
+ siteId: 'site-1',
320
+ updatedBy: 'system',
321
+ log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
322
+ };
323
+
324
+ await saveSuggestions(dataAccess, items, queueContext);
325
+
326
+ expect(queueContext.log.warn.calledOnce).to.be.true;
327
+ expect(queueContext.log.warn.firstCall.args[0]).to.include('SQS/IMPORT_WORKER_QUEUE_URL not configured');
328
+ expect(items[0].save.calledOnce).to.be.true;
329
+ });
330
+
331
+ it('falls back to direct save when sqs.sendMessage rejects', async () => {
332
+ const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
333
+ const items = Array.from({ length: 1701 }, (_, i) => ({
334
+ getId: () => `sugg-${i}`,
335
+ save: sinon.stub().resolves(),
336
+ }));
337
+ const queueContext = {
338
+ sqs: { sendMessage: sinon.stub().rejects(new Error('SQS unavailable')) },
339
+ queueUrl: 'https://sqs.example.com/queue',
340
+ siteId: 'site-1',
341
+ updatedBy: 'system',
342
+ log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
343
+ };
344
+
345
+ await saveSuggestions(dataAccess, items, queueContext);
346
+
347
+ expect(queueContext.log.error.calledOnce).to.be.true;
348
+ expect(queueContext.log.error.firstCall.args[0]).to.include('Failed to queue bulk update');
349
+ expect(queueContext.log.error.firstCall.args[0]).to.include('SQS unavailable');
350
+ expect(items[0].save.calledOnce).to.be.true;
351
+ expect(items[1700].save.calledOnce).to.be.true;
352
+ expect(dataAccess.Suggestion.saveMany.called).to.be.false;
353
+ });
354
+
355
+ it('throws when the direct-save fallback also fails after a queue failure', async () => {
356
+ const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
357
+ const items = Array.from({ length: 1701 }, (_, i) => ({
358
+ getId: () => `sugg-${i}`,
359
+ save: i === 0
360
+ ? sinon.stub().rejects(new Error('DB error'))
361
+ : sinon.stub().resolves(),
362
+ }));
363
+ const queueContext = {
364
+ sqs: { sendMessage: sinon.stub().rejects(new Error('SQS unavailable')) },
365
+ queueUrl: 'https://sqs.example.com/queue',
366
+ siteId: 'site-1',
367
+ updatedBy: 'system',
368
+ log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
369
+ };
370
+
371
+ try {
372
+ await saveSuggestions(dataAccess, items, queueContext);
373
+ expect.fail('should have thrown');
374
+ } catch (error) {
375
+ expect(error.message).to.include('1 of 1701 suggestions failed');
376
+ expect(error.message).to.include('DB error');
377
+ }
378
+ });
226
379
  });
227
380
  });