@adobe/spacecat-shared-tokowaka-client 1.20.0 → 1.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/package.json +1 -1
- package/src/index.js +29 -3
- package/src/utils/suggestion-utils.js +48 -5
- package/test/index.test.js +160 -0
- package/test/utils/suggestion-utils.test.js +154 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
## [@adobe/spacecat-shared-tokowaka-client-v1.21.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.20.0...@adobe/spacecat-shared-tokowaka-client-v1.21.0) (2026-07-14)
|
|
2
|
+
|
|
3
|
+
### Features
|
|
4
|
+
|
|
5
|
+
* **tokowaka-client:** offload covered-marking saves to import worker queue above threshold ([#1802](https://github.com/adobe/spacecat-shared/issues/1802)) ([ad1fe43](https://github.com/adobe/spacecat-shared/commit/ad1fe433b4628c38ef0d2d4f4008c2932b5981e0))
|
|
6
|
+
|
|
1
7
|
## [@adobe/spacecat-shared-tokowaka-client-v1.20.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.19.1...@adobe/spacecat-shared-tokowaka-client-v1.20.0) (2026-06-27)
|
|
2
8
|
|
|
3
9
|
### Features
|
package/package.json
CHANGED
package/src/index.js
CHANGED
|
@@ -50,6 +50,7 @@ import {
|
|
|
50
50
|
|
|
51
51
|
export { FastlyKVClient } from './fastly-kv-client.js';
|
|
52
52
|
export { calculateForwardedHost } from './utils/custom-html-utils.js';
|
|
53
|
+
export { SUGGESTION_BULK_UPDATE_TYPE } from './utils/suggestion-utils.js';
|
|
53
54
|
|
|
54
55
|
// CloudFront control-plane (free functions + constants).
|
|
55
56
|
export {
|
|
@@ -88,6 +89,7 @@ class TokowakaClient {
|
|
|
88
89
|
const {
|
|
89
90
|
TOKOWAKA_SITE_CONFIG_BUCKET: bucketName,
|
|
90
91
|
TOKOWAKA_PREVIEW_BUCKET: previewBucketName,
|
|
92
|
+
IMPORT_WORKER_QUEUE_URL: importWorkerQueueUrl,
|
|
91
93
|
} = env;
|
|
92
94
|
|
|
93
95
|
if (context.tokowakaClient) {
|
|
@@ -100,6 +102,8 @@ class TokowakaClient {
|
|
|
100
102
|
s3Client: s3?.s3Client ?? context.s3Client,
|
|
101
103
|
env,
|
|
102
104
|
dataAccess: context.dataAccess,
|
|
105
|
+
sqs: context.sqs,
|
|
106
|
+
importWorkerQueueUrl,
|
|
103
107
|
}, log);
|
|
104
108
|
context.tokowakaClient = client;
|
|
105
109
|
return client;
|
|
@@ -114,10 +118,13 @@ class TokowakaClient {
|
|
|
114
118
|
* @param {Object} config.env - Environment variables (for CDN credentials)
|
|
115
119
|
* @param {Object} [config.dataAccess] - Data access layer
|
|
116
120
|
* (provides Suggestion.saveMany for batch saves)
|
|
121
|
+
* @param {Object} [config.sqs] - SQS client (sendMessage(queueUrl, payload)). Used to offload
|
|
122
|
+
* very large covered-suggestion saves to the import worker instead of saving them directly.
|
|
123
|
+
* @param {string} [config.importWorkerQueueUrl] - Import worker SQS queue URL.
|
|
117
124
|
* @param {Object} log - Logger instance
|
|
118
125
|
*/
|
|
119
126
|
constructor({
|
|
120
|
-
bucketName, previewBucketName, s3Client, env = {}, dataAccess,
|
|
127
|
+
bucketName, previewBucketName, s3Client, env = {}, dataAccess, sqs, importWorkerQueueUrl,
|
|
121
128
|
}, log) {
|
|
122
129
|
this.log = log;
|
|
123
130
|
|
|
@@ -134,6 +141,8 @@ class TokowakaClient {
|
|
|
134
141
|
this.s3Client = s3Client;
|
|
135
142
|
this.env = env;
|
|
136
143
|
this.dataAccess = dataAccess;
|
|
144
|
+
this.sqs = sqs;
|
|
145
|
+
this.importWorkerQueueUrl = importWorkerQueueUrl;
|
|
137
146
|
|
|
138
147
|
this.mapperRegistry = new MapperRegistry(log);
|
|
139
148
|
this.cdnClientRegistry = new CdnClientRegistry(env, log);
|
|
@@ -1077,7 +1086,14 @@ class TokowakaClient {
|
|
|
1077
1086
|
this.log.info(`[edge-rollback] Cleaning ${covered.length} covered suggestion(s) for pattern ${suggestion.getId()} (isDomainWide=${isDomainWide}, fallback=${coveredFallback})`);
|
|
1078
1087
|
}
|
|
1079
1088
|
// eslint-disable-next-line no-await-in-loop, max-len
|
|
1080
|
-
await cleanupCoveredSuggestions(this.dataAccess, covered, coveredFallback, updatedBy, fieldsToStrip, this.log
|
|
1089
|
+
await cleanupCoveredSuggestions(this.dataAccess, covered, coveredFallback, updatedBy, fieldsToStrip, this.log, {
|
|
1090
|
+
sqs: this.sqs,
|
|
1091
|
+
queueUrl: this.importWorkerQueueUrl,
|
|
1092
|
+
siteId: site.getId(),
|
|
1093
|
+
unset: fieldsToStrip,
|
|
1094
|
+
updatedBy: updatedBy ?? coveredFallback,
|
|
1095
|
+
log: this.log,
|
|
1096
|
+
});
|
|
1081
1097
|
}
|
|
1082
1098
|
}
|
|
1083
1099
|
}
|
|
@@ -1504,6 +1520,7 @@ class TokowakaClient {
|
|
|
1504
1520
|
* @param {Array} allSuggestions - Full opportunity suggestion list
|
|
1505
1521
|
* @param {string} updatedBy
|
|
1506
1522
|
* @param {Array} coveredSuggestions - Accumulator (mutated)
|
|
1523
|
+
* @param {string} siteId
|
|
1507
1524
|
* @returns {Promise<void>} Throws on metaconfig upload failure
|
|
1508
1525
|
* @private
|
|
1509
1526
|
*/
|
|
@@ -1516,6 +1533,7 @@ class TokowakaClient {
|
|
|
1516
1533
|
allSuggestions,
|
|
1517
1534
|
updatedBy,
|
|
1518
1535
|
coveredSuggestions,
|
|
1536
|
+
siteId,
|
|
1519
1537
|
) {
|
|
1520
1538
|
const data = suggestion.getData();
|
|
1521
1539
|
const coverageField = data?.isDomainWide ? 'coveredByDomainWide' : 'coveredByPattern';
|
|
@@ -1586,7 +1604,14 @@ class TokowakaClient {
|
|
|
1586
1604
|
cs.setUpdatedBy(updatedBy);
|
|
1587
1605
|
});
|
|
1588
1606
|
try {
|
|
1589
|
-
await saveSuggestions(this.dataAccess, covered
|
|
1607
|
+
await saveSuggestions(this.dataAccess, covered, {
|
|
1608
|
+
sqs: this.sqs,
|
|
1609
|
+
queueUrl: this.importWorkerQueueUrl,
|
|
1610
|
+
siteId,
|
|
1611
|
+
set: { [coverageField]: suggestion.getId() },
|
|
1612
|
+
updatedBy,
|
|
1613
|
+
log: this.log,
|
|
1614
|
+
});
|
|
1590
1615
|
coveredSuggestions.push(...covered);
|
|
1591
1616
|
// eslint-disable-next-line max-len
|
|
1592
1617
|
this.log.info(`[edge-deploy] Marked ${covered.length} suggestions as ${coverageField}=${suggestion.getId()}`);
|
|
@@ -1679,6 +1704,7 @@ class TokowakaClient {
|
|
|
1679
1704
|
allSuggestions,
|
|
1680
1705
|
updatedBy,
|
|
1681
1706
|
coveredSuggestions,
|
|
1707
|
+
site.getId(),
|
|
1682
1708
|
);
|
|
1683
1709
|
deployedPatternSuggestions.push(suggestion);
|
|
1684
1710
|
} catch (error) {
|
|
@@ -92,26 +92,68 @@ export function filterEligibleSuggestions(suggestions, mapper) {
|
|
|
92
92
|
return { eligible, ineligible };
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
+
// Import worker job type for a bulk suggestion field update (agnostic set/unset by id).
|
|
96
|
+
export const SUGGESTION_BULK_UPDATE_TYPE = 'suggestion-bulk-update';
|
|
97
|
+
|
|
95
98
|
/**
|
|
96
99
|
* Batch-saves suggestions using the optimal strategy based on count.
|
|
97
100
|
*
|
|
98
101
|
* - <= 1700: sequential chunked upsert via saveMany (chunkSize 25).
|
|
99
102
|
* ~4s for 1700 suggestions from the DB layer, but ~11s observed end-to-end
|
|
100
103
|
* from the API due to serialization, network, and Lambda overhead.
|
|
101
|
-
* - > 1700: parallel individual .save() via Promise.allSettled to
|
|
102
|
-
* sequential chunk bottleneck at scale.
|
|
104
|
+
* - > 1700 with no queueContext: parallel individual .save() via Promise.allSettled to
|
|
105
|
+
* avoid sequential chunk bottleneck at scale.
|
|
106
|
+
* - > 1700 with queueContext: enqueues a SUGGESTION_BULK_UPDATE_TYPE job to the import
|
|
107
|
+
* worker instead of saving here (the worker fetches by id and applies
|
|
108
|
+
* queueContext.set/unset itself). Falls back to the Promise.allSettled path above
|
|
109
|
+
* if sqs/queueUrl aren't configured or sendMessage fails, so a queue outage
|
|
110
|
+
* degrades rather than silently drops the save.
|
|
103
111
|
*
|
|
104
112
|
* @param {Object} dataAccess - Data access layer
|
|
105
113
|
* @param {Array} suggestions - Suggestion entities to save
|
|
114
|
+
* @param {Object} [queueContext] - sqs, queueUrl, siteId, set/unset,
|
|
115
|
+
* updatedBy (already resolved), log
|
|
106
116
|
* @returns {Promise<void>}
|
|
107
117
|
*/
|
|
108
118
|
const PARALLEL_SAVE_THRESHOLD = 1700;
|
|
109
119
|
|
|
110
|
-
export async function saveSuggestions(dataAccess, suggestions) {
|
|
120
|
+
export async function saveSuggestions(dataAccess, suggestions, queueContext) {
|
|
111
121
|
if (suggestions.length === 0) {
|
|
112
122
|
return;
|
|
113
123
|
}
|
|
114
124
|
if (suggestions.length > PARALLEL_SAVE_THRESHOLD) {
|
|
125
|
+
if (queueContext) {
|
|
126
|
+
const {
|
|
127
|
+
sqs, queueUrl, siteId, set, unset, updatedBy, log,
|
|
128
|
+
} = queueContext;
|
|
129
|
+
|
|
130
|
+
if (queueUrl && sqs) {
|
|
131
|
+
try {
|
|
132
|
+
await sqs.sendMessage(queueUrl, {
|
|
133
|
+
type: SUGGESTION_BULK_UPDATE_TYPE,
|
|
134
|
+
siteId,
|
|
135
|
+
suggestionIds: suggestions.map((s) => s.getId()),
|
|
136
|
+
...(set && { set }),
|
|
137
|
+
...(unset && { unset }),
|
|
138
|
+
updatedBy,
|
|
139
|
+
});
|
|
140
|
+
log.info(
|
|
141
|
+
`[suggestion-bulk-update] Queued bulk update for ${suggestions.length} suggestion(s)`,
|
|
142
|
+
);
|
|
143
|
+
return;
|
|
144
|
+
} catch (error) {
|
|
145
|
+
log.error(
|
|
146
|
+
`[suggestion-bulk-update] Failed to queue bulk update, falling back to direct save: ${error.message}`,
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
} else {
|
|
150
|
+
log.warn(
|
|
151
|
+
'[suggestion-bulk-update] SQS/IMPORT_WORKER_QUEUE_URL not configured; '
|
|
152
|
+
+ `falling back to direct save for ${suggestions.length} suggestion(s)`,
|
|
153
|
+
);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
115
157
|
const results = await Promise.allSettled(suggestions.map((s) => s.save()));
|
|
116
158
|
const failed = results.filter((r) => r.status === 'rejected');
|
|
117
159
|
if (failed.length > 0) {
|
|
@@ -150,10 +192,11 @@ export function stripSuggestion(suggestion, actorFallback, updatedBy) {
|
|
|
150
192
|
* @param {string|undefined} updatedBy - Explicit actor
|
|
151
193
|
* @param {string[]} fieldsToStrip - Specific fields to remove
|
|
152
194
|
* @param {Object} log - Logger instance
|
|
195
|
+
* @param {Object} [queueContext] - Passed through to saveSuggestions unchanged
|
|
153
196
|
* @returns {Promise<void>}
|
|
154
197
|
*/
|
|
155
198
|
// eslint-disable-next-line max-len
|
|
156
|
-
export async function cleanupCoveredSuggestions(dataAccess, covered, actorFallback, updatedBy, fieldsToStrip, log) {
|
|
199
|
+
export async function cleanupCoveredSuggestions(dataAccess, covered, actorFallback, updatedBy, fieldsToStrip, log, queueContext) {
|
|
157
200
|
if (covered.length === 0) {
|
|
158
201
|
return;
|
|
159
202
|
}
|
|
@@ -162,7 +205,7 @@ export async function cleanupCoveredSuggestions(dataAccess, covered, actorFallba
|
|
|
162
205
|
cs.setUpdatedBy(updatedBy ?? actorFallback);
|
|
163
206
|
});
|
|
164
207
|
try {
|
|
165
|
-
await saveSuggestions(dataAccess, covered);
|
|
208
|
+
await saveSuggestions(dataAccess, covered, queueContext);
|
|
166
209
|
} catch (error) {
|
|
167
210
|
// eslint-disable-next-line max-len
|
|
168
211
|
log.error(`[edge-rollback-failed] Failed to clean ${covered.length} covered suggestion(s): ${error.message}`);
|
package/test/index.test.js
CHANGED
|
@@ -3197,6 +3197,57 @@ describe('TokowakaClient', () => {
|
|
|
3197
3197
|
);
|
|
3198
3198
|
});
|
|
3199
3199
|
|
|
3200
|
+
it('enqueues a suggestion-bulk-update job instead of saveMany when cleaning up more than the threshold (domain-wide rollback)', async () => {
|
|
3201
|
+
const prerenderOpportunity = { getId: () => 'opp-dw', getType: () => 'prerender' };
|
|
3202
|
+
const dwSuggestion = {
|
|
3203
|
+
getId: () => 'dw-1',
|
|
3204
|
+
getData: () => ({
|
|
3205
|
+
isDomainWide: true,
|
|
3206
|
+
allowedRegexPatterns: ['/*'],
|
|
3207
|
+
edgeDeployed: Date.now(),
|
|
3208
|
+
}),
|
|
3209
|
+
setData: sinon.stub(),
|
|
3210
|
+
setUpdatedBy: sinon.stub(),
|
|
3211
|
+
save: sinon.stub().resolves(),
|
|
3212
|
+
};
|
|
3213
|
+
const covered = Array.from({ length: 1701 }, (_, i) => ({
|
|
3214
|
+
getId: () => `covered-${i}`,
|
|
3215
|
+
getData: () => ({ url: `https://example.com/page${i}`, coveredByDomainWide: 'dw-1' }),
|
|
3216
|
+
setData: sinon.stub(),
|
|
3217
|
+
setUpdatedBy: sinon.stub(),
|
|
3218
|
+
save: sinon.stub().resolves(),
|
|
3219
|
+
}));
|
|
3220
|
+
|
|
3221
|
+
sinon.stub(client, 'fetchMetaconfig').resolves({
|
|
3222
|
+
siteId: 'site-123',
|
|
3223
|
+
prerender: { allowList: ['/*'] },
|
|
3224
|
+
});
|
|
3225
|
+
sinon.stub(client, 'uploadMetaconfig').resolves();
|
|
3226
|
+
client.sqs = { sendMessage: sinon.stub().resolves() };
|
|
3227
|
+
client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
|
|
3228
|
+
|
|
3229
|
+
await client.rollbackSuggestions(
|
|
3230
|
+
mockSite,
|
|
3231
|
+
prerenderOpportunity,
|
|
3232
|
+
[dwSuggestion],
|
|
3233
|
+
{ allSuggestions: [dwSuggestion, ...covered], updatedBy: 'test@example.com' },
|
|
3234
|
+
);
|
|
3235
|
+
|
|
3236
|
+
expect(client.sqs.sendMessage).to.have.been.calledOnce;
|
|
3237
|
+
const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
|
|
3238
|
+
expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
|
|
3239
|
+
expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-123' });
|
|
3240
|
+
expect(payload).to.not.have.property('opportunityId');
|
|
3241
|
+
expect(payload.suggestionIds).to.have.length(1701);
|
|
3242
|
+
expect(payload.unset).to.deep.equal(['coveredByDomainWide']);
|
|
3243
|
+
expect(payload.updatedBy).to.equal('test@example.com');
|
|
3244
|
+
// Not saved directly — the enqueue replaces the saveMany call for this batch.
|
|
3245
|
+
expect(client.dataAccess.Suggestion.saveMany).to.not.have.been.calledWith(
|
|
3246
|
+
sinon.match((arr) => arr.length === 1701),
|
|
3247
|
+
sinon.match.any,
|
|
3248
|
+
);
|
|
3249
|
+
});
|
|
3250
|
+
|
|
3200
3251
|
it('cleans up covered suggestions (coveredByPattern) when rolling back a path-level pattern', async () => {
|
|
3201
3252
|
const prerenderOpportunity = { getId: () => 'opp-p', getType: () => 'prerender' };
|
|
3202
3253
|
const pathSuggestion = {
|
|
@@ -3240,6 +3291,54 @@ describe('TokowakaClient', () => {
|
|
|
3240
3291
|
expect(coveredData).to.not.have.property('coveredByPattern');
|
|
3241
3292
|
});
|
|
3242
3293
|
|
|
3294
|
+
it('enqueues a suggestion-bulk-update job instead of saveMany when cleaning up more than the threshold (path-level rollback)', async () => {
|
|
3295
|
+
const prerenderOpportunity = { getId: () => 'opp-p', getType: () => 'prerender' };
|
|
3296
|
+
const pathSuggestion = {
|
|
3297
|
+
getId: () => 'path-1',
|
|
3298
|
+
getData: () => ({
|
|
3299
|
+
allowedRegexPatterns: ['/products/*'],
|
|
3300
|
+
edgeDeployed: Date.now(),
|
|
3301
|
+
}),
|
|
3302
|
+
setData: sinon.stub(),
|
|
3303
|
+
setUpdatedBy: sinon.stub(),
|
|
3304
|
+
save: sinon.stub().resolves(),
|
|
3305
|
+
};
|
|
3306
|
+
const covered = Array.from({ length: 1701 }, (_, i) => ({
|
|
3307
|
+
getId: () => `covered-${i}`,
|
|
3308
|
+
getData: () => ({
|
|
3309
|
+
url: `https://example.com/products/item-${i}`,
|
|
3310
|
+
edgeDeployed: Date.now(),
|
|
3311
|
+
coveredByPattern: 'path-1',
|
|
3312
|
+
}),
|
|
3313
|
+
setData: sinon.stub(),
|
|
3314
|
+
setUpdatedBy: sinon.stub(),
|
|
3315
|
+
save: sinon.stub().resolves(),
|
|
3316
|
+
}));
|
|
3317
|
+
|
|
3318
|
+
sinon.stub(client, 'fetchMetaconfig').resolves({
|
|
3319
|
+
siteId: 'site-123',
|
|
3320
|
+
prerender: { allowList: ['/products/*'] },
|
|
3321
|
+
});
|
|
3322
|
+
sinon.stub(client, 'uploadMetaconfig').resolves();
|
|
3323
|
+
client.sqs = { sendMessage: sinon.stub().resolves() };
|
|
3324
|
+
client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
|
|
3325
|
+
|
|
3326
|
+
await client.rollbackSuggestions(
|
|
3327
|
+
mockSite,
|
|
3328
|
+
prerenderOpportunity,
|
|
3329
|
+
[pathSuggestion],
|
|
3330
|
+
{ allSuggestions: [pathSuggestion, ...covered] },
|
|
3331
|
+
);
|
|
3332
|
+
|
|
3333
|
+
expect(client.sqs.sendMessage).to.have.been.calledOnce;
|
|
3334
|
+
const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
|
|
3335
|
+
expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
|
|
3336
|
+
expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-123' });
|
|
3337
|
+
expect(payload).to.not.have.property('opportunityId');
|
|
3338
|
+
expect(payload.suggestionIds).to.have.length(1701);
|
|
3339
|
+
expect(payload.unset).to.deep.equal(['edgeDeployed', 'tokowakaDeployed', 'coveredByPattern']);
|
|
3340
|
+
});
|
|
3341
|
+
|
|
3243
3342
|
it('uses domain-wide-rollback fallback for covered suggestions when updatedBy is not provided (domain-wide parent)', async () => {
|
|
3244
3343
|
const prerenderOpportunity = { getId: () => 'opp-dw', getType: () => 'prerender' };
|
|
3245
3344
|
const dwSuggestion = {
|
|
@@ -5965,6 +6064,37 @@ describe('TokowakaClient', () => {
|
|
|
5965
6064
|
expect(covered.getData()).to.have.property('coveredByDomainWide', 'dw1');
|
|
5966
6065
|
});
|
|
5967
6066
|
|
|
6067
|
+
it('enqueues a suggestion-bulk-update job instead of saveMany when covered count exceeds the threshold (domain-wide)', async () => {
|
|
6068
|
+
const dw = makeSuggestion('dw1', {
|
|
6069
|
+
isDomainWide: true,
|
|
6070
|
+
allowedRegexPatterns: ['^https://example\\.com/.*'],
|
|
6071
|
+
});
|
|
6072
|
+
const covered = Array.from({ length: 1701 }, (_, i) => makeSuggestion(
|
|
6073
|
+
`covered-${i}`,
|
|
6074
|
+
{ url: `https://example.com/page${i}` },
|
|
6075
|
+
));
|
|
6076
|
+
|
|
6077
|
+
fetchMetaconfigStub.resolves({ siteId: 'site-123' });
|
|
6078
|
+
client.sqs = { sendMessage: sinon.stub().resolves() };
|
|
6079
|
+
client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
|
|
6080
|
+
|
|
6081
|
+
const result = await client.deployToEdge({
|
|
6082
|
+
site: mockSite,
|
|
6083
|
+
opportunity: mockOpportunity,
|
|
6084
|
+
targetSuggestions: [dw],
|
|
6085
|
+
allSuggestions: [dw, ...covered],
|
|
6086
|
+
});
|
|
6087
|
+
|
|
6088
|
+
expect(result.coveredSuggestions).to.have.length(1701);
|
|
6089
|
+
expect(client.sqs.sendMessage).to.have.been.calledOnce;
|
|
6090
|
+
const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
|
|
6091
|
+
expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
|
|
6092
|
+
expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-123' });
|
|
6093
|
+
expect(payload).to.not.have.property('opportunityId');
|
|
6094
|
+
expect(payload.suggestionIds).to.have.length(1701);
|
|
6095
|
+
expect(payload.set).to.deep.equal({ coveredByDomainWide: 'dw1' });
|
|
6096
|
+
});
|
|
6097
|
+
|
|
5968
6098
|
it('should not mark already-domain-wide suggestions as covered', async () => {
|
|
5969
6099
|
const dw1 = makeSuggestion('dw1', {
|
|
5970
6100
|
isDomainWide: true,
|
|
@@ -6467,6 +6597,36 @@ describe('TokowakaClient', () => {
|
|
|
6467
6597
|
expect(result.coveredSuggestions).to.not.include(urlElsewhere);
|
|
6468
6598
|
});
|
|
6469
6599
|
|
|
6600
|
+
it('enqueues a suggestion-bulk-update job instead of saveMany when covered count exceeds the threshold (path-level)', async () => {
|
|
6601
|
+
const path = makeSuggestion('p1', {
|
|
6602
|
+
allowedRegexPatterns: ['/products/*'],
|
|
6603
|
+
});
|
|
6604
|
+
const covered = Array.from({ length: 1701 }, (_, i) => makeSuggestion(
|
|
6605
|
+
`covered-${i}`,
|
|
6606
|
+
{ url: `https://example.com/products/item-${i}` },
|
|
6607
|
+
));
|
|
6608
|
+
|
|
6609
|
+
fetchMetaconfigStub.resolves({ siteId: 'site-123' });
|
|
6610
|
+
client.sqs = { sendMessage: sinon.stub().resolves() };
|
|
6611
|
+
client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
|
|
6612
|
+
|
|
6613
|
+
const result = await client.deployToEdge({
|
|
6614
|
+
site: mockSite,
|
|
6615
|
+
opportunity: mockOpportunity,
|
|
6616
|
+
targetSuggestions: [path],
|
|
6617
|
+
allSuggestions: [path, ...covered],
|
|
6618
|
+
});
|
|
6619
|
+
|
|
6620
|
+
expect(result.coveredSuggestions).to.have.length(1701);
|
|
6621
|
+
expect(client.sqs.sendMessage).to.have.been.calledOnce;
|
|
6622
|
+
const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
|
|
6623
|
+
expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
|
|
6624
|
+
expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-123' });
|
|
6625
|
+
expect(payload).to.not.have.property('opportunityId');
|
|
6626
|
+
expect(payload.suggestionIds).to.have.length(1701);
|
|
6627
|
+
expect(payload.set).to.deep.equal({ coveredByPattern: 'p1' });
|
|
6628
|
+
});
|
|
6629
|
+
|
|
6470
6630
|
it('path deploy marks as failed with statusCode 500 when metaconfig upload fails', async () => {
|
|
6471
6631
|
const path = makeSuggestion('p1', {
|
|
6472
6632
|
allowedRegexPatterns: ['/products/*'],
|
|
@@ -12,7 +12,12 @@
|
|
|
12
12
|
|
|
13
13
|
import { expect } from 'chai';
|
|
14
14
|
import sinon from 'sinon';
|
|
15
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
groupSuggestionsByUrlPath,
|
|
17
|
+
filterEligibleSuggestions,
|
|
18
|
+
saveSuggestions,
|
|
19
|
+
SUGGESTION_BULK_UPDATE_TYPE,
|
|
20
|
+
} from '../../src/utils/suggestion-utils.js';
|
|
16
21
|
|
|
17
22
|
describe('Suggestion Utils', () => {
|
|
18
23
|
describe('groupSuggestionsByUrlPath', () => {
|
|
@@ -223,5 +228,153 @@ describe('Suggestion Utils', () => {
|
|
|
223
228
|
expect(error.message).to.include('DB error');
|
|
224
229
|
}
|
|
225
230
|
});
|
|
231
|
+
|
|
232
|
+
it('should enqueue a bulk update job instead of saving when queueContext is provided', async () => {
|
|
233
|
+
const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
|
|
234
|
+
const items = Array.from({ length: 1701 }, (_, i) => ({
|
|
235
|
+
getId: () => `sugg-${i}`,
|
|
236
|
+
save: sinon.stub().resolves(),
|
|
237
|
+
}));
|
|
238
|
+
const queueContext = {
|
|
239
|
+
sqs: { sendMessage: sinon.stub().resolves() },
|
|
240
|
+
queueUrl: 'https://sqs.example.com/queue',
|
|
241
|
+
siteId: 'site-1',
|
|
242
|
+
set: { coveredByDomainWide: 'pattern-sugg-id' },
|
|
243
|
+
updatedBy: 'system',
|
|
244
|
+
log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
await saveSuggestions(dataAccess, items, queueContext);
|
|
248
|
+
|
|
249
|
+
expect(dataAccess.Suggestion.saveMany.called).to.be.false;
|
|
250
|
+
expect(items[0].save.called).to.be.false;
|
|
251
|
+
expect(queueContext.sqs.sendMessage.calledOnce).to.be.true;
|
|
252
|
+
const [queueUrl, payload] = queueContext.sqs.sendMessage.firstCall.args;
|
|
253
|
+
expect(queueUrl).to.equal('https://sqs.example.com/queue');
|
|
254
|
+
expect(payload).to.deep.equal({
|
|
255
|
+
type: SUGGESTION_BULK_UPDATE_TYPE,
|
|
256
|
+
siteId: 'site-1',
|
|
257
|
+
suggestionIds: items.map((s) => s.getId()),
|
|
258
|
+
set: { coveredByDomainWide: 'pattern-sugg-id' },
|
|
259
|
+
updatedBy: 'system',
|
|
260
|
+
});
|
|
261
|
+
expect(queueContext.log.info.calledOnce).to.be.true;
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
it('should include unset instead of set when queueContext.unset is provided', async () => {
|
|
265
|
+
const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
|
|
266
|
+
const items = Array.from({ length: 1701 }, (_, i) => ({
|
|
267
|
+
getId: () => `sugg-${i}`,
|
|
268
|
+
save: sinon.stub().resolves(),
|
|
269
|
+
}));
|
|
270
|
+
const queueContext = {
|
|
271
|
+
sqs: { sendMessage: sinon.stub().resolves() },
|
|
272
|
+
queueUrl: 'https://sqs.example.com/queue',
|
|
273
|
+
siteId: 'site-1',
|
|
274
|
+
unset: ['coveredByDomainWide'],
|
|
275
|
+
updatedBy: 'system',
|
|
276
|
+
log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
|
|
277
|
+
};
|
|
278
|
+
|
|
279
|
+
await saveSuggestions(dataAccess, items, queueContext);
|
|
280
|
+
|
|
281
|
+
const [, payload] = queueContext.sqs.sendMessage.firstCall.args;
|
|
282
|
+
expect(payload.set).to.be.undefined;
|
|
283
|
+
expect(payload.unset).to.deep.equal(['coveredByDomainWide']);
|
|
284
|
+
});
|
|
285
|
+
|
|
286
|
+
it('falls back to direct save when queueUrl is not configured', async () => {
|
|
287
|
+
const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
|
|
288
|
+
const items = Array.from({ length: 1701 }, (_, i) => ({
|
|
289
|
+
getId: () => `sugg-${i}`,
|
|
290
|
+
save: sinon.stub().resolves(),
|
|
291
|
+
}));
|
|
292
|
+
const queueContext = {
|
|
293
|
+
sqs: { sendMessage: sinon.stub().resolves() },
|
|
294
|
+
queueUrl: undefined,
|
|
295
|
+
siteId: 'site-1',
|
|
296
|
+
updatedBy: 'system',
|
|
297
|
+
log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
|
|
298
|
+
};
|
|
299
|
+
|
|
300
|
+
await saveSuggestions(dataAccess, items, queueContext);
|
|
301
|
+
|
|
302
|
+
expect(queueContext.sqs.sendMessage.called).to.be.false;
|
|
303
|
+
expect(queueContext.log.warn.calledOnce).to.be.true;
|
|
304
|
+
expect(queueContext.log.warn.firstCall.args[0]).to.include('SQS/IMPORT_WORKER_QUEUE_URL not configured');
|
|
305
|
+
expect(items[0].save.calledOnce).to.be.true;
|
|
306
|
+
expect(items[1700].save.calledOnce).to.be.true;
|
|
307
|
+
expect(dataAccess.Suggestion.saveMany.called).to.be.false;
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
it('falls back to direct save when sqs is not configured', async () => {
|
|
311
|
+
const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
|
|
312
|
+
const items = Array.from({ length: 1701 }, (_, i) => ({
|
|
313
|
+
getId: () => `sugg-${i}`,
|
|
314
|
+
save: sinon.stub().resolves(),
|
|
315
|
+
}));
|
|
316
|
+
const queueContext = {
|
|
317
|
+
sqs: undefined,
|
|
318
|
+
queueUrl: 'https://sqs.example.com/queue',
|
|
319
|
+
siteId: 'site-1',
|
|
320
|
+
updatedBy: 'system',
|
|
321
|
+
log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
|
|
322
|
+
};
|
|
323
|
+
|
|
324
|
+
await saveSuggestions(dataAccess, items, queueContext);
|
|
325
|
+
|
|
326
|
+
expect(queueContext.log.warn.calledOnce).to.be.true;
|
|
327
|
+
expect(queueContext.log.warn.firstCall.args[0]).to.include('SQS/IMPORT_WORKER_QUEUE_URL not configured');
|
|
328
|
+
expect(items[0].save.calledOnce).to.be.true;
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
it('falls back to direct save when sqs.sendMessage rejects', async () => {
|
|
332
|
+
const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
|
|
333
|
+
const items = Array.from({ length: 1701 }, (_, i) => ({
|
|
334
|
+
getId: () => `sugg-${i}`,
|
|
335
|
+
save: sinon.stub().resolves(),
|
|
336
|
+
}));
|
|
337
|
+
const queueContext = {
|
|
338
|
+
sqs: { sendMessage: sinon.stub().rejects(new Error('SQS unavailable')) },
|
|
339
|
+
queueUrl: 'https://sqs.example.com/queue',
|
|
340
|
+
siteId: 'site-1',
|
|
341
|
+
updatedBy: 'system',
|
|
342
|
+
log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
|
|
343
|
+
};
|
|
344
|
+
|
|
345
|
+
await saveSuggestions(dataAccess, items, queueContext);
|
|
346
|
+
|
|
347
|
+
expect(queueContext.log.error.calledOnce).to.be.true;
|
|
348
|
+
expect(queueContext.log.error.firstCall.args[0]).to.include('Failed to queue bulk update');
|
|
349
|
+
expect(queueContext.log.error.firstCall.args[0]).to.include('SQS unavailable');
|
|
350
|
+
expect(items[0].save.calledOnce).to.be.true;
|
|
351
|
+
expect(items[1700].save.calledOnce).to.be.true;
|
|
352
|
+
expect(dataAccess.Suggestion.saveMany.called).to.be.false;
|
|
353
|
+
});
|
|
354
|
+
|
|
355
|
+
it('throws when the direct-save fallback also fails after a queue failure', async () => {
|
|
356
|
+
const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
|
|
357
|
+
const items = Array.from({ length: 1701 }, (_, i) => ({
|
|
358
|
+
getId: () => `sugg-${i}`,
|
|
359
|
+
save: i === 0
|
|
360
|
+
? sinon.stub().rejects(new Error('DB error'))
|
|
361
|
+
: sinon.stub().resolves(),
|
|
362
|
+
}));
|
|
363
|
+
const queueContext = {
|
|
364
|
+
sqs: { sendMessage: sinon.stub().rejects(new Error('SQS unavailable')) },
|
|
365
|
+
queueUrl: 'https://sqs.example.com/queue',
|
|
366
|
+
siteId: 'site-1',
|
|
367
|
+
updatedBy: 'system',
|
|
368
|
+
log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
|
|
369
|
+
};
|
|
370
|
+
|
|
371
|
+
try {
|
|
372
|
+
await saveSuggestions(dataAccess, items, queueContext);
|
|
373
|
+
expect.fail('should have thrown');
|
|
374
|
+
} catch (error) {
|
|
375
|
+
expect(error.message).to.include('1 of 1701 suggestions failed');
|
|
376
|
+
expect(error.message).to.include('DB error');
|
|
377
|
+
}
|
|
378
|
+
});
|
|
226
379
|
});
|
|
227
380
|
});
|