@adobe/spacecat-shared-tokowaka-client 1.20.0 → 1.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/package.json +1 -1
- package/src/index.js +95 -40
- package/src/utils/suggestion-utils.js +103 -5
- package/test/index.test.js +252 -0
- package/test/utils/suggestion-utils.test.js +154 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
## [@adobe/spacecat-shared-tokowaka-client-v1.21.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.21.0...@adobe/spacecat-shared-tokowaka-client-v1.21.1) (2026-07-15)
|
|
2
|
+
|
|
3
|
+
### Bug Fixes
|
|
4
|
+
|
|
5
|
+
* utility for marking suggestions as covered ([#1810](https://github.com/adobe/spacecat-shared/issues/1810)) ([2709d47](https://github.com/adobe/spacecat-shared/commit/2709d4741bfe2968dfb481ceedf3e283525f5e71))
|
|
6
|
+
|
|
7
|
+
## [@adobe/spacecat-shared-tokowaka-client-v1.21.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.20.0...@adobe/spacecat-shared-tokowaka-client-v1.21.0) (2026-07-14)
|
|
8
|
+
|
|
9
|
+
### Features
|
|
10
|
+
|
|
11
|
+
* **tokowaka-client:** offload covered-marking saves to import worker queue above threshold ([#1802](https://github.com/adobe/spacecat-shared/issues/1802)) ([ad1fe43](https://github.com/adobe/spacecat-shared/commit/ad1fe433b4628c38ef0d2d4f4008c2932b5981e0))
|
|
12
|
+
|
|
1
13
|
## [@adobe/spacecat-shared-tokowaka-client-v1.20.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.19.1...@adobe/spacecat-shared-tokowaka-client-v1.20.0) (2026-06-27)
|
|
2
14
|
|
|
3
15
|
### Features
|
package/package.json
CHANGED
package/src/index.js
CHANGED
|
@@ -27,7 +27,6 @@ import {
|
|
|
27
27
|
} from './utils/s3-utils.js';
|
|
28
28
|
import {
|
|
29
29
|
omitKeys,
|
|
30
|
-
isEdgeDeployableSuggestionStatus,
|
|
31
30
|
isPatternSuggestion,
|
|
32
31
|
groupSuggestionsByUrlPath,
|
|
33
32
|
filterEligibleSuggestions,
|
|
@@ -36,8 +35,8 @@ import {
|
|
|
36
35
|
cleanupCoveredSuggestions,
|
|
37
36
|
classifySuggestions,
|
|
38
37
|
filterBatchCoveredSuggestions,
|
|
38
|
+
findCoveredSuggestions,
|
|
39
39
|
} from './utils/suggestion-utils.js';
|
|
40
|
-
import { buildUrlMatcher } from './utils/pattern-utils.js';
|
|
41
40
|
import { getEffectiveBaseURL } from './utils/site-utils.js';
|
|
42
41
|
import { removePatternFromMetaconfig, addPatternsToMetaconfig } from './utils/metaconfig-utils.js';
|
|
43
42
|
import { fetchHtmlWithWarmup, calculateForwardedHost } from './utils/custom-html-utils.js';
|
|
@@ -50,6 +49,7 @@ import {
|
|
|
50
49
|
|
|
51
50
|
export { FastlyKVClient } from './fastly-kv-client.js';
|
|
52
51
|
export { calculateForwardedHost } from './utils/custom-html-utils.js';
|
|
52
|
+
export { SUGGESTION_BULK_UPDATE_TYPE } from './utils/suggestion-utils.js';
|
|
53
53
|
|
|
54
54
|
// CloudFront control-plane (free functions + constants).
|
|
55
55
|
export {
|
|
@@ -88,6 +88,7 @@ class TokowakaClient {
|
|
|
88
88
|
const {
|
|
89
89
|
TOKOWAKA_SITE_CONFIG_BUCKET: bucketName,
|
|
90
90
|
TOKOWAKA_PREVIEW_BUCKET: previewBucketName,
|
|
91
|
+
IMPORT_WORKER_QUEUE_URL: importWorkerQueueUrl,
|
|
91
92
|
} = env;
|
|
92
93
|
|
|
93
94
|
if (context.tokowakaClient) {
|
|
@@ -100,6 +101,8 @@ class TokowakaClient {
|
|
|
100
101
|
s3Client: s3?.s3Client ?? context.s3Client,
|
|
101
102
|
env,
|
|
102
103
|
dataAccess: context.dataAccess,
|
|
104
|
+
sqs: context.sqs,
|
|
105
|
+
importWorkerQueueUrl,
|
|
103
106
|
}, log);
|
|
104
107
|
context.tokowakaClient = client;
|
|
105
108
|
return client;
|
|
@@ -114,10 +117,13 @@ class TokowakaClient {
|
|
|
114
117
|
* @param {Object} config.env - Environment variables (for CDN credentials)
|
|
115
118
|
* @param {Object} [config.dataAccess] - Data access layer
|
|
116
119
|
* (provides Suggestion.saveMany for batch saves)
|
|
120
|
+
* @param {Object} [config.sqs] - SQS client (sendMessage(queueUrl, payload)). Used to offload
|
|
121
|
+
* very large covered-suggestion saves to the import worker instead of saving them directly.
|
|
122
|
+
* @param {string} [config.importWorkerQueueUrl] - Import worker SQS queue URL.
|
|
117
123
|
* @param {Object} log - Logger instance
|
|
118
124
|
*/
|
|
119
125
|
constructor({
|
|
120
|
-
bucketName, previewBucketName, s3Client, env = {}, dataAccess,
|
|
126
|
+
bucketName, previewBucketName, s3Client, env = {}, dataAccess, sqs, importWorkerQueueUrl,
|
|
121
127
|
}, log) {
|
|
122
128
|
this.log = log;
|
|
123
129
|
|
|
@@ -134,6 +140,8 @@ class TokowakaClient {
|
|
|
134
140
|
this.s3Client = s3Client;
|
|
135
141
|
this.env = env;
|
|
136
142
|
this.dataAccess = dataAccess;
|
|
143
|
+
this.sqs = sqs;
|
|
144
|
+
this.importWorkerQueueUrl = importWorkerQueueUrl;
|
|
137
145
|
|
|
138
146
|
this.mapperRegistry = new MapperRegistry(log);
|
|
139
147
|
this.cdnClientRegistry = new CdnClientRegistry(env, log);
|
|
@@ -1077,7 +1085,14 @@ class TokowakaClient {
|
|
|
1077
1085
|
this.log.info(`[edge-rollback] Cleaning ${covered.length} covered suggestion(s) for pattern ${suggestion.getId()} (isDomainWide=${isDomainWide}, fallback=${coveredFallback})`);
|
|
1078
1086
|
}
|
|
1079
1087
|
// eslint-disable-next-line no-await-in-loop, max-len
|
|
1080
|
-
await cleanupCoveredSuggestions(this.dataAccess, covered, coveredFallback, updatedBy, fieldsToStrip, this.log
|
|
1088
|
+
await cleanupCoveredSuggestions(this.dataAccess, covered, coveredFallback, updatedBy, fieldsToStrip, this.log, {
|
|
1089
|
+
sqs: this.sqs,
|
|
1090
|
+
queueUrl: this.importWorkerQueueUrl,
|
|
1091
|
+
siteId: site.getId(),
|
|
1092
|
+
unset: fieldsToStrip,
|
|
1093
|
+
updatedBy: updatedBy ?? coveredFallback,
|
|
1094
|
+
log: this.log,
|
|
1095
|
+
});
|
|
1081
1096
|
}
|
|
1082
1097
|
}
|
|
1083
1098
|
}
|
|
@@ -1504,6 +1519,7 @@ class TokowakaClient {
|
|
|
1504
1519
|
* @param {Array} allSuggestions - Full opportunity suggestion list
|
|
1505
1520
|
* @param {string} updatedBy
|
|
1506
1521
|
* @param {Array} coveredSuggestions - Accumulator (mutated)
|
|
1522
|
+
* @param {string} siteId
|
|
1507
1523
|
* @returns {Promise<void>} Throws on metaconfig upload failure
|
|
1508
1524
|
* @private
|
|
1509
1525
|
*/
|
|
@@ -1516,6 +1532,7 @@ class TokowakaClient {
|
|
|
1516
1532
|
allSuggestions,
|
|
1517
1533
|
updatedBy,
|
|
1518
1534
|
coveredSuggestions,
|
|
1535
|
+
siteId,
|
|
1519
1536
|
) {
|
|
1520
1537
|
const data = suggestion.getData();
|
|
1521
1538
|
const coverageField = data?.isDomainWide ? 'coveredByDomainWide' : 'coveredByPattern';
|
|
@@ -1541,41 +1558,13 @@ class TokowakaClient {
|
|
|
1541
1558
|
suggestion.setUpdatedBy(updatedBy);
|
|
1542
1559
|
// suggestion.save() is deferred — caller batches saves via saveSuggestions.
|
|
1543
1560
|
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
return m ? [m] : [];
|
|
1552
|
-
});
|
|
1553
|
-
|
|
1554
|
-
if (matchers.length === 0) {
|
|
1555
|
-
return;
|
|
1556
|
-
}
|
|
1557
|
-
|
|
1558
|
-
const covered = allSuggestions.filter((s) => {
|
|
1559
|
-
if (s.getId() === suggestion.getId()) {
|
|
1560
|
-
return false;
|
|
1561
|
-
}
|
|
1562
|
-
if (skippedInBatchIds.has(s.getId())) {
|
|
1563
|
-
return false;
|
|
1564
|
-
}
|
|
1565
|
-
if (!isEdgeDeployableSuggestionStatus(s.getStatus())) {
|
|
1566
|
-
return false;
|
|
1567
|
-
}
|
|
1568
|
-
if (s.getData()?.edgeDeployed) {
|
|
1569
|
-
return false;
|
|
1570
|
-
}
|
|
1571
|
-
// Path-level pattern suggestions (not domain-wide) are fully covered by a
|
|
1572
|
-
// domain-wide deployment. Other pattern suggestions (including other DW ones) are not.
|
|
1573
|
-
if (isPatternSuggestion(s)) {
|
|
1574
|
-
return coverageField === 'coveredByDomainWide' && !s.getData()?.isDomainWide;
|
|
1575
|
-
}
|
|
1576
|
-
const url = s.getData()?.url;
|
|
1577
|
-
return url && matchers.some((match) => match(url));
|
|
1578
|
-
});
|
|
1561
|
+
const covered = findCoveredSuggestions(
|
|
1562
|
+
suggestion,
|
|
1563
|
+
allowedRegexPatterns,
|
|
1564
|
+
allSuggestions,
|
|
1565
|
+
skippedInBatchIds,
|
|
1566
|
+
this.log,
|
|
1567
|
+
);
|
|
1579
1568
|
|
|
1580
1569
|
// eslint-disable-next-line max-len
|
|
1581
1570
|
this.log.info(`[edge-deploy] Pattern ${suggestion.getId()}: found ${covered.length} coverable per-URL suggestions (field=${coverageField})`);
|
|
@@ -1586,7 +1575,14 @@ class TokowakaClient {
|
|
|
1586
1575
|
cs.setUpdatedBy(updatedBy);
|
|
1587
1576
|
});
|
|
1588
1577
|
try {
|
|
1589
|
-
await saveSuggestions(this.dataAccess, covered
|
|
1578
|
+
await saveSuggestions(this.dataAccess, covered, {
|
|
1579
|
+
sqs: this.sqs,
|
|
1580
|
+
queueUrl: this.importWorkerQueueUrl,
|
|
1581
|
+
siteId,
|
|
1582
|
+
set: { [coverageField]: suggestion.getId() },
|
|
1583
|
+
updatedBy,
|
|
1584
|
+
log: this.log,
|
|
1585
|
+
});
|
|
1590
1586
|
coveredSuggestions.push(...covered);
|
|
1591
1587
|
// eslint-disable-next-line max-len
|
|
1592
1588
|
this.log.info(`[edge-deploy] Marked ${covered.length} suggestions as ${coverageField}=${suggestion.getId()}`);
|
|
@@ -1679,6 +1675,7 @@ class TokowakaClient {
|
|
|
1679
1675
|
allSuggestions,
|
|
1680
1676
|
updatedBy,
|
|
1681
1677
|
coveredSuggestions,
|
|
1678
|
+
site.getId(),
|
|
1682
1679
|
);
|
|
1683
1680
|
deployedPatternSuggestions.push(suggestion);
|
|
1684
1681
|
} catch (error) {
|
|
@@ -1733,6 +1730,64 @@ class TokowakaClient {
|
|
|
1733
1730
|
return { succeededSuggestions, failedSuggestions, coveredSuggestions };
|
|
1734
1731
|
}
|
|
1735
1732
|
|
|
1733
|
+
/**
|
|
1734
|
+
* Finds and marks the suggestions that fall within a pattern suggestion's
|
|
1735
|
+
* scope (domain-wide or segment/path) as covered and saves them.
|
|
1736
|
+
*
|
|
1737
|
+
* @param {Object} patternSuggestion - The domain-wide / segment pattern suggestion
|
|
1738
|
+
* @param {Array} allSuggestions - Full opportunity suggestion list to search for matches
|
|
1739
|
+
* @param {string} siteId - Site ID
|
|
1740
|
+
* @param {string} [updatedBy]
|
|
1741
|
+
* @returns {Promise<Array>} the marked suggestions
|
|
1742
|
+
*/
|
|
1743
|
+
async markPatternCoveredSuggestions(
|
|
1744
|
+
patternSuggestion,
|
|
1745
|
+
allSuggestions,
|
|
1746
|
+
siteId,
|
|
1747
|
+
updatedBy = 'edge-deploy',
|
|
1748
|
+
) {
|
|
1749
|
+
const allowedRegexPatterns = patternSuggestion.getData()?.allowedRegexPatterns;
|
|
1750
|
+
if (!Array.isArray(allowedRegexPatterns) || allowedRegexPatterns.length === 0) {
|
|
1751
|
+
this.log.warn(`[edge-deploy] Pattern suggestion ${patternSuggestion.getId()} has `
|
|
1752
|
+
+ 'no allowedRegexPatterns, skipping cover-marking');
|
|
1753
|
+
return [];
|
|
1754
|
+
}
|
|
1755
|
+
const covered = findCoveredSuggestions(
|
|
1756
|
+
patternSuggestion,
|
|
1757
|
+
allowedRegexPatterns,
|
|
1758
|
+
allSuggestions,
|
|
1759
|
+
new Set(),
|
|
1760
|
+
this.log,
|
|
1761
|
+
);
|
|
1762
|
+
if (covered.length === 0) {
|
|
1763
|
+
return [];
|
|
1764
|
+
}
|
|
1765
|
+
const coverageField = patternSuggestion.getData()?.isDomainWide
|
|
1766
|
+
? 'coveredByDomainWide' : 'coveredByPattern';
|
|
1767
|
+
covered.forEach((s) => {
|
|
1768
|
+
s.setData({ ...s.getData(), [coverageField]: patternSuggestion.getId() });
|
|
1769
|
+
s.setUpdatedBy(updatedBy);
|
|
1770
|
+
});
|
|
1771
|
+
try {
|
|
1772
|
+
await saveSuggestions(this.dataAccess, covered, {
|
|
1773
|
+
sqs: this.sqs,
|
|
1774
|
+
queueUrl: this.importWorkerQueueUrl,
|
|
1775
|
+
siteId,
|
|
1776
|
+
set: { [coverageField]: patternSuggestion.getId() },
|
|
1777
|
+
updatedBy,
|
|
1778
|
+
log: this.log,
|
|
1779
|
+
});
|
|
1780
|
+
this.log.info(`[edge-deploy] Marked ${covered.length} suggestions as ${coverageField}=${patternSuggestion.getId()}`);
|
|
1781
|
+
return covered;
|
|
1782
|
+
} catch (saveError) {
|
|
1783
|
+
this.log.warn(
|
|
1784
|
+
'[edge-deploy] Failed to mark covered suggestions for pattern suggestion '
|
|
1785
|
+
+ `${patternSuggestion.getId()}: ${saveError.message}`,
|
|
1786
|
+
);
|
|
1787
|
+
return [];
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1736
1791
|
/**
|
|
1737
1792
|
* Strips the `applyStale` flag from patches in S3 for the given per-URL suggestions.
|
|
1738
1793
|
* Called when an experiment completes so edge reverts to normal (non-stale) behaviour.
|
|
@@ -92,26 +92,68 @@ export function filterEligibleSuggestions(suggestions, mapper) {
|
|
|
92
92
|
return { eligible, ineligible };
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
+
// Import worker job type for a bulk suggestion field update (agnostic set/unset by id).
|
|
96
|
+
export const SUGGESTION_BULK_UPDATE_TYPE = 'suggestion-bulk-update';
|
|
97
|
+
|
|
95
98
|
/**
|
|
96
99
|
* Batch-saves suggestions using the optimal strategy based on count.
|
|
97
100
|
*
|
|
98
101
|
* - <= 1700: sequential chunked upsert via saveMany (chunkSize 25).
|
|
99
102
|
* ~4s for 1700 suggestions from the DB layer, but ~11s observed end-to-end
|
|
100
103
|
* from the API due to serialization, network, and Lambda overhead.
|
|
101
|
-
* - > 1700: parallel individual .save() via Promise.allSettled to
|
|
102
|
-
* sequential chunk bottleneck at scale.
|
|
104
|
+
* - > 1700 with no queueContext: parallel individual .save() via Promise.allSettled to
|
|
105
|
+
* avoid sequential chunk bottleneck at scale.
|
|
106
|
+
* - > 1700 with queueContext: enqueues a SUGGESTION_BULK_UPDATE_TYPE job to the import
|
|
107
|
+
* worker instead of saving here (the worker fetches by id and applies
|
|
108
|
+
* queueContext.set/unset itself). Falls back to the Promise.allSettled path above
|
|
109
|
+
* if sqs/queueUrl aren't configured or sendMessage fails, so a queue outage
|
|
110
|
+
* degrades rather than silently drops the save.
|
|
103
111
|
*
|
|
104
112
|
* @param {Object} dataAccess - Data access layer
|
|
105
113
|
* @param {Array} suggestions - Suggestion entities to save
|
|
114
|
+
* @param {Object} [queueContext] - sqs, queueUrl, siteId, set/unset,
|
|
115
|
+
* updatedBy (already resolved), log
|
|
106
116
|
* @returns {Promise<void>}
|
|
107
117
|
*/
|
|
108
118
|
const PARALLEL_SAVE_THRESHOLD = 1700;
|
|
109
119
|
|
|
110
|
-
export async function saveSuggestions(dataAccess, suggestions) {
|
|
120
|
+
export async function saveSuggestions(dataAccess, suggestions, queueContext) {
|
|
111
121
|
if (suggestions.length === 0) {
|
|
112
122
|
return;
|
|
113
123
|
}
|
|
114
124
|
if (suggestions.length > PARALLEL_SAVE_THRESHOLD) {
|
|
125
|
+
if (queueContext) {
|
|
126
|
+
const {
|
|
127
|
+
sqs, queueUrl, siteId, set, unset, updatedBy, log,
|
|
128
|
+
} = queueContext;
|
|
129
|
+
|
|
130
|
+
if (queueUrl && sqs) {
|
|
131
|
+
try {
|
|
132
|
+
await sqs.sendMessage(queueUrl, {
|
|
133
|
+
type: SUGGESTION_BULK_UPDATE_TYPE,
|
|
134
|
+
siteId,
|
|
135
|
+
suggestionIds: suggestions.map((s) => s.getId()),
|
|
136
|
+
...(set && { set }),
|
|
137
|
+
...(unset && { unset }),
|
|
138
|
+
updatedBy,
|
|
139
|
+
});
|
|
140
|
+
log.info(
|
|
141
|
+
`[suggestion-bulk-update] Queued bulk update for ${suggestions.length} suggestion(s)`,
|
|
142
|
+
);
|
|
143
|
+
return;
|
|
144
|
+
} catch (error) {
|
|
145
|
+
log.error(
|
|
146
|
+
`[suggestion-bulk-update] Failed to queue bulk update, falling back to direct save: ${error.message}`,
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
} else {
|
|
150
|
+
log.warn(
|
|
151
|
+
'[suggestion-bulk-update] SQS/IMPORT_WORKER_QUEUE_URL not configured; '
|
|
152
|
+
+ `falling back to direct save for ${suggestions.length} suggestion(s)`,
|
|
153
|
+
);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
115
157
|
const results = await Promise.allSettled(suggestions.map((s) => s.save()));
|
|
116
158
|
const failed = results.filter((r) => r.status === 'rejected');
|
|
117
159
|
if (failed.length > 0) {
|
|
@@ -150,10 +192,11 @@ export function stripSuggestion(suggestion, actorFallback, updatedBy) {
|
|
|
150
192
|
* @param {string|undefined} updatedBy - Explicit actor
|
|
151
193
|
* @param {string[]} fieldsToStrip - Specific fields to remove
|
|
152
194
|
* @param {Object} log - Logger instance
|
|
195
|
+
* @param {Object} [queueContext] - Passed through to saveSuggestions unchanged
|
|
153
196
|
* @returns {Promise<void>}
|
|
154
197
|
*/
|
|
155
198
|
// eslint-disable-next-line max-len
|
|
156
|
-
export async function cleanupCoveredSuggestions(dataAccess, covered, actorFallback, updatedBy, fieldsToStrip, log) {
|
|
199
|
+
export async function cleanupCoveredSuggestions(dataAccess, covered, actorFallback, updatedBy, fieldsToStrip, log, queueContext) {
|
|
157
200
|
if (covered.length === 0) {
|
|
158
201
|
return;
|
|
159
202
|
}
|
|
@@ -162,7 +205,7 @@ export async function cleanupCoveredSuggestions(dataAccess, covered, actorFallba
|
|
|
162
205
|
cs.setUpdatedBy(updatedBy ?? actorFallback);
|
|
163
206
|
});
|
|
164
207
|
try {
|
|
165
|
-
await saveSuggestions(dataAccess, covered);
|
|
208
|
+
await saveSuggestions(dataAccess, covered, queueContext);
|
|
166
209
|
} catch (error) {
|
|
167
210
|
// eslint-disable-next-line max-len
|
|
168
211
|
log.error(`[edge-rollback-failed] Failed to clean ${covered.length} covered suggestion(s): ${error.message}`);
|
|
@@ -241,3 +284,58 @@ export function filterBatchCoveredSuggestions(validSuggestions, patternSuggestio
|
|
|
241
284
|
|
|
242
285
|
return { remaining, skippedInBatch };
|
|
243
286
|
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Finds the suggestions in `allSuggestions` that are covered by a pattern suggestion's
|
|
290
|
+
* allowedRegexPatterns
|
|
291
|
+
*
|
|
292
|
+
* @param {Object} patternSuggestion - The domain-wide / segment pattern suggestion
|
|
293
|
+
* @param {Array<string>} allowedRegexPatterns
|
|
294
|
+
* @param {Array} allSuggestions - Full opportunity suggestion list to search
|
|
295
|
+
* @param {Set<string>} [excludeIds] - Suggestion IDs to skip (e.g. already handled in-batch)
|
|
296
|
+
* @param {Object} log - Logger instance
|
|
297
|
+
* @returns {Array} Suggestions covered by the pattern
|
|
298
|
+
*/
|
|
299
|
+
export function findCoveredSuggestions(
|
|
300
|
+
patternSuggestion,
|
|
301
|
+
allowedRegexPatterns,
|
|
302
|
+
allSuggestions,
|
|
303
|
+
excludeIds,
|
|
304
|
+
log,
|
|
305
|
+
) {
|
|
306
|
+
const isDomainWide = patternSuggestion.getData()?.isDomainWide === true;
|
|
307
|
+
const matchers = allowedRegexPatterns.flatMap((p) => {
|
|
308
|
+
const m = buildUrlMatcher(p);
|
|
309
|
+
if (!m) {
|
|
310
|
+
// eslint-disable-next-line max-len
|
|
311
|
+
log.warn(`[edge-deploy] Pattern '${p}' for suggestion ${patternSuggestion.getId()} is invalid, skipping`);
|
|
312
|
+
}
|
|
313
|
+
return m ? [m] : [];
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
if (matchers.length === 0) {
|
|
317
|
+
return [];
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
return allSuggestions.filter((s) => {
|
|
321
|
+
if (s.getId() === patternSuggestion.getId()) {
|
|
322
|
+
return false;
|
|
323
|
+
}
|
|
324
|
+
if (excludeIds?.has(s.getId())) {
|
|
325
|
+
return false;
|
|
326
|
+
}
|
|
327
|
+
if (!isEdgeDeployableSuggestionStatus(s.getStatus())) {
|
|
328
|
+
return false;
|
|
329
|
+
}
|
|
330
|
+
if (s.getData()?.edgeDeployed) {
|
|
331
|
+
return false;
|
|
332
|
+
}
|
|
333
|
+
// Path-level pattern suggestions (not domain-wide) are fully covered by a
|
|
334
|
+
// domain-wide deployment. Other pattern suggestions (including other DW ones) are not.
|
|
335
|
+
if (isPatternSuggestion(s)) {
|
|
336
|
+
return isDomainWide && !s.getData()?.isDomainWide;
|
|
337
|
+
}
|
|
338
|
+
const url = s.getData()?.url;
|
|
339
|
+
return url && matchers.some((match) => match(url));
|
|
340
|
+
});
|
|
341
|
+
}
|
package/test/index.test.js
CHANGED
|
@@ -147,6 +147,98 @@ describe('TokowakaClient', () => {
|
|
|
147
147
|
});
|
|
148
148
|
});
|
|
149
149
|
|
|
150
|
+
describe('markPatternCoveredSuggestions', () => {
|
|
151
|
+
const mkPattern = (isDomainWide) => ({
|
|
152
|
+
getId: () => (isDomainWide ? 'dw-1' : 'path-1'),
|
|
153
|
+
getData: () => (isDomainWide
|
|
154
|
+
? { isDomainWide: true, allowedRegexPatterns: ['/*'] }
|
|
155
|
+
: { allowedRegexPatterns: ['/foo/*'] }),
|
|
156
|
+
});
|
|
157
|
+
const mkCovered = (id, url = `https://example.com/foo/${id}`) => ({
|
|
158
|
+
getId: () => id,
|
|
159
|
+
getStatus: () => 'NEW',
|
|
160
|
+
getData: () => ({ url }),
|
|
161
|
+
setData: sinon.stub(),
|
|
162
|
+
setUpdatedBy: sinon.stub(),
|
|
163
|
+
save: sinon.stub().resolves(),
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
it('marks all matching suggestions in the opportunity with coveredByDomainWide', async () => {
|
|
167
|
+
const covered = [mkCovered('a'), mkCovered('b')];
|
|
168
|
+
|
|
169
|
+
const result = await client.markPatternCoveredSuggestions(
|
|
170
|
+
mkPattern(true),
|
|
171
|
+
covered,
|
|
172
|
+
'site-1',
|
|
173
|
+
'tester',
|
|
174
|
+
);
|
|
175
|
+
|
|
176
|
+
expect(result).to.deep.equal(covered);
|
|
177
|
+
covered.forEach((s) => {
|
|
178
|
+
expect(s.setData.calledWithMatch({ coveredByDomainWide: 'dw-1' })).to.be.true;
|
|
179
|
+
expect(s.setUpdatedBy.calledOnceWith('tester')).to.be.true;
|
|
180
|
+
});
|
|
181
|
+
expect(client.dataAccess.Suggestion.saveMany.calledOnceWith(covered)).to.be.true;
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
it('marks only URLs matching the segment/path pattern with coveredByPattern', async () => {
|
|
185
|
+
const matching = mkCovered('a', 'https://example.com/foo/a');
|
|
186
|
+
const nonMatching = mkCovered('b', 'https://example.com/bar/b');
|
|
187
|
+
|
|
188
|
+
const result = await client.markPatternCoveredSuggestions(
|
|
189
|
+
mkPattern(false),
|
|
190
|
+
[matching, nonMatching],
|
|
191
|
+
'site-1',
|
|
192
|
+
);
|
|
193
|
+
|
|
194
|
+
expect(result).to.deep.equal([matching]);
|
|
195
|
+
expect(matching.setData.calledWithMatch({ coveredByPattern: 'path-1' })).to.be.true;
|
|
196
|
+
expect(matching.setUpdatedBy.calledOnceWith('edge-deploy')).to.be.true;
|
|
197
|
+
expect(nonMatching.setData.called).to.be.false;
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
it('is a no-op when the pattern suggestion has no allowedRegexPatterns', async () => {
|
|
201
|
+
const noPattern = { getId: () => 'dw-1', getData: () => ({ isDomainWide: true }) };
|
|
202
|
+
|
|
203
|
+
expect(await client.markPatternCoveredSuggestions(noPattern, [mkCovered('a')], 'site-1'))
|
|
204
|
+
.to.deep.equal([]);
|
|
205
|
+
expect(client.dataAccess.Suggestion.saveMany.called).to.be.false;
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
it('is a no-op when nothing in allSuggestions matches', async () => {
|
|
209
|
+
const result = await client.markPatternCoveredSuggestions(mkPattern(true), [], 'site-1');
|
|
210
|
+
|
|
211
|
+
expect(result).to.deep.equal([]);
|
|
212
|
+
expect(client.dataAccess.Suggestion.saveMany.called).to.be.false;
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
it('enqueues a bulk-update job with the passed-in siteId when covered count is large', async () => {
|
|
216
|
+
const covered = Array.from({ length: 1701 }, (_, i) => mkCovered(`c${i}`));
|
|
217
|
+
client.sqs = { sendMessage: sinon.stub().resolves() };
|
|
218
|
+
client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
|
|
219
|
+
|
|
220
|
+
await client.markPatternCoveredSuggestions(mkPattern(true), covered, 'site-1', 'tester');
|
|
221
|
+
|
|
222
|
+
expect(client.sqs.sendMessage).to.have.been.calledOnce;
|
|
223
|
+
const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
|
|
224
|
+
expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
|
|
225
|
+
expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-1' });
|
|
226
|
+
expect(payload.set).to.deep.equal({ coveredByDomainWide: 'dw-1' });
|
|
227
|
+
expect(payload.updatedBy).to.equal('tester');
|
|
228
|
+
expect(client.dataAccess.Suggestion.saveMany.called).to.be.false;
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
it('warns and returns [] instead of throwing when saveSuggestions fails', async () => {
|
|
232
|
+
client.dataAccess.Suggestion.saveMany.rejects(new Error('DB down'));
|
|
233
|
+
const covered = [mkCovered('a')];
|
|
234
|
+
|
|
235
|
+
const result = await client.markPatternCoveredSuggestions(mkPattern(true), covered, 'site-1');
|
|
236
|
+
|
|
237
|
+
expect(result).to.deep.equal([]);
|
|
238
|
+
expect(client.log.warn.calledWithMatch(/Failed to mark covered suggestions/)).to.be.true;
|
|
239
|
+
});
|
|
240
|
+
});
|
|
241
|
+
|
|
150
242
|
describe('createFrom', () => {
|
|
151
243
|
it('should create client from context', () => {
|
|
152
244
|
const context = {
|
|
@@ -3197,6 +3289,57 @@ describe('TokowakaClient', () => {
|
|
|
3197
3289
|
);
|
|
3198
3290
|
});
|
|
3199
3291
|
|
|
3292
|
+
it('enqueues a suggestion-bulk-update job instead of saveMany when cleaning up more than the threshold (domain-wide rollback)', async () => {
|
|
3293
|
+
const prerenderOpportunity = { getId: () => 'opp-dw', getType: () => 'prerender' };
|
|
3294
|
+
const dwSuggestion = {
|
|
3295
|
+
getId: () => 'dw-1',
|
|
3296
|
+
getData: () => ({
|
|
3297
|
+
isDomainWide: true,
|
|
3298
|
+
allowedRegexPatterns: ['/*'],
|
|
3299
|
+
edgeDeployed: Date.now(),
|
|
3300
|
+
}),
|
|
3301
|
+
setData: sinon.stub(),
|
|
3302
|
+
setUpdatedBy: sinon.stub(),
|
|
3303
|
+
save: sinon.stub().resolves(),
|
|
3304
|
+
};
|
|
3305
|
+
const covered = Array.from({ length: 1701 }, (_, i) => ({
|
|
3306
|
+
getId: () => `covered-${i}`,
|
|
3307
|
+
getData: () => ({ url: `https://example.com/page${i}`, coveredByDomainWide: 'dw-1' }),
|
|
3308
|
+
setData: sinon.stub(),
|
|
3309
|
+
setUpdatedBy: sinon.stub(),
|
|
3310
|
+
save: sinon.stub().resolves(),
|
|
3311
|
+
}));
|
|
3312
|
+
|
|
3313
|
+
sinon.stub(client, 'fetchMetaconfig').resolves({
|
|
3314
|
+
siteId: 'site-123',
|
|
3315
|
+
prerender: { allowList: ['/*'] },
|
|
3316
|
+
});
|
|
3317
|
+
sinon.stub(client, 'uploadMetaconfig').resolves();
|
|
3318
|
+
client.sqs = { sendMessage: sinon.stub().resolves() };
|
|
3319
|
+
client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
|
|
3320
|
+
|
|
3321
|
+
await client.rollbackSuggestions(
|
|
3322
|
+
mockSite,
|
|
3323
|
+
prerenderOpportunity,
|
|
3324
|
+
[dwSuggestion],
|
|
3325
|
+
{ allSuggestions: [dwSuggestion, ...covered], updatedBy: 'test@example.com' },
|
|
3326
|
+
);
|
|
3327
|
+
|
|
3328
|
+
expect(client.sqs.sendMessage).to.have.been.calledOnce;
|
|
3329
|
+
const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
|
|
3330
|
+
expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
|
|
3331
|
+
expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-123' });
|
|
3332
|
+
expect(payload).to.not.have.property('opportunityId');
|
|
3333
|
+
expect(payload.suggestionIds).to.have.length(1701);
|
|
3334
|
+
expect(payload.unset).to.deep.equal(['coveredByDomainWide']);
|
|
3335
|
+
expect(payload.updatedBy).to.equal('test@example.com');
|
|
3336
|
+
// Not saved directly — the enqueue replaces the saveMany call for this batch.
|
|
3337
|
+
expect(client.dataAccess.Suggestion.saveMany).to.not.have.been.calledWith(
|
|
3338
|
+
sinon.match((arr) => arr.length === 1701),
|
|
3339
|
+
sinon.match.any,
|
|
3340
|
+
);
|
|
3341
|
+
});
|
|
3342
|
+
|
|
3200
3343
|
it('cleans up covered suggestions (coveredByPattern) when rolling back a path-level pattern', async () => {
|
|
3201
3344
|
const prerenderOpportunity = { getId: () => 'opp-p', getType: () => 'prerender' };
|
|
3202
3345
|
const pathSuggestion = {
|
|
@@ -3240,6 +3383,54 @@ describe('TokowakaClient', () => {
|
|
|
3240
3383
|
expect(coveredData).to.not.have.property('coveredByPattern');
|
|
3241
3384
|
});
|
|
3242
3385
|
|
|
3386
|
+
it('enqueues a suggestion-bulk-update job instead of saveMany when cleaning up more than the threshold (path-level rollback)', async () => {
|
|
3387
|
+
const prerenderOpportunity = { getId: () => 'opp-p', getType: () => 'prerender' };
|
|
3388
|
+
const pathSuggestion = {
|
|
3389
|
+
getId: () => 'path-1',
|
|
3390
|
+
getData: () => ({
|
|
3391
|
+
allowedRegexPatterns: ['/products/*'],
|
|
3392
|
+
edgeDeployed: Date.now(),
|
|
3393
|
+
}),
|
|
3394
|
+
setData: sinon.stub(),
|
|
3395
|
+
setUpdatedBy: sinon.stub(),
|
|
3396
|
+
save: sinon.stub().resolves(),
|
|
3397
|
+
};
|
|
3398
|
+
const covered = Array.from({ length: 1701 }, (_, i) => ({
|
|
3399
|
+
getId: () => `covered-${i}`,
|
|
3400
|
+
getData: () => ({
|
|
3401
|
+
url: `https://example.com/products/item-${i}`,
|
|
3402
|
+
edgeDeployed: Date.now(),
|
|
3403
|
+
coveredByPattern: 'path-1',
|
|
3404
|
+
}),
|
|
3405
|
+
setData: sinon.stub(),
|
|
3406
|
+
setUpdatedBy: sinon.stub(),
|
|
3407
|
+
save: sinon.stub().resolves(),
|
|
3408
|
+
}));
|
|
3409
|
+
|
|
3410
|
+
sinon.stub(client, 'fetchMetaconfig').resolves({
|
|
3411
|
+
siteId: 'site-123',
|
|
3412
|
+
prerender: { allowList: ['/products/*'] },
|
|
3413
|
+
});
|
|
3414
|
+
sinon.stub(client, 'uploadMetaconfig').resolves();
|
|
3415
|
+
client.sqs = { sendMessage: sinon.stub().resolves() };
|
|
3416
|
+
client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
|
|
3417
|
+
|
|
3418
|
+
await client.rollbackSuggestions(
|
|
3419
|
+
mockSite,
|
|
3420
|
+
prerenderOpportunity,
|
|
3421
|
+
[pathSuggestion],
|
|
3422
|
+
{ allSuggestions: [pathSuggestion, ...covered] },
|
|
3423
|
+
);
|
|
3424
|
+
|
|
3425
|
+
expect(client.sqs.sendMessage).to.have.been.calledOnce;
|
|
3426
|
+
const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
|
|
3427
|
+
expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
|
|
3428
|
+
expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-123' });
|
|
3429
|
+
expect(payload).to.not.have.property('opportunityId');
|
|
3430
|
+
expect(payload.suggestionIds).to.have.length(1701);
|
|
3431
|
+
expect(payload.unset).to.deep.equal(['edgeDeployed', 'tokowakaDeployed', 'coveredByPattern']);
|
|
3432
|
+
});
|
|
3433
|
+
|
|
3243
3434
|
it('uses domain-wide-rollback fallback for covered suggestions when updatedBy is not provided (domain-wide parent)', async () => {
|
|
3244
3435
|
const prerenderOpportunity = { getId: () => 'opp-dw', getType: () => 'prerender' };
|
|
3245
3436
|
const dwSuggestion = {
|
|
@@ -5965,6 +6156,37 @@ describe('TokowakaClient', () => {
|
|
|
5965
6156
|
expect(covered.getData()).to.have.property('coveredByDomainWide', 'dw1');
|
|
5966
6157
|
});
|
|
5967
6158
|
|
|
6159
|
+
it('enqueues a suggestion-bulk-update job instead of saveMany when covered count exceeds the threshold (domain-wide)', async () => {
|
|
6160
|
+
const dw = makeSuggestion('dw1', {
|
|
6161
|
+
isDomainWide: true,
|
|
6162
|
+
allowedRegexPatterns: ['^https://example\\.com/.*'],
|
|
6163
|
+
});
|
|
6164
|
+
const covered = Array.from({ length: 1701 }, (_, i) => makeSuggestion(
|
|
6165
|
+
`covered-${i}`,
|
|
6166
|
+
{ url: `https://example.com/page${i}` },
|
|
6167
|
+
));
|
|
6168
|
+
|
|
6169
|
+
fetchMetaconfigStub.resolves({ siteId: 'site-123' });
|
|
6170
|
+
client.sqs = { sendMessage: sinon.stub().resolves() };
|
|
6171
|
+
client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
|
|
6172
|
+
|
|
6173
|
+
const result = await client.deployToEdge({
|
|
6174
|
+
site: mockSite,
|
|
6175
|
+
opportunity: mockOpportunity,
|
|
6176
|
+
targetSuggestions: [dw],
|
|
6177
|
+
allSuggestions: [dw, ...covered],
|
|
6178
|
+
});
|
|
6179
|
+
|
|
6180
|
+
expect(result.coveredSuggestions).to.have.length(1701);
|
|
6181
|
+
expect(client.sqs.sendMessage).to.have.been.calledOnce;
|
|
6182
|
+
const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
|
|
6183
|
+
expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
|
|
6184
|
+
expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-123' });
|
|
6185
|
+
expect(payload).to.not.have.property('opportunityId');
|
|
6186
|
+
expect(payload.suggestionIds).to.have.length(1701);
|
|
6187
|
+
expect(payload.set).to.deep.equal({ coveredByDomainWide: 'dw1' });
|
|
6188
|
+
});
|
|
6189
|
+
|
|
5968
6190
|
it('should not mark already-domain-wide suggestions as covered', async () => {
|
|
5969
6191
|
const dw1 = makeSuggestion('dw1', {
|
|
5970
6192
|
isDomainWide: true,
|
|
@@ -6467,6 +6689,36 @@ describe('TokowakaClient', () => {
|
|
|
6467
6689
|
expect(result.coveredSuggestions).to.not.include(urlElsewhere);
|
|
6468
6690
|
});
|
|
6469
6691
|
|
|
6692
|
+
it('enqueues a suggestion-bulk-update job instead of saveMany when covered count exceeds the threshold (path-level)', async () => {
|
|
6693
|
+
const path = makeSuggestion('p1', {
|
|
6694
|
+
allowedRegexPatterns: ['/products/*'],
|
|
6695
|
+
});
|
|
6696
|
+
const covered = Array.from({ length: 1701 }, (_, i) => makeSuggestion(
|
|
6697
|
+
`covered-${i}`,
|
|
6698
|
+
{ url: `https://example.com/products/item-${i}` },
|
|
6699
|
+
));
|
|
6700
|
+
|
|
6701
|
+
fetchMetaconfigStub.resolves({ siteId: 'site-123' });
|
|
6702
|
+
client.sqs = { sendMessage: sinon.stub().resolves() };
|
|
6703
|
+
client.importWorkerQueueUrl = 'https://sqs.test/import-worker-queue';
|
|
6704
|
+
|
|
6705
|
+
const result = await client.deployToEdge({
|
|
6706
|
+
site: mockSite,
|
|
6707
|
+
opportunity: mockOpportunity,
|
|
6708
|
+
targetSuggestions: [path],
|
|
6709
|
+
allSuggestions: [path, ...covered],
|
|
6710
|
+
});
|
|
6711
|
+
|
|
6712
|
+
expect(result.coveredSuggestions).to.have.length(1701);
|
|
6713
|
+
expect(client.sqs.sendMessage).to.have.been.calledOnce;
|
|
6714
|
+
const [queueUrl, payload] = client.sqs.sendMessage.firstCall.args;
|
|
6715
|
+
expect(queueUrl).to.equal('https://sqs.test/import-worker-queue');
|
|
6716
|
+
expect(payload).to.include({ type: 'suggestion-bulk-update', siteId: 'site-123' });
|
|
6717
|
+
expect(payload).to.not.have.property('opportunityId');
|
|
6718
|
+
expect(payload.suggestionIds).to.have.length(1701);
|
|
6719
|
+
expect(payload.set).to.deep.equal({ coveredByPattern: 'p1' });
|
|
6720
|
+
});
|
|
6721
|
+
|
|
6470
6722
|
it('path deploy marks as failed with statusCode 500 when metaconfig upload fails', async () => {
|
|
6471
6723
|
const path = makeSuggestion('p1', {
|
|
6472
6724
|
allowedRegexPatterns: ['/products/*'],
|
|
@@ -12,7 +12,12 @@
|
|
|
12
12
|
|
|
13
13
|
import { expect } from 'chai';
|
|
14
14
|
import sinon from 'sinon';
|
|
15
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
groupSuggestionsByUrlPath,
|
|
17
|
+
filterEligibleSuggestions,
|
|
18
|
+
saveSuggestions,
|
|
19
|
+
SUGGESTION_BULK_UPDATE_TYPE,
|
|
20
|
+
} from '../../src/utils/suggestion-utils.js';
|
|
16
21
|
|
|
17
22
|
describe('Suggestion Utils', () => {
|
|
18
23
|
describe('groupSuggestionsByUrlPath', () => {
|
|
@@ -223,5 +228,153 @@ describe('Suggestion Utils', () => {
|
|
|
223
228
|
expect(error.message).to.include('DB error');
|
|
224
229
|
}
|
|
225
230
|
});
|
|
231
|
+
|
|
232
|
+
it('should enqueue a bulk update job instead of saving when queueContext is provided', async () => {
|
|
233
|
+
const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
|
|
234
|
+
const items = Array.from({ length: 1701 }, (_, i) => ({
|
|
235
|
+
getId: () => `sugg-${i}`,
|
|
236
|
+
save: sinon.stub().resolves(),
|
|
237
|
+
}));
|
|
238
|
+
const queueContext = {
|
|
239
|
+
sqs: { sendMessage: sinon.stub().resolves() },
|
|
240
|
+
queueUrl: 'https://sqs.example.com/queue',
|
|
241
|
+
siteId: 'site-1',
|
|
242
|
+
set: { coveredByDomainWide: 'pattern-sugg-id' },
|
|
243
|
+
updatedBy: 'system',
|
|
244
|
+
log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
await saveSuggestions(dataAccess, items, queueContext);
|
|
248
|
+
|
|
249
|
+
expect(dataAccess.Suggestion.saveMany.called).to.be.false;
|
|
250
|
+
expect(items[0].save.called).to.be.false;
|
|
251
|
+
expect(queueContext.sqs.sendMessage.calledOnce).to.be.true;
|
|
252
|
+
const [queueUrl, payload] = queueContext.sqs.sendMessage.firstCall.args;
|
|
253
|
+
expect(queueUrl).to.equal('https://sqs.example.com/queue');
|
|
254
|
+
expect(payload).to.deep.equal({
|
|
255
|
+
type: SUGGESTION_BULK_UPDATE_TYPE,
|
|
256
|
+
siteId: 'site-1',
|
|
257
|
+
suggestionIds: items.map((s) => s.getId()),
|
|
258
|
+
set: { coveredByDomainWide: 'pattern-sugg-id' },
|
|
259
|
+
updatedBy: 'system',
|
|
260
|
+
});
|
|
261
|
+
expect(queueContext.log.info.calledOnce).to.be.true;
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
it('should include unset instead of set when queueContext.unset is provided', async () => {
|
|
265
|
+
const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
|
|
266
|
+
const items = Array.from({ length: 1701 }, (_, i) => ({
|
|
267
|
+
getId: () => `sugg-${i}`,
|
|
268
|
+
save: sinon.stub().resolves(),
|
|
269
|
+
}));
|
|
270
|
+
const queueContext = {
|
|
271
|
+
sqs: { sendMessage: sinon.stub().resolves() },
|
|
272
|
+
queueUrl: 'https://sqs.example.com/queue',
|
|
273
|
+
siteId: 'site-1',
|
|
274
|
+
unset: ['coveredByDomainWide'],
|
|
275
|
+
updatedBy: 'system',
|
|
276
|
+
log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
|
|
277
|
+
};
|
|
278
|
+
|
|
279
|
+
await saveSuggestions(dataAccess, items, queueContext);
|
|
280
|
+
|
|
281
|
+
const [, payload] = queueContext.sqs.sendMessage.firstCall.args;
|
|
282
|
+
expect(payload.set).to.be.undefined;
|
|
283
|
+
expect(payload.unset).to.deep.equal(['coveredByDomainWide']);
|
|
284
|
+
});
|
|
285
|
+
|
|
286
|
+
it('falls back to direct save when queueUrl is not configured', async () => {
|
|
287
|
+
const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
|
|
288
|
+
const items = Array.from({ length: 1701 }, (_, i) => ({
|
|
289
|
+
getId: () => `sugg-${i}`,
|
|
290
|
+
save: sinon.stub().resolves(),
|
|
291
|
+
}));
|
|
292
|
+
const queueContext = {
|
|
293
|
+
sqs: { sendMessage: sinon.stub().resolves() },
|
|
294
|
+
queueUrl: undefined,
|
|
295
|
+
siteId: 'site-1',
|
|
296
|
+
updatedBy: 'system',
|
|
297
|
+
log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
|
|
298
|
+
};
|
|
299
|
+
|
|
300
|
+
await saveSuggestions(dataAccess, items, queueContext);
|
|
301
|
+
|
|
302
|
+
expect(queueContext.sqs.sendMessage.called).to.be.false;
|
|
303
|
+
expect(queueContext.log.warn.calledOnce).to.be.true;
|
|
304
|
+
expect(queueContext.log.warn.firstCall.args[0]).to.include('SQS/IMPORT_WORKER_QUEUE_URL not configured');
|
|
305
|
+
expect(items[0].save.calledOnce).to.be.true;
|
|
306
|
+
expect(items[1700].save.calledOnce).to.be.true;
|
|
307
|
+
expect(dataAccess.Suggestion.saveMany.called).to.be.false;
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
it('falls back to direct save when sqs is not configured', async () => {
|
|
311
|
+
const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
|
|
312
|
+
const items = Array.from({ length: 1701 }, (_, i) => ({
|
|
313
|
+
getId: () => `sugg-${i}`,
|
|
314
|
+
save: sinon.stub().resolves(),
|
|
315
|
+
}));
|
|
316
|
+
const queueContext = {
|
|
317
|
+
sqs: undefined,
|
|
318
|
+
queueUrl: 'https://sqs.example.com/queue',
|
|
319
|
+
siteId: 'site-1',
|
|
320
|
+
updatedBy: 'system',
|
|
321
|
+
log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
|
|
322
|
+
};
|
|
323
|
+
|
|
324
|
+
await saveSuggestions(dataAccess, items, queueContext);
|
|
325
|
+
|
|
326
|
+
expect(queueContext.log.warn.calledOnce).to.be.true;
|
|
327
|
+
expect(queueContext.log.warn.firstCall.args[0]).to.include('SQS/IMPORT_WORKER_QUEUE_URL not configured');
|
|
328
|
+
expect(items[0].save.calledOnce).to.be.true;
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
it('falls back to direct save when sqs.sendMessage rejects', async () => {
|
|
332
|
+
const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
|
|
333
|
+
const items = Array.from({ length: 1701 }, (_, i) => ({
|
|
334
|
+
getId: () => `sugg-${i}`,
|
|
335
|
+
save: sinon.stub().resolves(),
|
|
336
|
+
}));
|
|
337
|
+
const queueContext = {
|
|
338
|
+
sqs: { sendMessage: sinon.stub().rejects(new Error('SQS unavailable')) },
|
|
339
|
+
queueUrl: 'https://sqs.example.com/queue',
|
|
340
|
+
siteId: 'site-1',
|
|
341
|
+
updatedBy: 'system',
|
|
342
|
+
log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
|
|
343
|
+
};
|
|
344
|
+
|
|
345
|
+
await saveSuggestions(dataAccess, items, queueContext);
|
|
346
|
+
|
|
347
|
+
expect(queueContext.log.error.calledOnce).to.be.true;
|
|
348
|
+
expect(queueContext.log.error.firstCall.args[0]).to.include('Failed to queue bulk update');
|
|
349
|
+
expect(queueContext.log.error.firstCall.args[0]).to.include('SQS unavailable');
|
|
350
|
+
expect(items[0].save.calledOnce).to.be.true;
|
|
351
|
+
expect(items[1700].save.calledOnce).to.be.true;
|
|
352
|
+
expect(dataAccess.Suggestion.saveMany.called).to.be.false;
|
|
353
|
+
});
|
|
354
|
+
|
|
355
|
+
it('throws when the direct-save fallback also fails after a queue failure', async () => {
|
|
356
|
+
const dataAccess = { Suggestion: { saveMany: sinon.stub() } };
|
|
357
|
+
const items = Array.from({ length: 1701 }, (_, i) => ({
|
|
358
|
+
getId: () => `sugg-${i}`,
|
|
359
|
+
save: i === 0
|
|
360
|
+
? sinon.stub().rejects(new Error('DB error'))
|
|
361
|
+
: sinon.stub().resolves(),
|
|
362
|
+
}));
|
|
363
|
+
const queueContext = {
|
|
364
|
+
sqs: { sendMessage: sinon.stub().rejects(new Error('SQS unavailable')) },
|
|
365
|
+
queueUrl: 'https://sqs.example.com/queue',
|
|
366
|
+
siteId: 'site-1',
|
|
367
|
+
updatedBy: 'system',
|
|
368
|
+
log: { warn: sinon.stub(), info: sinon.stub(), error: sinon.stub() },
|
|
369
|
+
};
|
|
370
|
+
|
|
371
|
+
try {
|
|
372
|
+
await saveSuggestions(dataAccess, items, queueContext);
|
|
373
|
+
expect.fail('should have thrown');
|
|
374
|
+
} catch (error) {
|
|
375
|
+
expect(error.message).to.include('1 of 1701 suggestions failed');
|
|
376
|
+
expect(error.message).to.include('DB error');
|
|
377
|
+
}
|
|
378
|
+
});
|
|
226
379
|
});
|
|
227
380
|
});
|