@adobe/spacecat-shared-tokowaka-client 1.1.1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.releaserc.cjs +17 -0
- package/CHANGELOG.md +76 -1
- package/CODE_OF_CONDUCT.md +75 -0
- package/CONTRIBUTING.md +74 -0
- package/README.md +155 -15
- package/package.json +6 -6
- package/src/index.d.ts +120 -25
- package/src/index.js +481 -177
- package/src/mappers/base-mapper.js +41 -9
- package/src/mappers/content-summarization-mapper.js +38 -35
- package/src/mappers/faq-mapper.js +247 -0
- package/src/mappers/headings-mapper.js +37 -23
- package/src/mappers/mapper-registry.js +2 -0
- package/src/utils/custom-html-utils.js +195 -0
- package/src/utils/markdown-utils.js +24 -0
- package/src/utils/patch-utils.js +103 -0
- package/src/utils/s3-utils.js +117 -0
- package/src/utils/site-utils.js +25 -0
- package/src/utils/suggestion-utils.js +69 -0
- package/test/index.test.js +1268 -462
- package/test/mappers/base-mapper.test.js +250 -7
- package/test/mappers/content-mapper.test.js +26 -24
- package/test/mappers/faq-mapper.test.js +1428 -0
- package/test/mappers/headings-mapper.test.js +23 -17
- package/test/utils/html-utils.test.js +432 -0
- package/test/utils/patch-utils.test.js +409 -0
- package/test/utils/s3-utils.test.js +140 -0
- package/test/utils/site-utils.test.js +80 -0
- package/test/utils/suggestion-utils.test.js +187 -0
package/src/index.js
CHANGED
|
@@ -11,9 +11,14 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3';
|
|
14
|
-
import { hasText, isNonEmptyObject
|
|
14
|
+
import { hasText, isNonEmptyObject } from '@adobe/spacecat-shared-utils';
|
|
15
15
|
import MapperRegistry from './mappers/mapper-registry.js';
|
|
16
16
|
import CdnClientRegistry from './cdn/cdn-client-registry.js';
|
|
17
|
+
import { mergePatches } from './utils/patch-utils.js';
|
|
18
|
+
import { getTokowakaConfigS3Path, getTokowakaMetaconfigS3Path } from './utils/s3-utils.js';
|
|
19
|
+
import { groupSuggestionsByUrlPath, filterEligibleSuggestions } from './utils/suggestion-utils.js';
|
|
20
|
+
import { getEffectiveBaseURL } from './utils/site-utils.js';
|
|
21
|
+
import { fetchHtmlWithWarmup } from './utils/custom-html-utils.js';
|
|
17
22
|
|
|
18
23
|
const HTTP_BAD_REQUEST = 400;
|
|
19
24
|
const HTTP_INTERNAL_SERVER_ERROR = 500;
|
|
@@ -30,7 +35,10 @@ class TokowakaClient {
|
|
|
30
35
|
*/
|
|
31
36
|
static createFrom(context) {
|
|
32
37
|
const { env, log = console, s3 } = context;
|
|
33
|
-
const {
|
|
38
|
+
const {
|
|
39
|
+
TOKOWAKA_SITE_CONFIG_BUCKET: bucketName,
|
|
40
|
+
TOKOWAKA_PREVIEW_BUCKET: previewBucketName,
|
|
41
|
+
} = env;
|
|
34
42
|
|
|
35
43
|
if (context.tokowakaClient) {
|
|
36
44
|
return context.tokowakaClient;
|
|
@@ -39,6 +47,7 @@ class TokowakaClient {
|
|
|
39
47
|
// s3ClientWrapper puts s3Client at context.s3.s3Client, so check both locations
|
|
40
48
|
const client = new TokowakaClient({
|
|
41
49
|
bucketName,
|
|
50
|
+
previewBucketName,
|
|
42
51
|
s3Client: s3?.s3Client,
|
|
43
52
|
env,
|
|
44
53
|
}, log);
|
|
@@ -50,11 +59,14 @@ class TokowakaClient {
|
|
|
50
59
|
* Constructor
|
|
51
60
|
* @param {Object} config - Configuration object
|
|
52
61
|
* @param {string} config.bucketName - S3 bucket name for configs
|
|
62
|
+
* @param {string} config.previewBucketName - S3 bucket name for preview configs
|
|
53
63
|
* @param {Object} config.s3Client - AWS S3 client
|
|
54
64
|
* @param {Object} config.env - Environment variables (for CDN credentials)
|
|
55
65
|
* @param {Object} log - Logger instance
|
|
56
66
|
*/
|
|
57
|
-
constructor({
|
|
67
|
+
constructor({
|
|
68
|
+
bucketName, previewBucketName, s3Client, env = {},
|
|
69
|
+
}, log) {
|
|
58
70
|
this.log = log;
|
|
59
71
|
|
|
60
72
|
if (!hasText(bucketName)) {
|
|
@@ -65,7 +77,8 @@ class TokowakaClient {
|
|
|
65
77
|
throw this.#createError('S3 client is required', HTTP_BAD_REQUEST);
|
|
66
78
|
}
|
|
67
79
|
|
|
68
|
-
this.
|
|
80
|
+
this.deployBucketName = bucketName;
|
|
81
|
+
this.previewBucketName = previewBucketName;
|
|
69
82
|
this.s3Client = s3Client;
|
|
70
83
|
this.env = env;
|
|
71
84
|
|
|
@@ -80,23 +93,15 @@ class TokowakaClient {
|
|
|
80
93
|
}
|
|
81
94
|
|
|
82
95
|
/**
|
|
83
|
-
* Generates Tokowaka site configuration from suggestions
|
|
84
|
-
* @param {
|
|
96
|
+
* Generates Tokowaka site configuration from suggestions for a specific URL
|
|
97
|
+
* @param {string} url - Full URL for which to generate config
|
|
85
98
|
* @param {Object} opportunity - Opportunity entity
|
|
86
|
-
* @param {Array}
|
|
87
|
-
* @returns {Object} - Tokowaka configuration object
|
|
99
|
+
* @param {Array} suggestionsToDeploy - Array of suggestion entities to deploy
|
|
100
|
+
* @returns {Object} - Tokowaka configuration object for the URL
|
|
88
101
|
*/
|
|
89
|
-
generateConfig(
|
|
102
|
+
generateConfig(url, opportunity, suggestionsToDeploy) {
|
|
90
103
|
const opportunityType = opportunity.getType();
|
|
91
|
-
const siteId = site.getId();
|
|
92
|
-
|
|
93
|
-
// Get baseURL, respecting overrideBaseURL from fetchConfig if it exists
|
|
94
|
-
const overrideBaseURL = site.getConfig()?.getFetchConfig?.()?.overrideBaseURL;
|
|
95
|
-
const baseURL = (overrideBaseURL && isValidUrl(overrideBaseURL))
|
|
96
|
-
? overrideBaseURL
|
|
97
|
-
: site.getBaseURL();
|
|
98
104
|
|
|
99
|
-
// Get mapper for this opportunity type
|
|
100
105
|
const mapper = this.mapperRegistry.getMapper(opportunityType);
|
|
101
106
|
if (!mapper) {
|
|
102
107
|
throw this.#createError(
|
|
@@ -106,54 +111,27 @@ class TokowakaClient {
|
|
|
106
111
|
);
|
|
107
112
|
}
|
|
108
113
|
|
|
109
|
-
//
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
const url = data?.url;
|
|
113
|
-
|
|
114
|
-
if (!url) {
|
|
115
|
-
this.log.warn(`Suggestion ${suggestion.getId()} does not have a URL, skipping`);
|
|
116
|
-
return acc;
|
|
117
|
-
}
|
|
114
|
+
// Extract URL path from the full URL
|
|
115
|
+
const urlObj = new URL(url);
|
|
116
|
+
const urlPath = urlObj.pathname;
|
|
118
117
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
}
|
|
118
|
+
// Generate patches for the URL using the mapper
|
|
119
|
+
const patches = mapper.suggestionsToPatches(
|
|
120
|
+
urlPath,
|
|
121
|
+
suggestionsToDeploy,
|
|
122
|
+
opportunity.getId(),
|
|
123
|
+
);
|
|
126
124
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
acc[urlPath].push(suggestion);
|
|
131
|
-
return acc;
|
|
132
|
-
}, {});
|
|
133
|
-
|
|
134
|
-
// Generate patches for each URL using the mapper
|
|
135
|
-
const tokowakaOptimizations = {};
|
|
136
|
-
|
|
137
|
-
Object.entries(suggestionsByUrl).forEach(([urlPath, urlSuggestions]) => {
|
|
138
|
-
const patches = urlSuggestions.map((suggestion) => {
|
|
139
|
-
const patch = mapper.suggestionToPatch(suggestion, opportunity.getId());
|
|
140
|
-
return patch;
|
|
141
|
-
}).filter((patch) => patch !== null);
|
|
142
|
-
|
|
143
|
-
if (patches.length > 0) {
|
|
144
|
-
tokowakaOptimizations[urlPath] = {
|
|
145
|
-
prerender: mapper.requiresPrerender(),
|
|
146
|
-
patches,
|
|
147
|
-
};
|
|
148
|
-
}
|
|
149
|
-
});
|
|
125
|
+
if (patches.length === 0) {
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
150
128
|
|
|
151
129
|
return {
|
|
152
|
-
|
|
153
|
-
baseURL,
|
|
130
|
+
url,
|
|
154
131
|
version: '1.0',
|
|
155
|
-
|
|
156
|
-
|
|
132
|
+
forceFail: false,
|
|
133
|
+
prerender: mapper.requiresPrerender(),
|
|
134
|
+
patches,
|
|
157
135
|
};
|
|
158
136
|
}
|
|
159
137
|
|
|
@@ -174,20 +152,98 @@ class TokowakaClient {
|
|
|
174
152
|
}
|
|
175
153
|
|
|
176
154
|
/**
|
|
177
|
-
* Fetches
|
|
178
|
-
* @param {string}
|
|
155
|
+
* Fetches domain-level metaconfig from S3
|
|
156
|
+
* @param {string} url - Full URL (used to extract domain)
|
|
157
|
+
* @param {boolean} isPreview - Whether to fetch from preview path (default: false)
|
|
158
|
+
* @returns {Promise<Object|null>} - Metaconfig object or null if not found
|
|
159
|
+
*/
|
|
160
|
+
async fetchMetaconfig(url, isPreview = false) {
|
|
161
|
+
if (!hasText(url)) {
|
|
162
|
+
throw this.#createError('URL is required', HTTP_BAD_REQUEST);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const s3Path = getTokowakaMetaconfigS3Path(url, this.log, isPreview);
|
|
166
|
+
const bucketName = isPreview ? this.previewBucketName : this.deployBucketName;
|
|
167
|
+
|
|
168
|
+
try {
|
|
169
|
+
const command = new GetObjectCommand({
|
|
170
|
+
Bucket: bucketName,
|
|
171
|
+
Key: s3Path,
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
const response = await this.s3Client.send(command);
|
|
175
|
+
const bodyContents = await response.Body.transformToString();
|
|
176
|
+
const metaconfig = JSON.parse(bodyContents);
|
|
177
|
+
|
|
178
|
+
this.log.debug(`Successfully fetched metaconfig from s3://${bucketName}/${s3Path}`);
|
|
179
|
+
return metaconfig;
|
|
180
|
+
} catch (error) {
|
|
181
|
+
// If metaconfig doesn't exist (NoSuchKey), return null
|
|
182
|
+
if (error.name === 'NoSuchKey' || error.Code === 'NoSuchKey') {
|
|
183
|
+
this.log.debug(`No metaconfig found at s3://${bucketName}/${s3Path}`);
|
|
184
|
+
return null;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// For other errors, log and throw
|
|
188
|
+
this.log.error(`Failed to fetch metaconfig from S3: ${error.message}`, error);
|
|
189
|
+
throw this.#createError(`S3 fetch failed: ${error.message}`, HTTP_INTERNAL_SERVER_ERROR);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Uploads domain-level metaconfig to S3
|
|
195
|
+
* @param {string} url - Full URL (used to extract domain)
|
|
196
|
+
* @param {Object} metaconfig - Metaconfig object (siteId, prerender)
|
|
197
|
+
* @param {boolean} isPreview - Whether to upload to preview path (default: false)
|
|
198
|
+
* @returns {Promise<string>} - S3 key of uploaded metaconfig
|
|
199
|
+
*/
|
|
200
|
+
async uploadMetaconfig(url, metaconfig, isPreview = false) {
|
|
201
|
+
if (!hasText(url)) {
|
|
202
|
+
throw this.#createError('URL is required', HTTP_BAD_REQUEST);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
if (!isNonEmptyObject(metaconfig)) {
|
|
206
|
+
throw this.#createError('Metaconfig object is required', HTTP_BAD_REQUEST);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const s3Path = getTokowakaMetaconfigS3Path(url, this.log, isPreview);
|
|
210
|
+
const bucketName = isPreview ? this.previewBucketName : this.deployBucketName;
|
|
211
|
+
|
|
212
|
+
try {
|
|
213
|
+
const command = new PutObjectCommand({
|
|
214
|
+
Bucket: bucketName,
|
|
215
|
+
Key: s3Path,
|
|
216
|
+
Body: JSON.stringify(metaconfig, null, 2),
|
|
217
|
+
ContentType: 'application/json',
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
await this.s3Client.send(command);
|
|
221
|
+
this.log.info(`Successfully uploaded metaconfig to s3://${bucketName}/${s3Path}`);
|
|
222
|
+
|
|
223
|
+
return s3Path;
|
|
224
|
+
} catch (error) {
|
|
225
|
+
this.log.error(`Failed to upload metaconfig to S3: ${error.message}`, error);
|
|
226
|
+
throw this.#createError(`S3 upload failed: ${error.message}`, HTTP_INTERNAL_SERVER_ERROR);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Fetches existing Tokowaka configuration from S3 for a specific URL
|
|
232
|
+
* @param {string} url - Full URL (e.g., 'https://www.example.com/products/item')
|
|
233
|
+
* @param {boolean} isPreview - Whether to fetch from preview path (default: false)
|
|
179
234
|
* @returns {Promise<Object|null>} - Existing configuration object or null if not found
|
|
180
235
|
*/
|
|
181
|
-
async fetchConfig(
|
|
182
|
-
if (!hasText(
|
|
183
|
-
throw this.#createError('
|
|
236
|
+
async fetchConfig(url, isPreview = false) {
|
|
237
|
+
if (!hasText(url)) {
|
|
238
|
+
throw this.#createError('URL is required', HTTP_BAD_REQUEST);
|
|
184
239
|
}
|
|
185
240
|
|
|
186
|
-
const s3Path =
|
|
241
|
+
const s3Path = getTokowakaConfigS3Path(url, this.log, isPreview);
|
|
242
|
+
const bucketName = isPreview ? this.previewBucketName : this.deployBucketName;
|
|
187
243
|
|
|
188
244
|
try {
|
|
189
245
|
const command = new GetObjectCommand({
|
|
190
|
-
Bucket:
|
|
246
|
+
Bucket: bucketName,
|
|
191
247
|
Key: s3Path,
|
|
192
248
|
});
|
|
193
249
|
|
|
@@ -195,12 +251,12 @@ class TokowakaClient {
|
|
|
195
251
|
const bodyContents = await response.Body.transformToString();
|
|
196
252
|
const config = JSON.parse(bodyContents);
|
|
197
253
|
|
|
198
|
-
this.log.debug(`Successfully fetched existing Tokowaka config from s3://${
|
|
254
|
+
this.log.debug(`Successfully fetched existing Tokowaka config from s3://${bucketName}/${s3Path}`);
|
|
199
255
|
return config;
|
|
200
256
|
} catch (error) {
|
|
201
257
|
// If config doesn't exist (NoSuchKey), return null
|
|
202
258
|
if (error.name === 'NoSuchKey' || error.Code === 'NoSuchKey') {
|
|
203
|
-
this.log.debug(`No existing Tokowaka config found at s3://${
|
|
259
|
+
this.log.debug(`No existing Tokowaka config found at s3://${bucketName}/${s3Path}`);
|
|
204
260
|
return null;
|
|
205
261
|
}
|
|
206
262
|
|
|
@@ -212,7 +268,9 @@ class TokowakaClient {
|
|
|
212
268
|
|
|
213
269
|
/**
|
|
214
270
|
* Merges existing configuration with new configuration
|
|
215
|
-
*
|
|
271
|
+
* Checks patch key:
|
|
272
|
+
* - Patches are identified by opportunityId+suggestionId
|
|
273
|
+
* - Heading patches (no suggestionId) are identified by opportunityId
|
|
216
274
|
* - If exists: updates the patch
|
|
217
275
|
* - If not exists: adds new patch to the array
|
|
218
276
|
* @param {Object} existingConfig - Existing configuration from S3
|
|
@@ -224,92 +282,55 @@ class TokowakaClient {
|
|
|
224
282
|
return newConfig;
|
|
225
283
|
}
|
|
226
284
|
|
|
227
|
-
|
|
228
|
-
const
|
|
285
|
+
const existingPatches = existingConfig.patches || [];
|
|
286
|
+
const newPatches = newConfig.patches || [];
|
|
287
|
+
|
|
288
|
+
const { patches: mergedPatches, updateCount, addCount } = mergePatches(
|
|
289
|
+
existingPatches,
|
|
290
|
+
newPatches,
|
|
291
|
+
);
|
|
292
|
+
|
|
293
|
+
this.log.debug(`Merged patches: ${updateCount} updated, ${addCount} added`);
|
|
294
|
+
|
|
295
|
+
return {
|
|
229
296
|
...existingConfig,
|
|
230
|
-
|
|
297
|
+
url: newConfig.url,
|
|
231
298
|
version: newConfig.version,
|
|
232
|
-
|
|
299
|
+
forceFail: newConfig.forceFail,
|
|
300
|
+
prerender: newConfig.prerender,
|
|
301
|
+
patches: mergedPatches,
|
|
233
302
|
};
|
|
234
|
-
|
|
235
|
-
// Merge optimizations for each URL path
|
|
236
|
-
Object.entries(newConfig.tokowakaOptimizations).forEach(([urlPath, newOptimization]) => {
|
|
237
|
-
const existingOptimization = mergedConfig.tokowakaOptimizations[urlPath];
|
|
238
|
-
|
|
239
|
-
if (!existingOptimization) {
|
|
240
|
-
// URL path doesn't exist in existing config, add it entirely
|
|
241
|
-
mergedConfig.tokowakaOptimizations[urlPath] = newOptimization;
|
|
242
|
-
this.log.debug(`Added new URL path: ${urlPath}`);
|
|
243
|
-
} else {
|
|
244
|
-
// URL path exists, merge patches
|
|
245
|
-
const existingPatches = existingOptimization.patches || [];
|
|
246
|
-
const newPatches = newOptimization.patches || [];
|
|
247
|
-
|
|
248
|
-
// Create a map of existing patches by opportunityId+suggestionId
|
|
249
|
-
const patchMap = new Map();
|
|
250
|
-
existingPatches.forEach((patch, index) => {
|
|
251
|
-
const key = `${patch.opportunityId}:${patch.suggestionId}`;
|
|
252
|
-
patchMap.set(key, { patch, index });
|
|
253
|
-
});
|
|
254
|
-
|
|
255
|
-
// Process new patches
|
|
256
|
-
const mergedPatches = [...existingPatches];
|
|
257
|
-
let updateCount = 0;
|
|
258
|
-
let addCount = 0;
|
|
259
|
-
|
|
260
|
-
newPatches.forEach((newPatch) => {
|
|
261
|
-
const key = `${newPatch.opportunityId}:${newPatch.suggestionId}`;
|
|
262
|
-
const existing = patchMap.get(key);
|
|
263
|
-
|
|
264
|
-
if (existing) {
|
|
265
|
-
mergedPatches[existing.index] = newPatch;
|
|
266
|
-
updateCount += 1;
|
|
267
|
-
} else {
|
|
268
|
-
mergedPatches.push(newPatch);
|
|
269
|
-
addCount += 1;
|
|
270
|
-
}
|
|
271
|
-
});
|
|
272
|
-
|
|
273
|
-
mergedConfig.tokowakaOptimizations[urlPath] = {
|
|
274
|
-
...existingOptimization,
|
|
275
|
-
prerender: newOptimization.prerender,
|
|
276
|
-
patches: mergedPatches,
|
|
277
|
-
};
|
|
278
|
-
|
|
279
|
-
this.log.debug(`Merged patches for ${urlPath}: ${updateCount} updated, ${addCount} added`);
|
|
280
|
-
}
|
|
281
|
-
});
|
|
282
|
-
|
|
283
|
-
return mergedConfig;
|
|
284
303
|
}
|
|
285
304
|
|
|
286
305
|
/**
|
|
287
|
-
* Uploads Tokowaka configuration to S3
|
|
288
|
-
* @param {string}
|
|
306
|
+
* Uploads Tokowaka configuration to S3 for a specific URL
|
|
307
|
+
* @param {string} url - Full URL (e.g., 'https://www.example.com/products/item')
|
|
289
308
|
* @param {Object} config - Tokowaka configuration object
|
|
309
|
+
* @param {boolean} isPreview - Whether to upload to preview path (default: false)
|
|
290
310
|
* @returns {Promise<string>} - S3 key of uploaded config
|
|
291
311
|
*/
|
|
292
|
-
async uploadConfig(
|
|
293
|
-
if (!hasText(
|
|
294
|
-
throw this.#createError('
|
|
312
|
+
async uploadConfig(url, config, isPreview = false) {
|
|
313
|
+
if (!hasText(url)) {
|
|
314
|
+
throw this.#createError('URL is required', HTTP_BAD_REQUEST);
|
|
295
315
|
}
|
|
296
316
|
|
|
297
317
|
if (!isNonEmptyObject(config)) {
|
|
298
318
|
throw this.#createError('Config object is required', HTTP_BAD_REQUEST);
|
|
299
319
|
}
|
|
300
320
|
|
|
301
|
-
const s3Path =
|
|
321
|
+
const s3Path = getTokowakaConfigS3Path(url, this.log, isPreview);
|
|
322
|
+
const bucketName = isPreview ? this.previewBucketName : this.deployBucketName;
|
|
302
323
|
|
|
303
324
|
try {
|
|
304
325
|
const command = new PutObjectCommand({
|
|
305
|
-
Bucket:
|
|
326
|
+
Bucket: bucketName,
|
|
306
327
|
Key: s3Path,
|
|
307
328
|
Body: JSON.stringify(config, null, 2),
|
|
308
329
|
ContentType: 'application/json',
|
|
309
330
|
});
|
|
310
331
|
|
|
311
332
|
await this.s3Client.send(command);
|
|
312
|
-
this.log.info(`Successfully uploaded Tokowaka config to s3://${
|
|
333
|
+
this.log.info(`Successfully uploaded Tokowaka config to s3://${bucketName}/${s3Path}`);
|
|
313
334
|
|
|
314
335
|
return s3Path;
|
|
315
336
|
} catch (error) {
|
|
@@ -319,18 +340,19 @@ class TokowakaClient {
|
|
|
319
340
|
}
|
|
320
341
|
|
|
321
342
|
/**
|
|
322
|
-
* Invalidates CDN cache for the Tokowaka config
|
|
343
|
+
* Invalidates CDN cache for the Tokowaka config for a specific URL
|
|
323
344
|
* Currently supports CloudFront only
|
|
324
|
-
* @param {string}
|
|
345
|
+
* @param {string} url - Full URL (e.g., 'https://www.example.com/products/item')
|
|
325
346
|
* @param {string} provider - CDN provider name (default: 'cloudfront')
|
|
347
|
+
* @param {boolean} isPreview - Whether to invalidate preview path (default: false)
|
|
326
348
|
* @returns {Promise<Object|null>} - CDN invalidation result or null if skipped
|
|
327
349
|
*/
|
|
328
|
-
async invalidateCdnCache(
|
|
329
|
-
if (!hasText(
|
|
330
|
-
throw this.#createError('
|
|
350
|
+
async invalidateCdnCache(url, provider, isPreview = false) {
|
|
351
|
+
if (!hasText(url) || !hasText(provider)) {
|
|
352
|
+
throw this.#createError('URL and provider are required', HTTP_BAD_REQUEST);
|
|
331
353
|
}
|
|
332
354
|
try {
|
|
333
|
-
const pathsToInvalidate = [
|
|
355
|
+
const pathsToInvalidate = [`/${getTokowakaConfigS3Path(url, this.log, isPreview)}`];
|
|
334
356
|
this.log.debug(`Invalidating CDN cache for ${pathsToInvalidate.length} paths via ${provider}`);
|
|
335
357
|
const cdnClient = this.cdnClientRegistry.getClient(provider);
|
|
336
358
|
if (!cdnClient) {
|
|
@@ -351,18 +373,237 @@ class TokowakaClient {
|
|
|
351
373
|
|
|
352
374
|
/**
|
|
353
375
|
* Deploys suggestions to Tokowaka by generating config and uploading to S3
|
|
376
|
+
* Now creates one file per URL instead of a single file with all URLs
|
|
377
|
+
* Also creates/updates domain-level metadata if needed
|
|
354
378
|
* @param {Object} site - Site entity
|
|
355
379
|
* @param {Object} opportunity - Opportunity entity
|
|
356
|
-
* @param {Array} suggestions - Array of suggestion entities
|
|
380
|
+
* @param {Array} suggestions - Array of suggestion entities to deploy
|
|
357
381
|
* @returns {Promise<Object>} - Deployment result with succeeded/failed suggestions
|
|
358
382
|
*/
|
|
359
383
|
async deploySuggestions(site, opportunity, suggestions) {
|
|
360
|
-
|
|
361
|
-
const
|
|
384
|
+
const opportunityType = opportunity.getType();
|
|
385
|
+
const baseURL = getEffectiveBaseURL(site);
|
|
386
|
+
const siteId = site.getId();
|
|
387
|
+
const mapper = this.mapperRegistry.getMapper(opportunityType);
|
|
388
|
+
if (!mapper) {
|
|
389
|
+
throw this.#createError(
|
|
390
|
+
`No mapper found for opportunity type: ${opportunityType}. `
|
|
391
|
+
+ `Supported types: ${this.mapperRegistry.getSupportedOpportunityTypes().join(', ')}`,
|
|
392
|
+
HTTP_NOT_IMPLEMENTED,
|
|
393
|
+
);
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
// Validate which suggestions can be deployed using mapper's canDeploy method
|
|
397
|
+
const {
|
|
398
|
+
eligible: eligibleSuggestions,
|
|
399
|
+
ineligible: ineligibleSuggestions,
|
|
400
|
+
} = filterEligibleSuggestions(suggestions, mapper);
|
|
401
|
+
|
|
402
|
+
this.log.debug(
|
|
403
|
+
`Deploying ${eligibleSuggestions.length} eligible suggestions `
|
|
404
|
+
+ `(${ineligibleSuggestions.length} ineligible)`,
|
|
405
|
+
);
|
|
406
|
+
|
|
407
|
+
if (eligibleSuggestions.length === 0) {
|
|
408
|
+
this.log.warn('No eligible suggestions to deploy');
|
|
409
|
+
return {
|
|
410
|
+
succeededSuggestions: [],
|
|
411
|
+
failedSuggestions: ineligibleSuggestions,
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
// Group suggestions by URL
|
|
416
|
+
const suggestionsByUrl = groupSuggestionsByUrlPath(eligibleSuggestions, baseURL, this.log);
|
|
417
|
+
|
|
418
|
+
// Check/create domain-level metaconfig (only need to do this once per deployment)
|
|
419
|
+
const firstUrl = new URL(Object.keys(suggestionsByUrl)[0], baseURL).toString();
|
|
420
|
+
let metaconfig = await this.fetchMetaconfig(firstUrl);
|
|
421
|
+
|
|
422
|
+
if (!metaconfig) {
|
|
423
|
+
this.log.info('Creating domain-level metaconfig');
|
|
424
|
+
metaconfig = {
|
|
425
|
+
siteId,
|
|
426
|
+
prerender: mapper.requiresPrerender(),
|
|
427
|
+
};
|
|
428
|
+
await this.uploadMetaconfig(firstUrl, metaconfig);
|
|
429
|
+
} else {
|
|
430
|
+
this.log.debug('Domain-level metaconfig already exists');
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
// Process each URL separately
|
|
434
|
+
const s3Paths = [];
|
|
435
|
+
const cdnInvalidations = [];
|
|
436
|
+
|
|
437
|
+
for (const [urlPath, urlSuggestions] of Object.entries(suggestionsByUrl)) {
|
|
438
|
+
const fullUrl = new URL(urlPath, baseURL).toString();
|
|
439
|
+
this.log.debug(`Processing ${urlSuggestions.length} suggestions for URL: ${fullUrl}`);
|
|
440
|
+
|
|
441
|
+
// Fetch existing configuration for this URL from S3
|
|
442
|
+
// eslint-disable-next-line no-await-in-loop
|
|
443
|
+
const existingConfig = await this.fetchConfig(fullUrl);
|
|
444
|
+
|
|
445
|
+
// Generate configuration for this URL with eligible suggestions only
|
|
446
|
+
const newConfig = this.generateConfig(fullUrl, opportunity, urlSuggestions);
|
|
362
447
|
|
|
363
|
-
|
|
448
|
+
if (!newConfig || !newConfig.patches || newConfig.patches.length === 0) {
|
|
449
|
+
this.log.warn(`No eligible suggestions to deploy for URL: ${fullUrl}`);
|
|
450
|
+
// eslint-disable-next-line no-continue
|
|
451
|
+
continue;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// Merge with existing config for this URL
|
|
455
|
+
const config = this.mergeConfigs(existingConfig, newConfig);
|
|
456
|
+
|
|
457
|
+
// Upload to S3
|
|
458
|
+
// eslint-disable-next-line no-await-in-loop
|
|
459
|
+
const s3Path = await this.uploadConfig(fullUrl, config);
|
|
460
|
+
s3Paths.push(s3Path);
|
|
461
|
+
|
|
462
|
+
// Invalidate CDN cache (non-blocking, failures are logged but don't fail deployment)
|
|
463
|
+
// eslint-disable-next-line no-await-in-loop
|
|
464
|
+
const cdnInvalidationResult = await this.invalidateCdnCache(
|
|
465
|
+
fullUrl,
|
|
466
|
+
this.env.TOKOWAKA_CDN_PROVIDER,
|
|
467
|
+
);
|
|
468
|
+
cdnInvalidations.push(cdnInvalidationResult);
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
this.log.info(`Uploaded Tokowaka configs for ${s3Paths.length} URLs`);
|
|
472
|
+
|
|
473
|
+
return {
|
|
474
|
+
s3Paths,
|
|
475
|
+
cdnInvalidations,
|
|
476
|
+
succeededSuggestions: eligibleSuggestions,
|
|
477
|
+
failedSuggestions: ineligibleSuggestions,
|
|
478
|
+
};
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/**
|
|
482
|
+
* Rolls back deployed suggestions by removing their patches from the configuration
|
|
483
|
+
* Now updates one file per URL instead of a single file with all URLs
|
|
484
|
+
* @param {Object} site - Site entity
|
|
485
|
+
* @param {Object} opportunity - Opportunity entity
|
|
486
|
+
* @param {Array} suggestions - Array of suggestion entities to rollback
|
|
487
|
+
* @returns {Promise<Object>} - Rollback result with succeeded/failed suggestions
|
|
488
|
+
*/
|
|
489
|
+
async rollbackSuggestions(site, opportunity, suggestions) {
|
|
490
|
+
const opportunityType = opportunity.getType();
|
|
491
|
+
const baseURL = getEffectiveBaseURL(site);
|
|
492
|
+
const mapper = this.mapperRegistry.getMapper(opportunityType);
|
|
493
|
+
if (!mapper) {
|
|
364
494
|
throw this.#createError(
|
|
365
|
-
|
|
495
|
+
`No mapper found for opportunity type: ${opportunityType}. `
|
|
496
|
+
+ `Supported types: ${this.mapperRegistry.getSupportedOpportunityTypes().join(', ')}`,
|
|
497
|
+
HTTP_NOT_IMPLEMENTED,
|
|
498
|
+
);
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// Validate which suggestions can be rolled back
|
|
502
|
+
// For rollback, we use the same canDeploy check to ensure data integrity
|
|
503
|
+
const {
|
|
504
|
+
eligible: eligibleSuggestions,
|
|
505
|
+
ineligible: ineligibleSuggestions,
|
|
506
|
+
} = filterEligibleSuggestions(suggestions, mapper);
|
|
507
|
+
|
|
508
|
+
this.log.debug(
|
|
509
|
+
`Rolling back ${eligibleSuggestions.length} eligible suggestions `
|
|
510
|
+
+ `(${ineligibleSuggestions.length} ineligible)`,
|
|
511
|
+
);
|
|
512
|
+
|
|
513
|
+
if (eligibleSuggestions.length === 0) {
|
|
514
|
+
this.log.warn('No eligible suggestions to rollback');
|
|
515
|
+
return {
|
|
516
|
+
succeededSuggestions: [],
|
|
517
|
+
failedSuggestions: ineligibleSuggestions,
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
// Group suggestions by URL
|
|
522
|
+
const suggestionsByUrl = groupSuggestionsByUrlPath(eligibleSuggestions, baseURL, this.log);
|
|
523
|
+
|
|
524
|
+
// Process each URL separately
|
|
525
|
+
const s3Paths = [];
|
|
526
|
+
const cdnInvalidations = [];
|
|
527
|
+
let totalRemovedCount = 0;
|
|
528
|
+
|
|
529
|
+
for (const [urlPath, urlSuggestions] of Object.entries(suggestionsByUrl)) {
|
|
530
|
+
const fullUrl = new URL(urlPath, baseURL).toString();
|
|
531
|
+
this.log.debug(`Rolling back ${urlSuggestions.length} suggestions for URL: ${fullUrl}`);
|
|
532
|
+
|
|
533
|
+
// Fetch existing configuration for this URL from S3
|
|
534
|
+
// eslint-disable-next-line no-await-in-loop
|
|
535
|
+
const existingConfig = await this.fetchConfig(fullUrl);
|
|
536
|
+
|
|
537
|
+
if (!existingConfig || !existingConfig.patches) {
|
|
538
|
+
this.log.warn(`No existing configuration found for URL: ${fullUrl}`);
|
|
539
|
+
// eslint-disable-next-line no-continue
|
|
540
|
+
continue;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// Extract suggestion IDs to remove for this URL
|
|
544
|
+
const suggestionIdsToRemove = urlSuggestions.map((s) => s.getId());
|
|
545
|
+
|
|
546
|
+
// Use mapper to remove patches
|
|
547
|
+
const updatedConfig = mapper.rollbackPatches(
|
|
548
|
+
existingConfig,
|
|
549
|
+
suggestionIdsToRemove,
|
|
550
|
+
opportunity.getId(),
|
|
551
|
+
);
|
|
552
|
+
|
|
553
|
+
if (updatedConfig.removedCount === 0) {
|
|
554
|
+
this.log.warn(`No patches found for URL: ${fullUrl}`);
|
|
555
|
+
// eslint-disable-next-line no-continue
|
|
556
|
+
continue;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
this.log.info(`Removed ${updatedConfig.removedCount} patches for URL: ${fullUrl}`);
|
|
560
|
+
totalRemovedCount += updatedConfig.removedCount;
|
|
561
|
+
|
|
562
|
+
// Remove the removedCount property before uploading
|
|
563
|
+
delete updatedConfig.removedCount;
|
|
564
|
+
|
|
565
|
+
// Upload updated config to S3 for this URL
|
|
566
|
+
// eslint-disable-next-line no-await-in-loop
|
|
567
|
+
const s3Path = await this.uploadConfig(fullUrl, updatedConfig);
|
|
568
|
+
s3Paths.push(s3Path);
|
|
569
|
+
|
|
570
|
+
// Invalidate CDN cache (non-blocking, failures are logged but don't fail rollback)
|
|
571
|
+
// eslint-disable-next-line no-await-in-loop
|
|
572
|
+
const cdnInvalidationResult = await this.invalidateCdnCache(
|
|
573
|
+
fullUrl,
|
|
574
|
+
this.env.TOKOWAKA_CDN_PROVIDER,
|
|
575
|
+
);
|
|
576
|
+
cdnInvalidations.push(cdnInvalidationResult);
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
this.log.info(`Updated Tokowaka configs for ${s3Paths.length} URLs, removed ${totalRemovedCount} patches total`);
|
|
580
|
+
|
|
581
|
+
return {
|
|
582
|
+
s3Paths,
|
|
583
|
+
cdnInvalidations,
|
|
584
|
+
succeededSuggestions: eligibleSuggestions,
|
|
585
|
+
failedSuggestions: ineligibleSuggestions,
|
|
586
|
+
removedPatchesCount: totalRemovedCount,
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* Previews suggestions by generating config and uploading to preview path
|
|
592
|
+
* All suggestions must belong to the same URL
|
|
593
|
+
* @param {Object} site - Site entity
|
|
594
|
+
* @param {Object} opportunity - Opportunity entity
|
|
595
|
+
* @param {Array} suggestions - Array of suggestion entities to preview (must be same URL)
|
|
596
|
+
* @param {Object} options - Optional configuration for HTML fetching
|
|
597
|
+
* @returns {Promise<Object>} - Preview result with config and succeeded/failed suggestions
|
|
598
|
+
*/
|
|
599
|
+
async previewSuggestions(site, opportunity, suggestions, options = {}) {
|
|
600
|
+
// Get site's forwarded host for preview
|
|
601
|
+
const { forwardedHost, apiKey } = site.getConfig()?.getTokowakaConfig() || {};
|
|
602
|
+
|
|
603
|
+
if (!hasText(forwardedHost) || !hasText(apiKey)) {
|
|
604
|
+
throw this.#createError(
|
|
605
|
+
'Site does not have a Tokowaka API key or forwarded host configured. '
|
|
606
|
+
+ 'Please onboard the site to Tokowaka first.',
|
|
366
607
|
HTTP_BAD_REQUEST,
|
|
367
608
|
);
|
|
368
609
|
}
|
|
@@ -377,68 +618,131 @@ class TokowakaClient {
|
|
|
377
618
|
);
|
|
378
619
|
}
|
|
379
620
|
|
|
380
|
-
//
|
|
381
|
-
const
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
} else {
|
|
389
|
-
ineligibleSuggestions.push({
|
|
390
|
-
suggestion,
|
|
391
|
-
reason: eligibility.reason || 'Suggestion cannot be deployed',
|
|
392
|
-
});
|
|
393
|
-
}
|
|
394
|
-
});
|
|
621
|
+
// TOKOWAKA_EDGE_URL is mandatory for preview
|
|
622
|
+
const tokowakaEdgeUrl = this.env.TOKOWAKA_EDGE_URL;
|
|
623
|
+
if (!hasText(tokowakaEdgeUrl)) {
|
|
624
|
+
throw this.#createError(
|
|
625
|
+
'TOKOWAKA_EDGE_URL is required for preview functionality',
|
|
626
|
+
HTTP_INTERNAL_SERVER_ERROR,
|
|
627
|
+
);
|
|
628
|
+
}
|
|
395
629
|
|
|
396
|
-
|
|
630
|
+
// Validate which suggestions can be deployed using mapper's canDeploy method
|
|
631
|
+
const {
|
|
632
|
+
eligible: eligibleSuggestions,
|
|
633
|
+
ineligible: ineligibleSuggestions,
|
|
634
|
+
} = filterEligibleSuggestions(suggestions, mapper);
|
|
635
|
+
|
|
636
|
+
this.log.debug(
|
|
637
|
+
`Previewing ${eligibleSuggestions.length} eligible suggestions `
|
|
638
|
+
+ `(${ineligibleSuggestions.length} ineligible)`,
|
|
639
|
+
);
|
|
397
640
|
|
|
398
641
|
if (eligibleSuggestions.length === 0) {
|
|
399
|
-
this.log.warn('No eligible suggestions to
|
|
642
|
+
this.log.warn('No eligible suggestions to preview');
|
|
400
643
|
return {
|
|
644
|
+
config: null,
|
|
401
645
|
succeededSuggestions: [],
|
|
402
646
|
failedSuggestions: ineligibleSuggestions,
|
|
403
647
|
};
|
|
404
648
|
}
|
|
405
649
|
|
|
406
|
-
//
|
|
407
|
-
|
|
408
|
-
|
|
650
|
+
// Get the preview URL from the first suggestion
|
|
651
|
+
const previewUrl = eligibleSuggestions[0].getData()?.url;
|
|
652
|
+
if (!hasText(previewUrl)) {
|
|
653
|
+
throw this.#createError('Preview URL not found in suggestion data', HTTP_BAD_REQUEST);
|
|
654
|
+
}
|
|
409
655
|
|
|
410
|
-
//
|
|
411
|
-
this.log.debug(`
|
|
412
|
-
const
|
|
656
|
+
// Fetch existing deployed configuration for this URL from production S3
|
|
657
|
+
this.log.debug(`Fetching existing deployed Tokowaka config for URL: ${previewUrl}`);
|
|
658
|
+
const existingConfig = await this.fetchConfig(previewUrl, false);
|
|
413
659
|
|
|
414
|
-
|
|
415
|
-
|
|
660
|
+
// Generate configuration with eligible preview suggestions
|
|
661
|
+
this.log.debug(`Generating preview Tokowaka config for opportunity ${opportunity.getId()}`);
|
|
662
|
+
const newConfig = this.generateConfig(previewUrl, opportunity, eligibleSuggestions);
|
|
663
|
+
|
|
664
|
+
if (!newConfig || !newConfig.patches || newConfig.patches.length === 0) {
|
|
665
|
+
this.log.warn('No eligible suggestions to preview');
|
|
416
666
|
return {
|
|
667
|
+
config: null,
|
|
417
668
|
succeededSuggestions: [],
|
|
418
669
|
failedSuggestions: suggestions,
|
|
419
670
|
};
|
|
420
671
|
}
|
|
421
672
|
|
|
422
|
-
// Merge with existing config
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
673
|
+
// Merge with existing deployed config to include already-deployed patches for this URL
|
|
674
|
+
let config = newConfig;
|
|
675
|
+
if (existingConfig && existingConfig.patches?.length > 0) {
|
|
676
|
+
this.log.info(
|
|
677
|
+
`Found ${existingConfig.patches.length} deployed patches, merging with preview suggestions`,
|
|
678
|
+
);
|
|
426
679
|
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
const s3Path = await this.uploadConfig(apiKey, config);
|
|
680
|
+
// Merge the existing deployed patches with new preview suggestions
|
|
681
|
+
config = this.mergeConfigs(existingConfig, newConfig);
|
|
430
682
|
|
|
431
|
-
|
|
683
|
+
this.log.debug(
|
|
684
|
+
`Preview config now has ${config.patches.length} total patches`,
|
|
685
|
+
);
|
|
686
|
+
} else {
|
|
687
|
+
this.log.info('No deployed patches found, using only preview suggestions');
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
// Upload to preview S3 path for this URL
|
|
691
|
+
this.log.info(`Uploading preview Tokowaka config with ${eligibleSuggestions.length} new suggestions`);
|
|
692
|
+
const s3Path = await this.uploadConfig(previewUrl, config, true);
|
|
693
|
+
|
|
694
|
+
// Invalidate CDN cache for preview path
|
|
432
695
|
const cdnInvalidationResult = await this.invalidateCdnCache(
|
|
433
|
-
|
|
696
|
+
previewUrl,
|
|
434
697
|
this.env.TOKOWAKA_CDN_PROVIDER,
|
|
698
|
+
true,
|
|
435
699
|
);
|
|
436
700
|
|
|
701
|
+
// Fetch HTML content for preview
|
|
702
|
+
let originalHtml = null;
|
|
703
|
+
let optimizedHtml = null;
|
|
704
|
+
|
|
705
|
+
try {
|
|
706
|
+
// Fetch original HTML (without preview)
|
|
707
|
+
originalHtml = await fetchHtmlWithWarmup(
|
|
708
|
+
previewUrl,
|
|
709
|
+
apiKey,
|
|
710
|
+
forwardedHost,
|
|
711
|
+
tokowakaEdgeUrl,
|
|
712
|
+
this.log,
|
|
713
|
+
false,
|
|
714
|
+
options,
|
|
715
|
+
);
|
|
716
|
+
// Then fetch optimized HTML (with preview)
|
|
717
|
+
optimizedHtml = await fetchHtmlWithWarmup(
|
|
718
|
+
previewUrl,
|
|
719
|
+
apiKey,
|
|
720
|
+
forwardedHost,
|
|
721
|
+
tokowakaEdgeUrl,
|
|
722
|
+
this.log,
|
|
723
|
+
true,
|
|
724
|
+
options,
|
|
725
|
+
);
|
|
726
|
+
this.log.info('Successfully fetched both original and optimized HTML for preview');
|
|
727
|
+
} catch (error) {
|
|
728
|
+
this.log.error(`Failed to fetch HTML for preview: ${error.message}`);
|
|
729
|
+
throw this.#createError(
|
|
730
|
+
`Preview failed: Unable to fetch HTML - ${error.message}`,
|
|
731
|
+
HTTP_INTERNAL_SERVER_ERROR,
|
|
732
|
+
);
|
|
733
|
+
}
|
|
734
|
+
|
|
437
735
|
return {
|
|
438
736
|
s3Path,
|
|
737
|
+
config,
|
|
439
738
|
cdnInvalidation: cdnInvalidationResult,
|
|
440
739
|
succeededSuggestions: eligibleSuggestions,
|
|
441
740
|
failedSuggestions: ineligibleSuggestions,
|
|
741
|
+
html: {
|
|
742
|
+
url: previewUrl,
|
|
743
|
+
originalHtml,
|
|
744
|
+
optimizedHtml,
|
|
745
|
+
},
|
|
442
746
|
};
|
|
443
747
|
}
|
|
444
748
|
}
|