@adobe/spacecat-shared-tokowaka-client 1.7.4 → 1.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/package.json +1 -1
- package/src/index.js +51 -34
- package/src/utils/custom-html-utils.js +47 -31
- package/test/index.test.js +1 -1
- package/test/utils/html-utils.test.js +104 -5
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
# [@adobe/spacecat-shared-tokowaka-client-v1.7.6](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.7.5...@adobe/spacecat-shared-tokowaka-client-v1.7.6) (2026-02-09)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Bug Fixes
|
|
5
|
+
|
|
6
|
+
* edge-preview api: ignore warmup call failure ([#1327](https://github.com/adobe/spacecat-shared/issues/1327)) ([fbc9f65](https://github.com/adobe/spacecat-shared/commit/fbc9f65bf96e408c878fa3b8b37fbbe29444e08f))
|
|
7
|
+
|
|
8
|
+
# [@adobe/spacecat-shared-tokowaka-client-v1.7.5](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.7.4...@adobe/spacecat-shared-tokowaka-client-v1.7.5) (2026-02-05)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
* optimize edge-preview API ([#1319](https://github.com/adobe/spacecat-shared/issues/1319)) ([7f13b66](https://github.com/adobe/spacecat-shared/commit/7f13b666bd234a246a530e0f387feac143792b36))
|
|
14
|
+
|
|
1
15
|
# [@adobe/spacecat-shared-tokowaka-client-v1.7.4](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.7.3...@adobe/spacecat-shared-tokowaka-client-v1.7.4) (2026-02-05)
|
|
2
16
|
|
|
3
17
|
|
package/package.json
CHANGED
package/src/index.js
CHANGED
|
@@ -238,6 +238,7 @@ class TokowakaClient {
|
|
|
238
238
|
throw this.#createError('URL is required', HTTP_BAD_REQUEST);
|
|
239
239
|
}
|
|
240
240
|
|
|
241
|
+
const fetchStartTime = Date.now();
|
|
241
242
|
const s3Path = getTokowakaMetaconfigS3Path(url, this.log);
|
|
242
243
|
const bucketName = this.deployBucketName;
|
|
243
244
|
|
|
@@ -251,7 +252,7 @@ class TokowakaClient {
|
|
|
251
252
|
const bodyContents = await response.Body.transformToString();
|
|
252
253
|
const metaconfig = JSON.parse(bodyContents);
|
|
253
254
|
|
|
254
|
-
this.log.debug(`Successfully fetched metaconfig from s3://${bucketName}/${s3Path}`);
|
|
255
|
+
this.log.debug(`Successfully fetched metaconfig from s3://${bucketName}/${s3Path} in ${Date.now() - fetchStartTime}ms`);
|
|
255
256
|
return metaconfig;
|
|
256
257
|
} catch (error) {
|
|
257
258
|
// If metaconfig doesn't exist (NoSuchKey), return null
|
|
@@ -392,6 +393,7 @@ class TokowakaClient {
|
|
|
392
393
|
throw this.#createError('Metaconfig object is required', HTTP_BAD_REQUEST);
|
|
393
394
|
}
|
|
394
395
|
|
|
396
|
+
const uploadStartTime = Date.now();
|
|
395
397
|
const s3Path = getTokowakaMetaconfigS3Path(url, this.log);
|
|
396
398
|
const bucketName = this.deployBucketName;
|
|
397
399
|
|
|
@@ -411,7 +413,7 @@ class TokowakaClient {
|
|
|
411
413
|
const command = new PutObjectCommand(putObjectParams);
|
|
412
414
|
|
|
413
415
|
await this.s3Client.send(command);
|
|
414
|
-
this.log.info(`Successfully uploaded metaconfig to s3://${bucketName}/${s3Path}`);
|
|
416
|
+
this.log.info(`Successfully uploaded metaconfig to s3://${bucketName}/${s3Path} in ${Date.now() - uploadStartTime}ms`);
|
|
415
417
|
|
|
416
418
|
// Invalidate CDN cache for the metaconfig (both CloudFront and Fastly)
|
|
417
419
|
this.log.info('Invalidating CDN cache for uploaded metaconfig');
|
|
@@ -435,6 +437,7 @@ class TokowakaClient {
|
|
|
435
437
|
throw this.#createError('URL is required', HTTP_BAD_REQUEST);
|
|
436
438
|
}
|
|
437
439
|
|
|
440
|
+
const fetchStartTime = Date.now();
|
|
438
441
|
const s3Path = getTokowakaConfigS3Path(url, this.log, isPreview);
|
|
439
442
|
const bucketName = isPreview ? this.previewBucketName : this.deployBucketName;
|
|
440
443
|
|
|
@@ -448,7 +451,7 @@ class TokowakaClient {
|
|
|
448
451
|
const bodyContents = await response.Body.transformToString();
|
|
449
452
|
const config = JSON.parse(bodyContents);
|
|
450
453
|
|
|
451
|
-
this.log.debug(`Successfully fetched existing Tokowaka config from s3://${bucketName}/${s3Path}`);
|
|
454
|
+
this.log.debug(`Successfully fetched existing Tokowaka config from s3://${bucketName}/${s3Path} in ${Date.now() - fetchStartTime}ms`);
|
|
452
455
|
return config;
|
|
453
456
|
} catch (error) {
|
|
454
457
|
// If config doesn't exist (NoSuchKey), return null
|
|
@@ -514,6 +517,7 @@ class TokowakaClient {
|
|
|
514
517
|
if (!isNonEmptyObject(config)) {
|
|
515
518
|
throw this.#createError('Config object is required', HTTP_BAD_REQUEST);
|
|
516
519
|
}
|
|
520
|
+
const uploadStartTime = Date.now();
|
|
517
521
|
|
|
518
522
|
const s3Path = getTokowakaConfigS3Path(url, this.log, isPreview);
|
|
519
523
|
const bucketName = isPreview ? this.previewBucketName : this.deployBucketName;
|
|
@@ -527,7 +531,7 @@ class TokowakaClient {
|
|
|
527
531
|
});
|
|
528
532
|
|
|
529
533
|
await this.s3Client.send(command);
|
|
530
|
-
this.log.info(`Successfully uploaded Tokowaka config to s3://${bucketName}/${s3Path}`);
|
|
534
|
+
this.log.info(`Successfully uploaded Tokowaka config to s3://${bucketName}/${s3Path} in ${Date.now() - uploadStartTime}ms`);
|
|
531
535
|
|
|
532
536
|
return s3Path;
|
|
533
537
|
} catch (error) {
|
|
@@ -553,6 +557,7 @@ class TokowakaClient {
|
|
|
553
557
|
providers = this.#getCdnProviders(),
|
|
554
558
|
isPreview = false,
|
|
555
559
|
}) {
|
|
560
|
+
const invalidationStartTime = Date.now();
|
|
556
561
|
// Convert single provider to array for uniform handling
|
|
557
562
|
const providerList = Array.isArray(providers) ? providers : [providers].filter(Boolean);
|
|
558
563
|
|
|
@@ -596,11 +601,12 @@ class TokowakaClient {
|
|
|
596
601
|
continue;
|
|
597
602
|
}
|
|
598
603
|
|
|
604
|
+
const startTime = Date.now();
|
|
599
605
|
// eslint-disable-next-line no-await-in-loop
|
|
600
606
|
const result = await cdnClient.invalidateCache(pathsToInvalidate);
|
|
601
607
|
this.log.info(
|
|
602
608
|
`CDN cache invalidation completed for ${provider}: `
|
|
603
|
-
+ `${pathsToInvalidate.length} path(s)`,
|
|
609
|
+
+ `${pathsToInvalidate.length} path(s), and took ${Date.now() - startTime}ms`,
|
|
604
610
|
);
|
|
605
611
|
results.push(result);
|
|
606
612
|
} catch (error) {
|
|
@@ -612,6 +618,7 @@ class TokowakaClient {
|
|
|
612
618
|
});
|
|
613
619
|
}
|
|
614
620
|
}
|
|
621
|
+
this.log.info(`CDN cache invalidation completed in total ${Date.now() - invalidationStartTime}ms`);
|
|
615
622
|
return results;
|
|
616
623
|
}
|
|
617
624
|
|
|
@@ -866,8 +873,10 @@ class TokowakaClient {
|
|
|
866
873
|
* @param {Object} options - Optional configuration for HTML fetching
|
|
867
874
|
* @returns {Promise<Object>} - Preview result with config and succeeded/failed suggestions
|
|
868
875
|
*/
|
|
869
|
-
async previewSuggestions(site, opportunity, suggestions, options = {}) {
|
|
876
|
+
async previewSuggestions(site, opportunity, suggestions = [], options = {}) {
|
|
870
877
|
const opportunityType = opportunity.getType();
|
|
878
|
+
this.log.info(`Previewing ${suggestions.length} suggestions for `
|
|
879
|
+
+ `${opportunityType} opportunity and URL ${site.getBaseURL()}`);
|
|
871
880
|
const mapper = this.mapperRegistry.getMapper(opportunityType);
|
|
872
881
|
if (!mapper) {
|
|
873
882
|
throw this.#createError(
|
|
@@ -892,7 +901,7 @@ class TokowakaClient {
|
|
|
892
901
|
ineligible: ineligibleSuggestions,
|
|
893
902
|
} = filterEligibleSuggestions(suggestions, mapper);
|
|
894
903
|
|
|
895
|
-
this.log.
|
|
904
|
+
this.log.info(
|
|
896
905
|
`Previewing ${eligibleSuggestions.length} eligible suggestions `
|
|
897
906
|
+ `(${ineligibleSuggestions.length} ineligible)`,
|
|
898
907
|
);
|
|
@@ -912,22 +921,24 @@ class TokowakaClient {
|
|
|
912
921
|
throw this.#createError('Preview URL not found in suggestion data', HTTP_BAD_REQUEST);
|
|
913
922
|
}
|
|
914
923
|
|
|
915
|
-
// Fetch metaconfig
|
|
916
|
-
const metaconfig = await
|
|
924
|
+
// Fetch metaconfig and existing config in parallel
|
|
925
|
+
const [metaconfig, existingConfig] = await Promise.all([
|
|
926
|
+
this.fetchMetaconfig(previewUrl).catch(() => null),
|
|
927
|
+
this.fetchConfig(previewUrl, false),
|
|
928
|
+
]);
|
|
917
929
|
|
|
918
930
|
let apiKey;
|
|
919
|
-
if (Array.isArray(metaconfig.apiKeys) && metaconfig.apiKeys.length > 0) {
|
|
931
|
+
if (metaconfig && Array.isArray(metaconfig.apiKeys) && metaconfig.apiKeys.length > 0) {
|
|
920
932
|
[apiKey] = metaconfig.apiKeys;
|
|
933
|
+
this.log.info('Using API key from metaconfig');
|
|
934
|
+
} else {
|
|
935
|
+
this.log.info('Proceeding with preview without API key');
|
|
921
936
|
}
|
|
922
937
|
|
|
923
938
|
const forwardedHost = calculateForwardedHost(previewUrl, this.log);
|
|
924
939
|
|
|
925
|
-
// Fetch existing deployed configuration for this URL from production S3
|
|
926
|
-
this.log.debug(`Fetching existing deployed Tokowaka config for URL: ${previewUrl}`);
|
|
927
|
-
const existingConfig = await this.fetchConfig(previewUrl, false);
|
|
928
|
-
|
|
929
940
|
// Generate configuration with eligible preview suggestions
|
|
930
|
-
this.log.
|
|
941
|
+
this.log.info(`Generating preview Tokowaka config for opportunity ${opportunity.getId()}`);
|
|
931
942
|
const newConfig = this.generateConfig(previewUrl, opportunity, eligibleSuggestions);
|
|
932
943
|
|
|
933
944
|
if (!newConfig) {
|
|
@@ -965,7 +976,7 @@ class TokowakaClient {
|
|
|
965
976
|
// Merge the existing deployed patches with new preview suggestions
|
|
966
977
|
config = this.mergeConfigs(existingConfig, newConfig);
|
|
967
978
|
|
|
968
|
-
this.log.
|
|
979
|
+
this.log.info(
|
|
969
980
|
`Preview config now has ${config.patches.length} total patches`,
|
|
970
981
|
);
|
|
971
982
|
} else {
|
|
@@ -976,28 +987,34 @@ class TokowakaClient {
|
|
|
976
987
|
this.log.info(`Uploading preview Tokowaka config with ${eligibleSuggestions.length} new suggestions`);
|
|
977
988
|
const s3Path = await this.uploadConfig(previewUrl, config, true);
|
|
978
989
|
|
|
979
|
-
// Invalidate CDN cache for all providers in parallel (preview path)
|
|
980
|
-
const cdnInvalidationResults = await this.invalidateCdnCache({
|
|
981
|
-
urls: [previewUrl],
|
|
982
|
-
isPreview: true,
|
|
983
|
-
});
|
|
984
|
-
|
|
985
990
|
// Fetch HTML content for preview
|
|
986
991
|
let originalHtml = null;
|
|
987
992
|
let optimizedHtml = null;
|
|
993
|
+
let cdnInvalidationResults = null;
|
|
988
994
|
|
|
989
995
|
try {
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
996
|
+
const fetchAndInvalidateStartTime = Date.now();
|
|
997
|
+
// Fetch original HTML and invalidate CDN cache in parallel to save time
|
|
998
|
+
[originalHtml, cdnInvalidationResults] = await Promise.all([
|
|
999
|
+
fetchHtmlWithWarmup(
|
|
1000
|
+
previewUrl,
|
|
1001
|
+
apiKey,
|
|
1002
|
+
forwardedHost,
|
|
1003
|
+
edgeUrl,
|
|
1004
|
+
this.log,
|
|
1005
|
+
false,
|
|
1006
|
+
options,
|
|
1007
|
+
),
|
|
1008
|
+
this.invalidateCdnCache({
|
|
1009
|
+
urls: [previewUrl],
|
|
1010
|
+
isPreview: true,
|
|
1011
|
+
}),
|
|
1012
|
+
]);
|
|
1013
|
+
this.log.info('Successfully fetched original HTML and invalidated CDN cache'
|
|
1014
|
+
+ ` in ${Date.now() - fetchAndInvalidateStartTime}ms`);
|
|
1015
|
+
|
|
1016
|
+
// Step 2: Fetch optimized HTML after CDN invalidation and original fetch complete
|
|
1017
|
+
this.log.info('Fetching optimized HTML...');
|
|
1001
1018
|
optimizedHtml = await fetchHtmlWithWarmup(
|
|
1002
1019
|
previewUrl,
|
|
1003
1020
|
apiKey,
|
|
@@ -1007,7 +1024,7 @@ class TokowakaClient {
|
|
|
1007
1024
|
true,
|
|
1008
1025
|
options,
|
|
1009
1026
|
);
|
|
1010
|
-
this.log.info('Successfully fetched
|
|
1027
|
+
this.log.info('Successfully fetched optimized HTML');
|
|
1011
1028
|
} catch (error) {
|
|
1012
1029
|
this.log.error(`Failed to fetch HTML for preview: ${error.message}`);
|
|
1013
1030
|
throw this.#createError(
|
|
@@ -23,6 +23,20 @@ function sleep(ms) {
|
|
|
23
23
|
});
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
+
async function fetchWithTimeout(url, options = {}, timeout = 3000) {
|
|
27
|
+
try {
|
|
28
|
+
return await fetch(url, {
|
|
29
|
+
...options,
|
|
30
|
+
signal: AbortSignal.timeout(timeout),
|
|
31
|
+
});
|
|
32
|
+
} catch (error) {
|
|
33
|
+
if (error.name === 'TimeoutError') {
|
|
34
|
+
throw new Error(`Request timed out after ${timeout}ms`);
|
|
35
|
+
}
|
|
36
|
+
throw error;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
26
40
|
/**
|
|
27
41
|
* Makes an HTTP request with retry logic for both original and optimized HTML.
|
|
28
42
|
* Header validation logic (same for both):
|
|
@@ -40,12 +54,14 @@ function sleep(ms) {
|
|
|
40
54
|
async function fetchWithRetry(url, options, maxRetries, retryDelayMs, log, fetchType) {
|
|
41
55
|
for (let attempt = 1; attempt <= maxRetries + 1; attempt += 1) {
|
|
42
56
|
try {
|
|
43
|
-
log.
|
|
57
|
+
log.info(`[${fetchType}] Retry attempt ${attempt}/${maxRetries + 1} for ${fetchType} HTML`);
|
|
44
58
|
|
|
45
59
|
// eslint-disable-next-line no-await-in-loop
|
|
46
|
-
const response = await
|
|
60
|
+
const response = await fetchWithTimeout(url, options);
|
|
47
61
|
|
|
48
|
-
|
|
62
|
+
// Log request ID for debugging
|
|
63
|
+
const requestId = response?.headers?.get('x-edgeoptimize-request-id');
|
|
64
|
+
log.info(`[${fetchType}] Response status (attempt ${attempt}): ${response.status} ${response.statusText}, x-edgeoptimize-request-id: ${requestId || 'none'}`);
|
|
49
65
|
|
|
50
66
|
if (!response.ok) {
|
|
51
67
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
@@ -55,35 +71,35 @@ async function fetchWithRetry(url, options, maxRetries, retryDelayMs, log, fetch
|
|
|
55
71
|
const cacheHeader = response.headers.get('x-edgeoptimize-cache');
|
|
56
72
|
const proxyHeader = response.headers.get('x-edgeoptimize-proxy');
|
|
57
73
|
|
|
58
|
-
log.
|
|
74
|
+
log.info(`[${fetchType}] Headers - cache: ${cacheHeader || 'none'}, proxy: ${proxyHeader || 'none'}`);
|
|
59
75
|
|
|
60
76
|
// Case 1: Cache header present (regardless of proxy) -> Success
|
|
61
77
|
if (cacheHeader) {
|
|
62
|
-
log.
|
|
78
|
+
log.info(`[${fetchType}] Cache header found (x-edgeoptimize-cache: ${cacheHeader}), stopping retry logic`);
|
|
63
79
|
return response;
|
|
64
80
|
}
|
|
65
81
|
|
|
66
82
|
// Case 2: No cache header AND no proxy header -> Success (return immediately)
|
|
67
83
|
if (!proxyHeader) {
|
|
68
|
-
log.
|
|
84
|
+
log.info(`[${fetchType}] No edge optimize headers found, proceeding as successful flow`);
|
|
69
85
|
return response;
|
|
70
86
|
}
|
|
71
87
|
|
|
72
88
|
// Case 3: Proxy header present BUT no cache header -> Retry until cache found
|
|
73
|
-
log.
|
|
89
|
+
log.info(`[${fetchType}] Proxy header present without cache header, will retry...`);
|
|
74
90
|
|
|
75
91
|
// If we haven't exhausted retries, continue
|
|
76
92
|
if (attempt < maxRetries + 1) {
|
|
77
|
-
log.debug(`Waiting ${retryDelayMs}ms before retry...`);
|
|
93
|
+
log.debug(`[${fetchType}] Waiting ${retryDelayMs}ms before retry...`);
|
|
78
94
|
// eslint-disable-next-line no-await-in-loop
|
|
79
95
|
await sleep(retryDelayMs);
|
|
80
96
|
} else {
|
|
81
97
|
// Last attempt - throw error
|
|
82
|
-
log.error(`Max retries (${maxRetries}) exhausted. Proxy header present but cache header not found`);
|
|
98
|
+
log.error(`[${fetchType}] Max retries (${maxRetries}) exhausted. Proxy header present but cache header not found`);
|
|
83
99
|
throw new Error(`Cache header (x-edgeoptimize-cache) not found after ${maxRetries} retries`);
|
|
84
100
|
}
|
|
85
101
|
} catch (error) {
|
|
86
|
-
log.warn(`Attempt ${attempt} failed for ${fetchType} HTML, error: ${error.message}`);
|
|
102
|
+
log.warn(`[${fetchType}] Attempt ${attempt} failed for ${fetchType} HTML, error: ${error.message}`);
|
|
87
103
|
|
|
88
104
|
// If this was the last attempt, throw the error
|
|
89
105
|
if (attempt === maxRetries + 1) {
|
|
@@ -91,7 +107,7 @@ async function fetchWithRetry(url, options, maxRetries, retryDelayMs, log, fetch
|
|
|
91
107
|
}
|
|
92
108
|
|
|
93
109
|
// Wait before retrying
|
|
94
|
-
log.debug(`Waiting ${retryDelayMs}ms before retry...`);
|
|
110
|
+
log.debug(`[${fetchType}] Waiting ${retryDelayMs}ms before retry...`);
|
|
95
111
|
// eslint-disable-next-line no-await-in-loop
|
|
96
112
|
await sleep(retryDelayMs);
|
|
97
113
|
}
|
|
@@ -110,11 +126,11 @@ async function fetchWithRetry(url, options, maxRetries, retryDelayMs, log, fetch
|
|
|
110
126
|
* @param {boolean} isOptimized - Whether to fetch optimized HTML (with preview param)
|
|
111
127
|
* @param {Object} log - Logger instance
|
|
112
128
|
* @param {Object} options - Additional options
|
|
113
|
-
* @param {number} options.warmupDelayMs - Delay after warmup call (default:
|
|
114
|
-
* @param {number} options.maxRetries - Maximum number of retries for actual call (default:
|
|
129
|
+
* @param {number} options.warmupDelayMs - Delay after warmup call (default: 1000ms)
|
|
130
|
+
* @param {number} options.maxRetries - Maximum number of retries for actual call (default: 3)
|
|
115
131
|
* @param {number} options.retryDelayMs - Delay between retries (default: 1000ms)
|
|
116
132
|
* @returns {Promise<string>} - HTML content
|
|
117
|
-
* @throws {Error} - If validation fails or fetch fails after retries
|
|
133
|
+
* @throws {Error} - If validation fails or fetch fails after retries or timeout
|
|
118
134
|
*/
|
|
119
135
|
export async function fetchHtmlWithWarmup(
|
|
120
136
|
url,
|
|
@@ -140,7 +156,7 @@ export async function fetchHtmlWithWarmup(
|
|
|
140
156
|
|
|
141
157
|
// Default options
|
|
142
158
|
const {
|
|
143
|
-
warmupDelayMs =
|
|
159
|
+
warmupDelayMs = isOptimized ? 750 : 0, // milliseconds
|
|
144
160
|
maxRetries = 3,
|
|
145
161
|
retryDelayMs = 1000,
|
|
146
162
|
} = options;
|
|
@@ -150,7 +166,7 @@ export async function fetchHtmlWithWarmup(
|
|
|
150
166
|
// Parse the URL to extract path and construct full URL
|
|
151
167
|
const urlObj = new URL(url);
|
|
152
168
|
const urlPath = urlObj.pathname;
|
|
153
|
-
|
|
169
|
+
const fullUrl = `${edgeUrl}${urlPath}`;
|
|
154
170
|
|
|
155
171
|
const headers = {
|
|
156
172
|
'x-forwarded-host': forwardedHost,
|
|
@@ -160,9 +176,7 @@ export async function fetchHtmlWithWarmup(
|
|
|
160
176
|
};
|
|
161
177
|
|
|
162
178
|
if (isOptimized) {
|
|
163
|
-
|
|
164
|
-
fullUrl = `${fullUrl}?tokowakaPreview=true`;
|
|
165
|
-
headers['x-edgeoptimize-url'] = `${urlPath}?tokowakaPreview=true`;
|
|
179
|
+
headers['x-edgeoptimize-preview'] = 1;
|
|
166
180
|
}
|
|
167
181
|
|
|
168
182
|
const fetchOptions = {
|
|
@@ -172,21 +186,23 @@ export async function fetchHtmlWithWarmup(
|
|
|
172
186
|
|
|
173
187
|
try {
|
|
174
188
|
// Warmup call (no retry logic for warmup)
|
|
175
|
-
log.
|
|
176
|
-
|
|
177
|
-
const warmupResponse = await
|
|
178
|
-
|
|
179
|
-
log.debug(`Warmup response status: ${warmupResponse.status} ${warmupResponse.statusText}`);
|
|
180
|
-
// Consume the response body to free up the connection
|
|
189
|
+
log.info(`[${fetchType}] Making warmup call for ${fetchType} HTML with URL: ${fullUrl}`);
|
|
190
|
+
const warmupStartTime = Date.now();
|
|
191
|
+
const warmupResponse = await fetchWithTimeout(fullUrl, fetchOptions);
|
|
192
|
+
log.debug(`[${fetchType}] Warmup response status: ${warmupResponse.status} ${warmupResponse.statusText}`);
|
|
181
193
|
await warmupResponse.text();
|
|
182
|
-
log.
|
|
194
|
+
log.info(`[${fetchType}] Warmup call completed in ${Date.now() - warmupStartTime}ms, waiting ${warmupDelayMs}ms...`);
|
|
195
|
+
} catch (warmupError) {
|
|
196
|
+
log.warn(`[${fetchType}] Warmup call failed (ignored): ${warmupError.message}`);
|
|
197
|
+
}
|
|
183
198
|
|
|
184
|
-
|
|
185
|
-
await sleep(warmupDelayMs);
|
|
199
|
+
await sleep(warmupDelayMs);
|
|
186
200
|
|
|
201
|
+
try {
|
|
187
202
|
// Actual call with retry logic
|
|
188
|
-
log.
|
|
203
|
+
log.info(`[${fetchType}] Making actual call for ${fetchType} HTML (max ${maxRetries} retries) with URL: ${fullUrl}`);
|
|
189
204
|
|
|
205
|
+
const fetchStartTime = Date.now();
|
|
190
206
|
const response = await fetchWithRetry(
|
|
191
207
|
fullUrl,
|
|
192
208
|
fetchOptions,
|
|
@@ -197,10 +213,10 @@ export async function fetchHtmlWithWarmup(
|
|
|
197
213
|
);
|
|
198
214
|
|
|
199
215
|
const html = await response.text();
|
|
200
|
-
log.
|
|
216
|
+
log.info(`[${fetchType}] Successfully fetched ${fetchType} HTML (${html.length} bytes) in ${Date.now() - fetchStartTime}ms`);
|
|
201
217
|
return html;
|
|
202
218
|
} catch (error) {
|
|
203
|
-
const errorMsg = `Failed to fetch ${fetchType} HTML after ${maxRetries} retries: ${error.message}`;
|
|
219
|
+
const errorMsg = `[${fetchType}] Failed to fetch ${fetchType} HTML after ${maxRetries} retries: ${error.message}`;
|
|
204
220
|
log.error(errorMsg);
|
|
205
221
|
throw new Error(errorMsg);
|
|
206
222
|
}
|
package/test/index.test.js
CHANGED
|
@@ -3153,7 +3153,7 @@ describe('TokowakaClient', () => {
|
|
|
3153
3153
|
mockSite,
|
|
3154
3154
|
mockOpportunity,
|
|
3155
3155
|
mockSuggestions,
|
|
3156
|
-
{ warmupDelayMs: 0 },
|
|
3156
|
+
{ warmupDelayMs: 0, maxRetries: 0, retryDelayMs: 0 },
|
|
3157
3157
|
);
|
|
3158
3158
|
expect.fail('Should have thrown error');
|
|
3159
3159
|
} catch (error) {
|
|
@@ -160,11 +160,68 @@ describe('HTML Utils', () => {
|
|
|
160
160
|
expect(html).to.equal('<html>Optimized HTML</html>');
|
|
161
161
|
expect(fetchStub.callCount).to.equal(2); // warmup + actual
|
|
162
162
|
|
|
163
|
-
// Verify original query params are dropped and
|
|
163
|
+
// Verify original query params are dropped and preview header is set
|
|
164
164
|
const actualUrl = fetchStub.secondCall.args[0];
|
|
165
|
+
const actualOptions = fetchStub.secondCall.args[1];
|
|
165
166
|
expect(actualUrl).to.not.include('param=value');
|
|
166
|
-
expect(actualUrl).to.
|
|
167
|
-
expect(
|
|
167
|
+
expect(actualUrl).to.equal('https://edge.example.com/page');
|
|
168
|
+
expect(actualOptions.headers['x-edgeoptimize-preview']).to.equal(1);
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
it('should use default warmup delay of 0ms for original HTML', async () => {
|
|
172
|
+
fetchStub.resolves({
|
|
173
|
+
ok: true,
|
|
174
|
+
status: 200,
|
|
175
|
+
statusText: 'OK',
|
|
176
|
+
headers: {
|
|
177
|
+
get: (name) => (name === 'x-edgeoptimize-cache' ? 'HIT' : null),
|
|
178
|
+
},
|
|
179
|
+
text: async () => '<html>Test HTML</html>',
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
const startTime = Date.now();
|
|
183
|
+
const html = await fetchHtmlWithWarmup(
|
|
184
|
+
'https://example.com/page',
|
|
185
|
+
'api-key',
|
|
186
|
+
'host',
|
|
187
|
+
'https://edge.example.com',
|
|
188
|
+
log,
|
|
189
|
+
false, // isOptimized = false, should use 750ms default
|
|
190
|
+
{}, // No warmupDelayMs provided - should use default
|
|
191
|
+
);
|
|
192
|
+
|
|
193
|
+
const elapsed = Date.now() - startTime;
|
|
194
|
+
expect(html).to.equal('<html>Test HTML</html>');
|
|
195
|
+
expect(fetchStub.callCount).to.equal(2); // warmup + actual
|
|
196
|
+
expect(elapsed).to.be.below(700); // Should have waited ~750ms
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it('should use default warmup delay of 750ms for optimized HTML', async () => {
|
|
200
|
+
fetchStub.resolves({
|
|
201
|
+
ok: true,
|
|
202
|
+
status: 200,
|
|
203
|
+
statusText: 'OK',
|
|
204
|
+
headers: {
|
|
205
|
+
get: (name) => (name === 'x-edgeoptimize-cache' ? 'HIT' : null),
|
|
206
|
+
},
|
|
207
|
+
text: async () => '<html>Optimized HTML</html>',
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
const startTime = Date.now();
|
|
211
|
+
const html = await fetchHtmlWithWarmup(
|
|
212
|
+
'https://example.com/page',
|
|
213
|
+
'api-key',
|
|
214
|
+
'host',
|
|
215
|
+
'https://edge.example.com',
|
|
216
|
+
log,
|
|
217
|
+
true, // isOptimized = true, should use 0ms default
|
|
218
|
+
{}, // No warmupDelayMs provided - should use default
|
|
219
|
+
);
|
|
220
|
+
|
|
221
|
+
const elapsed = Date.now() - startTime;
|
|
222
|
+
expect(html).to.equal('<html>Optimized HTML</html>');
|
|
223
|
+
expect(fetchStub.callCount).to.equal(2); // warmup + actual
|
|
224
|
+
expect(elapsed).to.be.at.least(750); // Should not have waited (0ms warmup)
|
|
168
225
|
});
|
|
169
226
|
|
|
170
227
|
it('should return immediately for optimized HTML when no headers present', async () => {
|
|
@@ -370,10 +427,15 @@ describe('HTML Utils', () => {
|
|
|
370
427
|
'https://edge.example.com',
|
|
371
428
|
log,
|
|
372
429
|
false,
|
|
373
|
-
{
|
|
430
|
+
{
|
|
431
|
+
warmupDelayMs: 0,
|
|
432
|
+
maxRetries: 2,
|
|
433
|
+
retryDelayMs: 0,
|
|
434
|
+
},
|
|
374
435
|
);
|
|
375
436
|
expect.fail('Should have thrown error');
|
|
376
437
|
} catch (error) {
|
|
438
|
+
expect(error.message).to.include('original HTML');
|
|
377
439
|
expect(error.message).to.include('Failed to fetch original HTML');
|
|
378
440
|
expect(error.message).to.include('Network error');
|
|
379
441
|
}
|
|
@@ -401,10 +463,11 @@ describe('HTML Utils', () => {
|
|
|
401
463
|
'https://edge.example.com',
|
|
402
464
|
log,
|
|
403
465
|
false,
|
|
404
|
-
{ warmupDelayMs: 0, maxRetries: 0 },
|
|
466
|
+
{ warmupDelayMs: 0, maxRetries: 0, timeoutMs: 10000 },
|
|
405
467
|
);
|
|
406
468
|
expect.fail('Should have thrown error');
|
|
407
469
|
} catch (error) {
|
|
470
|
+
expect(error.message).to.include('original HTML');
|
|
408
471
|
expect(error.message).to.include('Network error');
|
|
409
472
|
}
|
|
410
473
|
|
|
@@ -442,6 +505,42 @@ describe('HTML Utils', () => {
|
|
|
442
505
|
}
|
|
443
506
|
});
|
|
444
507
|
|
|
508
|
+
it('should handle timeout errors properly', async () => {
|
|
509
|
+
// Create a timeout error
|
|
510
|
+
const timeoutError = new Error('The operation was aborted');
|
|
511
|
+
timeoutError.name = 'TimeoutError';
|
|
512
|
+
|
|
513
|
+
// Warmup succeeds
|
|
514
|
+
fetchStub.onCall(0).resolves({
|
|
515
|
+
ok: true,
|
|
516
|
+
status: 200,
|
|
517
|
+
statusText: 'OK',
|
|
518
|
+
headers: {
|
|
519
|
+
get: () => null,
|
|
520
|
+
},
|
|
521
|
+
text: async () => 'warmup',
|
|
522
|
+
});
|
|
523
|
+
|
|
524
|
+
// Actual call times out
|
|
525
|
+
fetchStub.onCall(1).rejects(timeoutError);
|
|
526
|
+
|
|
527
|
+
try {
|
|
528
|
+
await fetchHtmlWithWarmup(
|
|
529
|
+
'https://example.com/page',
|
|
530
|
+
'api-key',
|
|
531
|
+
'host',
|
|
532
|
+
'https://edge.example.com',
|
|
533
|
+
log,
|
|
534
|
+
false,
|
|
535
|
+
{ warmupDelayMs: 0, maxRetries: 0, timeoutMs: 100 },
|
|
536
|
+
);
|
|
537
|
+
expect.fail('Should have thrown error');
|
|
538
|
+
} catch (error) {
|
|
539
|
+
expect(error.message).to.include('Failed to fetch original HTML');
|
|
540
|
+
expect(error.message).to.include('Request timed out after');
|
|
541
|
+
}
|
|
542
|
+
});
|
|
543
|
+
|
|
445
544
|
it('should return immediately when no edge optimize headers are present', async () => {
|
|
446
545
|
// Warmup succeeds
|
|
447
546
|
fetchStub.onCall(0).resolves({
|