@adobe/spacecat-shared-tokowaka-client 1.4.7 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/package.json +1 -1
- package/src/fastly-kv-client.js +197 -0
- package/src/index.d.ts +19 -0
- package/src/index.js +7 -5
- package/src/utils/custom-html-utils.js +15 -15
- package/test/fastly-kv-client.test.js +254 -0
- package/test/index.test.js +1 -1
- package/test/mappers/faq-mapper.test.js +1 -2
- package/test/utils/html-utils.test.js +7 -7
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
# [@adobe/spacecat-shared-tokowaka-client-v1.5.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.5.0...@adobe/spacecat-shared-tokowaka-client-v1.5.1) (2026-01-20)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Bug Fixes
|
|
5
|
+
|
|
6
|
+
* rename tokowaka to edge-optimize in preview api ([#1269](https://github.com/adobe/spacecat-shared/issues/1269)) ([d40caa4](https://github.com/adobe/spacecat-shared/commit/d40caa46d2bb72506e45081a263f4513fce4ce41))
|
|
7
|
+
|
|
8
|
+
# [@adobe/spacecat-shared-tokowaka-client-v1.5.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.4.7...@adobe/spacecat-shared-tokowaka-client-v1.5.0) (2026-01-15)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Features
|
|
12
|
+
|
|
13
|
+
* add TokowakaKVClient for fetching stale suggestions from edge o… ([#1262](https://github.com/adobe/spacecat-shared/issues/1262)) ([e8d440b](https://github.com/adobe/spacecat-shared/commit/e8d440bcc462419b4c82d3415f3a83dfde5195f1))
|
|
14
|
+
|
|
1
15
|
# [@adobe/spacecat-shared-tokowaka-client-v1.4.7](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-tokowaka-client-v1.4.6...@adobe/spacecat-shared-tokowaka-client-v1.4.7) (2026-01-14)
|
|
2
16
|
|
|
3
17
|
|
package/package.json
CHANGED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2026 Adobe. All rights reserved.
|
|
3
|
+
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
* you may not use this file except in compliance with the License. You may obtain a copy
|
|
5
|
+
* of the License at http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
*
|
|
7
|
+
* Unless required by applicable law or agreed to in writing, software distributed under
|
|
8
|
+
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
|
|
9
|
+
* OF ANY KIND, either express or implied. See the License for the specific language
|
|
10
|
+
* governing permissions and limitations under the License.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import {
|
|
14
|
+
tracingFetch as fetch,
|
|
15
|
+
hasText,
|
|
16
|
+
isNonEmptyArray,
|
|
17
|
+
} from '@adobe/spacecat-shared-utils';
|
|
18
|
+
|
|
19
|
+
const FASTLY_KV_API_BASE = 'https://api.fastly.com/resources/stores/kv';
|
|
20
|
+
const DEFAULT_PAGE_SIZE = 100;
|
|
21
|
+
const DEFAULT_TIMEOUT = 30000;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Client for interacting with Fastly KV Store used by Tokowaka.
|
|
25
|
+
* Used to fetch stale suggestion IDs for cleanup.
|
|
26
|
+
*
|
|
27
|
+
* Key format: `${suggestionId}`
|
|
28
|
+
* Value format: { url: string, status: 'stale' | 'live' }
|
|
29
|
+
*/
|
|
30
|
+
export class FastlyKVClient {
|
|
31
|
+
/**
|
|
32
|
+
* Creates a new FastlyKVClient instance.
|
|
33
|
+
* @param {object} env - Environment variables containing FASTLY_KV_STORE_ID and FASTLY_API_TOKEN
|
|
34
|
+
* @param {object} log - Logger instance
|
|
35
|
+
*/
|
|
36
|
+
constructor(env, log) {
|
|
37
|
+
if (!hasText(env?.FASTLY_KV_STORE_ID)) {
|
|
38
|
+
throw new Error('FASTLY_KV_STORE_ID environment variable is required');
|
|
39
|
+
}
|
|
40
|
+
if (!hasText(env?.FASTLY_API_TOKEN)) {
|
|
41
|
+
throw new Error('FASTLY_API_TOKEN environment variable is required');
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
this.storeId = env.FASTLY_KV_STORE_ID;
|
|
45
|
+
this.apiToken = env.FASTLY_API_TOKEN;
|
|
46
|
+
this.log = log;
|
|
47
|
+
this.timeout = env.FASTLY_KV_TIMEOUT || DEFAULT_TIMEOUT;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Creates the authorization headers for Fastly API requests.
|
|
52
|
+
* @returns {object} Headers object
|
|
53
|
+
*/
|
|
54
|
+
#getHeaders() {
|
|
55
|
+
return {
|
|
56
|
+
'Fastly-Key': this.apiToken,
|
|
57
|
+
Accept: 'application/json',
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Retrieves the value for a specific key from the KV store.
|
|
63
|
+
*
|
|
64
|
+
* @param {string} key - The key to retrieve
|
|
65
|
+
* @returns {Promise<{url: string, status: string}|null>} The parsed value or null if not found
|
|
66
|
+
*/
|
|
67
|
+
async #getValue(key) {
|
|
68
|
+
const url = `${FASTLY_KV_API_BASE}/${this.storeId}/keys/${encodeURIComponent(key)}`;
|
|
69
|
+
|
|
70
|
+
const response = await fetch(url, {
|
|
71
|
+
method: 'GET',
|
|
72
|
+
headers: this.#getHeaders(),
|
|
73
|
+
timeout: this.timeout,
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
if (response.status === 404) {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
if (!response.ok) {
|
|
81
|
+
const errorText = await response.text();
|
|
82
|
+
throw new Error(`Failed to get value from KV Store: ${response.status} - ${errorText}`);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const text = await response.text();
|
|
86
|
+
|
|
87
|
+
try {
|
|
88
|
+
return JSON.parse(text);
|
|
89
|
+
} catch {
|
|
90
|
+
this.log.warn(`Failed to parse value for key ${key} as JSON`);
|
|
91
|
+
return { url: text, status: 'unknown' };
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Lists stale keys from a single page of the KV store.
|
|
97
|
+
*
|
|
98
|
+
* @param {object} [options] - Options for listing keys
|
|
99
|
+
* @param {number} [options.pageSize=100] - Number of keys to fetch per page
|
|
100
|
+
* @param {string} [options.cursor] - Cursor for pagination
|
|
101
|
+
* @returns {Promise<{keys: Array<object>, cursor: string|null}>}
|
|
102
|
+
*/
|
|
103
|
+
async #listStaleKeysPage(options = {}) {
|
|
104
|
+
const { pageSize = DEFAULT_PAGE_SIZE, cursor } = options;
|
|
105
|
+
const staleEntries = [];
|
|
106
|
+
|
|
107
|
+
const url = new URL(`${FASTLY_KV_API_BASE}/${this.storeId}/keys`);
|
|
108
|
+
url.searchParams.set('limit', pageSize.toString());
|
|
109
|
+
if (cursor) {
|
|
110
|
+
url.searchParams.set('cursor', cursor);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const response = await fetch(url.toString(), {
|
|
114
|
+
method: 'GET',
|
|
115
|
+
headers: this.#getHeaders(),
|
|
116
|
+
timeout: this.timeout,
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
if (!response.ok) {
|
|
120
|
+
const errorText = await response.text();
|
|
121
|
+
throw new Error(`Failed to list keys from KV Store: ${response.status} - ${errorText}`);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const result = await response.json();
|
|
125
|
+
const { data: keys = [], meta = {} } = result;
|
|
126
|
+
|
|
127
|
+
for (const keyName of keys) {
|
|
128
|
+
try {
|
|
129
|
+
// eslint-disable-next-line no-await-in-loop
|
|
130
|
+
const value = await this.#getValue(keyName);
|
|
131
|
+
|
|
132
|
+
const normalizedStatus = typeof value?.status === 'string'
|
|
133
|
+
? value.status.trim().toLowerCase()
|
|
134
|
+
: '';
|
|
135
|
+
|
|
136
|
+
if (normalizedStatus === 'stale' && hasText(keyName)) {
|
|
137
|
+
staleEntries.push({
|
|
138
|
+
key: keyName,
|
|
139
|
+
suggestionId: keyName,
|
|
140
|
+
url: value.url,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
} catch (error) {
|
|
144
|
+
this.log.warn(`Failed to fetch value for key ${keyName}: ${error.message}`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
return {
|
|
149
|
+
keys: staleEntries,
|
|
150
|
+
cursor: meta.next_cursor || null,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Lists all stale suggestion IDs from the KV store, handling pagination automatically.
|
|
156
|
+
*
|
|
157
|
+
* @param {object} [options] - Options for listing keys
|
|
158
|
+
* @param {number} [options.pageSize=100] - Number of keys to fetch per page
|
|
159
|
+
* @param {number} [options.maxPages=100] - Maximum number of pages to fetch (safety limit)
|
|
160
|
+
* @returns {Promise<Array<{key: string, suggestionId: string, url: string}>>}
|
|
161
|
+
*/
|
|
162
|
+
async listAllStaleKeys(options = {}) {
|
|
163
|
+
const { pageSize = DEFAULT_PAGE_SIZE, maxPages = 100 } = options;
|
|
164
|
+
const allStaleKeys = [];
|
|
165
|
+
let cursor = null;
|
|
166
|
+
let pageCount = 0;
|
|
167
|
+
|
|
168
|
+
this.log.info('Starting to fetch stale keys from Fastly KV Store');
|
|
169
|
+
|
|
170
|
+
do {
|
|
171
|
+
// eslint-disable-next-line no-await-in-loop
|
|
172
|
+
const result = await this.#listStaleKeysPage({ pageSize, cursor });
|
|
173
|
+
|
|
174
|
+
if (isNonEmptyArray(result.keys)) {
|
|
175
|
+
allStaleKeys.push(...result.keys);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
cursor = result.cursor;
|
|
179
|
+
pageCount += 1;
|
|
180
|
+
|
|
181
|
+
this.log.debug(
|
|
182
|
+
`Fetched page ${pageCount}, found ${result.keys.length} stale keys, total: ${allStaleKeys.length}`,
|
|
183
|
+
);
|
|
184
|
+
|
|
185
|
+
if (pageCount >= maxPages) {
|
|
186
|
+
this.log.warn(`Reached maximum page limit (${maxPages}), stopping pagination`);
|
|
187
|
+
break;
|
|
188
|
+
}
|
|
189
|
+
} while (cursor);
|
|
190
|
+
|
|
191
|
+
this.log.info(`Completed fetching stale keys: ${allStaleKeys.length} total from ${pageCount} pages`);
|
|
192
|
+
|
|
193
|
+
return allStaleKeys;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export default FastlyKVClient;
|
package/src/index.d.ts
CHANGED
|
@@ -128,6 +128,25 @@ export interface Suggestion {
|
|
|
128
128
|
getUpdatedAt(): string;
|
|
129
129
|
}
|
|
130
130
|
|
|
131
|
+
export interface FastlyKVEntry {
|
|
132
|
+
key: string;
|
|
133
|
+
suggestionId: string;
|
|
134
|
+
url: string;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export class FastlyKVClient {
|
|
138
|
+
constructor(env: {
|
|
139
|
+
FASTLY_KV_STORE_ID: string;
|
|
140
|
+
FASTLY_API_TOKEN: string;
|
|
141
|
+
FASTLY_KV_TIMEOUT?: number;
|
|
142
|
+
}, log: any);
|
|
143
|
+
|
|
144
|
+
listAllStaleKeys(options?: {
|
|
145
|
+
pageSize?: number;
|
|
146
|
+
maxPages?: number;
|
|
147
|
+
}): Promise<FastlyKVEntry[]>;
|
|
148
|
+
}
|
|
149
|
+
|
|
131
150
|
/**
|
|
132
151
|
* Base class for opportunity mappers
|
|
133
152
|
* Extend this class to create custom mappers for new opportunity types
|
package/src/index.js
CHANGED
|
@@ -27,6 +27,8 @@ import { groupSuggestionsByUrlPath, filterEligibleSuggestions } from './utils/su
|
|
|
27
27
|
import { getEffectiveBaseURL } from './utils/site-utils.js';
|
|
28
28
|
import { fetchHtmlWithWarmup, calculateForwardedHost } from './utils/custom-html-utils.js';
|
|
29
29
|
|
|
30
|
+
export { FastlyKVClient } from './fastly-kv-client.js';
|
|
31
|
+
|
|
30
32
|
const HTTP_BAD_REQUEST = 400;
|
|
31
33
|
const HTTP_INTERNAL_SERVER_ERROR = 500;
|
|
32
34
|
const HTTP_NOT_IMPLEMENTED = 501;
|
|
@@ -304,7 +306,7 @@ class TokowakaClient {
|
|
|
304
306
|
siteId,
|
|
305
307
|
apiKeys: [apiKey],
|
|
306
308
|
tokowakaEnabled: options.tokowakaEnabled ?? true,
|
|
307
|
-
enhancements: false,
|
|
309
|
+
enhancements: options.enhancements ?? false,
|
|
308
310
|
};
|
|
309
311
|
|
|
310
312
|
const s3Path = await this.uploadMetaconfig(url, metaconfig);
|
|
@@ -808,8 +810,8 @@ class TokowakaClient {
|
|
|
808
810
|
}
|
|
809
811
|
|
|
810
812
|
// TOKOWAKA_EDGE_URL is mandatory for preview
|
|
811
|
-
const
|
|
812
|
-
if (!hasText(
|
|
813
|
+
const edgeUrl = this.env.TOKOWAKA_EDGE_URL;
|
|
814
|
+
if (!hasText(edgeUrl)) {
|
|
813
815
|
throw this.#createError(
|
|
814
816
|
'TOKOWAKA_EDGE_URL is required for preview functionality',
|
|
815
817
|
HTTP_INTERNAL_SERVER_ERROR,
|
|
@@ -929,7 +931,7 @@ class TokowakaClient {
|
|
|
929
931
|
previewUrl,
|
|
930
932
|
apiKey,
|
|
931
933
|
forwardedHost,
|
|
932
|
-
|
|
934
|
+
edgeUrl,
|
|
933
935
|
this.log,
|
|
934
936
|
false,
|
|
935
937
|
options,
|
|
@@ -939,7 +941,7 @@ class TokowakaClient {
|
|
|
939
941
|
previewUrl,
|
|
940
942
|
apiKey,
|
|
941
943
|
forwardedHost,
|
|
942
|
-
|
|
944
|
+
edgeUrl,
|
|
943
945
|
this.log,
|
|
944
946
|
true,
|
|
945
947
|
options,
|
|
@@ -25,7 +25,7 @@ function sleep(ms) {
|
|
|
25
25
|
|
|
26
26
|
/**
|
|
27
27
|
* Makes an HTTP request with retry logic
|
|
28
|
-
* Retries until max retries are exhausted or x-
|
|
28
|
+
* Retries until max retries are exhausted or x-edge-optimize-cache header is present
|
|
29
29
|
* @param {string} url - URL to fetch
|
|
30
30
|
* @param {Object} options - Fetch options
|
|
31
31
|
* @param {number} maxRetries - Maximum number of retries
|
|
@@ -48,10 +48,10 @@ async function fetchWithRetry(url, options, maxRetries, retryDelayMs, log, fetch
|
|
|
48
48
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
// Check for x-
|
|
52
|
-
const cacheHeader = response.headers.get('x-
|
|
51
|
+
// Check for x-edge-optimize-cache header - if present, stop retrying
|
|
52
|
+
const cacheHeader = response.headers.get('x-edge-optimize-cache');
|
|
53
53
|
if (cacheHeader) {
|
|
54
|
-
log.debug(`Cache header found (x-
|
|
54
|
+
log.debug(`Cache header found (x-edge-optimize-cache: ${cacheHeader}), stopping retry logic`);
|
|
55
55
|
return response;
|
|
56
56
|
}
|
|
57
57
|
|
|
@@ -65,7 +65,7 @@ async function fetchWithRetry(url, options, maxRetries, retryDelayMs, log, fetch
|
|
|
65
65
|
} else {
|
|
66
66
|
// Last attempt without cache header - throw error
|
|
67
67
|
log.error(`Max retries (${maxRetries}) exhausted without cache header`);
|
|
68
|
-
throw new Error(`Cache header (x-
|
|
68
|
+
throw new Error(`Cache header (x-edge-optimize-cache) not found after ${maxRetries} retries`);
|
|
69
69
|
}
|
|
70
70
|
} catch (error) {
|
|
71
71
|
log.warn(`Attempt ${attempt} failed for ${fetchType} HTML, error: ${error.message}`);
|
|
@@ -86,12 +86,12 @@ async function fetchWithRetry(url, options, maxRetries, retryDelayMs, log, fetch
|
|
|
86
86
|
}
|
|
87
87
|
|
|
88
88
|
/**
|
|
89
|
-
* Fetches HTML content from
|
|
89
|
+
* Fetches HTML content from edge with warmup call and retry logic
|
|
90
90
|
* Makes an initial warmup call, waits, then makes the actual call with retries
|
|
91
91
|
* @param {string} url - Full URL to fetch
|
|
92
|
-
* @param {string} apiKey -
|
|
92
|
+
* @param {string} apiKey - Edge Optimize API key
|
|
93
93
|
* @param {string} forwardedHost - Host to forward in x-forwarded-host header
|
|
94
|
-
* @param {string}
|
|
94
|
+
* @param {string} edgeUrl - Edge URL
|
|
95
95
|
* @param {boolean} isOptimized - Whether to fetch optimized HTML (with preview param)
|
|
96
96
|
* @param {Object} log - Logger instance
|
|
97
97
|
* @param {Object} options - Additional options
|
|
@@ -105,7 +105,7 @@ export async function fetchHtmlWithWarmup(
|
|
|
105
105
|
url,
|
|
106
106
|
apiKey,
|
|
107
107
|
forwardedHost,
|
|
108
|
-
|
|
108
|
+
edgeUrl,
|
|
109
109
|
log,
|
|
110
110
|
isOptimized = false,
|
|
111
111
|
options = {},
|
|
@@ -116,14 +116,14 @@ export async function fetchHtmlWithWarmup(
|
|
|
116
116
|
}
|
|
117
117
|
|
|
118
118
|
if (!hasText(apiKey)) {
|
|
119
|
-
throw new Error('
|
|
119
|
+
throw new Error('Edge Optimize API key is required for fetching HTML');
|
|
120
120
|
}
|
|
121
121
|
|
|
122
122
|
if (!hasText(forwardedHost)) {
|
|
123
123
|
throw new Error('Forwarded host is required for fetching HTML');
|
|
124
124
|
}
|
|
125
125
|
|
|
126
|
-
if (!hasText(
|
|
126
|
+
if (!hasText(edgeUrl)) {
|
|
127
127
|
throw new Error('TOKOWAKA_EDGE_URL is not configured');
|
|
128
128
|
}
|
|
129
129
|
|
|
@@ -139,18 +139,18 @@ export async function fetchHtmlWithWarmup(
|
|
|
139
139
|
// Parse the URL to extract path and construct full URL
|
|
140
140
|
const urlObj = new URL(url);
|
|
141
141
|
const urlPath = urlObj.pathname;
|
|
142
|
-
let fullUrl = `${
|
|
142
|
+
let fullUrl = `${edgeUrl}${urlPath}`;
|
|
143
143
|
|
|
144
144
|
const headers = {
|
|
145
145
|
'x-forwarded-host': forwardedHost,
|
|
146
|
-
'x-
|
|
147
|
-
'x-
|
|
146
|
+
'x-edge-optimize-api-key': apiKey,
|
|
147
|
+
'x-edge-optimize-url': urlPath,
|
|
148
148
|
};
|
|
149
149
|
|
|
150
150
|
if (isOptimized) {
|
|
151
151
|
// Add tokowakaPreview param for optimized HTML
|
|
152
152
|
fullUrl = `${fullUrl}?tokowakaPreview=true`;
|
|
153
|
-
headers['x-
|
|
153
|
+
headers['x-edge-optimize-url'] = `${urlPath}?tokowakaPreview=true`;
|
|
154
154
|
}
|
|
155
155
|
|
|
156
156
|
const fetchOptions = {
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2026 Adobe. All rights reserved.
|
|
3
|
+
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
* you may not use this file except in compliance with the License. You may obtain a copy
|
|
5
|
+
* of the License at http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
*
|
|
7
|
+
* Unless required by applicable law or agreed to in writing, software distributed under
|
|
8
|
+
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
|
|
9
|
+
* OF ANY KIND, either express or implied. See the License for the specific language
|
|
10
|
+
* governing permissions and limitations under the License.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/* eslint-env mocha */
|
|
14
|
+
|
|
15
|
+
import { expect } from 'chai';
|
|
16
|
+
import sinon from 'sinon';
|
|
17
|
+
import nock from 'nock';
|
|
18
|
+
import { FastlyKVClient } from '../src/fastly-kv-client.js';
|
|
19
|
+
|
|
20
|
+
describe('FastlyKVClient', () => {
|
|
21
|
+
let sandbox;
|
|
22
|
+
let log;
|
|
23
|
+
let env;
|
|
24
|
+
|
|
25
|
+
const FASTLY_KV_API_BASE = 'https://api.fastly.com/resources/stores/kv';
|
|
26
|
+
const TEST_STORE_ID = 'test-store-id';
|
|
27
|
+
const TEST_API_TOKEN = 'test-api-token';
|
|
28
|
+
|
|
29
|
+
beforeEach(() => {
|
|
30
|
+
sandbox = sinon.createSandbox();
|
|
31
|
+
log = {
|
|
32
|
+
debug: sandbox.spy(),
|
|
33
|
+
info: sandbox.spy(),
|
|
34
|
+
warn: sandbox.spy(),
|
|
35
|
+
error: sandbox.spy(),
|
|
36
|
+
};
|
|
37
|
+
env = {
|
|
38
|
+
FASTLY_KV_STORE_ID: TEST_STORE_ID,
|
|
39
|
+
FASTLY_API_TOKEN: TEST_API_TOKEN,
|
|
40
|
+
};
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
afterEach(() => {
|
|
44
|
+
sandbox.restore();
|
|
45
|
+
nock.cleanAll();
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
describe('constructor', () => {
|
|
49
|
+
it('should create a client with valid configuration', () => {
|
|
50
|
+
const client = new FastlyKVClient(env, log);
|
|
51
|
+
expect(client.storeId).to.equal(TEST_STORE_ID);
|
|
52
|
+
expect(client.apiToken).to.equal(TEST_API_TOKEN);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it('should throw error if FASTLY_KV_STORE_ID is missing', () => {
|
|
56
|
+
delete env.FASTLY_KV_STORE_ID;
|
|
57
|
+
expect(() => new FastlyKVClient(env, log))
|
|
58
|
+
.to.throw('FASTLY_KV_STORE_ID environment variable is required');
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('should throw error if FASTLY_API_TOKEN is missing', () => {
|
|
62
|
+
delete env.FASTLY_API_TOKEN;
|
|
63
|
+
expect(() => new FastlyKVClient(env, log))
|
|
64
|
+
.to.throw('FASTLY_API_TOKEN environment variable is required');
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('should throw error if env is null', () => {
|
|
68
|
+
expect(() => new FastlyKVClient(null, log))
|
|
69
|
+
.to.throw('FASTLY_KV_STORE_ID environment variable is required');
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
describe('listAllStaleKeys', () => {
|
|
74
|
+
it('should fetch all pages of stale keys', async () => {
|
|
75
|
+
const client = new FastlyKVClient(env, log);
|
|
76
|
+
const keysPage1 = ['sugg-1'];
|
|
77
|
+
const keysPage2 = ['sugg-2'];
|
|
78
|
+
const staleValue = { url: 'https://example.com', status: 'stale' };
|
|
79
|
+
|
|
80
|
+
// Page 1
|
|
81
|
+
nock(FASTLY_KV_API_BASE)
|
|
82
|
+
.get(`/${TEST_STORE_ID}/keys`)
|
|
83
|
+
.query({ limit: '100' })
|
|
84
|
+
.reply(200, { data: keysPage1, meta: { next_cursor: 'cursor-1' } });
|
|
85
|
+
|
|
86
|
+
nock(FASTLY_KV_API_BASE)
|
|
87
|
+
.get(`/${TEST_STORE_ID}/keys/${encodeURIComponent(keysPage1[0])}`)
|
|
88
|
+
.reply(200, JSON.stringify(staleValue));
|
|
89
|
+
|
|
90
|
+
// Page 2
|
|
91
|
+
nock(FASTLY_KV_API_BASE)
|
|
92
|
+
.get(`/${TEST_STORE_ID}/keys`)
|
|
93
|
+
.query({ limit: '100', cursor: 'cursor-1' })
|
|
94
|
+
.reply(200, { data: keysPage2, meta: {} });
|
|
95
|
+
|
|
96
|
+
nock(FASTLY_KV_API_BASE)
|
|
97
|
+
.get(`/${TEST_STORE_ID}/keys/${encodeURIComponent(keysPage2[0])}`)
|
|
98
|
+
.reply(200, JSON.stringify(staleValue));
|
|
99
|
+
|
|
100
|
+
const result = await client.listAllStaleKeys();
|
|
101
|
+
|
|
102
|
+
expect(result).to.have.lengthOf(2);
|
|
103
|
+
expect(result[0].suggestionId).to.equal('sugg-1');
|
|
104
|
+
expect(result[1].suggestionId).to.equal('sugg-2');
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it('should filter out non-stale keys', async () => {
|
|
108
|
+
const client = new FastlyKVClient(env, log);
|
|
109
|
+
const keys = ['sugg-stale', 'sugg-live'];
|
|
110
|
+
|
|
111
|
+
nock(FASTLY_KV_API_BASE)
|
|
112
|
+
.get(`/${TEST_STORE_ID}/keys`)
|
|
113
|
+
.query({ limit: '100' })
|
|
114
|
+
.reply(200, { data: keys, meta: {} });
|
|
115
|
+
|
|
116
|
+
nock(FASTLY_KV_API_BASE)
|
|
117
|
+
.get(`/${TEST_STORE_ID}/keys/${encodeURIComponent(keys[0])}`)
|
|
118
|
+
.reply(200, JSON.stringify({ url: 'https://example.com/stale', status: 'stale' }));
|
|
119
|
+
|
|
120
|
+
nock(FASTLY_KV_API_BASE)
|
|
121
|
+
.get(`/${TEST_STORE_ID}/keys/${encodeURIComponent(keys[1])}`)
|
|
122
|
+
.reply(200, JSON.stringify({ url: 'https://example.com/live', status: 'live' }));
|
|
123
|
+
|
|
124
|
+
const result = await client.listAllStaleKeys();
|
|
125
|
+
|
|
126
|
+
expect(result).to.have.lengthOf(1);
|
|
127
|
+
expect(result[0].suggestionId).to.equal('sugg-stale');
|
|
128
|
+
expect(result[0].url).to.equal('https://example.com/stale');
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it('should respect maxPages limit', async () => {
|
|
132
|
+
const client = new FastlyKVClient(env, log);
|
|
133
|
+
const staleValue = { url: 'https://example.com', status: 'stale' };
|
|
134
|
+
|
|
135
|
+
// Always return a next cursor to simulate infinite pages
|
|
136
|
+
nock(FASTLY_KV_API_BASE)
|
|
137
|
+
.get(`/${TEST_STORE_ID}/keys`)
|
|
138
|
+
.query(true)
|
|
139
|
+
.times(3)
|
|
140
|
+
.reply(200, { data: ['sugg-1'], meta: { next_cursor: 'next' } });
|
|
141
|
+
|
|
142
|
+
nock(FASTLY_KV_API_BASE)
|
|
143
|
+
.get(`/${TEST_STORE_ID}/keys/${encodeURIComponent('sugg-1')}`)
|
|
144
|
+
.times(3)
|
|
145
|
+
.reply(200, JSON.stringify(staleValue));
|
|
146
|
+
|
|
147
|
+
const result = await client.listAllStaleKeys({ maxPages: 3 });
|
|
148
|
+
|
|
149
|
+
expect(result).to.have.lengthOf(3);
|
|
150
|
+
expect(log.warn.calledWith(sinon.match(/Reached maximum page limit/))).to.be.true;
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it('should return empty array if no stale keys found', async () => {
|
|
154
|
+
const client = new FastlyKVClient(env, log);
|
|
155
|
+
|
|
156
|
+
nock(FASTLY_KV_API_BASE)
|
|
157
|
+
.get(`/${TEST_STORE_ID}/keys`)
|
|
158
|
+
.query({ limit: '100' })
|
|
159
|
+
.reply(200, { data: [], meta: {} });
|
|
160
|
+
|
|
161
|
+
const result = await client.listAllStaleKeys();
|
|
162
|
+
|
|
163
|
+
expect(result).to.have.lengthOf(0);
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
it('should continue processing if individual key fetch fails', async () => {
|
|
167
|
+
const client = new FastlyKVClient(env, log);
|
|
168
|
+
const keys = ['sugg-fail', 'sugg-success'];
|
|
169
|
+
|
|
170
|
+
nock(FASTLY_KV_API_BASE)
|
|
171
|
+
.get(`/${TEST_STORE_ID}/keys`)
|
|
172
|
+
.query({ limit: '100' })
|
|
173
|
+
.reply(200, { data: keys, meta: {} });
|
|
174
|
+
|
|
175
|
+
// First key fails
|
|
176
|
+
nock(FASTLY_KV_API_BASE)
|
|
177
|
+
.get(`/${TEST_STORE_ID}/keys/${encodeURIComponent(keys[0])}`)
|
|
178
|
+
.reply(500, 'Error');
|
|
179
|
+
|
|
180
|
+
// Second key succeeds
|
|
181
|
+
nock(FASTLY_KV_API_BASE)
|
|
182
|
+
.get(`/${TEST_STORE_ID}/keys/${encodeURIComponent(keys[1])}`)
|
|
183
|
+
.reply(200, JSON.stringify({ url: 'https://example.com', status: 'stale' }));
|
|
184
|
+
|
|
185
|
+
const result = await client.listAllStaleKeys();
|
|
186
|
+
|
|
187
|
+
expect(result).to.have.lengthOf(1);
|
|
188
|
+
expect(result[0].suggestionId).to.equal('sugg-success');
|
|
189
|
+
expect(log.warn.called).to.be.true;
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
it('should handle non-JSON response gracefully', async () => {
|
|
193
|
+
const client = new FastlyKVClient(env, log);
|
|
194
|
+
const keys = ['sugg-1'];
|
|
195
|
+
|
|
196
|
+
nock(FASTLY_KV_API_BASE)
|
|
197
|
+
.get(`/${TEST_STORE_ID}/keys`)
|
|
198
|
+
.query({ limit: '100' })
|
|
199
|
+
.reply(200, { data: keys, meta: {} });
|
|
200
|
+
|
|
201
|
+
nock(FASTLY_KV_API_BASE)
|
|
202
|
+
.get(`/${TEST_STORE_ID}/keys/${encodeURIComponent(keys[0])}`)
|
|
203
|
+
.reply(200, 'plain-text-value');
|
|
204
|
+
|
|
205
|
+
const result = await client.listAllStaleKeys();
|
|
206
|
+
|
|
207
|
+
// Non-JSON response gets status 'unknown', which is not 'stale', so filtered out
|
|
208
|
+
expect(result).to.have.lengthOf(0);
|
|
209
|
+
expect(log.warn.called).to.be.true;
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
it('should skip keys that return 404', async () => {
|
|
213
|
+
const client = new FastlyKVClient(env, log);
|
|
214
|
+
const keys = ['sugg-deleted', 'sugg-exists'];
|
|
215
|
+
|
|
216
|
+
nock(FASTLY_KV_API_BASE)
|
|
217
|
+
.get(`/${TEST_STORE_ID}/keys`)
|
|
218
|
+
.query({ limit: '100' })
|
|
219
|
+
.reply(200, { data: keys, meta: {} });
|
|
220
|
+
|
|
221
|
+
// First key returns 404
|
|
222
|
+
nock(FASTLY_KV_API_BASE)
|
|
223
|
+
.get(`/${TEST_STORE_ID}/keys/${encodeURIComponent(keys[0])}`)
|
|
224
|
+
.reply(404, 'Not Found');
|
|
225
|
+
|
|
226
|
+
// Second key exists and is stale
|
|
227
|
+
nock(FASTLY_KV_API_BASE)
|
|
228
|
+
.get(`/${TEST_STORE_ID}/keys/${encodeURIComponent(keys[1])}`)
|
|
229
|
+
.reply(200, JSON.stringify({ url: 'https://example.com', status: 'stale' }));
|
|
230
|
+
|
|
231
|
+
const result = await client.listAllStaleKeys();
|
|
232
|
+
|
|
233
|
+
// Only the existing stale key should be returned
|
|
234
|
+
expect(result).to.have.lengthOf(1);
|
|
235
|
+
expect(result[0].suggestionId).to.equal('sugg-exists');
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
it('should throw error when listing keys fails', async () => {
|
|
239
|
+
const client = new FastlyKVClient(env, log);
|
|
240
|
+
|
|
241
|
+
nock(FASTLY_KV_API_BASE)
|
|
242
|
+
.get(`/${TEST_STORE_ID}/keys`)
|
|
243
|
+
.query({ limit: '100' })
|
|
244
|
+
.reply(500, 'Internal Server Error');
|
|
245
|
+
|
|
246
|
+
try {
|
|
247
|
+
await client.listAllStaleKeys();
|
|
248
|
+
expect.fail('Should have thrown an error');
|
|
249
|
+
} catch (error) {
|
|
250
|
+
expect(error.message).to.include('Failed to list keys from KV Store');
|
|
251
|
+
}
|
|
252
|
+
});
|
|
253
|
+
});
|
|
254
|
+
});
|
package/test/index.test.js
CHANGED
|
@@ -1924,7 +1924,7 @@ describe('TokowakaClient', () => {
|
|
|
1924
1924
|
status: 200,
|
|
1925
1925
|
statusText: 'OK',
|
|
1926
1926
|
headers: {
|
|
1927
|
-
get: (name) => (name === 'x-
|
|
1927
|
+
get: (name) => (name === 'x-edge-optimize-cache' ? 'HIT' : null),
|
|
1928
1928
|
},
|
|
1929
1929
|
text: async () => '<html><body>Test HTML</body></html>',
|
|
1930
1930
|
});
|
|
@@ -744,7 +744,6 @@ Overall, Bulk positions itself as a better choice for sports nutrition through i
|
|
|
744
744
|
question: 'Old Q?',
|
|
745
745
|
answer: 'Old A.',
|
|
746
746
|
},
|
|
747
|
-
tokowakaDeployed: 1704884400000,
|
|
748
747
|
transformRules: {
|
|
749
748
|
action: 'appendChild',
|
|
750
749
|
selector: 'main',
|
|
@@ -1154,7 +1153,7 @@ Overall, Bulk positions itself as a better choice for sports nutrition through i
|
|
|
1154
1153
|
});
|
|
1155
1154
|
});
|
|
1156
1155
|
|
|
1157
|
-
describe('
|
|
1156
|
+
describe('edgeDeployed filtering', () => {
|
|
1158
1157
|
it('should always create heading patch even when FAQ already deployed for URL', () => {
|
|
1159
1158
|
const newSuggestion = {
|
|
1160
1159
|
getId: () => 'sugg-new-1',
|
|
@@ -94,7 +94,7 @@ describe('HTML Utils', () => {
|
|
|
94
94
|
);
|
|
95
95
|
expect.fail('Should have thrown error');
|
|
96
96
|
} catch (error) {
|
|
97
|
-
expect(error.message).to.equal('
|
|
97
|
+
expect(error.message).to.equal('Edge Optimize API key is required for fetching HTML');
|
|
98
98
|
}
|
|
99
99
|
});
|
|
100
100
|
|
|
@@ -104,7 +104,7 @@ describe('HTML Utils', () => {
|
|
|
104
104
|
status: 200,
|
|
105
105
|
statusText: 'OK',
|
|
106
106
|
headers: {
|
|
107
|
-
get: (name) => (name === 'x-
|
|
107
|
+
get: (name) => (name === 'x-edge-optimize-cache' ? 'HIT' : null),
|
|
108
108
|
},
|
|
109
109
|
text: async () => '<html>Test HTML</html>',
|
|
110
110
|
});
|
|
@@ -129,7 +129,7 @@ describe('HTML Utils', () => {
|
|
|
129
129
|
status: 200,
|
|
130
130
|
statusText: 'OK',
|
|
131
131
|
headers: {
|
|
132
|
-
get: (name) => (name === 'x-
|
|
132
|
+
get: (name) => (name === 'x-edge-optimize-cache' ? 'HIT' : null),
|
|
133
133
|
},
|
|
134
134
|
text: async () => '<html>Optimized HTML</html>',
|
|
135
135
|
});
|
|
@@ -285,7 +285,7 @@ describe('HTML Utils', () => {
|
|
|
285
285
|
}
|
|
286
286
|
});
|
|
287
287
|
|
|
288
|
-
it('should stop retrying when x-
|
|
288
|
+
it('should stop retrying when x-edge-optimize-cache header is found', async () => {
|
|
289
289
|
// Warmup succeeds
|
|
290
290
|
fetchStub.onCall(0).resolves({
|
|
291
291
|
ok: true,
|
|
@@ -312,7 +312,7 @@ describe('HTML Utils', () => {
|
|
|
312
312
|
status: 200,
|
|
313
313
|
statusText: 'OK',
|
|
314
314
|
headers: {
|
|
315
|
-
get: (name) => (name === 'x-
|
|
315
|
+
get: (name) => (name === 'x-edge-optimize-cache' ? 'HIT' : null),
|
|
316
316
|
},
|
|
317
317
|
text: async () => '<html>Cached HTML</html>',
|
|
318
318
|
});
|
|
@@ -385,7 +385,7 @@ describe('HTML Utils', () => {
|
|
|
385
385
|
expect.fail('Should have thrown error');
|
|
386
386
|
} catch (error) {
|
|
387
387
|
expect(error.message).to.include('Failed to fetch original HTML');
|
|
388
|
-
expect(error.message).to.include('Cache header (x-
|
|
388
|
+
expect(error.message).to.include('Cache header (x-edge-optimize-cache) not found after 2 retries');
|
|
389
389
|
}
|
|
390
390
|
|
|
391
391
|
// Should have tried 3 times (initial + 2 retries) plus warmup
|
|
@@ -409,7 +409,7 @@ describe('HTML Utils', () => {
|
|
|
409
409
|
status: 200,
|
|
410
410
|
statusText: 'OK',
|
|
411
411
|
headers: {
|
|
412
|
-
get: (name) => (name === 'x-
|
|
412
|
+
get: (name) => (name === 'x-edge-optimize-cache' ? 'HIT' : null),
|
|
413
413
|
},
|
|
414
414
|
text: async () => '<html>Cached HTML</html>',
|
|
415
415
|
});
|