@gravito/constellation 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -251
- package/README.zh-TW.md +130 -13
- package/dist/{DiskSitemapStorage-7ZZMGC4K.js → DiskSitemapStorage-VLN5I24C.js} +1 -1
- package/dist/chunk-3IZTXYU7.js +166 -0
- package/dist/index.cjs +1893 -196
- package/dist/index.d.cts +1597 -171
- package/dist/index.d.ts +1597 -171
- package/dist/index.js +1774 -199
- package/package.json +7 -5
- package/dist/chunk-7WHLC3OJ.js +0 -56
package/dist/index.cjs
CHANGED
|
@@ -30,6 +30,49 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
30
30
|
));
|
|
31
31
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
32
32
|
|
|
33
|
+
// src/utils/Compression.ts
|
|
34
|
+
async function compressToBuffer(source, config) {
|
|
35
|
+
const level = config?.level ?? 6;
|
|
36
|
+
if (level < 1 || level > 9) {
|
|
37
|
+
throw new Error(`Invalid compression level: ${level}. Must be between 1 and 9.`);
|
|
38
|
+
}
|
|
39
|
+
const chunks = [];
|
|
40
|
+
const gzip = (0, import_node_zlib.createGzip)({ level });
|
|
41
|
+
const readable = import_node_stream.Readable.from(source, { encoding: "utf-8" });
|
|
42
|
+
try {
|
|
43
|
+
readable.pipe(gzip);
|
|
44
|
+
for await (const chunk of gzip) {
|
|
45
|
+
chunks.push(chunk);
|
|
46
|
+
}
|
|
47
|
+
return Buffer.concat(chunks);
|
|
48
|
+
} catch (error) {
|
|
49
|
+
readable.destroy();
|
|
50
|
+
gzip.destroy();
|
|
51
|
+
throw error;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
function createCompressionStream(config) {
|
|
55
|
+
const level = config?.level ?? 6;
|
|
56
|
+
if (level < 1 || level > 9) {
|
|
57
|
+
throw new Error(`Invalid compression level: ${level}. Must be between 1 and 9.`);
|
|
58
|
+
}
|
|
59
|
+
return (0, import_node_zlib.createGzip)({ level });
|
|
60
|
+
}
|
|
61
|
+
function toGzipFilename(filename) {
|
|
62
|
+
return filename.endsWith(".gz") ? filename : `${filename}.gz`;
|
|
63
|
+
}
|
|
64
|
+
function fromGzipFilename(filename) {
|
|
65
|
+
return filename.endsWith(".gz") ? filename.slice(0, -3) : filename;
|
|
66
|
+
}
|
|
67
|
+
var import_node_stream, import_node_zlib;
|
|
68
|
+
var init_Compression = __esm({
|
|
69
|
+
"src/utils/Compression.ts"() {
|
|
70
|
+
"use strict";
|
|
71
|
+
import_node_stream = require("stream");
|
|
72
|
+
import_node_zlib = require("zlib");
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
|
|
33
76
|
// src/storage/DiskSitemapStorage.ts
|
|
34
77
|
var DiskSitemapStorage_exports = {};
|
|
35
78
|
__export(DiskSitemapStorage_exports, {
|
|
@@ -50,22 +93,61 @@ function sanitizeFilename(filename) {
|
|
|
50
93
|
}
|
|
51
94
|
return filename;
|
|
52
95
|
}
|
|
53
|
-
var import_promises, import_node_path, DiskSitemapStorage;
|
|
96
|
+
var import_node_fs, import_promises, import_node_path, import_node_stream2, import_promises2, DiskSitemapStorage;
|
|
54
97
|
var init_DiskSitemapStorage = __esm({
|
|
55
98
|
"src/storage/DiskSitemapStorage.ts"() {
|
|
56
99
|
"use strict";
|
|
100
|
+
import_node_fs = require("fs");
|
|
57
101
|
import_promises = __toESM(require("fs/promises"), 1);
|
|
58
102
|
import_node_path = __toESM(require("path"), 1);
|
|
103
|
+
import_node_stream2 = require("stream");
|
|
104
|
+
import_promises2 = require("stream/promises");
|
|
105
|
+
init_Compression();
|
|
59
106
|
DiskSitemapStorage = class {
|
|
60
107
|
constructor(outDir, baseUrl) {
|
|
61
108
|
this.outDir = outDir;
|
|
62
109
|
this.baseUrl = baseUrl;
|
|
63
110
|
}
|
|
111
|
+
/**
|
|
112
|
+
* Writes sitemap content to a file on the local disk.
|
|
113
|
+
*
|
|
114
|
+
* @param filename - The name of the file to write.
|
|
115
|
+
* @param content - The XML or JSON content.
|
|
116
|
+
*/
|
|
64
117
|
async write(filename, content) {
|
|
65
118
|
const safeName = sanitizeFilename(filename);
|
|
66
119
|
await import_promises.default.mkdir(this.outDir, { recursive: true });
|
|
67
120
|
await import_promises.default.writeFile(import_node_path.default.join(this.outDir, safeName), content);
|
|
68
121
|
}
|
|
122
|
+
/**
|
|
123
|
+
* 使用串流方式寫入 sitemap 檔案,可選擇性啟用 gzip 壓縮。
|
|
124
|
+
* 此方法可大幅降低大型 sitemap 的記憶體峰值。
|
|
125
|
+
*
|
|
126
|
+
* @param filename - 檔案名稱
|
|
127
|
+
* @param stream - XML 內容的 AsyncIterable
|
|
128
|
+
* @param options - 寫入選項(如壓縮、content type)
|
|
129
|
+
*
|
|
130
|
+
* @since 3.1.0
|
|
131
|
+
*/
|
|
132
|
+
async writeStream(filename, stream, options) {
|
|
133
|
+
const safeName = sanitizeFilename(options?.compress ? toGzipFilename(filename) : filename);
|
|
134
|
+
await import_promises.default.mkdir(this.outDir, { recursive: true });
|
|
135
|
+
const filePath = import_node_path.default.join(this.outDir, safeName);
|
|
136
|
+
const writeStream = (0, import_node_fs.createWriteStream)(filePath);
|
|
137
|
+
const readable = import_node_stream2.Readable.from(stream, { encoding: "utf-8" });
|
|
138
|
+
if (options?.compress) {
|
|
139
|
+
const gzip = createCompressionStream();
|
|
140
|
+
await (0, import_promises2.pipeline)(readable, gzip, writeStream);
|
|
141
|
+
} else {
|
|
142
|
+
await (0, import_promises2.pipeline)(readable, writeStream);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Reads sitemap content from a file on the local disk.
|
|
147
|
+
*
|
|
148
|
+
* @param filename - The name of the file to read.
|
|
149
|
+
* @returns A promise resolving to the file content as a string, or null if not found.
|
|
150
|
+
*/
|
|
69
151
|
async read(filename) {
|
|
70
152
|
try {
|
|
71
153
|
const safeName = sanitizeFilename(filename);
|
|
@@ -74,6 +156,29 @@ var init_DiskSitemapStorage = __esm({
|
|
|
74
156
|
return null;
|
|
75
157
|
}
|
|
76
158
|
}
|
|
159
|
+
/**
|
|
160
|
+
* Returns a readable stream for a sitemap file on the local disk.
|
|
161
|
+
*
|
|
162
|
+
* @param filename - The name of the file to stream.
|
|
163
|
+
* @returns A promise resolving to an async iterable of file chunks, or null if not found.
|
|
164
|
+
*/
|
|
165
|
+
async readStream(filename) {
|
|
166
|
+
try {
|
|
167
|
+
const safeName = sanitizeFilename(filename);
|
|
168
|
+
const fullPath = import_node_path.default.join(this.outDir, safeName);
|
|
169
|
+
await import_promises.default.access(fullPath);
|
|
170
|
+
const stream = (0, import_node_fs.createReadStream)(fullPath, { encoding: "utf-8" });
|
|
171
|
+
return stream;
|
|
172
|
+
} catch {
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Checks if a sitemap file exists on the local disk.
|
|
178
|
+
*
|
|
179
|
+
* @param filename - The name of the file to check.
|
|
180
|
+
* @returns A promise resolving to true if the file exists, false otherwise.
|
|
181
|
+
*/
|
|
77
182
|
async exists(filename) {
|
|
78
183
|
try {
|
|
79
184
|
const safeName = sanitizeFilename(filename);
|
|
@@ -83,6 +188,12 @@ var init_DiskSitemapStorage = __esm({
|
|
|
83
188
|
return false;
|
|
84
189
|
}
|
|
85
190
|
}
|
|
191
|
+
/**
|
|
192
|
+
* Returns the full public URL for a sitemap file.
|
|
193
|
+
*
|
|
194
|
+
* @param filename - The name of the sitemap file.
|
|
195
|
+
* @returns The public URL as a string.
|
|
196
|
+
*/
|
|
86
197
|
getUrl(filename) {
|
|
87
198
|
const safeName = sanitizeFilename(filename);
|
|
88
199
|
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
@@ -102,6 +213,7 @@ __export(index_exports, {
|
|
|
102
213
|
GenerateSitemapJob: () => GenerateSitemapJob,
|
|
103
214
|
IncrementalGenerator: () => IncrementalGenerator,
|
|
104
215
|
MemoryChangeTracker: () => MemoryChangeTracker,
|
|
216
|
+
MemoryLock: () => MemoryLock,
|
|
105
217
|
MemoryProgressStorage: () => MemoryProgressStorage,
|
|
106
218
|
MemoryRedirectManager: () => MemoryRedirectManager,
|
|
107
219
|
MemorySitemapStorage: () => MemorySitemapStorage,
|
|
@@ -110,6 +222,7 @@ __export(index_exports, {
|
|
|
110
222
|
RedirectDetector: () => RedirectDetector,
|
|
111
223
|
RedirectHandler: () => RedirectHandler,
|
|
112
224
|
RedisChangeTracker: () => RedisChangeTracker,
|
|
225
|
+
RedisLock: () => RedisLock,
|
|
113
226
|
RedisProgressStorage: () => RedisProgressStorage,
|
|
114
227
|
RedisRedirectManager: () => RedisRedirectManager,
|
|
115
228
|
RouteScanner: () => RouteScanner,
|
|
@@ -118,39 +231,88 @@ __export(index_exports, {
|
|
|
118
231
|
SitemapGenerator: () => SitemapGenerator,
|
|
119
232
|
SitemapIndex: () => SitemapIndex,
|
|
120
233
|
SitemapStream: () => SitemapStream,
|
|
234
|
+
compressToBuffer: () => compressToBuffer,
|
|
235
|
+
createCompressionStream: () => createCompressionStream,
|
|
236
|
+
fromGzipFilename: () => fromGzipFilename,
|
|
121
237
|
generateI18nEntries: () => generateI18nEntries,
|
|
122
|
-
routeScanner: () => routeScanner
|
|
238
|
+
routeScanner: () => routeScanner,
|
|
239
|
+
toGzipFilename: () => toGzipFilename
|
|
123
240
|
});
|
|
124
241
|
module.exports = __toCommonJS(index_exports);
|
|
125
242
|
|
|
126
243
|
// src/core/ChangeTracker.ts
|
|
127
244
|
var MemoryChangeTracker = class {
|
|
128
245
|
changes = [];
|
|
246
|
+
urlIndex = /* @__PURE__ */ new Map();
|
|
129
247
|
maxChanges;
|
|
130
248
|
constructor(options = {}) {
|
|
131
249
|
this.maxChanges = options.maxChanges || 1e5;
|
|
132
250
|
}
|
|
251
|
+
/**
|
|
252
|
+
* Record a new site structure change in memory.
|
|
253
|
+
*
|
|
254
|
+
* @param change - The change event to record.
|
|
255
|
+
*/
|
|
133
256
|
async track(change) {
|
|
134
257
|
this.changes.push(change);
|
|
258
|
+
const urlChanges = this.urlIndex.get(change.url) || [];
|
|
259
|
+
urlChanges.push(change);
|
|
260
|
+
this.urlIndex.set(change.url, urlChanges);
|
|
135
261
|
if (this.changes.length > this.maxChanges) {
|
|
136
|
-
|
|
262
|
+
const removed = this.changes.shift();
|
|
263
|
+
if (removed) {
|
|
264
|
+
const changes = this.urlIndex.get(removed.url);
|
|
265
|
+
if (changes) {
|
|
266
|
+
const index = changes.indexOf(removed);
|
|
267
|
+
if (index > -1) {
|
|
268
|
+
changes.splice(index, 1);
|
|
269
|
+
}
|
|
270
|
+
if (changes.length === 0) {
|
|
271
|
+
this.urlIndex.delete(removed.url);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
137
275
|
}
|
|
138
276
|
}
|
|
277
|
+
/**
|
|
278
|
+
* Retrieve all changes recorded in memory since a specific time.
|
|
279
|
+
*
|
|
280
|
+
* @param since - Optional start date for the query.
|
|
281
|
+
* @returns An array of change events.
|
|
282
|
+
*/
|
|
139
283
|
async getChanges(since) {
|
|
140
284
|
if (!since) {
|
|
141
285
|
return [...this.changes];
|
|
142
286
|
}
|
|
143
287
|
return this.changes.filter((change) => change.timestamp >= since);
|
|
144
288
|
}
|
|
289
|
+
/**
|
|
290
|
+
* Retrieve the full change history for a specific URL from memory.
|
|
291
|
+
*
|
|
292
|
+
* @param url - The URL to query history for.
|
|
293
|
+
* @returns An array of change events.
|
|
294
|
+
*/
|
|
145
295
|
async getChangesByUrl(url) {
|
|
146
|
-
return this.
|
|
296
|
+
return this.urlIndex.get(url) || [];
|
|
147
297
|
}
|
|
298
|
+
/**
|
|
299
|
+
* Purge old change records from memory storage.
|
|
300
|
+
*
|
|
301
|
+
* @param since - If provided, only records older than this date will be cleared.
|
|
302
|
+
*/
|
|
148
303
|
async clear(since) {
|
|
149
304
|
if (!since) {
|
|
150
305
|
this.changes = [];
|
|
306
|
+
this.urlIndex.clear();
|
|
151
307
|
return;
|
|
152
308
|
}
|
|
153
309
|
this.changes = this.changes.filter((change) => change.timestamp < since);
|
|
310
|
+
this.urlIndex.clear();
|
|
311
|
+
for (const change of this.changes) {
|
|
312
|
+
const urlChanges = this.urlIndex.get(change.url) || [];
|
|
313
|
+
urlChanges.push(change);
|
|
314
|
+
this.urlIndex.set(change.url, urlChanges);
|
|
315
|
+
}
|
|
154
316
|
}
|
|
155
317
|
};
|
|
156
318
|
var RedisChangeTracker = class {
|
|
@@ -168,6 +330,11 @@ var RedisChangeTracker = class {
|
|
|
168
330
|
getListKey() {
|
|
169
331
|
return `${this.keyPrefix}list`;
|
|
170
332
|
}
|
|
333
|
+
/**
|
|
334
|
+
* Record a new site structure change in Redis.
|
|
335
|
+
*
|
|
336
|
+
* @param change - The change event to record.
|
|
337
|
+
*/
|
|
171
338
|
async track(change) {
|
|
172
339
|
const key = this.getKey(change.url);
|
|
173
340
|
const listKey = this.getListKey();
|
|
@@ -177,6 +344,12 @@ var RedisChangeTracker = class {
|
|
|
177
344
|
await this.client.zadd(listKey, score, change.url);
|
|
178
345
|
await this.client.expire(listKey, this.ttl);
|
|
179
346
|
}
|
|
347
|
+
/**
|
|
348
|
+
* Retrieve all changes recorded in Redis since a specific time.
|
|
349
|
+
*
|
|
350
|
+
* @param since - Optional start date for the query.
|
|
351
|
+
* @returns An array of change events.
|
|
352
|
+
*/
|
|
180
353
|
async getChanges(since) {
|
|
181
354
|
try {
|
|
182
355
|
const listKey = this.getListKey();
|
|
@@ -197,6 +370,12 @@ var RedisChangeTracker = class {
|
|
|
197
370
|
return [];
|
|
198
371
|
}
|
|
199
372
|
}
|
|
373
|
+
/**
|
|
374
|
+
* Retrieve the full change history for a specific URL from Redis.
|
|
375
|
+
*
|
|
376
|
+
* @param url - The URL to query history for.
|
|
377
|
+
* @returns An array of change events.
|
|
378
|
+
*/
|
|
200
379
|
async getChangesByUrl(url) {
|
|
201
380
|
try {
|
|
202
381
|
const key = this.getKey(url);
|
|
@@ -211,6 +390,11 @@ var RedisChangeTracker = class {
|
|
|
211
390
|
return [];
|
|
212
391
|
}
|
|
213
392
|
}
|
|
393
|
+
/**
|
|
394
|
+
* Purge old change records from Redis storage.
|
|
395
|
+
*
|
|
396
|
+
* @param since - If provided, only records older than this date will be cleared.
|
|
397
|
+
*/
|
|
214
398
|
async clear(since) {
|
|
215
399
|
try {
|
|
216
400
|
const listKey = this.getListKey();
|
|
@@ -240,7 +424,11 @@ var DiffCalculator = class {
|
|
|
240
424
|
this.batchSize = options.batchSize || 1e4;
|
|
241
425
|
}
|
|
242
426
|
/**
|
|
243
|
-
*
|
|
427
|
+
* Calculates the difference between two sets of sitemap entries.
|
|
428
|
+
*
|
|
429
|
+
* @param oldEntries - The previous set of sitemap entries.
|
|
430
|
+
* @param newEntries - The current set of sitemap entries.
|
|
431
|
+
* @returns A DiffResult containing added, updated, and removed entries.
|
|
244
432
|
*/
|
|
245
433
|
calculate(oldEntries, newEntries) {
|
|
246
434
|
const oldMap = /* @__PURE__ */ new Map();
|
|
@@ -270,7 +458,11 @@ var DiffCalculator = class {
|
|
|
270
458
|
return { added, updated, removed };
|
|
271
459
|
}
|
|
272
460
|
/**
|
|
273
|
-
*
|
|
461
|
+
* Batch calculates differences for large datasets using async iterables.
|
|
462
|
+
*
|
|
463
|
+
* @param oldEntries - An async iterable of the previous sitemap entries.
|
|
464
|
+
* @param newEntries - An async iterable of the current sitemap entries.
|
|
465
|
+
* @returns A promise resolving to the DiffResult.
|
|
274
466
|
*/
|
|
275
467
|
async calculateBatch(oldEntries, newEntries) {
|
|
276
468
|
const oldMap = /* @__PURE__ */ new Map();
|
|
@@ -284,7 +476,11 @@ var DiffCalculator = class {
|
|
|
284
476
|
return this.calculate(Array.from(oldMap.values()), Array.from(newMap.values()));
|
|
285
477
|
}
|
|
286
478
|
/**
|
|
287
|
-
*
|
|
479
|
+
* Calculates differences based on a sequence of change records.
|
|
480
|
+
*
|
|
481
|
+
* @param baseEntries - The base set of sitemap entries.
|
|
482
|
+
* @param changes - An array of change records to apply to the base set.
|
|
483
|
+
* @returns A DiffResult comparing the base set with the applied changes.
|
|
288
484
|
*/
|
|
289
485
|
calculateFromChanges(baseEntries, changes) {
|
|
290
486
|
const entryMap = /* @__PURE__ */ new Map();
|
|
@@ -304,7 +500,11 @@ var DiffCalculator = class {
|
|
|
304
500
|
return this.calculate(baseEntries, newEntries);
|
|
305
501
|
}
|
|
306
502
|
/**
|
|
307
|
-
*
|
|
503
|
+
* Checks if a sitemap entry has changed by comparing its key properties.
|
|
504
|
+
*
|
|
505
|
+
* @param oldEntry - The previous sitemap entry.
|
|
506
|
+
* @param newEntry - The current sitemap entry.
|
|
507
|
+
* @returns True if the entry has changed, false otherwise.
|
|
308
508
|
*/
|
|
309
509
|
hasChanged(oldEntry, newEntry) {
|
|
310
510
|
if (oldEntry.lastmod !== newEntry.lastmod) {
|
|
@@ -325,55 +525,92 @@ var DiffCalculator = class {
|
|
|
325
525
|
}
|
|
326
526
|
};
|
|
327
527
|
|
|
528
|
+
// src/utils/Mutex.ts
|
|
529
|
+
var Mutex = class {
|
|
530
|
+
queue = Promise.resolve();
|
|
531
|
+
/**
|
|
532
|
+
* Executes a function exclusively, ensuring no other task can run it concurrently.
|
|
533
|
+
*
|
|
534
|
+
* @param fn - The async function to execute.
|
|
535
|
+
* @returns A promise resolving to the result of the function.
|
|
536
|
+
*/
|
|
537
|
+
async runExclusive(fn) {
|
|
538
|
+
const next = this.queue.then(() => fn());
|
|
539
|
+
this.queue = next.then(
|
|
540
|
+
() => {
|
|
541
|
+
},
|
|
542
|
+
() => {
|
|
543
|
+
}
|
|
544
|
+
);
|
|
545
|
+
return next;
|
|
546
|
+
}
|
|
547
|
+
};
|
|
548
|
+
|
|
549
|
+
// src/core/SitemapGenerator.ts
|
|
550
|
+
init_Compression();
|
|
551
|
+
|
|
328
552
|
// src/core/ShadowProcessor.ts
|
|
329
553
|
var ShadowProcessor = class {
|
|
330
554
|
options;
|
|
331
555
|
shadowId;
|
|
332
556
|
operations = [];
|
|
557
|
+
mutex = new Mutex();
|
|
333
558
|
constructor(options) {
|
|
334
559
|
this.options = options;
|
|
335
|
-
this.shadowId = `shadow-${Date.now()}-${
|
|
560
|
+
this.shadowId = `shadow-${Date.now()}-${crypto.randomUUID()}`;
|
|
336
561
|
}
|
|
337
562
|
/**
|
|
338
|
-
*
|
|
563
|
+
* Adds a single file write operation to the current shadow session.
|
|
564
|
+
*
|
|
565
|
+
* If shadow processing is disabled, the file is written directly to the
|
|
566
|
+
* final destination in storage. Otherwise, it is written to the shadow staging area.
|
|
567
|
+
*
|
|
568
|
+
* @param operation - The shadow write operation details.
|
|
339
569
|
*/
|
|
340
570
|
async addOperation(operation) {
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
571
|
+
return this.mutex.runExclusive(async () => {
|
|
572
|
+
if (!this.options.enabled) {
|
|
573
|
+
await this.options.storage.write(operation.filename, operation.content);
|
|
574
|
+
return;
|
|
575
|
+
}
|
|
576
|
+
this.operations.push({
|
|
577
|
+
...operation,
|
|
578
|
+
shadowId: operation.shadowId || this.shadowId
|
|
579
|
+
});
|
|
580
|
+
if (this.options.storage.writeShadow) {
|
|
581
|
+
await this.options.storage.writeShadow(operation.filename, operation.content, this.shadowId);
|
|
582
|
+
} else {
|
|
583
|
+
await this.options.storage.write(operation.filename, operation.content);
|
|
584
|
+
}
|
|
348
585
|
});
|
|
349
|
-
if (this.options.storage.writeShadow) {
|
|
350
|
-
await this.options.storage.writeShadow(operation.filename, operation.content, this.shadowId);
|
|
351
|
-
} else {
|
|
352
|
-
await this.options.storage.write(operation.filename, operation.content);
|
|
353
|
-
}
|
|
354
586
|
}
|
|
355
587
|
/**
|
|
356
|
-
*
|
|
588
|
+
* Commits all staged shadow operations to the final production location.
|
|
589
|
+
*
|
|
590
|
+
* Depending on the `mode`, this will either perform an atomic swap of all files
|
|
591
|
+
* or commit each file individually (potentially creating new versions).
|
|
357
592
|
*/
|
|
358
593
|
async commit() {
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
if (this.options.mode === "atomic") {
|
|
363
|
-
if (this.options.storage.commitShadow) {
|
|
364
|
-
await this.options.storage.commitShadow(this.shadowId);
|
|
594
|
+
return this.mutex.runExclusive(async () => {
|
|
595
|
+
if (!this.options.enabled) {
|
|
596
|
+
return;
|
|
365
597
|
}
|
|
366
|
-
|
|
367
|
-
for (const operation of this.operations) {
|
|
598
|
+
if (this.options.mode === "atomic") {
|
|
368
599
|
if (this.options.storage.commitShadow) {
|
|
369
|
-
await this.options.storage.commitShadow(
|
|
600
|
+
await this.options.storage.commitShadow(this.shadowId);
|
|
601
|
+
}
|
|
602
|
+
} else {
|
|
603
|
+
for (const operation of this.operations) {
|
|
604
|
+
if (this.options.storage.commitShadow) {
|
|
605
|
+
await this.options.storage.commitShadow(operation.shadowId || this.shadowId);
|
|
606
|
+
}
|
|
370
607
|
}
|
|
371
608
|
}
|
|
372
|
-
|
|
373
|
-
|
|
609
|
+
this.operations = [];
|
|
610
|
+
});
|
|
374
611
|
}
|
|
375
612
|
/**
|
|
376
|
-
*
|
|
613
|
+
* Cancels all staged shadow operations without committing them.
|
|
377
614
|
*/
|
|
378
615
|
async rollback() {
|
|
379
616
|
if (!this.options.enabled) {
|
|
@@ -382,13 +619,13 @@ var ShadowProcessor = class {
|
|
|
382
619
|
this.operations = [];
|
|
383
620
|
}
|
|
384
621
|
/**
|
|
385
|
-
*
|
|
622
|
+
* Returns the unique identifier for the current shadow session.
|
|
386
623
|
*/
|
|
387
624
|
getShadowId() {
|
|
388
625
|
return this.shadowId;
|
|
389
626
|
}
|
|
390
627
|
/**
|
|
391
|
-
*
|
|
628
|
+
* Returns an array of all staged shadow operations.
|
|
392
629
|
*/
|
|
393
630
|
getOperations() {
|
|
394
631
|
return [...this.operations];
|
|
@@ -405,6 +642,12 @@ var SitemapIndex = class {
|
|
|
405
642
|
this.options.baseUrl = this.options.baseUrl.slice(0, -1);
|
|
406
643
|
}
|
|
407
644
|
}
|
|
645
|
+
/**
|
|
646
|
+
* Adds a single entry to the sitemap index.
|
|
647
|
+
*
|
|
648
|
+
* @param entry - A sitemap filename or a `SitemapIndexEntry` object.
|
|
649
|
+
* @returns The `SitemapIndex` instance for chaining.
|
|
650
|
+
*/
|
|
408
651
|
add(entry) {
|
|
409
652
|
if (typeof entry === "string") {
|
|
410
653
|
this.entries.push({ url: entry });
|
|
@@ -413,12 +656,23 @@ var SitemapIndex = class {
|
|
|
413
656
|
}
|
|
414
657
|
return this;
|
|
415
658
|
}
|
|
659
|
+
/**
|
|
660
|
+
* Adds multiple entries to the sitemap index.
|
|
661
|
+
*
|
|
662
|
+
* @param entries - An array of sitemap filenames or `SitemapIndexEntry` objects.
|
|
663
|
+
* @returns The `SitemapIndex` instance for chaining.
|
|
664
|
+
*/
|
|
416
665
|
addAll(entries) {
|
|
417
666
|
for (const entry of entries) {
|
|
418
667
|
this.add(entry);
|
|
419
668
|
}
|
|
420
669
|
return this;
|
|
421
670
|
}
|
|
671
|
+
/**
|
|
672
|
+
* Generates the sitemap index XML content.
|
|
673
|
+
*
|
|
674
|
+
* @returns The complete XML string for the sitemap index.
|
|
675
|
+
*/
|
|
422
676
|
toXML() {
|
|
423
677
|
const { baseUrl, pretty } = this.options;
|
|
424
678
|
let xml = `<?xml version="1.0" encoding="UTF-8"?>
|
|
@@ -447,6 +701,9 @@ var SitemapIndex = class {
|
|
|
447
701
|
xml += `</sitemapindex>`;
|
|
448
702
|
return xml;
|
|
449
703
|
}
|
|
704
|
+
/**
|
|
705
|
+
* Escapes special XML characters in a string.
|
|
706
|
+
*/
|
|
450
707
|
escape(str) {
|
|
451
708
|
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
452
709
|
}
|
|
@@ -456,55 +713,137 @@ var SitemapIndex = class {
|
|
|
456
713
|
var SitemapStream = class {
|
|
457
714
|
options;
|
|
458
715
|
entries = [];
|
|
716
|
+
flags = {
|
|
717
|
+
hasImages: false,
|
|
718
|
+
hasVideos: false,
|
|
719
|
+
hasNews: false,
|
|
720
|
+
hasAlternates: false
|
|
721
|
+
};
|
|
459
722
|
constructor(options) {
|
|
460
723
|
this.options = { ...options };
|
|
461
724
|
if (this.options.baseUrl.endsWith("/")) {
|
|
462
725
|
this.options.baseUrl = this.options.baseUrl.slice(0, -1);
|
|
463
726
|
}
|
|
464
727
|
}
|
|
728
|
+
/**
|
|
729
|
+
* Adds a single entry to the sitemap stream.
|
|
730
|
+
*
|
|
731
|
+
* @param entry - A URL string or a complete `SitemapEntry` object.
|
|
732
|
+
* @returns The `SitemapStream` instance for chaining.
|
|
733
|
+
*/
|
|
465
734
|
add(entry) {
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
this.
|
|
735
|
+
const e = typeof entry === "string" ? { url: entry } : entry;
|
|
736
|
+
this.entries.push(e);
|
|
737
|
+
if (e.images && e.images.length > 0) {
|
|
738
|
+
this.flags.hasImages = true;
|
|
739
|
+
}
|
|
740
|
+
if (e.videos && e.videos.length > 0) {
|
|
741
|
+
this.flags.hasVideos = true;
|
|
742
|
+
}
|
|
743
|
+
if (e.news) {
|
|
744
|
+
this.flags.hasNews = true;
|
|
745
|
+
}
|
|
746
|
+
if (e.alternates && e.alternates.length > 0) {
|
|
747
|
+
this.flags.hasAlternates = true;
|
|
470
748
|
}
|
|
471
749
|
return this;
|
|
472
750
|
}
|
|
751
|
+
/**
|
|
752
|
+
* Adds multiple entries to the sitemap stream.
|
|
753
|
+
*
|
|
754
|
+
* @param entries - An array of URL strings or `SitemapEntry` objects.
|
|
755
|
+
* @returns The `SitemapStream` instance for chaining.
|
|
756
|
+
*/
|
|
473
757
|
addAll(entries) {
|
|
474
758
|
for (const entry of entries) {
|
|
475
759
|
this.add(entry);
|
|
476
760
|
}
|
|
477
761
|
return this;
|
|
478
762
|
}
|
|
763
|
+
/**
|
|
764
|
+
* Generates the sitemap XML content.
|
|
765
|
+
*
|
|
766
|
+
* Automatically includes the necessary XML namespaces for images, videos, news,
|
|
767
|
+
* and internationalization if the entries contain such metadata.
|
|
768
|
+
*
|
|
769
|
+
* @returns The complete XML string for the sitemap.
|
|
770
|
+
*/
|
|
479
771
|
toXML() {
|
|
772
|
+
const parts = [];
|
|
773
|
+
for (const chunk of this.toSyncIterable()) {
|
|
774
|
+
parts.push(chunk);
|
|
775
|
+
}
|
|
776
|
+
return parts.join("");
|
|
777
|
+
}
|
|
778
|
+
/**
|
|
779
|
+
* 以 AsyncGenerator 方式產生 XML 內容。
|
|
780
|
+
* 每次 yield 一個邏輯區塊,適合串流寫入場景,可減少記憶體峰值。
|
|
781
|
+
*
|
|
782
|
+
* @returns AsyncGenerator 產生 XML 字串片段
|
|
783
|
+
*
|
|
784
|
+
* @example
|
|
785
|
+
* ```typescript
|
|
786
|
+
* const stream = new SitemapStream({ baseUrl: 'https://example.com' })
|
|
787
|
+
* stream.add({ url: '/page1' })
|
|
788
|
+
* stream.add({ url: '/page2' })
|
|
789
|
+
*
|
|
790
|
+
* for await (const chunk of stream.toAsyncIterable()) {
|
|
791
|
+
* process.stdout.write(chunk)
|
|
792
|
+
* }
|
|
793
|
+
* ```
|
|
794
|
+
*
|
|
795
|
+
* @since 3.1.0
|
|
796
|
+
*/
|
|
797
|
+
async *toAsyncIterable() {
|
|
798
|
+
yield '<?xml version="1.0" encoding="UTF-8"?>\n';
|
|
799
|
+
yield this.buildUrlsetOpenTag();
|
|
480
800
|
const { baseUrl, pretty } = this.options;
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
xml += `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
|
|
484
|
-
if (this.hasImages()) {
|
|
485
|
-
xml += ` xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"`;
|
|
801
|
+
for (const entry of this.entries) {
|
|
802
|
+
yield this.renderUrl(entry, baseUrl, pretty);
|
|
486
803
|
}
|
|
487
|
-
|
|
488
|
-
|
|
804
|
+
yield "</urlset>";
|
|
805
|
+
}
|
|
806
|
+
/**
|
|
807
|
+
* 同步版本的 iterable,供 toXML() 使用。
|
|
808
|
+
*/
|
|
809
|
+
*toSyncIterable() {
|
|
810
|
+
yield '<?xml version="1.0" encoding="UTF-8"?>\n';
|
|
811
|
+
yield this.buildUrlsetOpenTag();
|
|
812
|
+
const { baseUrl, pretty } = this.options;
|
|
813
|
+
for (const entry of this.entries) {
|
|
814
|
+
yield this.renderUrl(entry, baseUrl, pretty);
|
|
489
815
|
}
|
|
490
|
-
|
|
491
|
-
|
|
816
|
+
yield "</urlset>";
|
|
817
|
+
}
|
|
818
|
+
/**
|
|
819
|
+
* 建立 urlset 開標籤與所有必要的 XML 命名空間。
|
|
820
|
+
*/
|
|
821
|
+
buildUrlsetOpenTag() {
|
|
822
|
+
const parts = [];
|
|
823
|
+
parts.push('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"');
|
|
824
|
+
if (this.flags.hasImages) {
|
|
825
|
+
parts.push(' xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"');
|
|
492
826
|
}
|
|
493
|
-
if (this.
|
|
494
|
-
|
|
827
|
+
if (this.flags.hasVideos) {
|
|
828
|
+
parts.push(' xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"');
|
|
495
829
|
}
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
for (const entry of this.entries) {
|
|
499
|
-
xml += this.renderUrl(entry, baseUrl, pretty);
|
|
830
|
+
if (this.flags.hasNews) {
|
|
831
|
+
parts.push(' xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"');
|
|
500
832
|
}
|
|
501
|
-
|
|
502
|
-
|
|
833
|
+
if (this.flags.hasAlternates) {
|
|
834
|
+
parts.push(' xmlns:xhtml="http://www.w3.org/1999/xhtml"');
|
|
835
|
+
}
|
|
836
|
+
parts.push(">\n");
|
|
837
|
+
return parts.join("");
|
|
503
838
|
}
|
|
839
|
+
/**
|
|
840
|
+
* Renders a single sitemap entry into its XML representation.
|
|
841
|
+
*/
|
|
504
842
|
renderUrl(entry, baseUrl, pretty) {
|
|
505
843
|
const indent = pretty ? " " : "";
|
|
506
844
|
const subIndent = pretty ? " " : "";
|
|
507
845
|
const nl = pretty ? "\n" : "";
|
|
846
|
+
const parts = [];
|
|
508
847
|
let loc = entry.url;
|
|
509
848
|
if (!loc.startsWith("http")) {
|
|
510
849
|
if (!loc.startsWith("/")) {
|
|
@@ -512,17 +851,17 @@ var SitemapStream = class {
|
|
|
512
851
|
}
|
|
513
852
|
loc = baseUrl + loc;
|
|
514
853
|
}
|
|
515
|
-
|
|
516
|
-
|
|
854
|
+
parts.push(`${indent}<url>${nl}`);
|
|
855
|
+
parts.push(`${subIndent}<loc>${this.escape(loc)}</loc>${nl}`);
|
|
517
856
|
if (entry.lastmod) {
|
|
518
857
|
const date = entry.lastmod instanceof Date ? entry.lastmod : new Date(entry.lastmod);
|
|
519
|
-
|
|
858
|
+
parts.push(`${subIndent}<lastmod>${date.toISOString().split("T")[0]}</lastmod>${nl}`);
|
|
520
859
|
}
|
|
521
860
|
if (entry.changefreq) {
|
|
522
|
-
|
|
861
|
+
parts.push(`${subIndent}<changefreq>${entry.changefreq}</changefreq>${nl}`);
|
|
523
862
|
}
|
|
524
863
|
if (entry.priority !== void 0) {
|
|
525
|
-
|
|
864
|
+
parts.push(`${subIndent}<priority>${entry.priority.toFixed(1)}</priority>${nl}`);
|
|
526
865
|
}
|
|
527
866
|
if (entry.alternates) {
|
|
528
867
|
for (const alt of entry.alternates) {
|
|
@@ -533,7 +872,9 @@ var SitemapStream = class {
|
|
|
533
872
|
}
|
|
534
873
|
altLoc = baseUrl + altLoc;
|
|
535
874
|
}
|
|
536
|
-
|
|
875
|
+
parts.push(
|
|
876
|
+
`${subIndent}<xhtml:link rel="alternate" hreflang="${alt.lang}" href="${this.escape(altLoc)}"/>${nl}`
|
|
877
|
+
);
|
|
537
878
|
}
|
|
538
879
|
}
|
|
539
880
|
if (entry.redirect?.canonical) {
|
|
@@ -544,10 +885,14 @@ var SitemapStream = class {
|
|
|
544
885
|
}
|
|
545
886
|
canonicalUrl = baseUrl + canonicalUrl;
|
|
546
887
|
}
|
|
547
|
-
|
|
888
|
+
parts.push(
|
|
889
|
+
`${subIndent}<xhtml:link rel="canonical" href="${this.escape(canonicalUrl)}"/>${nl}`
|
|
890
|
+
);
|
|
548
891
|
}
|
|
549
892
|
if (entry.redirect && !entry.redirect.canonical) {
|
|
550
|
-
|
|
893
|
+
parts.push(
|
|
894
|
+
`${subIndent}<!-- Redirect: ${entry.redirect.from} \u2192 ${entry.redirect.to} (${entry.redirect.type}) -->${nl}`
|
|
895
|
+
);
|
|
551
896
|
}
|
|
552
897
|
if (entry.images) {
|
|
553
898
|
for (const img of entry.images) {
|
|
@@ -558,91 +903,118 @@ var SitemapStream = class {
|
|
|
558
903
|
}
|
|
559
904
|
loc2 = baseUrl + loc2;
|
|
560
905
|
}
|
|
561
|
-
|
|
562
|
-
|
|
906
|
+
parts.push(`${subIndent}<image:image>${nl}`);
|
|
907
|
+
parts.push(`${subIndent} <image:loc>${this.escape(loc2)}</image:loc>${nl}`);
|
|
563
908
|
if (img.title) {
|
|
564
|
-
|
|
909
|
+
parts.push(`${subIndent} <image:title>${this.escape(img.title)}</image:title>${nl}`);
|
|
565
910
|
}
|
|
566
911
|
if (img.caption) {
|
|
567
|
-
|
|
912
|
+
parts.push(
|
|
913
|
+
`${subIndent} <image:caption>${this.escape(img.caption)}</image:caption>${nl}`
|
|
914
|
+
);
|
|
568
915
|
}
|
|
569
916
|
if (img.geo_location) {
|
|
570
|
-
|
|
917
|
+
parts.push(
|
|
918
|
+
`${subIndent} <image:geo_location>${this.escape(img.geo_location)}</image:geo_location>${nl}`
|
|
919
|
+
);
|
|
571
920
|
}
|
|
572
921
|
if (img.license) {
|
|
573
|
-
|
|
922
|
+
parts.push(
|
|
923
|
+
`${subIndent} <image:license>${this.escape(img.license)}</image:license>${nl}`
|
|
924
|
+
);
|
|
574
925
|
}
|
|
575
|
-
|
|
926
|
+
parts.push(`${subIndent}</image:image>${nl}`);
|
|
576
927
|
}
|
|
577
928
|
}
|
|
578
929
|
if (entry.videos) {
|
|
579
930
|
for (const video of entry.videos) {
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
931
|
+
parts.push(`${subIndent}<video:video>${nl}`);
|
|
932
|
+
parts.push(
|
|
933
|
+
`${subIndent} <video:thumbnail_loc>${this.escape(video.thumbnail_loc)}</video:thumbnail_loc>${nl}`
|
|
934
|
+
);
|
|
935
|
+
parts.push(`${subIndent} <video:title>${this.escape(video.title)}</video:title>${nl}`);
|
|
936
|
+
parts.push(
|
|
937
|
+
`${subIndent} <video:description>${this.escape(video.description)}</video:description>${nl}`
|
|
938
|
+
);
|
|
584
939
|
if (video.content_loc) {
|
|
585
|
-
|
|
940
|
+
parts.push(
|
|
941
|
+
`${subIndent} <video:content_loc>${this.escape(video.content_loc)}</video:content_loc>${nl}`
|
|
942
|
+
);
|
|
586
943
|
}
|
|
587
944
|
if (video.player_loc) {
|
|
588
|
-
|
|
945
|
+
parts.push(
|
|
946
|
+
`${subIndent} <video:player_loc>${this.escape(video.player_loc)}</video:player_loc>${nl}`
|
|
947
|
+
);
|
|
589
948
|
}
|
|
590
949
|
if (video.duration) {
|
|
591
|
-
|
|
950
|
+
parts.push(`${subIndent} <video:duration>${video.duration}</video:duration>${nl}`);
|
|
592
951
|
}
|
|
593
952
|
if (video.view_count) {
|
|
594
|
-
|
|
953
|
+
parts.push(`${subIndent} <video:view_count>${video.view_count}</video:view_count>${nl}`);
|
|
595
954
|
}
|
|
596
955
|
if (video.publication_date) {
|
|
597
956
|
const pubDate = video.publication_date instanceof Date ? video.publication_date : new Date(video.publication_date);
|
|
598
|
-
|
|
957
|
+
parts.push(
|
|
958
|
+
`${subIndent} <video:publication_date>${pubDate.toISOString()}</video:publication_date>${nl}`
|
|
959
|
+
);
|
|
599
960
|
}
|
|
600
961
|
if (video.family_friendly) {
|
|
601
|
-
|
|
962
|
+
parts.push(
|
|
963
|
+
`${subIndent} <video:family_friendly>${video.family_friendly}</video:family_friendly>${nl}`
|
|
964
|
+
);
|
|
602
965
|
}
|
|
603
966
|
if (video.tag) {
|
|
604
967
|
for (const tag of video.tag) {
|
|
605
|
-
|
|
968
|
+
parts.push(`${subIndent} <video:tag>${this.escape(tag)}</video:tag>${nl}`);
|
|
606
969
|
}
|
|
607
970
|
}
|
|
608
|
-
|
|
971
|
+
parts.push(`${subIndent}</video:video>${nl}`);
|
|
609
972
|
}
|
|
610
973
|
}
|
|
611
974
|
if (entry.news) {
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
975
|
+
parts.push(`${subIndent}<news:news>${nl}`);
|
|
976
|
+
parts.push(`${subIndent} <news:publication>${nl}`);
|
|
977
|
+
parts.push(
|
|
978
|
+
`${subIndent} <news:name>${this.escape(entry.news.publication.name)}</news:name>${nl}`
|
|
979
|
+
);
|
|
980
|
+
parts.push(
|
|
981
|
+
`${subIndent} <news:language>${this.escape(entry.news.publication.language)}</news:language>${nl}`
|
|
982
|
+
);
|
|
983
|
+
parts.push(`${subIndent} </news:publication>${nl}`);
|
|
617
984
|
const pubDate = entry.news.publication_date instanceof Date ? entry.news.publication_date : new Date(entry.news.publication_date);
|
|
618
|
-
|
|
619
|
-
|
|
985
|
+
parts.push(
|
|
986
|
+
`${subIndent} <news:publication_date>${pubDate.toISOString()}</news:publication_date>${nl}`
|
|
987
|
+
);
|
|
988
|
+
parts.push(`${subIndent} <news:title>${this.escape(entry.news.title)}</news:title>${nl}`);
|
|
620
989
|
if (entry.news.genres) {
|
|
621
|
-
|
|
990
|
+
parts.push(
|
|
991
|
+
`${subIndent} <news:genres>${this.escape(entry.news.genres)}</news:genres>${nl}`
|
|
992
|
+
);
|
|
622
993
|
}
|
|
623
994
|
if (entry.news.keywords) {
|
|
624
|
-
|
|
995
|
+
parts.push(
|
|
996
|
+
`${subIndent} <news:keywords>${entry.news.keywords.map((k) => this.escape(k)).join(", ")}</news:keywords>${nl}`
|
|
997
|
+
);
|
|
625
998
|
}
|
|
626
|
-
|
|
999
|
+
parts.push(`${subIndent}</news:news>${nl}`);
|
|
627
1000
|
}
|
|
628
|
-
|
|
629
|
-
return
|
|
630
|
-
}
|
|
631
|
-
hasImages() {
|
|
632
|
-
return this.entries.some((e) => e.images && e.images.length > 0);
|
|
633
|
-
}
|
|
634
|
-
hasVideos() {
|
|
635
|
-
return this.entries.some((e) => e.videos && e.videos.length > 0);
|
|
636
|
-
}
|
|
637
|
-
hasNews() {
|
|
638
|
-
return this.entries.some((e) => !!e.news);
|
|
639
|
-
}
|
|
640
|
-
hasAlternates() {
|
|
641
|
-
return this.entries.some((e) => e.alternates && e.alternates.length > 0);
|
|
1001
|
+
parts.push(`${indent}</url>${nl}`);
|
|
1002
|
+
return parts.join("");
|
|
642
1003
|
}
|
|
1004
|
+
/**
|
|
1005
|
+
* Escapes special XML characters in a string.
|
|
1006
|
+
*/
|
|
643
1007
|
escape(str) {
|
|
644
1008
|
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
645
1009
|
}
|
|
1010
|
+
/**
|
|
1011
|
+
* Returns all entries currently in the stream.
|
|
1012
|
+
*
|
|
1013
|
+
* @returns An array of `SitemapEntry` objects.
|
|
1014
|
+
*/
|
|
1015
|
+
getEntries() {
|
|
1016
|
+
return this.entries;
|
|
1017
|
+
}
|
|
646
1018
|
};
|
|
647
1019
|
|
|
648
1020
|
// src/core/SitemapGenerator.ts
|
|
@@ -663,6 +1035,12 @@ var SitemapGenerator = class {
|
|
|
663
1035
|
});
|
|
664
1036
|
}
|
|
665
1037
|
}
|
|
1038
|
+
/**
|
|
1039
|
+
* Orchestrates the sitemap generation process.
|
|
1040
|
+
*
|
|
1041
|
+
* This method scans all providers, handles sharding, generates the XML files,
|
|
1042
|
+
* and optionally creates a sitemap index and manifest.
|
|
1043
|
+
*/
|
|
666
1044
|
async run() {
|
|
667
1045
|
let shardIndex = 1;
|
|
668
1046
|
let currentCount = 0;
|
|
@@ -674,18 +1052,29 @@ var SitemapGenerator = class {
|
|
|
674
1052
|
baseUrl: this.options.baseUrl,
|
|
675
1053
|
pretty: this.options.pretty
|
|
676
1054
|
});
|
|
1055
|
+
const shards = [];
|
|
677
1056
|
let isMultiFile = false;
|
|
678
1057
|
const flushShard = async () => {
|
|
679
1058
|
isMultiFile = true;
|
|
680
1059
|
const baseName = this.options.filename?.replace(/\.xml$/, "");
|
|
681
1060
|
const filename = `${baseName}-${shardIndex}.xml`;
|
|
682
|
-
const
|
|
1061
|
+
const entries = currentStream.getEntries();
|
|
1062
|
+
let actualFilename;
|
|
683
1063
|
if (this.shadowProcessor) {
|
|
684
|
-
|
|
1064
|
+
actualFilename = this.options.compression?.enabled ? toGzipFilename(filename) : filename;
|
|
1065
|
+
const xml = currentStream.toXML();
|
|
1066
|
+
await this.shadowProcessor.addOperation({ filename: actualFilename, content: xml });
|
|
685
1067
|
} else {
|
|
686
|
-
await this.
|
|
1068
|
+
actualFilename = await this.writeSitemap(currentStream, filename);
|
|
687
1069
|
}
|
|
688
|
-
|
|
1070
|
+
shards.push({
|
|
1071
|
+
filename: actualFilename,
|
|
1072
|
+
from: this.normalizeUrl(entries[0].url),
|
|
1073
|
+
to: this.normalizeUrl(entries[entries.length - 1].url),
|
|
1074
|
+
count: entries.length,
|
|
1075
|
+
lastmod: /* @__PURE__ */ new Date()
|
|
1076
|
+
});
|
|
1077
|
+
const url = this.options.storage.getUrl(actualFilename);
|
|
689
1078
|
index.add({
|
|
690
1079
|
url,
|
|
691
1080
|
lastmod: /* @__PURE__ */ new Date()
|
|
@@ -717,16 +1106,51 @@ var SitemapGenerator = class {
|
|
|
717
1106
|
}
|
|
718
1107
|
}
|
|
719
1108
|
}
|
|
1109
|
+
const writeManifest = async () => {
|
|
1110
|
+
if (!this.options.generateManifest) {
|
|
1111
|
+
return;
|
|
1112
|
+
}
|
|
1113
|
+
const manifest = {
|
|
1114
|
+
version: 1,
|
|
1115
|
+
generatedAt: /* @__PURE__ */ new Date(),
|
|
1116
|
+
baseUrl: this.options.baseUrl,
|
|
1117
|
+
maxEntriesPerShard: this.options.maxEntriesPerFile,
|
|
1118
|
+
sort: "url-lex",
|
|
1119
|
+
shards
|
|
1120
|
+
};
|
|
1121
|
+
const manifestFilename = this.options.filename?.replace(/\.xml$/, "-manifest.json") || "sitemap-manifest.json";
|
|
1122
|
+
const content = JSON.stringify(manifest, null, this.options.pretty ? 2 : 0);
|
|
1123
|
+
if (this.shadowProcessor) {
|
|
1124
|
+
await this.shadowProcessor.addOperation({ filename: manifestFilename, content });
|
|
1125
|
+
} else {
|
|
1126
|
+
await this.options.storage.write(manifestFilename, content);
|
|
1127
|
+
}
|
|
1128
|
+
};
|
|
720
1129
|
if (!isMultiFile) {
|
|
721
|
-
const
|
|
1130
|
+
const entries = currentStream.getEntries();
|
|
1131
|
+
let actualFilename;
|
|
722
1132
|
if (this.shadowProcessor) {
|
|
1133
|
+
actualFilename = this.options.compression?.enabled ? toGzipFilename(this.options.filename) : this.options.filename;
|
|
1134
|
+
const xml = currentStream.toXML();
|
|
723
1135
|
await this.shadowProcessor.addOperation({
|
|
724
|
-
filename:
|
|
1136
|
+
filename: actualFilename,
|
|
725
1137
|
content: xml
|
|
726
1138
|
});
|
|
1139
|
+
} else {
|
|
1140
|
+
actualFilename = await this.writeSitemap(currentStream, this.options.filename);
|
|
1141
|
+
}
|
|
1142
|
+
shards.push({
|
|
1143
|
+
filename: actualFilename,
|
|
1144
|
+
from: entries[0] ? this.normalizeUrl(entries[0].url) : "",
|
|
1145
|
+
to: entries[entries.length - 1] ? this.normalizeUrl(entries[entries.length - 1].url) : "",
|
|
1146
|
+
count: entries.length,
|
|
1147
|
+
lastmod: /* @__PURE__ */ new Date()
|
|
1148
|
+
});
|
|
1149
|
+
if (this.shadowProcessor) {
|
|
1150
|
+
await writeManifest();
|
|
727
1151
|
await this.shadowProcessor.commit();
|
|
728
1152
|
} else {
|
|
729
|
-
await
|
|
1153
|
+
await writeManifest();
|
|
730
1154
|
}
|
|
731
1155
|
return;
|
|
732
1156
|
}
|
|
@@ -739,35 +1163,203 @@ var SitemapGenerator = class {
|
|
|
739
1163
|
filename: this.options.filename,
|
|
740
1164
|
content: indexXml
|
|
741
1165
|
});
|
|
1166
|
+
await writeManifest();
|
|
742
1167
|
await this.shadowProcessor.commit();
|
|
743
1168
|
} else {
|
|
744
1169
|
await this.options.storage.write(this.options.filename, indexXml);
|
|
1170
|
+
await writeManifest();
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
/**
|
|
1174
|
+
* 統一的 sitemap 寫入方法,優先使用串流寫入以降低記憶體使用。
|
|
1175
|
+
*
|
|
1176
|
+
* @param stream - SitemapStream 實例
|
|
1177
|
+
* @param filename - 檔案名稱(不含 .gz)
|
|
1178
|
+
* @returns 實際寫入的檔名(可能包含 .gz)
|
|
1179
|
+
* @since 3.1.0
|
|
1180
|
+
*/
|
|
1181
|
+
async writeSitemap(stream, filename) {
|
|
1182
|
+
const { storage, compression } = this.options;
|
|
1183
|
+
const compress = compression?.enabled ?? false;
|
|
1184
|
+
if (storage.writeStream) {
|
|
1185
|
+
await storage.writeStream(filename, stream.toAsyncIterable(), {
|
|
1186
|
+
compress,
|
|
1187
|
+
contentType: "application/xml"
|
|
1188
|
+
});
|
|
1189
|
+
} else {
|
|
1190
|
+
const xml = stream.toXML();
|
|
1191
|
+
if (compress) {
|
|
1192
|
+
const buffer = await compressToBuffer(
|
|
1193
|
+
(async function* () {
|
|
1194
|
+
yield xml;
|
|
1195
|
+
})(),
|
|
1196
|
+
{ level: compression?.level }
|
|
1197
|
+
);
|
|
1198
|
+
await storage.write(toGzipFilename(filename), buffer.toString("base64"));
|
|
1199
|
+
} else {
|
|
1200
|
+
await storage.write(filename, xml);
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
return compress ? toGzipFilename(filename) : filename;
|
|
1204
|
+
}
|
|
1205
|
+
/**
|
|
1206
|
+
* Normalizes a URL to an absolute URL using the base URL.
|
|
1207
|
+
*/
|
|
1208
|
+
normalizeUrl(url) {
|
|
1209
|
+
if (url.startsWith("http")) {
|
|
1210
|
+
return url;
|
|
745
1211
|
}
|
|
1212
|
+
const { baseUrl } = this.options;
|
|
1213
|
+
const normalizedBase = baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
|
|
1214
|
+
const normalizedPath = url.startsWith("/") ? url : `/${url}`;
|
|
1215
|
+
return normalizedBase + normalizedPath;
|
|
746
1216
|
}
|
|
747
1217
|
/**
|
|
748
|
-
*
|
|
1218
|
+
* Returns the shadow processor instance if enabled.
|
|
749
1219
|
*/
|
|
750
1220
|
getShadowProcessor() {
|
|
751
1221
|
return this.shadowProcessor;
|
|
752
1222
|
}
|
|
753
1223
|
};
|
|
754
1224
|
|
|
1225
|
+
// src/core/SitemapParser.ts
|
|
1226
|
+
var SitemapParser = class {
|
|
1227
|
+
/**
|
|
1228
|
+
* Parses a sitemap XML string into an array of entries.
|
|
1229
|
+
*
|
|
1230
|
+
* @param xml - The raw sitemap XML content.
|
|
1231
|
+
* @returns An array of `SitemapEntry` objects.
|
|
1232
|
+
*/
|
|
1233
|
+
static parse(xml) {
|
|
1234
|
+
const entries = [];
|
|
1235
|
+
const urlRegex = /<url>([\s\S]*?)<\/url>/g;
|
|
1236
|
+
let match;
|
|
1237
|
+
while ((match = urlRegex.exec(xml)) !== null) {
|
|
1238
|
+
const entry = this.parseEntry(match[1]);
|
|
1239
|
+
if (entry) {
|
|
1240
|
+
entries.push(entry);
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
return entries;
|
|
1244
|
+
}
|
|
1245
|
+
/**
|
|
1246
|
+
* Parses a sitemap XML stream into an async iterable of entries.
|
|
1247
|
+
*
|
|
1248
|
+
* Useful for large sitemap files that should not be fully loaded into memory.
|
|
1249
|
+
*
|
|
1250
|
+
* @param stream - An async iterable of XML chunks.
|
|
1251
|
+
* @returns An async iterable of `SitemapEntry` objects.
|
|
1252
|
+
*/
|
|
1253
|
+
static async *parseStream(stream) {
|
|
1254
|
+
let buffer = "";
|
|
1255
|
+
const urlRegex = /<url>([\s\S]*?)<\/url>/g;
|
|
1256
|
+
for await (const chunk of stream) {
|
|
1257
|
+
buffer += chunk;
|
|
1258
|
+
let match;
|
|
1259
|
+
while ((match = urlRegex.exec(buffer)) !== null) {
|
|
1260
|
+
const entry = this.parseEntry(match[1]);
|
|
1261
|
+
if (entry) {
|
|
1262
|
+
yield entry;
|
|
1263
|
+
}
|
|
1264
|
+
buffer = buffer.slice(match.index + match[0].length);
|
|
1265
|
+
urlRegex.lastIndex = 0;
|
|
1266
|
+
}
|
|
1267
|
+
if (buffer.length > 1024 * 1024) {
|
|
1268
|
+
const lastUrlStart = buffer.lastIndexOf("<url>");
|
|
1269
|
+
buffer = lastUrlStart !== -1 ? buffer.slice(lastUrlStart) : "";
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
/**
|
|
1274
|
+
* Parses a single `<url>` tag content into a `SitemapEntry`.
|
|
1275
|
+
*/
|
|
1276
|
+
static parseEntry(urlContent) {
|
|
1277
|
+
const entry = { url: "" };
|
|
1278
|
+
const locMatch = /<loc>(.*?)<\/loc>/.exec(urlContent);
|
|
1279
|
+
if (locMatch) {
|
|
1280
|
+
entry.url = this.unescape(locMatch[1]);
|
|
1281
|
+
} else {
|
|
1282
|
+
return null;
|
|
1283
|
+
}
|
|
1284
|
+
const lastmodMatch = /<lastmod>(.*?)<\/lastmod>/.exec(urlContent);
|
|
1285
|
+
if (lastmodMatch) {
|
|
1286
|
+
entry.lastmod = new Date(lastmodMatch[1]);
|
|
1287
|
+
}
|
|
1288
|
+
const priorityMatch = /<priority>(.*?)<\/priority>/.exec(urlContent);
|
|
1289
|
+
if (priorityMatch) {
|
|
1290
|
+
entry.priority = parseFloat(priorityMatch[1]);
|
|
1291
|
+
}
|
|
1292
|
+
const changefreqMatch = /<changefreq>(.*?)<\/changefreq>/.exec(urlContent);
|
|
1293
|
+
if (changefreqMatch) {
|
|
1294
|
+
entry.changefreq = changefreqMatch[1];
|
|
1295
|
+
}
|
|
1296
|
+
return entry;
|
|
1297
|
+
}
|
|
1298
|
+
/**
|
|
1299
|
+
* Parses a sitemap index XML string into an array of sitemap URLs.
|
|
1300
|
+
*
|
|
1301
|
+
* @param xml - The raw sitemap index XML content.
|
|
1302
|
+
* @returns An array of sub-sitemap URLs.
|
|
1303
|
+
*/
|
|
1304
|
+
static parseIndex(xml) {
|
|
1305
|
+
const urls = [];
|
|
1306
|
+
const sitemapRegex = /<sitemap>([\s\S]*?)<\/sitemap>/g;
|
|
1307
|
+
let match;
|
|
1308
|
+
while ((match = sitemapRegex.exec(xml)) !== null) {
|
|
1309
|
+
const content = match[1];
|
|
1310
|
+
const locMatch = /<loc>(.*?)<\/loc>/.exec(content);
|
|
1311
|
+
if (locMatch) {
|
|
1312
|
+
urls.push(this.unescape(locMatch[1]));
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
return urls;
|
|
1316
|
+
}
|
|
1317
|
+
/**
|
|
1318
|
+
* Unescapes special XML entities in a string.
|
|
1319
|
+
*/
|
|
1320
|
+
static unescape(str) {
|
|
1321
|
+
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'");
|
|
1322
|
+
}
|
|
1323
|
+
};
|
|
1324
|
+
|
|
755
1325
|
// src/core/IncrementalGenerator.ts
|
|
756
1326
|
var IncrementalGenerator = class {
|
|
757
1327
|
options;
|
|
758
1328
|
changeTracker;
|
|
759
1329
|
diffCalculator;
|
|
760
1330
|
generator;
|
|
1331
|
+
mutex = new Mutex();
|
|
761
1332
|
constructor(options) {
|
|
762
|
-
this.options =
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
1333
|
+
this.options = {
|
|
1334
|
+
autoTrack: true,
|
|
1335
|
+
generateManifest: true,
|
|
1336
|
+
...options
|
|
1337
|
+
};
|
|
1338
|
+
this.changeTracker = this.options.changeTracker;
|
|
1339
|
+
this.diffCalculator = this.options.diffCalculator || new DiffCalculator();
|
|
1340
|
+
this.generator = new SitemapGenerator(this.options);
|
|
766
1341
|
}
|
|
767
1342
|
/**
|
|
768
|
-
*
|
|
1343
|
+
* Performs a full sitemap generation and optionally records all entries in the change tracker.
|
|
769
1344
|
*/
|
|
770
1345
|
async generateFull() {
|
|
1346
|
+
return this.mutex.runExclusive(() => this.performFullGeneration());
|
|
1347
|
+
}
|
|
1348
|
+
/**
|
|
1349
|
+
* Performs an incremental sitemap update based on changes recorded since a specific time.
|
|
1350
|
+
*
|
|
1351
|
+
* If the number of changes exceeds a certain threshold (e.g., 30% of total URLs),
|
|
1352
|
+
* a full generation is triggered instead to ensure consistency.
|
|
1353
|
+
*
|
|
1354
|
+
* @param since - Optional start date for the incremental update.
|
|
1355
|
+
*/
|
|
1356
|
+
async generateIncremental(since) {
|
|
1357
|
+
return this.mutex.runExclusive(() => this.performIncrementalGeneration(since));
|
|
1358
|
+
}
|
|
1359
|
+
/**
|
|
1360
|
+
* Internal implementation of full sitemap generation.
|
|
1361
|
+
*/
|
|
1362
|
+
async performFullGeneration() {
|
|
771
1363
|
await this.generator.run();
|
|
772
1364
|
if (this.options.autoTrack) {
|
|
773
1365
|
const { providers } = this.options;
|
|
@@ -786,50 +1378,149 @@ var IncrementalGenerator = class {
|
|
|
786
1378
|
}
|
|
787
1379
|
}
|
|
788
1380
|
/**
|
|
789
|
-
*
|
|
1381
|
+
* Internal implementation of incremental sitemap generation.
|
|
790
1382
|
*/
|
|
791
|
-
async
|
|
1383
|
+
async performIncrementalGeneration(since) {
|
|
792
1384
|
const changes = await this.changeTracker.getChanges(since);
|
|
793
1385
|
if (changes.length === 0) {
|
|
794
1386
|
return;
|
|
795
1387
|
}
|
|
796
|
-
const
|
|
797
|
-
|
|
798
|
-
|
|
1388
|
+
const manifest = await this.loadManifest();
|
|
1389
|
+
if (!manifest) {
|
|
1390
|
+
await this.performFullGeneration();
|
|
1391
|
+
return;
|
|
1392
|
+
}
|
|
1393
|
+
const totalCount = manifest.shards.reduce((acc, s) => acc + s.count, 0);
|
|
1394
|
+
const changeRatio = totalCount > 0 ? changes.length / totalCount : 1;
|
|
1395
|
+
if (changeRatio > 0.3) {
|
|
1396
|
+
await this.performFullGeneration();
|
|
1397
|
+
return;
|
|
1398
|
+
}
|
|
1399
|
+
const affectedShards = this.getAffectedShards(manifest, changes);
|
|
1400
|
+
if (affectedShards.size / manifest.shards.length > 0.5) {
|
|
1401
|
+
await this.performFullGeneration();
|
|
1402
|
+
return;
|
|
1403
|
+
}
|
|
1404
|
+
await this.updateShards(manifest, affectedShards);
|
|
799
1405
|
}
|
|
800
1406
|
/**
|
|
801
|
-
*
|
|
1407
|
+
* Normalizes a URL to an absolute URL using the base URL.
|
|
802
1408
|
*/
|
|
803
|
-
|
|
804
|
-
|
|
1409
|
+
normalizeUrl(url) {
|
|
1410
|
+
if (url.startsWith("http")) {
|
|
1411
|
+
return url;
|
|
1412
|
+
}
|
|
1413
|
+
const { baseUrl } = this.options;
|
|
1414
|
+
const normalizedBase = baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
|
|
1415
|
+
const normalizedPath = url.startsWith("/") ? url : `/${url}`;
|
|
1416
|
+
return normalizedBase + normalizedPath;
|
|
805
1417
|
}
|
|
806
1418
|
/**
|
|
807
|
-
*
|
|
1419
|
+
* Loads the sitemap shard manifest from storage.
|
|
808
1420
|
*/
|
|
809
|
-
async
|
|
810
|
-
|
|
1421
|
+
async loadManifest() {
|
|
1422
|
+
const filename = this.options.filename?.replace(/\.xml$/, "-manifest.json") || "sitemap-manifest.json";
|
|
1423
|
+
const content = await this.options.storage.read(filename);
|
|
1424
|
+
if (!content) {
|
|
1425
|
+
return null;
|
|
1426
|
+
}
|
|
1427
|
+
try {
|
|
1428
|
+
return JSON.parse(content);
|
|
1429
|
+
} catch {
|
|
1430
|
+
return null;
|
|
1431
|
+
}
|
|
811
1432
|
}
|
|
812
1433
|
/**
|
|
813
|
-
*
|
|
1434
|
+
* Identifies which shards are affected by the given set of changes.
|
|
814
1435
|
*/
|
|
815
|
-
|
|
816
|
-
const
|
|
817
|
-
const
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
1436
|
+
getAffectedShards(manifest, changes) {
|
|
1437
|
+
const affected = /* @__PURE__ */ new Map();
|
|
1438
|
+
for (const change of changes) {
|
|
1439
|
+
const normalizedUrl = this.normalizeUrl(change.url);
|
|
1440
|
+
let shard = manifest.shards.find((s) => {
|
|
1441
|
+
return normalizedUrl >= s.from && normalizedUrl <= s.to;
|
|
1442
|
+
});
|
|
1443
|
+
if (!shard) {
|
|
1444
|
+
shard = manifest.shards.find((s) => normalizedUrl <= s.to);
|
|
1445
|
+
if (!shard) {
|
|
1446
|
+
shard = manifest.shards[manifest.shards.length - 1];
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
1449
|
+
if (shard) {
|
|
1450
|
+
const shardChanges = affected.get(shard.filename) || [];
|
|
1451
|
+
shardChanges.push(change);
|
|
1452
|
+
affected.set(shard.filename, shardChanges);
|
|
1453
|
+
}
|
|
822
1454
|
}
|
|
823
|
-
return
|
|
1455
|
+
return affected;
|
|
824
1456
|
}
|
|
825
1457
|
/**
|
|
826
|
-
*
|
|
1458
|
+
* Updates the affected shards in storage.
|
|
827
1459
|
*/
|
|
828
|
-
async
|
|
829
|
-
|
|
1460
|
+
async updateShards(manifest, affectedShards) {
|
|
1461
|
+
for (const [filename, shardChanges] of affectedShards) {
|
|
1462
|
+
const entries = [];
|
|
1463
|
+
const stream = await this.options.storage.readStream?.(filename);
|
|
1464
|
+
if (stream) {
|
|
1465
|
+
for await (const entry of SitemapParser.parseStream(stream)) {
|
|
1466
|
+
entries.push(entry);
|
|
1467
|
+
}
|
|
1468
|
+
} else {
|
|
1469
|
+
const xml = await this.options.storage.read(filename);
|
|
1470
|
+
if (!xml) {
|
|
1471
|
+
continue;
|
|
1472
|
+
}
|
|
1473
|
+
entries.push(...SitemapParser.parse(xml));
|
|
1474
|
+
}
|
|
1475
|
+
const updatedEntries = this.applyChanges(entries, shardChanges);
|
|
1476
|
+
const outStream = new SitemapStream({
|
|
1477
|
+
baseUrl: this.options.baseUrl,
|
|
1478
|
+
pretty: this.options.pretty
|
|
1479
|
+
});
|
|
1480
|
+
outStream.addAll(updatedEntries);
|
|
1481
|
+
const newXml = outStream.toXML();
|
|
1482
|
+
await this.options.storage.write(filename, newXml);
|
|
1483
|
+
const shardInfo = manifest.shards.find((s) => s.filename === filename);
|
|
1484
|
+
if (shardInfo) {
|
|
1485
|
+
shardInfo.count = updatedEntries.length;
|
|
1486
|
+
shardInfo.lastmod = /* @__PURE__ */ new Date();
|
|
1487
|
+
shardInfo.from = this.normalizeUrl(updatedEntries[0].url);
|
|
1488
|
+
shardInfo.to = this.normalizeUrl(updatedEntries[updatedEntries.length - 1].url);
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
const manifestFilename = this.options.filename?.replace(/\.xml$/, "-manifest.json") || "sitemap-manifest.json";
|
|
1492
|
+
await this.options.storage.write(
|
|
1493
|
+
manifestFilename,
|
|
1494
|
+
JSON.stringify(manifest, null, this.options.pretty ? 2 : 0)
|
|
1495
|
+
);
|
|
830
1496
|
}
|
|
831
1497
|
/**
|
|
832
|
-
*
|
|
1498
|
+
* Applies changes to a set of sitemap entries and returns the updated, sorted list.
|
|
1499
|
+
*/
|
|
1500
|
+
applyChanges(entries, changes) {
|
|
1501
|
+
const entryMap = /* @__PURE__ */ new Map();
|
|
1502
|
+
for (const entry of entries) {
|
|
1503
|
+
entryMap.set(this.normalizeUrl(entry.url), entry);
|
|
1504
|
+
}
|
|
1505
|
+
for (const change of changes) {
|
|
1506
|
+
const normalizedUrl = this.normalizeUrl(change.url);
|
|
1507
|
+
if (change.type === "add" || change.type === "update") {
|
|
1508
|
+
if (change.entry) {
|
|
1509
|
+
entryMap.set(normalizedUrl, {
|
|
1510
|
+
...change.entry,
|
|
1511
|
+
url: normalizedUrl
|
|
1512
|
+
});
|
|
1513
|
+
}
|
|
1514
|
+
} else if (change.type === "remove") {
|
|
1515
|
+
entryMap.delete(normalizedUrl);
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1518
|
+
return Array.from(entryMap.values()).sort(
|
|
1519
|
+
(a, b) => this.normalizeUrl(a.url).localeCompare(this.normalizeUrl(b.url))
|
|
1520
|
+
);
|
|
1521
|
+
}
|
|
1522
|
+
/**
|
|
1523
|
+
* Helper to convert an async iterable into an array.
|
|
833
1524
|
*/
|
|
834
1525
|
async toArray(iterable) {
|
|
835
1526
|
const array = [];
|
|
@@ -851,7 +1542,10 @@ var ProgressTracker = class {
|
|
|
851
1542
|
this.updateInterval = options.updateInterval || 1e3;
|
|
852
1543
|
}
|
|
853
1544
|
/**
|
|
854
|
-
*
|
|
1545
|
+
* Initializes progress tracking for a new job.
|
|
1546
|
+
*
|
|
1547
|
+
* @param jobId - Unique identifier for the generation job.
|
|
1548
|
+
* @param total - Total number of entries to be processed.
|
|
855
1549
|
*/
|
|
856
1550
|
async init(jobId, total) {
|
|
857
1551
|
this.currentProgress = {
|
|
@@ -865,7 +1559,13 @@ var ProgressTracker = class {
|
|
|
865
1559
|
await this.storage.set(jobId, this.currentProgress);
|
|
866
1560
|
}
|
|
867
1561
|
/**
|
|
868
|
-
*
|
|
1562
|
+
* Updates the current progress of the job.
|
|
1563
|
+
*
|
|
1564
|
+
* Updates are debounced and flushed to storage at regular intervals
|
|
1565
|
+
* specified by `updateInterval` to avoid excessive write operations.
|
|
1566
|
+
*
|
|
1567
|
+
* @param processed - Number of entries processed so far.
|
|
1568
|
+
* @param status - Optional new status for the job.
|
|
869
1569
|
*/
|
|
870
1570
|
async update(processed, status) {
|
|
871
1571
|
if (!this.currentProgress) {
|
|
@@ -887,7 +1587,7 @@ var ProgressTracker = class {
|
|
|
887
1587
|
}
|
|
888
1588
|
}
|
|
889
1589
|
/**
|
|
890
|
-
*
|
|
1590
|
+
* Marks the current job as successfully completed.
|
|
891
1591
|
*/
|
|
892
1592
|
async complete() {
|
|
893
1593
|
if (!this.currentProgress) {
|
|
@@ -900,7 +1600,9 @@ var ProgressTracker = class {
|
|
|
900
1600
|
this.stop();
|
|
901
1601
|
}
|
|
902
1602
|
/**
|
|
903
|
-
*
|
|
1603
|
+
* Marks the current job as failed with an error message.
|
|
1604
|
+
*
|
|
1605
|
+
* @param error - The error message describing why the job failed.
|
|
904
1606
|
*/
|
|
905
1607
|
async fail(error) {
|
|
906
1608
|
if (!this.currentProgress) {
|
|
@@ -913,7 +1615,7 @@ var ProgressTracker = class {
|
|
|
913
1615
|
this.stop();
|
|
914
1616
|
}
|
|
915
1617
|
/**
|
|
916
|
-
*
|
|
1618
|
+
* Flushes the current progress state to the storage backend.
|
|
917
1619
|
*/
|
|
918
1620
|
async flush() {
|
|
919
1621
|
if (!this.currentProgress) {
|
|
@@ -928,7 +1630,7 @@ var ProgressTracker = class {
|
|
|
928
1630
|
});
|
|
929
1631
|
}
|
|
930
1632
|
/**
|
|
931
|
-
*
|
|
1633
|
+
* Stops the periodic update timer.
|
|
932
1634
|
*/
|
|
933
1635
|
stop() {
|
|
934
1636
|
if (this.updateTimer) {
|
|
@@ -937,7 +1639,9 @@ var ProgressTracker = class {
|
|
|
937
1639
|
}
|
|
938
1640
|
}
|
|
939
1641
|
/**
|
|
940
|
-
*
|
|
1642
|
+
* Returns a copy of the current progress state.
|
|
1643
|
+
*
|
|
1644
|
+
* @returns The current SitemapProgress object, or null if no job is active.
|
|
941
1645
|
*/
|
|
942
1646
|
getCurrentProgress() {
|
|
943
1647
|
return this.currentProgress ? { ...this.currentProgress } : null;
|
|
@@ -975,6 +1679,12 @@ var GenerateSitemapJob = class extends import_stream.Job {
|
|
|
975
1679
|
this.options = options;
|
|
976
1680
|
this.generator = new SitemapGenerator(options.generatorOptions);
|
|
977
1681
|
}
|
|
1682
|
+
/**
|
|
1683
|
+
* Main entry point for the job execution.
|
|
1684
|
+
*
|
|
1685
|
+
* Orchestrates the full lifecycle of sitemap generation, including progress
|
|
1686
|
+
* initialization, generation, shadow commit, and error handling.
|
|
1687
|
+
*/
|
|
978
1688
|
async handle() {
|
|
979
1689
|
const { progressTracker, onComplete, onError } = this.options;
|
|
980
1690
|
try {
|
|
@@ -1005,7 +1715,9 @@ var GenerateSitemapJob = class extends import_stream.Job {
|
|
|
1005
1715
|
}
|
|
1006
1716
|
}
|
|
1007
1717
|
/**
|
|
1008
|
-
*
|
|
1718
|
+
* Calculates the total number of URL entries from all providers.
|
|
1719
|
+
*
|
|
1720
|
+
* @returns A promise resolving to the total entry count.
|
|
1009
1721
|
*/
|
|
1010
1722
|
async calculateTotal() {
|
|
1011
1723
|
let total = 0;
|
|
@@ -1020,30 +1732,580 @@ var GenerateSitemapJob = class extends import_stream.Job {
|
|
|
1020
1732
|
}
|
|
1021
1733
|
}
|
|
1022
1734
|
}
|
|
1023
|
-
return total;
|
|
1735
|
+
return total;
|
|
1736
|
+
}
|
|
1737
|
+
/**
|
|
1738
|
+
* Performs sitemap generation while reporting progress to the tracker and callback.
|
|
1739
|
+
*/
|
|
1740
|
+
async generateWithProgress() {
|
|
1741
|
+
const { progressTracker, onProgress } = this.options;
|
|
1742
|
+
await this.generator.run();
|
|
1743
|
+
this.processedEntries = this.totalEntries;
|
|
1744
|
+
if (progressTracker) {
|
|
1745
|
+
await progressTracker.update(this.processedEntries, "processing");
|
|
1746
|
+
}
|
|
1747
|
+
if (onProgress) {
|
|
1748
|
+
onProgress({
|
|
1749
|
+
processed: this.processedEntries,
|
|
1750
|
+
total: this.totalEntries,
|
|
1751
|
+
percentage: this.totalEntries > 0 ? Math.round(this.processedEntries / this.totalEntries * 100) : 100
|
|
1752
|
+
});
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1755
|
+
};
|
|
1756
|
+
|
|
1757
|
+
// src/locks/MemoryLock.ts
|
|
1758
|
+
var MemoryLock = class {
|
|
1759
|
+
/**
|
|
1760
|
+
* Internal map storing resource identifiers to their lock expiration timestamps.
|
|
1761
|
+
*
|
|
1762
|
+
* Keys represent unique resource identifiers (e.g., 'sitemap-generation').
|
|
1763
|
+
* Values are Unix timestamps in milliseconds representing when the lock expires.
|
|
1764
|
+
* Expired locks are automatically cleaned up during `acquire()` and `isLocked()` calls.
|
|
1765
|
+
*/
|
|
1766
|
+
locks = /* @__PURE__ */ new Map();
|
|
1767
|
+
/**
|
|
1768
|
+
* Attempts to acquire an exclusive lock on the specified resource.
|
|
1769
|
+
*
|
|
1770
|
+
* Uses a test-and-set approach: checks if the lock exists and is valid, then atomically
|
|
1771
|
+
* sets the lock if available. Expired locks are treated as available and automatically
|
|
1772
|
+
* replaced during acquisition.
|
|
1773
|
+
*
|
|
1774
|
+
* **Behavior:**
|
|
1775
|
+
* - Returns `true` if lock was successfully acquired
|
|
1776
|
+
* - Returns `false` if resource is already locked by another caller
|
|
1777
|
+
* - Automatically replaces expired locks (acts as self-healing mechanism)
|
|
1778
|
+
* - Lock automatically expires after TTL milliseconds
|
|
1779
|
+
*
|
|
1780
|
+
* **Race condition handling:**
|
|
1781
|
+
* Safe within a single process due to JavaScript's single-threaded event loop.
|
|
1782
|
+
* NOT safe across multiple processes or instances (use RedisLock for that).
|
|
1783
|
+
*
|
|
1784
|
+
* @param resource - Unique identifier for the resource to lock (e.g., 'sitemap-generation', 'blog-index').
|
|
1785
|
+
* Should be consistent across all callers attempting to lock the same resource.
|
|
1786
|
+
* @param ttl - Time-to-live in milliseconds. Lock automatically expires after this duration.
|
|
1787
|
+
* Recommended: 2-5x the expected operation duration to prevent premature expiration.
|
|
1788
|
+
* @returns Promise resolving to `true` if lock acquired, `false` if already locked.
|
|
1789
|
+
*
|
|
1790
|
+
* @example Preventing concurrent sitemap generation
|
|
1791
|
+
* ```typescript
|
|
1792
|
+
* const lock = new MemoryLock()
|
|
1793
|
+
* const acquired = await lock.acquire('sitemap-generation', 60000)
|
|
1794
|
+
*
|
|
1795
|
+
* if (!acquired) {
|
|
1796
|
+
* console.log('Another process is already generating the sitemap')
|
|
1797
|
+
* return new Response('Generation in progress', { status: 503 })
|
|
1798
|
+
* }
|
|
1799
|
+
*
|
|
1800
|
+
* try {
|
|
1801
|
+
* await generateSitemap()
|
|
1802
|
+
* } finally {
|
|
1803
|
+
* await lock.release('sitemap-generation')
|
|
1804
|
+
* }
|
|
1805
|
+
* ```
|
|
1806
|
+
*
|
|
1807
|
+
* @example Setting appropriate TTL
|
|
1808
|
+
* ```typescript
|
|
1809
|
+
* // For fast operations (< 1 second), use short TTL
|
|
1810
|
+
* await lock.acquire('cache-refresh', 5000)
|
|
1811
|
+
*
|
|
1812
|
+
* // For slow operations (minutes), use longer TTL
|
|
1813
|
+
* await lock.acquire('full-reindex', 300000) // 5 minutes
|
|
1814
|
+
* ```
|
|
1815
|
+
*/
|
|
1816
|
+
async acquire(resource, ttl) {
|
|
1817
|
+
const now = Date.now();
|
|
1818
|
+
const expiresAt = this.locks.get(resource);
|
|
1819
|
+
if (expiresAt && expiresAt > now) {
|
|
1820
|
+
return false;
|
|
1821
|
+
}
|
|
1822
|
+
this.locks.set(resource, now + ttl);
|
|
1823
|
+
return true;
|
|
1824
|
+
}
|
|
1825
|
+
/**
|
|
1826
|
+
* Releases the lock on the specified resource, allowing others to acquire it.
|
|
1827
|
+
*
|
|
1828
|
+
* Immediately removes the lock from memory without any ownership validation.
|
|
1829
|
+
* Unlike RedisLock, this does NOT verify that the caller is the lock owner,
|
|
1830
|
+
* so callers must ensure they only release locks they acquired.
|
|
1831
|
+
*
|
|
1832
|
+
* **Best practices:**
|
|
1833
|
+
* - Always call `release()` in a `finally` block to prevent lock leakage
|
|
1834
|
+
* - Only release locks you successfully acquired
|
|
1835
|
+
* - If operation fails, still release the lock to allow retry
|
|
1836
|
+
*
|
|
1837
|
+
* **Idempotency:**
|
|
1838
|
+
* Safe to call multiple times on the same resource. Releasing a non-existent
|
|
1839
|
+
* lock is a no-op.
|
|
1840
|
+
*
|
|
1841
|
+
* @param resource - The resource identifier to unlock. Must match the identifier
|
|
1842
|
+
* used in the corresponding `acquire()` call.
|
|
1843
|
+
*
|
|
1844
|
+
* @example Proper release pattern with try-finally
|
|
1845
|
+
* ```typescript
|
|
1846
|
+
* const acquired = await lock.acquire('sitemap-generation', 60000)
|
|
1847
|
+
* if (!acquired) return
|
|
1848
|
+
*
|
|
1849
|
+
* try {
|
|
1850
|
+
* await generateSitemap()
|
|
1851
|
+
* } finally {
|
|
1852
|
+
* // Always release, even if operation throws
|
|
1853
|
+
* await lock.release('sitemap-generation')
|
|
1854
|
+
* }
|
|
1855
|
+
* ```
|
|
1856
|
+
*
|
|
1857
|
+
* @example Handling operation failures
|
|
1858
|
+
* ```typescript
|
|
1859
|
+
* const acquired = await lock.acquire('data-import', 120000)
|
|
1860
|
+
* if (!acquired) return
|
|
1861
|
+
*
|
|
1862
|
+
* try {
|
|
1863
|
+
* await importData()
|
|
1864
|
+
* } catch (error) {
|
|
1865
|
+
* console.error('Import failed:', error)
|
|
1866
|
+
* // Lock still released in finally block
|
|
1867
|
+
* throw error
|
|
1868
|
+
* } finally {
|
|
1869
|
+
* await lock.release('data-import')
|
|
1870
|
+
* }
|
|
1871
|
+
* ```
|
|
1872
|
+
*/
|
|
1873
|
+
async release(resource) {
|
|
1874
|
+
this.locks.delete(resource);
|
|
1875
|
+
}
|
|
1876
|
+
/**
|
|
1877
|
+
* Checks whether a resource is currently locked and has not expired.
|
|
1878
|
+
*
|
|
1879
|
+
* Performs automatic cleanup by removing expired locks during the check,
|
|
1880
|
+
* ensuring the internal map doesn't accumulate stale entries over time.
|
|
1881
|
+
*
|
|
1882
|
+
* **Use cases:**
|
|
1883
|
+
* - Pre-flight checks before attempting expensive operations
|
|
1884
|
+
* - Status monitoring and health checks
|
|
1885
|
+
* - Implementing custom retry logic
|
|
1886
|
+
* - Debugging and testing
|
|
1887
|
+
*
|
|
1888
|
+
* **Side effects:**
|
|
1889
|
+
* Automatically deletes expired locks as a garbage collection mechanism.
|
|
1890
|
+
* This is intentional to prevent memory leaks from abandoned locks.
|
|
1891
|
+
*
|
|
1892
|
+
* @param resource - The resource identifier to check for lock status.
|
|
1893
|
+
* @returns Promise resolving to `true` if resource is actively locked (not expired),
|
|
1894
|
+
* `false` if unlocked or lock has expired.
|
|
1895
|
+
*
|
|
1896
|
+
* @example Pre-flight check before starting work
|
|
1897
|
+
* ```typescript
|
|
1898
|
+
* const lock = new MemoryLock()
|
|
1899
|
+
*
|
|
1900
|
+
* if (await lock.isLocked('sitemap-generation')) {
|
|
1901
|
+
* console.log('Sitemap generation already in progress')
|
|
1902
|
+
* return
|
|
1903
|
+
* }
|
|
1904
|
+
*
|
|
1905
|
+
* // Safe to proceed
|
|
1906
|
+
* await lock.acquire('sitemap-generation', 60000)
|
|
1907
|
+
* ```
|
|
1908
|
+
*
|
|
1909
|
+
* @example Health check endpoint
|
|
1910
|
+
* ```typescript
|
|
1911
|
+
* app.get('/health/locks', async (c) => {
|
|
1912
|
+
* const isGenerating = await lock.isLocked('sitemap-generation')
|
|
1913
|
+
* const isIndexing = await lock.isLocked('search-indexing')
|
|
1914
|
+
*
|
|
1915
|
+
* return c.json({
|
|
1916
|
+
* sitemapGeneration: isGenerating ? 'in-progress' : 'idle',
|
|
1917
|
+
* searchIndexing: isIndexing ? 'in-progress' : 'idle'
|
|
1918
|
+
* })
|
|
1919
|
+
* })
|
|
1920
|
+
* ```
|
|
1921
|
+
*
|
|
1922
|
+
* @example Custom retry logic
|
|
1923
|
+
* ```typescript
|
|
1924
|
+
* let attempts = 0
|
|
1925
|
+
* while (attempts < 5) {
|
|
1926
|
+
* if (!await lock.isLocked('resource')) {
|
|
1927
|
+
* const acquired = await lock.acquire('resource', 10000)
|
|
1928
|
+
* if (acquired) break
|
|
1929
|
+
* }
|
|
1930
|
+
* await sleep(1000)
|
|
1931
|
+
* attempts++
|
|
1932
|
+
* }
|
|
1933
|
+
* ```
|
|
1934
|
+
*/
|
|
1935
|
+
async isLocked(resource) {
|
|
1936
|
+
const expiresAt = this.locks.get(resource);
|
|
1937
|
+
if (!expiresAt) {
|
|
1938
|
+
return false;
|
|
1939
|
+
}
|
|
1940
|
+
const now = Date.now();
|
|
1941
|
+
if (expiresAt <= now) {
|
|
1942
|
+
this.locks.delete(resource);
|
|
1943
|
+
return false;
|
|
1944
|
+
}
|
|
1945
|
+
return true;
|
|
1946
|
+
}
|
|
1947
|
+
/**
|
|
1948
|
+
* Clears all locks from memory, including both active and expired locks.
|
|
1949
|
+
*
|
|
1950
|
+
* **Use cases:**
|
|
1951
|
+
* - Test cleanup between test cases to ensure isolation
|
|
1952
|
+
* - Application shutdown to release all resources
|
|
1953
|
+
* - Manual intervention during debugging
|
|
1954
|
+
* - Resetting state after catastrophic errors
|
|
1955
|
+
*
|
|
1956
|
+
* **Warning:**
|
|
1957
|
+
* This forcibly releases ALL locks without any ownership validation.
|
|
1958
|
+
* Should not be called during normal operation in production environments.
|
|
1959
|
+
*
|
|
1960
|
+
* @example Test cleanup with beforeEach hook
|
|
1961
|
+
* ```typescript
|
|
1962
|
+
* import { describe, beforeEach, test } from 'vitest'
|
|
1963
|
+
*
|
|
1964
|
+
* const lock = new MemoryLock()
|
|
1965
|
+
*
|
|
1966
|
+
* beforeEach(async () => {
|
|
1967
|
+
* await lock.clear() // Ensure clean state for each test
|
|
1968
|
+
* })
|
|
1969
|
+
*
|
|
1970
|
+
* test('lock acquisition', async () => {
|
|
1971
|
+
* const acquired = await lock.acquire('test-resource', 5000)
|
|
1972
|
+
* expect(acquired).toBe(true)
|
|
1973
|
+
* })
|
|
1974
|
+
* ```
|
|
1975
|
+
*
|
|
1976
|
+
* @example Graceful shutdown handler
|
|
1977
|
+
* ```typescript
|
|
1978
|
+
* process.on('SIGTERM', async () => {
|
|
1979
|
+
* console.log('Shutting down, releasing all locks...')
|
|
1980
|
+
* await lock.clear()
|
|
1981
|
+
* process.exit(0)
|
|
1982
|
+
* })
|
|
1983
|
+
* ```
|
|
1984
|
+
*/
|
|
1985
|
+
async clear() {
|
|
1986
|
+
this.locks.clear();
|
|
1987
|
+
}
|
|
1988
|
+
/**
|
|
1989
|
+
* Returns the number of lock entries currently stored in memory.
|
|
1990
|
+
*
|
|
1991
|
+
* **Important:** This includes BOTH active and expired locks. Expired locks
|
|
1992
|
+
* are only cleaned up during `acquire()` or `isLocked()` calls, so this count
|
|
1993
|
+
* may include stale entries.
|
|
1994
|
+
*
|
|
1995
|
+
* **Use cases:**
|
|
1996
|
+
* - Monitoring memory usage and lock accumulation
|
|
1997
|
+
* - Debugging lock leakage issues
|
|
1998
|
+
* - Testing lock lifecycle behavior
|
|
1999
|
+
* - Detecting abnormal lock retention patterns
|
|
2000
|
+
*
|
|
2001
|
+
* **Not suitable for:**
|
|
2002
|
+
* - Determining number of ACTIVE locks (use `isLocked()` on each resource)
|
|
2003
|
+
* - Production health checks (includes expired locks)
|
|
2004
|
+
*
|
|
2005
|
+
* @returns The total number of lock entries in the internal Map, including expired ones.
|
|
2006
|
+
*
|
|
2007
|
+
* @example Monitoring lock accumulation
|
|
2008
|
+
* ```typescript
|
|
2009
|
+
* const lock = new MemoryLock()
|
|
2010
|
+
*
|
|
2011
|
+
* setInterval(() => {
|
|
2012
|
+
* const count = lock.size()
|
|
2013
|
+
* if (count > 100) {
|
|
2014
|
+
* console.warn(`High lock count detected: ${count}`)
|
|
2015
|
+
* // May indicate lock leakage or missing release() calls
|
|
2016
|
+
* }
|
|
2017
|
+
* }, 60000)
|
|
2018
|
+
* ```
|
|
2019
|
+
*
|
|
2020
|
+
* @example Testing lock cleanup behavior
|
|
2021
|
+
* ```typescript
|
|
2022
|
+
* import { test, expect } from 'vitest'
|
|
2023
|
+
*
|
|
2024
|
+
* test('expired locks are cleaned up', async () => {
|
|
2025
|
+
* const lock = new MemoryLock()
|
|
2026
|
+
*
|
|
2027
|
+
* await lock.acquire('resource', 10)
|
|
2028
|
+
* expect(lock.size()).toBe(1)
|
|
2029
|
+
*
|
|
2030
|
+
* await sleep(20) // Wait for expiration
|
|
2031
|
+
* expect(lock.size()).toBe(1) // Still in map (not cleaned yet)
|
|
2032
|
+
*
|
|
2033
|
+
* await lock.isLocked('resource') // Triggers cleanup
|
|
2034
|
+
* expect(lock.size()).toBe(0) // Now removed
|
|
2035
|
+
* })
|
|
2036
|
+
* ```
|
|
2037
|
+
*/
|
|
2038
|
+
size() {
|
|
2039
|
+
return this.locks.size;
|
|
2040
|
+
}
|
|
2041
|
+
};
|
|
2042
|
+
|
|
2043
|
+
// src/locks/RedisLock.ts
|
|
2044
|
+
var import_node_crypto = require("crypto");
|
|
2045
|
+
var RedisLock = class {
|
|
2046
|
+
/**
|
|
2047
|
+
* Constructs a new RedisLock instance with the specified configuration.
|
|
2048
|
+
*
|
|
2049
|
+
* @param options - Configuration including Redis client and retry parameters.
|
|
2050
|
+
*
|
|
2051
|
+
* @example With custom retry strategy
|
|
2052
|
+
* ```typescript
|
|
2053
|
+
* const lock = new RedisLock({
|
|
2054
|
+
* client: redisClient,
|
|
2055
|
+
* keyPrefix: 'app:locks:',
|
|
2056
|
+
* retryCount: 10, // More retries for high-contention scenarios
|
|
2057
|
+
* retryDelay: 50 // Shorter delay for low-latency requirements
|
|
2058
|
+
* })
|
|
2059
|
+
* ```
|
|
2060
|
+
*/
|
|
2061
|
+
constructor(options) {
|
|
2062
|
+
this.options = options;
|
|
2063
|
+
this.keyPrefix = options.keyPrefix || "sitemap:lock:";
|
|
2064
|
+
this.retryCount = options.retryCount ?? 0;
|
|
2065
|
+
this.retryDelay = options.retryDelay ?? 100;
|
|
2066
|
+
}
|
|
2067
|
+
/**
|
|
2068
|
+
* Unique identifier for this lock instance.
|
|
2069
|
+
*
|
|
2070
|
+
* Generated once during construction and used for all locks acquired by this instance.
|
|
2071
|
+
* Enables ownership validation: only the instance that acquired the lock can release it.
|
|
2072
|
+
*
|
|
2073
|
+
* **Security consideration:**
|
|
2074
|
+
* UUIDs are sufficiently random to prevent lock hijacking across instances.
|
|
2075
|
+
* However, they are stored in plain text in Redis (not encrypted).
|
|
2076
|
+
*/
|
|
2077
|
+
lockId = (0, import_node_crypto.randomUUID)();
|
|
2078
|
+
/**
|
|
2079
|
+
* Redis key prefix for all locks acquired through this instance.
|
|
2080
|
+
*
|
|
2081
|
+
* Combined with resource name to form full Redis key (e.g., 'sitemap:lock:generation').
|
|
2082
|
+
* Allows namespace isolation and easier debugging in Redis CLI.
|
|
2083
|
+
*/
|
|
2084
|
+
keyPrefix;
|
|
2085
|
+
/**
|
|
2086
|
+
* Maximum number of retry attempts when lock is held by another instance.
|
|
2087
|
+
*
|
|
2088
|
+
* Set to 0 for fail-fast behavior. Higher values increase acquisition success
|
|
2089
|
+
* rate but also increase latency under contention.
|
|
2090
|
+
*/
|
|
2091
|
+
retryCount;
|
|
2092
|
+
/**
|
|
2093
|
+
* Delay in milliseconds between consecutive retry attempts.
|
|
2094
|
+
*
|
|
2095
|
+
* Should be tuned based on expected lock hold time. Typical values: 50-500ms.
|
|
2096
|
+
*/
|
|
2097
|
+
retryDelay;
|
|
2098
|
+
/**
|
|
2099
|
+
* Attempts to acquire a distributed lock using Redis SET NX EX command.
|
|
2100
|
+
*
|
|
2101
|
+
* Uses atomic Redis operations to ensure only one instance across your entire
|
|
2102
|
+
* infrastructure can hold the lock at any given time. Implements retry logic
|
|
2103
|
+
* with exponential backoff for handling transient contention.
|
|
2104
|
+
*
|
|
2105
|
+
* **Algorithm:**
|
|
2106
|
+
* 1. Convert TTL from milliseconds to seconds (Redis requirement)
|
|
2107
|
+
* 2. Attempt Redis SET key lockId EX ttl NX (atomic operation)
|
|
2108
|
+
* 3. If successful (returns 'OK'), lock acquired
|
|
2109
|
+
* 4. If failed (returns null), retry up to retryCount times with retryDelay
|
|
2110
|
+
* 5. Return true if acquired, false if all attempts exhausted
|
|
2111
|
+
*
|
|
2112
|
+
* **Atomicity guarantee:**
|
|
2113
|
+
* The combination of NX (set if Not eXists) and EX (set EXpiration) in a single
|
|
2114
|
+
* Redis command ensures no race conditions. Either the lock is acquired or it isn't.
|
|
2115
|
+
*
|
|
2116
|
+
* **Error handling:**
|
|
2117
|
+
* Redis connection errors are caught, logged to console, and treated as acquisition
|
|
2118
|
+
* failure (returns false). This fail-safe behavior prevents exceptions from bubbling
|
|
2119
|
+
* up to application code.
|
|
2120
|
+
*
|
|
2121
|
+
* **Performance:**
|
|
2122
|
+
* - Single instance: O(1) Redis operation
|
|
2123
|
+
* - With retry: O(retryCount) worst case
|
|
2124
|
+
* - Network latency: ~1-5ms per attempt (depends on Redis location)
|
|
2125
|
+
*
|
|
2126
|
+
* @param resource - Unique identifier for the resource to lock.
|
|
2127
|
+
* Combined with keyPrefix to form Redis key.
|
|
2128
|
+
* @param ttl - Time-to-live in milliseconds. Lock auto-expires after this duration.
|
|
2129
|
+
* Recommended: 2-5x expected operation time to handle slowdowns.
|
|
2130
|
+
* Minimum: 1000ms (1 second) for practical use.
|
|
2131
|
+
* @returns Promise resolving to `true` if lock acquired, `false` if held by another instance
|
|
2132
|
+
* or Redis connection failed.
|
|
2133
|
+
*
|
|
2134
|
+
* @example Basic acquisition in distributed environment
|
|
2135
|
+
* ```typescript
|
|
2136
|
+
* const lock = new RedisLock({ client: redisClient })
|
|
2137
|
+
* const acquired = await lock.acquire('sitemap-generation', 60000)
|
|
2138
|
+
*
|
|
2139
|
+
* if (!acquired) {
|
|
2140
|
+
* // Another instance (e.g., different Kubernetes pod) holds the lock
|
|
2141
|
+
* console.log('Sitemap generation in progress on another instance')
|
|
2142
|
+
* return new Response('Service busy', {
|
|
2143
|
+
* status: 503,
|
|
2144
|
+
* headers: { 'Retry-After': '30' }
|
|
2145
|
+
* })
|
|
2146
|
+
* }
|
|
2147
|
+
*
|
|
2148
|
+
* try {
|
|
2149
|
+
* await generateSitemap()
|
|
2150
|
+
* } finally {
|
|
2151
|
+
* await lock.release('sitemap-generation')
|
|
2152
|
+
* }
|
|
2153
|
+
* ```
|
|
2154
|
+
*
|
|
2155
|
+
* @example With retry logic for transient contention
|
|
2156
|
+
* ```typescript
|
|
2157
|
+
* const lock = new RedisLock({
|
|
2158
|
+
* client: redisClient,
|
|
2159
|
+
* retryCount: 5,
|
|
2160
|
+
* retryDelay: 200
|
|
2161
|
+
* })
|
|
2162
|
+
*
|
|
2163
|
+
* // Will retry 5 times with 200ms delay between attempts
|
|
2164
|
+
* const acquired = await lock.acquire('data-import', 120000)
|
|
2165
|
+
* ```
|
|
2166
|
+
*
|
|
2167
|
+
* @example Setting appropriate TTL
|
|
2168
|
+
* ```typescript
|
|
2169
|
+
* // Fast operation: short TTL
|
|
2170
|
+
* await lock.acquire('cache-rebuild', 10000) // 10 seconds
|
|
2171
|
+
*
|
|
2172
|
+
* // Slow operation: longer TTL with buffer
|
|
2173
|
+
* await lock.acquire('full-sitemap', 300000) // 5 minutes
|
|
2174
|
+
*
|
|
2175
|
+
* // Very slow operation: generous TTL
|
|
2176
|
+
* await lock.acquire('data-migration', 1800000) // 30 minutes
|
|
2177
|
+
* ```
|
|
2178
|
+
*/
|
|
2179
|
+
async acquire(resource, ttl) {
|
|
2180
|
+
const key = this.keyPrefix + resource;
|
|
2181
|
+
const ttlSeconds = Math.ceil(ttl / 1e3);
|
|
2182
|
+
let attempts = 0;
|
|
2183
|
+
while (attempts <= this.retryCount) {
|
|
2184
|
+
try {
|
|
2185
|
+
const result = await this.options.client.set(key, this.lockId, "EX", ttlSeconds, "NX");
|
|
2186
|
+
if (result === "OK") {
|
|
2187
|
+
return true;
|
|
2188
|
+
}
|
|
2189
|
+
} catch (error) {
|
|
2190
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
2191
|
+
console.error(`[RedisLock] Failed to acquire lock for ${resource}:`, err.message);
|
|
2192
|
+
}
|
|
2193
|
+
attempts++;
|
|
2194
|
+
if (attempts <= this.retryCount) {
|
|
2195
|
+
await this.sleep(this.retryDelay);
|
|
2196
|
+
}
|
|
2197
|
+
}
|
|
2198
|
+
return false;
|
|
1024
2199
|
}
|
|
1025
2200
|
/**
|
|
1026
|
-
*
|
|
2201
|
+
* Releases the distributed lock using Lua script for atomic ownership validation.
|
|
2202
|
+
*
|
|
2203
|
+
* Uses a Lua script to atomically check if the current instance owns the lock
|
|
2204
|
+
* (by comparing lockId) and delete it if so. This prevents accidentally releasing
|
|
2205
|
+
* locks held by other instances, which could cause data corruption in distributed systems.
|
|
2206
|
+
*
|
|
2207
|
+
* **Lua script logic:**
|
|
2208
|
+
* ```lua
|
|
2209
|
+
* if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
2210
|
+
* return redis.call("del", KEYS[1]) -- Delete only if owner matches
|
|
2211
|
+
* else
|
|
2212
|
+
* return 0 -- Not owner or lock expired, do nothing
|
|
2213
|
+
* end
|
|
2214
|
+
* ```
|
|
2215
|
+
*
|
|
2216
|
+
* **Why Lua scripts?**
|
|
2217
|
+
* - Atomicity: GET + comparison + DEL execute as one atomic operation
|
|
2218
|
+
* - Prevents race conditions: Lock cannot change between check and delete
|
|
2219
|
+
* - Server-side execution: No network round trips between steps
|
|
2220
|
+
*
|
|
2221
|
+
* **Error handling:**
|
|
2222
|
+
* Errors during release (e.g., Redis connection loss) are logged but do NOT throw.
|
|
2223
|
+
* This is intentional: if instance crashed or TTL expired, lock is already released.
|
|
2224
|
+
* Silent failure here prevents cascading errors in finally blocks.
|
|
2225
|
+
*
|
|
2226
|
+
* **Idempotency:**
|
|
2227
|
+
* Safe to call multiple times. Releasing an already-released or expired lock is a no-op.
|
|
2228
|
+
*
|
|
2229
|
+
* @param resource - The resource identifier to unlock. Must match the identifier
|
|
2230
|
+
* used in the corresponding `acquire()` call.
|
|
2231
|
+
*
|
|
2232
|
+
* @example Proper release pattern with try-finally
|
|
2233
|
+
* ```typescript
|
|
2234
|
+
* const acquired = await lock.acquire('sitemap-generation', 60000)
|
|
2235
|
+
* if (!acquired) return
|
|
2236
|
+
*
|
|
2237
|
+
* try {
|
|
2238
|
+
* await generateSitemap()
|
|
2239
|
+
* } finally {
|
|
2240
|
+
* // Always release, even if operation throws
|
|
2241
|
+
* await lock.release('sitemap-generation')
|
|
2242
|
+
* }
|
|
2243
|
+
* ```
|
|
2244
|
+
*
|
|
2245
|
+
* @example Handling operation failures
|
|
2246
|
+
* ```typescript
|
|
2247
|
+
* const acquired = await lock.acquire('data-processing', 120000)
|
|
2248
|
+
* if (!acquired) {
|
|
2249
|
+
* throw new Error('Could not acquire lock')
|
|
2250
|
+
* }
|
|
2251
|
+
*
|
|
2252
|
+
* try {
|
|
2253
|
+
* await processData()
|
|
2254
|
+
* } catch (error) {
|
|
2255
|
+
* console.error('Processing failed:', error)
|
|
2256
|
+
* // Lock still released in finally block
|
|
2257
|
+
* throw error
|
|
2258
|
+
* } finally {
|
|
2259
|
+
* await lock.release('data-processing')
|
|
2260
|
+
* }
|
|
2261
|
+
* ```
|
|
2262
|
+
*
|
|
2263
|
+
* @example Why ownership validation matters
|
|
2264
|
+
* ```typescript
|
|
2265
|
+
* // Instance A acquires lock with 10-second TTL
|
|
2266
|
+
* const lockA = new RedisLock({ client: redisClientA })
|
|
2267
|
+
* await lockA.acquire('task', 10000)
|
|
2268
|
+
*
|
|
2269
|
+
* // ... 11 seconds pass, lock auto-expires ...
|
|
2270
|
+
*
|
|
2271
|
+
* // Instance B acquires the now-expired lock
|
|
2272
|
+
* const lockB = new RedisLock({ client: redisClientB })
|
|
2273
|
+
* await lockB.acquire('task', 10000)
|
|
2274
|
+
*
|
|
2275
|
+
* // Instance A tries to release (after slowdown/GC pause)
|
|
2276
|
+
* await lockA.release('task')
|
|
2277
|
+
* // ✅ Lua script detects lockId mismatch, does NOT delete B's lock
|
|
2278
|
+
* // ❌ Without Lua: Would delete B's lock, causing data corruption
|
|
2279
|
+
* ```
|
|
1027
2280
|
*/
|
|
1028
|
-
async
|
|
1029
|
-
const
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
2281
|
+
async release(resource) {
|
|
2282
|
+
const key = this.keyPrefix + resource;
|
|
2283
|
+
try {
|
|
2284
|
+
const script = `
|
|
2285
|
+
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
2286
|
+
return redis.call("del", KEYS[1])
|
|
2287
|
+
else
|
|
2288
|
+
return 0
|
|
2289
|
+
end
|
|
2290
|
+
`;
|
|
2291
|
+
await this.options.client.eval(script, 1, key, this.lockId);
|
|
2292
|
+
} catch (error) {
|
|
2293
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
2294
|
+
console.error(`[RedisLock] Failed to release lock for ${resource}:`, err.message);
|
|
1041
2295
|
}
|
|
1042
2296
|
}
|
|
2297
|
+
/**
|
|
2298
|
+
* Internal utility for sleeping between retry attempts.
|
|
2299
|
+
*
|
|
2300
|
+
* @param ms - Duration to sleep in milliseconds
|
|
2301
|
+
*/
|
|
2302
|
+
sleep(ms) {
|
|
2303
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2304
|
+
}
|
|
1043
2305
|
};
|
|
1044
2306
|
|
|
1045
2307
|
// src/OrbitSitemap.ts
|
|
1046
|
-
var
|
|
2308
|
+
var import_node_crypto2 = require("crypto");
|
|
1047
2309
|
|
|
1048
2310
|
// src/redirect/RedirectHandler.ts
|
|
1049
2311
|
var RedirectHandler = class {
|
|
@@ -1052,7 +2314,10 @@ var RedirectHandler = class {
|
|
|
1052
2314
|
this.options = options;
|
|
1053
2315
|
}
|
|
1054
2316
|
/**
|
|
1055
|
-
*
|
|
2317
|
+
* Processes a list of sitemap entries and handles redirects according to the configured strategy.
|
|
2318
|
+
*
|
|
2319
|
+
* @param entries - The original list of sitemap entries.
|
|
2320
|
+
* @returns A promise resolving to the processed list of entries.
|
|
1056
2321
|
*/
|
|
1057
2322
|
async processEntries(entries) {
|
|
1058
2323
|
const { manager, strategy, followChains, maxChainLength } = this.options;
|
|
@@ -1083,7 +2348,7 @@ var RedirectHandler = class {
|
|
|
1083
2348
|
}
|
|
1084
2349
|
}
|
|
1085
2350
|
/**
|
|
1086
|
-
*
|
|
2351
|
+
* Strategy 1: Remove old URL and add the new destination URL.
|
|
1087
2352
|
*/
|
|
1088
2353
|
handleRemoveOldAddNew(entries, redirectMap) {
|
|
1089
2354
|
const processed = [];
|
|
@@ -1108,7 +2373,7 @@ var RedirectHandler = class {
|
|
|
1108
2373
|
return processed;
|
|
1109
2374
|
}
|
|
1110
2375
|
/**
|
|
1111
|
-
*
|
|
2376
|
+
* Strategy 2: Keep the original URL but mark the destination as canonical.
|
|
1112
2377
|
*/
|
|
1113
2378
|
handleKeepRelation(entries, redirectMap) {
|
|
1114
2379
|
const processed = [];
|
|
@@ -1131,7 +2396,7 @@ var RedirectHandler = class {
|
|
|
1131
2396
|
return processed;
|
|
1132
2397
|
}
|
|
1133
2398
|
/**
|
|
1134
|
-
*
|
|
2399
|
+
* Strategy 3: Silently update the URL to the destination.
|
|
1135
2400
|
*/
|
|
1136
2401
|
handleUpdateUrl(entries, redirectMap) {
|
|
1137
2402
|
return entries.map((entry) => {
|
|
@@ -1151,7 +2416,7 @@ var RedirectHandler = class {
|
|
|
1151
2416
|
});
|
|
1152
2417
|
}
|
|
1153
2418
|
/**
|
|
1154
|
-
*
|
|
2419
|
+
* Strategy 4: Include both the original and destination URLs.
|
|
1155
2420
|
*/
|
|
1156
2421
|
handleDualMark(entries, redirectMap) {
|
|
1157
2422
|
const processed = [];
|
|
@@ -1188,20 +2453,88 @@ var RedirectHandler = class {
|
|
|
1188
2453
|
};
|
|
1189
2454
|
|
|
1190
2455
|
// src/storage/MemorySitemapStorage.ts
|
|
2456
|
+
init_Compression();
|
|
1191
2457
|
var MemorySitemapStorage = class {
|
|
1192
2458
|
constructor(baseUrl) {
|
|
1193
2459
|
this.baseUrl = baseUrl;
|
|
1194
2460
|
}
|
|
1195
2461
|
files = /* @__PURE__ */ new Map();
|
|
2462
|
+
/**
|
|
2463
|
+
* Writes sitemap content to memory.
|
|
2464
|
+
*
|
|
2465
|
+
* @param filename - The name of the file to store.
|
|
2466
|
+
* @param content - The XML or JSON content.
|
|
2467
|
+
*/
|
|
1196
2468
|
async write(filename, content) {
|
|
1197
2469
|
this.files.set(filename, content);
|
|
1198
2470
|
}
|
|
2471
|
+
/**
|
|
2472
|
+
* 使用串流方式寫入 sitemap 至記憶體,可選擇性啟用 gzip 壓縮。
|
|
2473
|
+
* 記憶體儲存會收集串流為完整字串。
|
|
2474
|
+
*
|
|
2475
|
+
* @param filename - 檔案名稱
|
|
2476
|
+
* @param stream - XML 內容的 AsyncIterable
|
|
2477
|
+
* @param options - 寫入選項(如壓縮)
|
|
2478
|
+
*
|
|
2479
|
+
* @since 3.1.0
|
|
2480
|
+
*/
|
|
2481
|
+
async writeStream(filename, stream, options) {
|
|
2482
|
+
const chunks = [];
|
|
2483
|
+
for await (const chunk of stream) {
|
|
2484
|
+
chunks.push(chunk);
|
|
2485
|
+
}
|
|
2486
|
+
const content = chunks.join("");
|
|
2487
|
+
const key = options?.compress ? toGzipFilename(filename) : filename;
|
|
2488
|
+
if (options?.compress) {
|
|
2489
|
+
const compressed = await compressToBuffer(
|
|
2490
|
+
(async function* () {
|
|
2491
|
+
yield content;
|
|
2492
|
+
})()
|
|
2493
|
+
);
|
|
2494
|
+
this.files.set(key, compressed.toString("base64"));
|
|
2495
|
+
} else {
|
|
2496
|
+
this.files.set(key, content);
|
|
2497
|
+
}
|
|
2498
|
+
}
|
|
2499
|
+
/**
|
|
2500
|
+
* Reads sitemap content from memory.
|
|
2501
|
+
*
|
|
2502
|
+
* @param filename - The name of the file to read.
|
|
2503
|
+
* @returns A promise resolving to the file content as a string, or null if not found.
|
|
2504
|
+
*/
|
|
1199
2505
|
async read(filename) {
|
|
1200
2506
|
return this.files.get(filename) || null;
|
|
1201
2507
|
}
|
|
2508
|
+
/**
|
|
2509
|
+
* Returns a readable stream for a sitemap file in memory.
|
|
2510
|
+
*
|
|
2511
|
+
* @param filename - The name of the file to stream.
|
|
2512
|
+
* @returns A promise resolving to an async iterable of file chunks, or null if not found.
|
|
2513
|
+
*/
|
|
2514
|
+
async readStream(filename) {
|
|
2515
|
+
const content = this.files.get(filename);
|
|
2516
|
+
if (content === void 0) {
|
|
2517
|
+
return null;
|
|
2518
|
+
}
|
|
2519
|
+
return (async function* () {
|
|
2520
|
+
yield content;
|
|
2521
|
+
})();
|
|
2522
|
+
}
|
|
2523
|
+
/**
|
|
2524
|
+
* Checks if a sitemap file exists in memory.
|
|
2525
|
+
*
|
|
2526
|
+
* @param filename - The name of the file to check.
|
|
2527
|
+
* @returns A promise resolving to true if the file exists, false otherwise.
|
|
2528
|
+
*/
|
|
1202
2529
|
async exists(filename) {
|
|
1203
2530
|
return this.files.has(filename);
|
|
1204
2531
|
}
|
|
2532
|
+
/**
|
|
2533
|
+
* Returns the full public URL for a sitemap file.
|
|
2534
|
+
*
|
|
2535
|
+
* @param filename - The name of the sitemap file.
|
|
2536
|
+
* @returns The public URL as a string.
|
|
2537
|
+
*/
|
|
1205
2538
|
getUrl(filename) {
|
|
1206
2539
|
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
1207
2540
|
const file = filename.startsWith("/") ? filename.slice(1) : filename;
|
|
@@ -1257,7 +2590,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1257
2590
|
});
|
|
1258
2591
|
}
|
|
1259
2592
|
/**
|
|
1260
|
-
*
|
|
2593
|
+
* Installs the sitemap module into PlanetCore.
|
|
1261
2594
|
*
|
|
1262
2595
|
* @param core - The PlanetCore instance.
|
|
1263
2596
|
*/
|
|
@@ -1268,6 +2601,9 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1268
2601
|
core.logger.info("[OrbitSitemap] Static mode configured. Use generate() to build sitemaps.");
|
|
1269
2602
|
}
|
|
1270
2603
|
}
|
|
2604
|
+
/**
|
|
2605
|
+
* Internal method to set up dynamic sitemap routes.
|
|
2606
|
+
*/
|
|
1271
2607
|
installDynamic(core) {
|
|
1272
2608
|
const opts = this.options;
|
|
1273
2609
|
const storage = opts.storage ?? new MemorySitemapStorage(opts.baseUrl);
|
|
@@ -1325,7 +2661,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1325
2661
|
core.router.get(shardRoute, handler);
|
|
1326
2662
|
}
|
|
1327
2663
|
/**
|
|
1328
|
-
*
|
|
2664
|
+
* Generates the sitemap (static mode only).
|
|
1329
2665
|
*
|
|
1330
2666
|
* @returns A promise that resolves when generation is complete.
|
|
1331
2667
|
* @throws {Error} If called in dynamic mode.
|
|
@@ -1367,7 +2703,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1367
2703
|
console.log(`[OrbitSitemap] Generated sitemap in ${opts.outDir}`);
|
|
1368
2704
|
}
|
|
1369
2705
|
/**
|
|
1370
|
-
*
|
|
2706
|
+
* Generates incremental sitemap updates (static mode only).
|
|
1371
2707
|
*
|
|
1372
2708
|
* @param since - Only include items modified since this date.
|
|
1373
2709
|
* @returns A promise that resolves when incremental generation is complete.
|
|
@@ -1398,7 +2734,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1398
2734
|
console.log(`[OrbitSitemap] Generated incremental sitemap in ${opts.outDir}`);
|
|
1399
2735
|
}
|
|
1400
2736
|
/**
|
|
1401
|
-
*
|
|
2737
|
+
* Generates sitemap asynchronously in the background (static mode only).
|
|
1402
2738
|
*
|
|
1403
2739
|
* @param options - Options for the async generation job.
|
|
1404
2740
|
* @returns A promise resolving to the job ID.
|
|
@@ -1409,7 +2745,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1409
2745
|
throw new Error("generateAsync() can only be called in static mode");
|
|
1410
2746
|
}
|
|
1411
2747
|
const opts = this.options;
|
|
1412
|
-
const jobId = (0,
|
|
2748
|
+
const jobId = (0, import_node_crypto2.randomUUID)();
|
|
1413
2749
|
let storage = opts.storage;
|
|
1414
2750
|
if (!storage) {
|
|
1415
2751
|
const { DiskSitemapStorage: DiskSitemapStorage2 } = await Promise.resolve().then(() => (init_DiskSitemapStorage(), DiskSitemapStorage_exports));
|
|
@@ -1459,7 +2795,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1459
2795
|
return jobId;
|
|
1460
2796
|
}
|
|
1461
2797
|
/**
|
|
1462
|
-
*
|
|
2798
|
+
* Installs API endpoints for triggering and monitoring sitemap generation.
|
|
1463
2799
|
*
|
|
1464
2800
|
* @param core - The PlanetCore instance.
|
|
1465
2801
|
* @param basePath - The base path for the API endpoints (default: '/admin/sitemap').
|
|
@@ -1532,9 +2868,12 @@ var RouteScanner = class {
|
|
|
1532
2868
|
};
|
|
1533
2869
|
}
|
|
1534
2870
|
/**
|
|
1535
|
-
*
|
|
2871
|
+
* Scans the router and returns discovered static GET routes as sitemap entries.
|
|
1536
2872
|
*
|
|
1537
|
-
*
|
|
2873
|
+
* This method iterates through all registered routes in the Gravito router,
|
|
2874
|
+
* applying inclusion/exclusion filters and defaulting metadata for matching routes.
|
|
2875
|
+
*
|
|
2876
|
+
* @returns An array of `SitemapEntry` objects.
|
|
1538
2877
|
*/
|
|
1539
2878
|
getEntries() {
|
|
1540
2879
|
const entries = [];
|
|
@@ -1596,7 +2935,10 @@ var RedirectDetector = class {
|
|
|
1596
2935
|
this.options = options;
|
|
1597
2936
|
}
|
|
1598
2937
|
/**
|
|
1599
|
-
*
|
|
2938
|
+
* Detects redirects for a single URL using multiple strategies.
|
|
2939
|
+
*
|
|
2940
|
+
* @param url - The URL path to probe for redirects.
|
|
2941
|
+
* @returns A promise resolving to a `RedirectRule` if a redirect is found, or null.
|
|
1600
2942
|
*/
|
|
1601
2943
|
async detect(url) {
|
|
1602
2944
|
if (this.options.autoDetect?.cache) {
|
|
@@ -1628,7 +2970,10 @@ var RedirectDetector = class {
|
|
|
1628
2970
|
return null;
|
|
1629
2971
|
}
|
|
1630
2972
|
/**
|
|
1631
|
-
*
|
|
2973
|
+
* Batch detects redirects for multiple URLs with concurrency control.
|
|
2974
|
+
*
|
|
2975
|
+
* @param urls - An array of URL paths to probe.
|
|
2976
|
+
* @returns A promise resolving to a Map of URLs to their respective `RedirectRule` or null.
|
|
1632
2977
|
*/
|
|
1633
2978
|
async detectBatch(urls) {
|
|
1634
2979
|
const results = /* @__PURE__ */ new Map();
|
|
@@ -1648,7 +2993,7 @@ var RedirectDetector = class {
|
|
|
1648
2993
|
return results;
|
|
1649
2994
|
}
|
|
1650
2995
|
/**
|
|
1651
|
-
*
|
|
2996
|
+
* Detects a redirect from the configured database table.
|
|
1652
2997
|
*/
|
|
1653
2998
|
async detectFromDatabase(url) {
|
|
1654
2999
|
const { database } = this.options;
|
|
@@ -1666,14 +3011,14 @@ var RedirectDetector = class {
|
|
|
1666
3011
|
return {
|
|
1667
3012
|
from: row[columns.from],
|
|
1668
3013
|
to: row[columns.to],
|
|
1669
|
-
type: parseInt(row[columns.type], 10)
|
|
3014
|
+
type: Number.parseInt(row[columns.type], 10)
|
|
1670
3015
|
};
|
|
1671
3016
|
} catch {
|
|
1672
3017
|
return null;
|
|
1673
3018
|
}
|
|
1674
3019
|
}
|
|
1675
3020
|
/**
|
|
1676
|
-
*
|
|
3021
|
+
* Detects a redirect from a static JSON configuration file.
|
|
1677
3022
|
*/
|
|
1678
3023
|
async detectFromConfig(url) {
|
|
1679
3024
|
const { config } = this.options;
|
|
@@ -1691,7 +3036,7 @@ var RedirectDetector = class {
|
|
|
1691
3036
|
}
|
|
1692
3037
|
}
|
|
1693
3038
|
/**
|
|
1694
|
-
*
|
|
3039
|
+
* Auto-detects a redirect by sending an HTTP HEAD request.
|
|
1695
3040
|
*/
|
|
1696
3041
|
async detectAuto(url) {
|
|
1697
3042
|
const { autoDetect, baseUrl } = this.options;
|
|
@@ -1708,7 +3053,7 @@ var RedirectDetector = class {
|
|
|
1708
3053
|
method: "HEAD",
|
|
1709
3054
|
signal: controller.signal,
|
|
1710
3055
|
redirect: "manual"
|
|
1711
|
-
//
|
|
3056
|
+
// Handle redirects manually
|
|
1712
3057
|
});
|
|
1713
3058
|
clearTimeout(timeoutId);
|
|
1714
3059
|
if (response.status === 301 || response.status === 302) {
|
|
@@ -1732,7 +3077,7 @@ var RedirectDetector = class {
|
|
|
1732
3077
|
return null;
|
|
1733
3078
|
}
|
|
1734
3079
|
/**
|
|
1735
|
-
*
|
|
3080
|
+
* Caches the detection result for a URL.
|
|
1736
3081
|
*/
|
|
1737
3082
|
cacheResult(url, rule) {
|
|
1738
3083
|
if (!this.options.autoDetect?.cache) {
|
|
@@ -1753,6 +3098,11 @@ var MemoryRedirectManager = class {
|
|
|
1753
3098
|
constructor(options = {}) {
|
|
1754
3099
|
this.maxRules = options.maxRules || 1e5;
|
|
1755
3100
|
}
|
|
3101
|
+
/**
|
|
3102
|
+
* Registers a single redirect rule in memory.
|
|
3103
|
+
*
|
|
3104
|
+
* @param redirect - The redirect rule to add.
|
|
3105
|
+
*/
|
|
1756
3106
|
async register(redirect) {
|
|
1757
3107
|
this.rules.set(redirect.from, redirect);
|
|
1758
3108
|
if (this.rules.size > this.maxRules) {
|
|
@@ -1762,17 +3112,41 @@ var MemoryRedirectManager = class {
|
|
|
1762
3112
|
}
|
|
1763
3113
|
}
|
|
1764
3114
|
}
|
|
3115
|
+
/**
|
|
3116
|
+
* Registers multiple redirect rules in memory.
|
|
3117
|
+
*
|
|
3118
|
+
* @param redirects - An array of redirect rules.
|
|
3119
|
+
*/
|
|
1765
3120
|
async registerBatch(redirects) {
|
|
1766
3121
|
for (const redirect of redirects) {
|
|
1767
3122
|
await this.register(redirect);
|
|
1768
3123
|
}
|
|
1769
3124
|
}
|
|
3125
|
+
/**
|
|
3126
|
+
* Retrieves a specific redirect rule by its source path from memory.
|
|
3127
|
+
*
|
|
3128
|
+
* @param from - The source path.
|
|
3129
|
+
* @returns A promise resolving to the redirect rule, or null if not found.
|
|
3130
|
+
*/
|
|
1770
3131
|
async get(from) {
|
|
1771
3132
|
return this.rules.get(from) || null;
|
|
1772
3133
|
}
|
|
3134
|
+
/**
|
|
3135
|
+
* Retrieves all registered redirect rules from memory.
|
|
3136
|
+
*
|
|
3137
|
+
* @returns A promise resolving to an array of all redirect rules.
|
|
3138
|
+
*/
|
|
1773
3139
|
async getAll() {
|
|
1774
3140
|
return Array.from(this.rules.values());
|
|
1775
3141
|
}
|
|
3142
|
+
/**
|
|
3143
|
+
* Resolves a URL to its final destination through the redirect table.
|
|
3144
|
+
*
|
|
3145
|
+
* @param url - The URL to resolve.
|
|
3146
|
+
* @param followChains - Whether to recursively resolve chained redirects.
|
|
3147
|
+
* @param maxChainLength - Maximum depth for chain resolution.
|
|
3148
|
+
* @returns A promise resolving to the final destination URL.
|
|
3149
|
+
*/
|
|
1776
3150
|
async resolve(url, followChains = false, maxChainLength = 5) {
|
|
1777
3151
|
let current = url;
|
|
1778
3152
|
let chainLength = 0;
|
|
@@ -1805,6 +3179,11 @@ var RedisRedirectManager = class {
|
|
|
1805
3179
|
getListKey() {
|
|
1806
3180
|
return `${this.keyPrefix}list`;
|
|
1807
3181
|
}
|
|
3182
|
+
/**
|
|
3183
|
+
* Registers a single redirect rule in Redis.
|
|
3184
|
+
*
|
|
3185
|
+
* @param redirect - The redirect rule to add.
|
|
3186
|
+
*/
|
|
1808
3187
|
async register(redirect) {
|
|
1809
3188
|
const key = this.getKey(redirect.from);
|
|
1810
3189
|
const listKey = this.getListKey();
|
|
@@ -1816,11 +3195,22 @@ var RedisRedirectManager = class {
|
|
|
1816
3195
|
}
|
|
1817
3196
|
await this.client.sadd(listKey, redirect.from);
|
|
1818
3197
|
}
|
|
3198
|
+
/**
|
|
3199
|
+
* Registers multiple redirect rules in Redis.
|
|
3200
|
+
*
|
|
3201
|
+
* @param redirects - An array of redirect rules.
|
|
3202
|
+
*/
|
|
1819
3203
|
async registerBatch(redirects) {
|
|
1820
3204
|
for (const redirect of redirects) {
|
|
1821
3205
|
await this.register(redirect);
|
|
1822
3206
|
}
|
|
1823
3207
|
}
|
|
3208
|
+
/**
|
|
3209
|
+
* Retrieves a specific redirect rule by its source path from Redis.
|
|
3210
|
+
*
|
|
3211
|
+
* @param from - The source path.
|
|
3212
|
+
* @returns A promise resolving to the redirect rule, or null if not found.
|
|
3213
|
+
*/
|
|
1824
3214
|
async get(from) {
|
|
1825
3215
|
try {
|
|
1826
3216
|
const key = this.getKey(from);
|
|
@@ -1837,6 +3227,11 @@ var RedisRedirectManager = class {
|
|
|
1837
3227
|
return null;
|
|
1838
3228
|
}
|
|
1839
3229
|
}
|
|
3230
|
+
/**
|
|
3231
|
+
* Retrieves all registered redirect rules from Redis.
|
|
3232
|
+
*
|
|
3233
|
+
* @returns A promise resolving to an array of all redirect rules.
|
|
3234
|
+
*/
|
|
1840
3235
|
async getAll() {
|
|
1841
3236
|
try {
|
|
1842
3237
|
const listKey = this.getListKey();
|
|
@@ -1853,6 +3248,14 @@ var RedisRedirectManager = class {
|
|
|
1853
3248
|
return [];
|
|
1854
3249
|
}
|
|
1855
3250
|
}
|
|
3251
|
+
/**
|
|
3252
|
+
* Resolves a URL to its final destination through the Redis redirect table.
|
|
3253
|
+
*
|
|
3254
|
+
* @param url - The URL to resolve.
|
|
3255
|
+
* @param followChains - Whether to recursively resolve chained redirects.
|
|
3256
|
+
* @param maxChainLength - Maximum depth for chain resolution.
|
|
3257
|
+
* @returns A promise resolving to the final destination URL.
|
|
3258
|
+
*/
|
|
1856
3259
|
async resolve(url, followChains = false, maxChainLength = 5) {
|
|
1857
3260
|
let current = url;
|
|
1858
3261
|
let chainLength = 0;
|
|
@@ -1875,6 +3278,9 @@ var RedisRedirectManager = class {
|
|
|
1875
3278
|
init_DiskSitemapStorage();
|
|
1876
3279
|
|
|
1877
3280
|
// src/storage/GCPSitemapStorage.ts
|
|
3281
|
+
var import_node_stream3 = require("stream");
|
|
3282
|
+
var import_promises3 = require("stream/promises");
|
|
3283
|
+
init_Compression();
|
|
1878
3284
|
var GCPSitemapStorage = class {
|
|
1879
3285
|
bucket;
|
|
1880
3286
|
prefix;
|
|
@@ -1913,6 +3319,12 @@ var GCPSitemapStorage = class {
|
|
|
1913
3319
|
const cleanPrefix = this.prefix.endsWith("/") ? this.prefix.slice(0, -1) : this.prefix;
|
|
1914
3320
|
return cleanPrefix ? `${cleanPrefix}/${filename}` : filename;
|
|
1915
3321
|
}
|
|
3322
|
+
/**
|
|
3323
|
+
* Writes sitemap content to a Google Cloud Storage object.
|
|
3324
|
+
*
|
|
3325
|
+
* @param filename - The name of the file to write.
|
|
3326
|
+
* @param content - The XML or JSON content.
|
|
3327
|
+
*/
|
|
1916
3328
|
async write(filename, content) {
|
|
1917
3329
|
const { bucket } = await this.getStorageClient();
|
|
1918
3330
|
const key = this.getKey(filename);
|
|
@@ -1924,6 +3336,41 @@ var GCPSitemapStorage = class {
|
|
|
1924
3336
|
}
|
|
1925
3337
|
});
|
|
1926
3338
|
}
|
|
3339
|
+
/**
|
|
3340
|
+
* 使用串流方式寫入 sitemap 至 GCP Cloud Storage,可選擇性啟用 gzip 壓縮。
|
|
3341
|
+
*
|
|
3342
|
+
* @param filename - 檔案名稱
|
|
3343
|
+
* @param stream - XML 內容的 AsyncIterable
|
|
3344
|
+
* @param options - 寫入選項(如壓縮、content type)
|
|
3345
|
+
*
|
|
3346
|
+
* @since 3.1.0
|
|
3347
|
+
*/
|
|
3348
|
+
async writeStream(filename, stream, options) {
|
|
3349
|
+
const { bucket } = await this.getStorageClient();
|
|
3350
|
+
const key = this.getKey(options?.compress ? toGzipFilename(filename) : filename);
|
|
3351
|
+
const file = bucket.file(key);
|
|
3352
|
+
const writeStream = file.createWriteStream({
|
|
3353
|
+
resumable: false,
|
|
3354
|
+
contentType: "application/xml",
|
|
3355
|
+
metadata: {
|
|
3356
|
+
contentEncoding: options?.compress ? "gzip" : void 0,
|
|
3357
|
+
cacheControl: "public, max-age=3600"
|
|
3358
|
+
}
|
|
3359
|
+
});
|
|
3360
|
+
const readable = import_node_stream3.Readable.from(stream, { encoding: "utf-8" });
|
|
3361
|
+
if (options?.compress) {
|
|
3362
|
+
const gzip = createCompressionStream();
|
|
3363
|
+
await (0, import_promises3.pipeline)(readable, gzip, writeStream);
|
|
3364
|
+
} else {
|
|
3365
|
+
await (0, import_promises3.pipeline)(readable, writeStream);
|
|
3366
|
+
}
|
|
3367
|
+
}
|
|
3368
|
+
/**
|
|
3369
|
+
* Reads sitemap content from a Google Cloud Storage object.
|
|
3370
|
+
*
|
|
3371
|
+
* @param filename - The name of the file to read.
|
|
3372
|
+
* @returns A promise resolving to the file content as a string, or null if not found.
|
|
3373
|
+
*/
|
|
1927
3374
|
async read(filename) {
|
|
1928
3375
|
try {
|
|
1929
3376
|
const { bucket } = await this.getStorageClient();
|
|
@@ -1942,6 +3389,42 @@ var GCPSitemapStorage = class {
|
|
|
1942
3389
|
throw error;
|
|
1943
3390
|
}
|
|
1944
3391
|
}
|
|
3392
|
+
/**
|
|
3393
|
+
* Returns a readable stream for a Google Cloud Storage object.
|
|
3394
|
+
*
|
|
3395
|
+
* @param filename - The name of the file to stream.
|
|
3396
|
+
* @returns A promise resolving to an async iterable of file chunks, or null if not found.
|
|
3397
|
+
*/
|
|
3398
|
+
async readStream(filename) {
|
|
3399
|
+
try {
|
|
3400
|
+
const { bucket } = await this.getStorageClient();
|
|
3401
|
+
const key = this.getKey(filename);
|
|
3402
|
+
const file = bucket.file(key);
|
|
3403
|
+
const [exists] = await file.exists();
|
|
3404
|
+
if (!exists) {
|
|
3405
|
+
return null;
|
|
3406
|
+
}
|
|
3407
|
+
const stream = file.createReadStream();
|
|
3408
|
+
return (async function* () {
|
|
3409
|
+
const decoder = new TextDecoder();
|
|
3410
|
+
for await (const chunk of stream) {
|
|
3411
|
+
yield decoder.decode(chunk, { stream: true });
|
|
3412
|
+
}
|
|
3413
|
+
yield decoder.decode();
|
|
3414
|
+
})();
|
|
3415
|
+
} catch (error) {
|
|
3416
|
+
if (error.code === 404) {
|
|
3417
|
+
return null;
|
|
3418
|
+
}
|
|
3419
|
+
throw error;
|
|
3420
|
+
}
|
|
3421
|
+
}
|
|
3422
|
+
/**
|
|
3423
|
+
* Checks if a Google Cloud Storage object exists.
|
|
3424
|
+
*
|
|
3425
|
+
* @param filename - The name of the file to check.
|
|
3426
|
+
* @returns A promise resolving to true if the file exists, false otherwise.
|
|
3427
|
+
*/
|
|
1945
3428
|
async exists(filename) {
|
|
1946
3429
|
try {
|
|
1947
3430
|
const { bucket } = await this.getStorageClient();
|
|
@@ -1953,18 +3436,30 @@ var GCPSitemapStorage = class {
|
|
|
1953
3436
|
return false;
|
|
1954
3437
|
}
|
|
1955
3438
|
}
|
|
3439
|
+
/**
|
|
3440
|
+
* Returns the full public URL for a Google Cloud Storage object.
|
|
3441
|
+
*
|
|
3442
|
+
* @param filename - The name of the sitemap file.
|
|
3443
|
+
* @returns The public URL as a string.
|
|
3444
|
+
*/
|
|
1956
3445
|
getUrl(filename) {
|
|
1957
3446
|
const key = this.getKey(filename);
|
|
1958
3447
|
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
1959
3448
|
return `${base}/${key}`;
|
|
1960
3449
|
}
|
|
1961
|
-
|
|
3450
|
+
/**
|
|
3451
|
+
* Writes content to a shadow (staged) location in Google Cloud Storage.
|
|
3452
|
+
*
|
|
3453
|
+
* @param filename - The name of the file to write.
|
|
3454
|
+
* @param content - The XML or JSON content.
|
|
3455
|
+
* @param shadowId - Optional unique session identifier.
|
|
3456
|
+
*/
|
|
1962
3457
|
async writeShadow(filename, content, shadowId) {
|
|
1963
3458
|
if (!this.shadowEnabled) {
|
|
1964
3459
|
return this.write(filename, content);
|
|
1965
3460
|
}
|
|
1966
3461
|
const { bucket } = await this.getStorageClient();
|
|
1967
|
-
const id = shadowId || `shadow-${Date.now()}-${
|
|
3462
|
+
const id = shadowId || `shadow-${Date.now()}-${crypto.randomUUID()}`;
|
|
1968
3463
|
const shadowKey = this.getKey(`${filename}.shadow.${id}`);
|
|
1969
3464
|
const file = bucket.file(shadowKey);
|
|
1970
3465
|
await file.save(content, {
|
|
@@ -1974,6 +3469,11 @@ var GCPSitemapStorage = class {
|
|
|
1974
3469
|
}
|
|
1975
3470
|
});
|
|
1976
3471
|
}
|
|
3472
|
+
/**
|
|
3473
|
+
* Commits all staged shadow objects in a session to production in Google Cloud Storage.
|
|
3474
|
+
*
|
|
3475
|
+
* @param shadowId - The identifier of the session to commit.
|
|
3476
|
+
*/
|
|
1977
3477
|
async commitShadow(shadowId) {
|
|
1978
3478
|
if (!this.shadowEnabled) {
|
|
1979
3479
|
return;
|
|
@@ -2000,6 +3500,12 @@ var GCPSitemapStorage = class {
|
|
|
2000
3500
|
}
|
|
2001
3501
|
}
|
|
2002
3502
|
}
|
|
3503
|
+
/**
|
|
3504
|
+
* Lists all archived versions of a specific sitemap in Google Cloud Storage.
|
|
3505
|
+
*
|
|
3506
|
+
* @param filename - The sitemap filename.
|
|
3507
|
+
* @returns A promise resolving to an array of version identifiers.
|
|
3508
|
+
*/
|
|
2003
3509
|
async listVersions(filename) {
|
|
2004
3510
|
if (this.shadowMode !== "versioned") {
|
|
2005
3511
|
return [];
|
|
@@ -2021,6 +3527,12 @@ var GCPSitemapStorage = class {
|
|
|
2021
3527
|
return [];
|
|
2022
3528
|
}
|
|
2023
3529
|
}
|
|
3530
|
+
/**
|
|
3531
|
+
* Reverts a sitemap to a previously archived version in Google Cloud Storage.
|
|
3532
|
+
*
|
|
3533
|
+
* @param filename - The sitemap filename.
|
|
3534
|
+
* @param version - The version identifier to switch to.
|
|
3535
|
+
*/
|
|
2024
3536
|
async switchVersion(filename, version) {
|
|
2025
3537
|
if (this.shadowMode !== "versioned") {
|
|
2026
3538
|
throw new Error("Version switching is only available in versioned mode");
|
|
@@ -2040,22 +3552,51 @@ var GCPSitemapStorage = class {
|
|
|
2040
3552
|
// src/storage/MemoryProgressStorage.ts
|
|
2041
3553
|
var MemoryProgressStorage = class {
|
|
2042
3554
|
storage = /* @__PURE__ */ new Map();
|
|
3555
|
+
/**
|
|
3556
|
+
* Retrieves the progress of a specific generation job from memory.
|
|
3557
|
+
*
|
|
3558
|
+
* @param jobId - Unique identifier for the job.
|
|
3559
|
+
* @returns A promise resolving to the `SitemapProgress` object, or null if not found.
|
|
3560
|
+
*/
|
|
2043
3561
|
async get(jobId) {
|
|
2044
3562
|
const progress = this.storage.get(jobId);
|
|
2045
3563
|
return progress ? { ...progress } : null;
|
|
2046
3564
|
}
|
|
3565
|
+
/**
|
|
3566
|
+
* Initializes or overwrites a progress record in memory.
|
|
3567
|
+
*
|
|
3568
|
+
* @param jobId - Unique identifier for the job.
|
|
3569
|
+
* @param progress - The initial or current state of the job progress.
|
|
3570
|
+
*/
|
|
2047
3571
|
async set(jobId, progress) {
|
|
2048
3572
|
this.storage.set(jobId, { ...progress });
|
|
2049
3573
|
}
|
|
3574
|
+
/**
|
|
3575
|
+
* Updates specific fields of an existing progress record in memory.
|
|
3576
|
+
*
|
|
3577
|
+
* @param jobId - Unique identifier for the job.
|
|
3578
|
+
* @param updates - Object containing the fields to update.
|
|
3579
|
+
*/
|
|
2050
3580
|
async update(jobId, updates) {
|
|
2051
3581
|
const existing = this.storage.get(jobId);
|
|
2052
3582
|
if (existing) {
|
|
2053
3583
|
this.storage.set(jobId, { ...existing, ...updates });
|
|
2054
3584
|
}
|
|
2055
3585
|
}
|
|
3586
|
+
/**
|
|
3587
|
+
* Deletes a progress record from memory.
|
|
3588
|
+
*
|
|
3589
|
+
* @param jobId - Unique identifier for the job to remove.
|
|
3590
|
+
*/
|
|
2056
3591
|
async delete(jobId) {
|
|
2057
3592
|
this.storage.delete(jobId);
|
|
2058
3593
|
}
|
|
3594
|
+
/**
|
|
3595
|
+
* Lists the most recent sitemap generation jobs from memory.
|
|
3596
|
+
*
|
|
3597
|
+
* @param limit - Maximum number of records to return.
|
|
3598
|
+
* @returns A promise resolving to an array of `SitemapProgress` objects, sorted by start time.
|
|
3599
|
+
*/
|
|
2059
3600
|
async list(limit) {
|
|
2060
3601
|
const all = Array.from(this.storage.values());
|
|
2061
3602
|
const sorted = all.sort((a, b) => {
|
|
@@ -2083,6 +3624,12 @@ var RedisProgressStorage = class {
|
|
|
2083
3624
|
getListKey() {
|
|
2084
3625
|
return `${this.keyPrefix}list`;
|
|
2085
3626
|
}
|
|
3627
|
+
/**
|
|
3628
|
+
* Retrieves the progress of a specific generation job from Redis.
|
|
3629
|
+
*
|
|
3630
|
+
* @param jobId - Unique identifier for the job.
|
|
3631
|
+
* @returns A promise resolving to the `SitemapProgress` object, or null if not found.
|
|
3632
|
+
*/
|
|
2086
3633
|
async get(jobId) {
|
|
2087
3634
|
try {
|
|
2088
3635
|
const key = this.getKey(jobId);
|
|
@@ -2102,6 +3649,12 @@ var RedisProgressStorage = class {
|
|
|
2102
3649
|
return null;
|
|
2103
3650
|
}
|
|
2104
3651
|
}
|
|
3652
|
+
/**
|
|
3653
|
+
* Initializes or overwrites a progress record in Redis.
|
|
3654
|
+
*
|
|
3655
|
+
* @param jobId - Unique identifier for the job.
|
|
3656
|
+
* @param progress - The initial or current state of the job progress.
|
|
3657
|
+
*/
|
|
2105
3658
|
async set(jobId, progress) {
|
|
2106
3659
|
const key = this.getKey(jobId);
|
|
2107
3660
|
const listKey = this.getListKey();
|
|
@@ -2110,18 +3663,35 @@ var RedisProgressStorage = class {
|
|
|
2110
3663
|
await this.client.zadd(listKey, Date.now(), jobId);
|
|
2111
3664
|
await this.client.expire(listKey, this.ttl);
|
|
2112
3665
|
}
|
|
3666
|
+
/**
|
|
3667
|
+
* Updates specific fields of an existing progress record in Redis.
|
|
3668
|
+
*
|
|
3669
|
+
* @param jobId - Unique identifier for the job.
|
|
3670
|
+
* @param updates - Object containing the fields to update.
|
|
3671
|
+
*/
|
|
2113
3672
|
async update(jobId, updates) {
|
|
2114
3673
|
const existing = await this.get(jobId);
|
|
2115
3674
|
if (existing) {
|
|
2116
3675
|
await this.set(jobId, { ...existing, ...updates });
|
|
2117
3676
|
}
|
|
2118
3677
|
}
|
|
3678
|
+
/**
|
|
3679
|
+
* Deletes a progress record from Redis.
|
|
3680
|
+
*
|
|
3681
|
+
* @param jobId - Unique identifier for the job to remove.
|
|
3682
|
+
*/
|
|
2119
3683
|
async delete(jobId) {
|
|
2120
3684
|
const key = this.getKey(jobId);
|
|
2121
3685
|
const listKey = this.getListKey();
|
|
2122
3686
|
await this.client.del(key);
|
|
2123
3687
|
await this.client.zrem(listKey, jobId);
|
|
2124
3688
|
}
|
|
3689
|
+
/**
|
|
3690
|
+
* Lists the most recent sitemap generation jobs from Redis.
|
|
3691
|
+
*
|
|
3692
|
+
* @param limit - Maximum number of records to return.
|
|
3693
|
+
* @returns A promise resolving to an array of `SitemapProgress` objects, sorted by start time.
|
|
3694
|
+
*/
|
|
2125
3695
|
async list(limit) {
|
|
2126
3696
|
try {
|
|
2127
3697
|
const listKey = this.getListKey();
|
|
@@ -2141,6 +3711,7 @@ var RedisProgressStorage = class {
|
|
|
2141
3711
|
};
|
|
2142
3712
|
|
|
2143
3713
|
// src/storage/S3SitemapStorage.ts
|
|
3714
|
+
init_Compression();
|
|
2144
3715
|
var S3SitemapStorage = class {
|
|
2145
3716
|
bucket;
|
|
2146
3717
|
region;
|
|
@@ -2198,6 +3769,12 @@ var S3SitemapStorage = class {
|
|
|
2198
3769
|
const cleanPrefix = this.prefix.endsWith("/") ? this.prefix.slice(0, -1) : this.prefix;
|
|
2199
3770
|
return cleanPrefix ? `${cleanPrefix}/${filename}` : filename;
|
|
2200
3771
|
}
|
|
3772
|
+
/**
|
|
3773
|
+
* Writes sitemap content to an S3 object.
|
|
3774
|
+
*
|
|
3775
|
+
* @param filename - The name of the file to write.
|
|
3776
|
+
* @param content - The XML or JSON content.
|
|
3777
|
+
*/
|
|
2201
3778
|
async write(filename, content) {
|
|
2202
3779
|
const s3 = await this.getS3Client();
|
|
2203
3780
|
const key = this.getKey(filename);
|
|
@@ -2210,6 +3787,48 @@ var S3SitemapStorage = class {
|
|
|
2210
3787
|
})
|
|
2211
3788
|
);
|
|
2212
3789
|
}
|
|
3790
|
+
/**
|
|
3791
|
+
* 使用串流方式寫入 sitemap 至 S3,可選擇性啟用 gzip 壓縮。
|
|
3792
|
+
* S3 需要知道 Content-Length,因此會先收集串流為 Buffer。
|
|
3793
|
+
*
|
|
3794
|
+
* @param filename - 檔案名稱
|
|
3795
|
+
* @param stream - XML 內容的 AsyncIterable
|
|
3796
|
+
* @param options - 寫入選項(如壓縮、content type)
|
|
3797
|
+
*
|
|
3798
|
+
* @since 3.1.0
|
|
3799
|
+
*/
|
|
3800
|
+
async writeStream(filename, stream, options) {
|
|
3801
|
+
const s3 = await this.getS3Client();
|
|
3802
|
+
const key = this.getKey(options?.compress ? toGzipFilename(filename) : filename);
|
|
3803
|
+
let body;
|
|
3804
|
+
const contentType = options?.contentType || "application/xml";
|
|
3805
|
+
let contentEncoding;
|
|
3806
|
+
if (options?.compress) {
|
|
3807
|
+
body = await compressToBuffer(stream);
|
|
3808
|
+
contentEncoding = "gzip";
|
|
3809
|
+
} else {
|
|
3810
|
+
const chunks = [];
|
|
3811
|
+
for await (const chunk of stream) {
|
|
3812
|
+
chunks.push(chunk);
|
|
3813
|
+
}
|
|
3814
|
+
body = chunks.join("");
|
|
3815
|
+
}
|
|
3816
|
+
await s3.client.send(
|
|
3817
|
+
new s3.PutObjectCommand({
|
|
3818
|
+
Bucket: this.bucket,
|
|
3819
|
+
Key: key,
|
|
3820
|
+
Body: body,
|
|
3821
|
+
ContentType: contentType,
|
|
3822
|
+
ContentEncoding: contentEncoding
|
|
3823
|
+
})
|
|
3824
|
+
);
|
|
3825
|
+
}
|
|
3826
|
+
/**
|
|
3827
|
+
* Reads sitemap content from an S3 object.
|
|
3828
|
+
*
|
|
3829
|
+
* @param filename - The name of the file to read.
|
|
3830
|
+
* @returns A promise resolving to the file content as a string, or null if not found.
|
|
3831
|
+
*/
|
|
2213
3832
|
async read(filename) {
|
|
2214
3833
|
try {
|
|
2215
3834
|
const s3 = await this.getS3Client();
|
|
@@ -2236,6 +3855,46 @@ var S3SitemapStorage = class {
|
|
|
2236
3855
|
throw error;
|
|
2237
3856
|
}
|
|
2238
3857
|
}
|
|
3858
|
+
/**
|
|
3859
|
+
* Returns a readable stream for an S3 object.
|
|
3860
|
+
*
|
|
3861
|
+
* @param filename - The name of the file to stream.
|
|
3862
|
+
* @returns A promise resolving to an async iterable of file chunks, or null if not found.
|
|
3863
|
+
*/
|
|
3864
|
+
async readStream(filename) {
|
|
3865
|
+
try {
|
|
3866
|
+
const s3 = await this.getS3Client();
|
|
3867
|
+
const key = this.getKey(filename);
|
|
3868
|
+
const response = await s3.client.send(
|
|
3869
|
+
new s3.GetObjectCommand({
|
|
3870
|
+
Bucket: this.bucket,
|
|
3871
|
+
Key: key
|
|
3872
|
+
})
|
|
3873
|
+
);
|
|
3874
|
+
if (!response.Body) {
|
|
3875
|
+
return null;
|
|
3876
|
+
}
|
|
3877
|
+
const body = response.Body;
|
|
3878
|
+
return (async function* () {
|
|
3879
|
+
const decoder = new TextDecoder();
|
|
3880
|
+
for await (const chunk of body) {
|
|
3881
|
+
yield decoder.decode(chunk, { stream: true });
|
|
3882
|
+
}
|
|
3883
|
+
yield decoder.decode();
|
|
3884
|
+
})();
|
|
3885
|
+
} catch (error) {
|
|
3886
|
+
if (error.name === "NoSuchKey" || error.$metadata?.httpStatusCode === 404) {
|
|
3887
|
+
return null;
|
|
3888
|
+
}
|
|
3889
|
+
throw error;
|
|
3890
|
+
}
|
|
3891
|
+
}
|
|
3892
|
+
/**
|
|
3893
|
+
* Checks if an S3 object exists.
|
|
3894
|
+
*
|
|
3895
|
+
* @param filename - The name of the file to check.
|
|
3896
|
+
* @returns A promise resolving to true if the file exists, false otherwise.
|
|
3897
|
+
*/
|
|
2239
3898
|
async exists(filename) {
|
|
2240
3899
|
try {
|
|
2241
3900
|
const s3 = await this.getS3Client();
|
|
@@ -2254,18 +3913,30 @@ var S3SitemapStorage = class {
|
|
|
2254
3913
|
throw error;
|
|
2255
3914
|
}
|
|
2256
3915
|
}
|
|
3916
|
+
/**
|
|
3917
|
+
* Returns the full public URL for an S3 object.
|
|
3918
|
+
*
|
|
3919
|
+
* @param filename - The name of the sitemap file.
|
|
3920
|
+
* @returns The public URL as a string.
|
|
3921
|
+
*/
|
|
2257
3922
|
getUrl(filename) {
|
|
2258
3923
|
const key = this.getKey(filename);
|
|
2259
3924
|
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
2260
3925
|
return `${base}/${key}`;
|
|
2261
3926
|
}
|
|
2262
|
-
|
|
3927
|
+
/**
|
|
3928
|
+
* Writes content to a shadow (staged) location in S3.
|
|
3929
|
+
*
|
|
3930
|
+
* @param filename - The name of the file to write.
|
|
3931
|
+
* @param content - The XML or JSON content.
|
|
3932
|
+
* @param shadowId - Optional unique session identifier.
|
|
3933
|
+
*/
|
|
2263
3934
|
async writeShadow(filename, content, shadowId) {
|
|
2264
3935
|
if (!this.shadowEnabled) {
|
|
2265
3936
|
return this.write(filename, content);
|
|
2266
3937
|
}
|
|
2267
3938
|
const s3 = await this.getS3Client();
|
|
2268
|
-
const id = shadowId || `shadow-${Date.now()}-${
|
|
3939
|
+
const id = shadowId || `shadow-${Date.now()}-${crypto.randomUUID()}`;
|
|
2269
3940
|
const shadowKey = this.getKey(`${filename}.shadow.${id}`);
|
|
2270
3941
|
await s3.client.send(
|
|
2271
3942
|
new s3.PutObjectCommand({
|
|
@@ -2276,6 +3947,11 @@ var S3SitemapStorage = class {
|
|
|
2276
3947
|
})
|
|
2277
3948
|
);
|
|
2278
3949
|
}
|
|
3950
|
+
/**
|
|
3951
|
+
* Commits all staged shadow objects in a session to production.
|
|
3952
|
+
*
|
|
3953
|
+
* @param shadowId - The identifier of the session to commit.
|
|
3954
|
+
*/
|
|
2279
3955
|
async commitShadow(shadowId) {
|
|
2280
3956
|
if (!this.shadowEnabled) {
|
|
2281
3957
|
return;
|
|
@@ -2343,6 +4019,12 @@ var S3SitemapStorage = class {
|
|
|
2343
4019
|
}
|
|
2344
4020
|
}
|
|
2345
4021
|
}
|
|
4022
|
+
/**
|
|
4023
|
+
* Lists all archived versions of a specific sitemap in S3.
|
|
4024
|
+
*
|
|
4025
|
+
* @param filename - The sitemap filename.
|
|
4026
|
+
* @returns A promise resolving to an array of version identifiers.
|
|
4027
|
+
*/
|
|
2346
4028
|
async listVersions(filename) {
|
|
2347
4029
|
if (this.shadowMode !== "versioned") {
|
|
2348
4030
|
return [];
|
|
@@ -2375,6 +4057,12 @@ var S3SitemapStorage = class {
|
|
|
2375
4057
|
return [];
|
|
2376
4058
|
}
|
|
2377
4059
|
}
|
|
4060
|
+
/**
|
|
4061
|
+
* Reverts a sitemap to a previously archived version in S3.
|
|
4062
|
+
*
|
|
4063
|
+
* @param filename - The sitemap filename.
|
|
4064
|
+
* @param version - The version identifier to switch to.
|
|
4065
|
+
*/
|
|
2378
4066
|
async switchVersion(filename, version) {
|
|
2379
4067
|
if (this.shadowMode !== "versioned") {
|
|
2380
4068
|
throw new Error("Version switching is only available in versioned mode");
|
|
@@ -2396,6 +4084,9 @@ var S3SitemapStorage = class {
|
|
|
2396
4084
|
);
|
|
2397
4085
|
}
|
|
2398
4086
|
};
|
|
4087
|
+
|
|
4088
|
+
// src/index.ts
|
|
4089
|
+
init_Compression();
|
|
2399
4090
|
// Annotate the CommonJS export names for ESM import in node:
|
|
2400
4091
|
0 && (module.exports = {
|
|
2401
4092
|
DiffCalculator,
|
|
@@ -2404,6 +4095,7 @@ var S3SitemapStorage = class {
|
|
|
2404
4095
|
GenerateSitemapJob,
|
|
2405
4096
|
IncrementalGenerator,
|
|
2406
4097
|
MemoryChangeTracker,
|
|
4098
|
+
MemoryLock,
|
|
2407
4099
|
MemoryProgressStorage,
|
|
2408
4100
|
MemoryRedirectManager,
|
|
2409
4101
|
MemorySitemapStorage,
|
|
@@ -2412,6 +4104,7 @@ var S3SitemapStorage = class {
|
|
|
2412
4104
|
RedirectDetector,
|
|
2413
4105
|
RedirectHandler,
|
|
2414
4106
|
RedisChangeTracker,
|
|
4107
|
+
RedisLock,
|
|
2415
4108
|
RedisProgressStorage,
|
|
2416
4109
|
RedisRedirectManager,
|
|
2417
4110
|
RouteScanner,
|
|
@@ -2420,6 +4113,10 @@ var S3SitemapStorage = class {
|
|
|
2420
4113
|
SitemapGenerator,
|
|
2421
4114
|
SitemapIndex,
|
|
2422
4115
|
SitemapStream,
|
|
4116
|
+
compressToBuffer,
|
|
4117
|
+
createCompressionStream,
|
|
4118
|
+
fromGzipFilename,
|
|
2423
4119
|
generateI18nEntries,
|
|
2424
|
-
routeScanner
|
|
4120
|
+
routeScanner,
|
|
4121
|
+
toGzipFilename
|
|
2425
4122
|
});
|