@gravito/constellation 3.0.2 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -251
- package/README.zh-TW.md +130 -13
- package/dist/{DiskSitemapStorage-WP6RITUN.js → DiskSitemapStorage-VLN5I24C.js} +1 -1
- package/dist/chunk-3IZTXYU7.js +166 -0
- package/dist/index.cjs +1400 -67
- package/dist/index.d.cts +1577 -148
- package/dist/index.d.ts +1577 -148
- package/dist/index.js +1293 -70
- package/package.json +7 -5
- package/dist/chunk-IS2H7U6M.js +0 -68
package/dist/index.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
|
-
DiskSitemapStorage
|
|
3
|
-
|
|
2
|
+
DiskSitemapStorage,
|
|
3
|
+
compressToBuffer,
|
|
4
|
+
createCompressionStream,
|
|
5
|
+
fromGzipFilename,
|
|
6
|
+
toGzipFilename
|
|
7
|
+
} from "./chunk-3IZTXYU7.js";
|
|
4
8
|
|
|
5
9
|
// src/core/ChangeTracker.ts
|
|
6
10
|
var MemoryChangeTracker = class {
|
|
@@ -10,6 +14,11 @@ var MemoryChangeTracker = class {
|
|
|
10
14
|
constructor(options = {}) {
|
|
11
15
|
this.maxChanges = options.maxChanges || 1e5;
|
|
12
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* Record a new site structure change in memory.
|
|
19
|
+
*
|
|
20
|
+
* @param change - The change event to record.
|
|
21
|
+
*/
|
|
13
22
|
async track(change) {
|
|
14
23
|
this.changes.push(change);
|
|
15
24
|
const urlChanges = this.urlIndex.get(change.url) || [];
|
|
@@ -31,15 +40,32 @@ var MemoryChangeTracker = class {
|
|
|
31
40
|
}
|
|
32
41
|
}
|
|
33
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Retrieve all changes recorded in memory since a specific time.
|
|
45
|
+
*
|
|
46
|
+
* @param since - Optional start date for the query.
|
|
47
|
+
* @returns An array of change events.
|
|
48
|
+
*/
|
|
34
49
|
async getChanges(since) {
|
|
35
50
|
if (!since) {
|
|
36
51
|
return [...this.changes];
|
|
37
52
|
}
|
|
38
53
|
return this.changes.filter((change) => change.timestamp >= since);
|
|
39
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* Retrieve the full change history for a specific URL from memory.
|
|
57
|
+
*
|
|
58
|
+
* @param url - The URL to query history for.
|
|
59
|
+
* @returns An array of change events.
|
|
60
|
+
*/
|
|
40
61
|
async getChangesByUrl(url) {
|
|
41
62
|
return this.urlIndex.get(url) || [];
|
|
42
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* Purge old change records from memory storage.
|
|
66
|
+
*
|
|
67
|
+
* @param since - If provided, only records older than this date will be cleared.
|
|
68
|
+
*/
|
|
43
69
|
async clear(since) {
|
|
44
70
|
if (!since) {
|
|
45
71
|
this.changes = [];
|
|
@@ -70,6 +96,11 @@ var RedisChangeTracker = class {
|
|
|
70
96
|
getListKey() {
|
|
71
97
|
return `${this.keyPrefix}list`;
|
|
72
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* Record a new site structure change in Redis.
|
|
101
|
+
*
|
|
102
|
+
* @param change - The change event to record.
|
|
103
|
+
*/
|
|
73
104
|
async track(change) {
|
|
74
105
|
const key = this.getKey(change.url);
|
|
75
106
|
const listKey = this.getListKey();
|
|
@@ -79,6 +110,12 @@ var RedisChangeTracker = class {
|
|
|
79
110
|
await this.client.zadd(listKey, score, change.url);
|
|
80
111
|
await this.client.expire(listKey, this.ttl);
|
|
81
112
|
}
|
|
113
|
+
/**
|
|
114
|
+
* Retrieve all changes recorded in Redis since a specific time.
|
|
115
|
+
*
|
|
116
|
+
* @param since - Optional start date for the query.
|
|
117
|
+
* @returns An array of change events.
|
|
118
|
+
*/
|
|
82
119
|
async getChanges(since) {
|
|
83
120
|
try {
|
|
84
121
|
const listKey = this.getListKey();
|
|
@@ -99,6 +136,12 @@ var RedisChangeTracker = class {
|
|
|
99
136
|
return [];
|
|
100
137
|
}
|
|
101
138
|
}
|
|
139
|
+
/**
|
|
140
|
+
* Retrieve the full change history for a specific URL from Redis.
|
|
141
|
+
*
|
|
142
|
+
* @param url - The URL to query history for.
|
|
143
|
+
* @returns An array of change events.
|
|
144
|
+
*/
|
|
102
145
|
async getChangesByUrl(url) {
|
|
103
146
|
try {
|
|
104
147
|
const key = this.getKey(url);
|
|
@@ -113,6 +156,11 @@ var RedisChangeTracker = class {
|
|
|
113
156
|
return [];
|
|
114
157
|
}
|
|
115
158
|
}
|
|
159
|
+
/**
|
|
160
|
+
* Purge old change records from Redis storage.
|
|
161
|
+
*
|
|
162
|
+
* @param since - If provided, only records older than this date will be cleared.
|
|
163
|
+
*/
|
|
116
164
|
async clear(since) {
|
|
117
165
|
try {
|
|
118
166
|
const listKey = this.getListKey();
|
|
@@ -142,7 +190,11 @@ var DiffCalculator = class {
|
|
|
142
190
|
this.batchSize = options.batchSize || 1e4;
|
|
143
191
|
}
|
|
144
192
|
/**
|
|
145
|
-
*
|
|
193
|
+
* Calculates the difference between two sets of sitemap entries.
|
|
194
|
+
*
|
|
195
|
+
* @param oldEntries - The previous set of sitemap entries.
|
|
196
|
+
* @param newEntries - The current set of sitemap entries.
|
|
197
|
+
* @returns A DiffResult containing added, updated, and removed entries.
|
|
146
198
|
*/
|
|
147
199
|
calculate(oldEntries, newEntries) {
|
|
148
200
|
const oldMap = /* @__PURE__ */ new Map();
|
|
@@ -172,7 +224,11 @@ var DiffCalculator = class {
|
|
|
172
224
|
return { added, updated, removed };
|
|
173
225
|
}
|
|
174
226
|
/**
|
|
175
|
-
*
|
|
227
|
+
* Batch calculates differences for large datasets using async iterables.
|
|
228
|
+
*
|
|
229
|
+
* @param oldEntries - An async iterable of the previous sitemap entries.
|
|
230
|
+
* @param newEntries - An async iterable of the current sitemap entries.
|
|
231
|
+
* @returns A promise resolving to the DiffResult.
|
|
176
232
|
*/
|
|
177
233
|
async calculateBatch(oldEntries, newEntries) {
|
|
178
234
|
const oldMap = /* @__PURE__ */ new Map();
|
|
@@ -186,7 +242,11 @@ var DiffCalculator = class {
|
|
|
186
242
|
return this.calculate(Array.from(oldMap.values()), Array.from(newMap.values()));
|
|
187
243
|
}
|
|
188
244
|
/**
|
|
189
|
-
*
|
|
245
|
+
* Calculates differences based on a sequence of change records.
|
|
246
|
+
*
|
|
247
|
+
* @param baseEntries - The base set of sitemap entries.
|
|
248
|
+
* @param changes - An array of change records to apply to the base set.
|
|
249
|
+
* @returns A DiffResult comparing the base set with the applied changes.
|
|
190
250
|
*/
|
|
191
251
|
calculateFromChanges(baseEntries, changes) {
|
|
192
252
|
const entryMap = /* @__PURE__ */ new Map();
|
|
@@ -206,7 +266,11 @@ var DiffCalculator = class {
|
|
|
206
266
|
return this.calculate(baseEntries, newEntries);
|
|
207
267
|
}
|
|
208
268
|
/**
|
|
209
|
-
*
|
|
269
|
+
* Checks if a sitemap entry has changed by comparing its key properties.
|
|
270
|
+
*
|
|
271
|
+
* @param oldEntry - The previous sitemap entry.
|
|
272
|
+
* @param newEntry - The current sitemap entry.
|
|
273
|
+
* @returns True if the entry has changed, false otherwise.
|
|
210
274
|
*/
|
|
211
275
|
hasChanged(oldEntry, newEntry) {
|
|
212
276
|
if (oldEntry.lastmod !== newEntry.lastmod) {
|
|
@@ -230,6 +294,12 @@ var DiffCalculator = class {
|
|
|
230
294
|
// src/utils/Mutex.ts
|
|
231
295
|
var Mutex = class {
|
|
232
296
|
queue = Promise.resolve();
|
|
297
|
+
/**
|
|
298
|
+
* Executes a function exclusively, ensuring no other task can run it concurrently.
|
|
299
|
+
*
|
|
300
|
+
* @param fn - The async function to execute.
|
|
301
|
+
* @returns A promise resolving to the result of the function.
|
|
302
|
+
*/
|
|
233
303
|
async runExclusive(fn) {
|
|
234
304
|
const next = this.queue.then(() => fn());
|
|
235
305
|
this.queue = next.then(
|
|
@@ -253,7 +323,12 @@ var ShadowProcessor = class {
|
|
|
253
323
|
this.shadowId = `shadow-${Date.now()}-${crypto.randomUUID()}`;
|
|
254
324
|
}
|
|
255
325
|
/**
|
|
256
|
-
*
|
|
326
|
+
* Adds a single file write operation to the current shadow session.
|
|
327
|
+
*
|
|
328
|
+
* If shadow processing is disabled, the file is written directly to the
|
|
329
|
+
* final destination in storage. Otherwise, it is written to the shadow staging area.
|
|
330
|
+
*
|
|
331
|
+
* @param operation - The shadow write operation details.
|
|
257
332
|
*/
|
|
258
333
|
async addOperation(operation) {
|
|
259
334
|
return this.mutex.runExclusive(async () => {
|
|
@@ -273,7 +348,10 @@ var ShadowProcessor = class {
|
|
|
273
348
|
});
|
|
274
349
|
}
|
|
275
350
|
/**
|
|
276
|
-
*
|
|
351
|
+
* Commits all staged shadow operations to the final production location.
|
|
352
|
+
*
|
|
353
|
+
* Depending on the `mode`, this will either perform an atomic swap of all files
|
|
354
|
+
* or commit each file individually (potentially creating new versions).
|
|
277
355
|
*/
|
|
278
356
|
async commit() {
|
|
279
357
|
return this.mutex.runExclusive(async () => {
|
|
@@ -295,7 +373,7 @@ var ShadowProcessor = class {
|
|
|
295
373
|
});
|
|
296
374
|
}
|
|
297
375
|
/**
|
|
298
|
-
*
|
|
376
|
+
* Cancels all staged shadow operations without committing them.
|
|
299
377
|
*/
|
|
300
378
|
async rollback() {
|
|
301
379
|
if (!this.options.enabled) {
|
|
@@ -304,13 +382,13 @@ var ShadowProcessor = class {
|
|
|
304
382
|
this.operations = [];
|
|
305
383
|
}
|
|
306
384
|
/**
|
|
307
|
-
*
|
|
385
|
+
* Returns the unique identifier for the current shadow session.
|
|
308
386
|
*/
|
|
309
387
|
getShadowId() {
|
|
310
388
|
return this.shadowId;
|
|
311
389
|
}
|
|
312
390
|
/**
|
|
313
|
-
*
|
|
391
|
+
* Returns an array of all staged shadow operations.
|
|
314
392
|
*/
|
|
315
393
|
getOperations() {
|
|
316
394
|
return [...this.operations];
|
|
@@ -327,6 +405,12 @@ var SitemapIndex = class {
|
|
|
327
405
|
this.options.baseUrl = this.options.baseUrl.slice(0, -1);
|
|
328
406
|
}
|
|
329
407
|
}
|
|
408
|
+
/**
|
|
409
|
+
* Adds a single entry to the sitemap index.
|
|
410
|
+
*
|
|
411
|
+
* @param entry - A sitemap filename or a `SitemapIndexEntry` object.
|
|
412
|
+
* @returns The `SitemapIndex` instance for chaining.
|
|
413
|
+
*/
|
|
330
414
|
add(entry) {
|
|
331
415
|
if (typeof entry === "string") {
|
|
332
416
|
this.entries.push({ url: entry });
|
|
@@ -335,12 +419,23 @@ var SitemapIndex = class {
|
|
|
335
419
|
}
|
|
336
420
|
return this;
|
|
337
421
|
}
|
|
422
|
+
/**
|
|
423
|
+
* Adds multiple entries to the sitemap index.
|
|
424
|
+
*
|
|
425
|
+
* @param entries - An array of sitemap filenames or `SitemapIndexEntry` objects.
|
|
426
|
+
* @returns The `SitemapIndex` instance for chaining.
|
|
427
|
+
*/
|
|
338
428
|
addAll(entries) {
|
|
339
429
|
for (const entry of entries) {
|
|
340
430
|
this.add(entry);
|
|
341
431
|
}
|
|
342
432
|
return this;
|
|
343
433
|
}
|
|
434
|
+
/**
|
|
435
|
+
* Generates the sitemap index XML content.
|
|
436
|
+
*
|
|
437
|
+
* @returns The complete XML string for the sitemap index.
|
|
438
|
+
*/
|
|
344
439
|
toXML() {
|
|
345
440
|
const { baseUrl, pretty } = this.options;
|
|
346
441
|
let xml = `<?xml version="1.0" encoding="UTF-8"?>
|
|
@@ -369,6 +464,9 @@ var SitemapIndex = class {
|
|
|
369
464
|
xml += `</sitemapindex>`;
|
|
370
465
|
return xml;
|
|
371
466
|
}
|
|
467
|
+
/**
|
|
468
|
+
* Escapes special XML characters in a string.
|
|
469
|
+
*/
|
|
372
470
|
escape(str) {
|
|
373
471
|
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
374
472
|
}
|
|
@@ -390,6 +488,12 @@ var SitemapStream = class {
|
|
|
390
488
|
this.options.baseUrl = this.options.baseUrl.slice(0, -1);
|
|
391
489
|
}
|
|
392
490
|
}
|
|
491
|
+
/**
|
|
492
|
+
* Adds a single entry to the sitemap stream.
|
|
493
|
+
*
|
|
494
|
+
* @param entry - A URL string or a complete `SitemapEntry` object.
|
|
495
|
+
* @returns The `SitemapStream` instance for chaining.
|
|
496
|
+
*/
|
|
393
497
|
add(entry) {
|
|
394
498
|
const e = typeof entry === "string" ? { url: entry } : entry;
|
|
395
499
|
this.entries.push(e);
|
|
@@ -407,16 +511,78 @@ var SitemapStream = class {
|
|
|
407
511
|
}
|
|
408
512
|
return this;
|
|
409
513
|
}
|
|
514
|
+
/**
|
|
515
|
+
* Adds multiple entries to the sitemap stream.
|
|
516
|
+
*
|
|
517
|
+
* @param entries - An array of URL strings or `SitemapEntry` objects.
|
|
518
|
+
* @returns The `SitemapStream` instance for chaining.
|
|
519
|
+
*/
|
|
410
520
|
addAll(entries) {
|
|
411
521
|
for (const entry of entries) {
|
|
412
522
|
this.add(entry);
|
|
413
523
|
}
|
|
414
524
|
return this;
|
|
415
525
|
}
|
|
526
|
+
/**
|
|
527
|
+
* Generates the sitemap XML content.
|
|
528
|
+
*
|
|
529
|
+
* Automatically includes the necessary XML namespaces for images, videos, news,
|
|
530
|
+
* and internationalization if the entries contain such metadata.
|
|
531
|
+
*
|
|
532
|
+
* @returns The complete XML string for the sitemap.
|
|
533
|
+
*/
|
|
416
534
|
toXML() {
|
|
535
|
+
const parts = [];
|
|
536
|
+
for (const chunk of this.toSyncIterable()) {
|
|
537
|
+
parts.push(chunk);
|
|
538
|
+
}
|
|
539
|
+
return parts.join("");
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* 以 AsyncGenerator 方式產生 XML 內容。
|
|
543
|
+
* 每次 yield 一個邏輯區塊,適合串流寫入場景,可減少記憶體峰值。
|
|
544
|
+
*
|
|
545
|
+
* @returns AsyncGenerator 產生 XML 字串片段
|
|
546
|
+
*
|
|
547
|
+
* @example
|
|
548
|
+
* ```typescript
|
|
549
|
+
* const stream = new SitemapStream({ baseUrl: 'https://example.com' })
|
|
550
|
+
* stream.add({ url: '/page1' })
|
|
551
|
+
* stream.add({ url: '/page2' })
|
|
552
|
+
*
|
|
553
|
+
* for await (const chunk of stream.toAsyncIterable()) {
|
|
554
|
+
* process.stdout.write(chunk)
|
|
555
|
+
* }
|
|
556
|
+
* ```
|
|
557
|
+
*
|
|
558
|
+
* @since 3.1.0
|
|
559
|
+
*/
|
|
560
|
+
async *toAsyncIterable() {
|
|
561
|
+
yield '<?xml version="1.0" encoding="UTF-8"?>\n';
|
|
562
|
+
yield this.buildUrlsetOpenTag();
|
|
563
|
+
const { baseUrl, pretty } = this.options;
|
|
564
|
+
for (const entry of this.entries) {
|
|
565
|
+
yield this.renderUrl(entry, baseUrl, pretty);
|
|
566
|
+
}
|
|
567
|
+
yield "</urlset>";
|
|
568
|
+
}
|
|
569
|
+
/**
|
|
570
|
+
* 同步版本的 iterable,供 toXML() 使用。
|
|
571
|
+
*/
|
|
572
|
+
*toSyncIterable() {
|
|
573
|
+
yield '<?xml version="1.0" encoding="UTF-8"?>\n';
|
|
574
|
+
yield this.buildUrlsetOpenTag();
|
|
417
575
|
const { baseUrl, pretty } = this.options;
|
|
576
|
+
for (const entry of this.entries) {
|
|
577
|
+
yield this.renderUrl(entry, baseUrl, pretty);
|
|
578
|
+
}
|
|
579
|
+
yield "</urlset>";
|
|
580
|
+
}
|
|
581
|
+
/**
|
|
582
|
+
* 建立 urlset 開標籤與所有必要的 XML 命名空間。
|
|
583
|
+
*/
|
|
584
|
+
buildUrlsetOpenTag() {
|
|
418
585
|
const parts = [];
|
|
419
|
-
parts.push('<?xml version="1.0" encoding="UTF-8"?>\n');
|
|
420
586
|
parts.push('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"');
|
|
421
587
|
if (this.flags.hasImages) {
|
|
422
588
|
parts.push(' xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"');
|
|
@@ -431,12 +597,11 @@ var SitemapStream = class {
|
|
|
431
597
|
parts.push(' xmlns:xhtml="http://www.w3.org/1999/xhtml"');
|
|
432
598
|
}
|
|
433
599
|
parts.push(">\n");
|
|
434
|
-
for (const entry of this.entries) {
|
|
435
|
-
parts.push(this.renderUrl(entry, baseUrl, pretty));
|
|
436
|
-
}
|
|
437
|
-
parts.push("</urlset>");
|
|
438
600
|
return parts.join("");
|
|
439
601
|
}
|
|
602
|
+
/**
|
|
603
|
+
* Renders a single sitemap entry into its XML representation.
|
|
604
|
+
*/
|
|
440
605
|
renderUrl(entry, baseUrl, pretty) {
|
|
441
606
|
const indent = pretty ? " " : "";
|
|
442
607
|
const subIndent = pretty ? " " : "";
|
|
@@ -599,9 +764,17 @@ var SitemapStream = class {
|
|
|
599
764
|
parts.push(`${indent}</url>${nl}`);
|
|
600
765
|
return parts.join("");
|
|
601
766
|
}
|
|
767
|
+
/**
|
|
768
|
+
* Escapes special XML characters in a string.
|
|
769
|
+
*/
|
|
602
770
|
escape(str) {
|
|
603
771
|
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
604
772
|
}
|
|
773
|
+
/**
|
|
774
|
+
* Returns all entries currently in the stream.
|
|
775
|
+
*
|
|
776
|
+
* @returns An array of `SitemapEntry` objects.
|
|
777
|
+
*/
|
|
605
778
|
getEntries() {
|
|
606
779
|
return this.entries;
|
|
607
780
|
}
|
|
@@ -625,6 +798,12 @@ var SitemapGenerator = class {
|
|
|
625
798
|
});
|
|
626
799
|
}
|
|
627
800
|
}
|
|
801
|
+
/**
|
|
802
|
+
* Orchestrates the sitemap generation process.
|
|
803
|
+
*
|
|
804
|
+
* This method scans all providers, handles sharding, generates the XML files,
|
|
805
|
+
* and optionally creates a sitemap index and manifest.
|
|
806
|
+
*/
|
|
628
807
|
async run() {
|
|
629
808
|
let shardIndex = 1;
|
|
630
809
|
let currentCount = 0;
|
|
@@ -642,21 +821,23 @@ var SitemapGenerator = class {
|
|
|
642
821
|
isMultiFile = true;
|
|
643
822
|
const baseName = this.options.filename?.replace(/\.xml$/, "");
|
|
644
823
|
const filename = `${baseName}-${shardIndex}.xml`;
|
|
645
|
-
const xml = currentStream.toXML();
|
|
646
824
|
const entries = currentStream.getEntries();
|
|
825
|
+
let actualFilename;
|
|
826
|
+
if (this.shadowProcessor) {
|
|
827
|
+
actualFilename = this.options.compression?.enabled ? toGzipFilename(filename) : filename;
|
|
828
|
+
const xml = currentStream.toXML();
|
|
829
|
+
await this.shadowProcessor.addOperation({ filename: actualFilename, content: xml });
|
|
830
|
+
} else {
|
|
831
|
+
actualFilename = await this.writeSitemap(currentStream, filename);
|
|
832
|
+
}
|
|
647
833
|
shards.push({
|
|
648
|
-
filename,
|
|
834
|
+
filename: actualFilename,
|
|
649
835
|
from: this.normalizeUrl(entries[0].url),
|
|
650
836
|
to: this.normalizeUrl(entries[entries.length - 1].url),
|
|
651
837
|
count: entries.length,
|
|
652
838
|
lastmod: /* @__PURE__ */ new Date()
|
|
653
839
|
});
|
|
654
|
-
|
|
655
|
-
await this.shadowProcessor.addOperation({ filename, content: xml });
|
|
656
|
-
} else {
|
|
657
|
-
await this.options.storage.write(filename, xml);
|
|
658
|
-
}
|
|
659
|
-
const url = this.options.storage.getUrl(filename);
|
|
840
|
+
const url = this.options.storage.getUrl(actualFilename);
|
|
660
841
|
index.add({
|
|
661
842
|
url,
|
|
662
843
|
lastmod: /* @__PURE__ */ new Date()
|
|
@@ -689,7 +870,9 @@ var SitemapGenerator = class {
|
|
|
689
870
|
}
|
|
690
871
|
}
|
|
691
872
|
const writeManifest = async () => {
|
|
692
|
-
if (!this.options.generateManifest)
|
|
873
|
+
if (!this.options.generateManifest) {
|
|
874
|
+
return;
|
|
875
|
+
}
|
|
693
876
|
const manifest = {
|
|
694
877
|
version: 1,
|
|
695
878
|
generatedAt: /* @__PURE__ */ new Date(),
|
|
@@ -707,24 +890,29 @@ var SitemapGenerator = class {
|
|
|
707
890
|
}
|
|
708
891
|
};
|
|
709
892
|
if (!isMultiFile) {
|
|
710
|
-
const xml = currentStream.toXML();
|
|
711
893
|
const entries = currentStream.getEntries();
|
|
894
|
+
let actualFilename;
|
|
895
|
+
if (this.shadowProcessor) {
|
|
896
|
+
actualFilename = this.options.compression?.enabled ? toGzipFilename(this.options.filename) : this.options.filename;
|
|
897
|
+
const xml = currentStream.toXML();
|
|
898
|
+
await this.shadowProcessor.addOperation({
|
|
899
|
+
filename: actualFilename,
|
|
900
|
+
content: xml
|
|
901
|
+
});
|
|
902
|
+
} else {
|
|
903
|
+
actualFilename = await this.writeSitemap(currentStream, this.options.filename);
|
|
904
|
+
}
|
|
712
905
|
shards.push({
|
|
713
|
-
filename:
|
|
906
|
+
filename: actualFilename,
|
|
714
907
|
from: entries[0] ? this.normalizeUrl(entries[0].url) : "",
|
|
715
908
|
to: entries[entries.length - 1] ? this.normalizeUrl(entries[entries.length - 1].url) : "",
|
|
716
909
|
count: entries.length,
|
|
717
910
|
lastmod: /* @__PURE__ */ new Date()
|
|
718
911
|
});
|
|
719
912
|
if (this.shadowProcessor) {
|
|
720
|
-
await this.shadowProcessor.addOperation({
|
|
721
|
-
filename: this.options.filename,
|
|
722
|
-
content: xml
|
|
723
|
-
});
|
|
724
913
|
await writeManifest();
|
|
725
914
|
await this.shadowProcessor.commit();
|
|
726
915
|
} else {
|
|
727
|
-
await this.options.storage.write(this.options.filename, xml);
|
|
728
916
|
await writeManifest();
|
|
729
917
|
}
|
|
730
918
|
return;
|
|
@@ -745,6 +933,41 @@ var SitemapGenerator = class {
|
|
|
745
933
|
await writeManifest();
|
|
746
934
|
}
|
|
747
935
|
}
|
|
936
|
+
/**
|
|
937
|
+
* 統一的 sitemap 寫入方法,優先使用串流寫入以降低記憶體使用。
|
|
938
|
+
*
|
|
939
|
+
* @param stream - SitemapStream 實例
|
|
940
|
+
* @param filename - 檔案名稱(不含 .gz)
|
|
941
|
+
* @returns 實際寫入的檔名(可能包含 .gz)
|
|
942
|
+
* @since 3.1.0
|
|
943
|
+
*/
|
|
944
|
+
async writeSitemap(stream, filename) {
|
|
945
|
+
const { storage, compression } = this.options;
|
|
946
|
+
const compress = compression?.enabled ?? false;
|
|
947
|
+
if (storage.writeStream) {
|
|
948
|
+
await storage.writeStream(filename, stream.toAsyncIterable(), {
|
|
949
|
+
compress,
|
|
950
|
+
contentType: "application/xml"
|
|
951
|
+
});
|
|
952
|
+
} else {
|
|
953
|
+
const xml = stream.toXML();
|
|
954
|
+
if (compress) {
|
|
955
|
+
const buffer = await compressToBuffer(
|
|
956
|
+
(async function* () {
|
|
957
|
+
yield xml;
|
|
958
|
+
})(),
|
|
959
|
+
{ level: compression?.level }
|
|
960
|
+
);
|
|
961
|
+
await storage.write(toGzipFilename(filename), buffer.toString("base64"));
|
|
962
|
+
} else {
|
|
963
|
+
await storage.write(filename, xml);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
return compress ? toGzipFilename(filename) : filename;
|
|
967
|
+
}
|
|
968
|
+
/**
|
|
969
|
+
* Normalizes a URL to an absolute URL using the base URL.
|
|
970
|
+
*/
|
|
748
971
|
normalizeUrl(url) {
|
|
749
972
|
if (url.startsWith("http")) {
|
|
750
973
|
return url;
|
|
@@ -755,7 +978,7 @@ var SitemapGenerator = class {
|
|
|
755
978
|
return normalizedBase + normalizedPath;
|
|
756
979
|
}
|
|
757
980
|
/**
|
|
758
|
-
*
|
|
981
|
+
* Returns the shadow processor instance if enabled.
|
|
759
982
|
*/
|
|
760
983
|
getShadowProcessor() {
|
|
761
984
|
return this.shadowProcessor;
|
|
@@ -764,6 +987,12 @@ var SitemapGenerator = class {
|
|
|
764
987
|
|
|
765
988
|
// src/core/SitemapParser.ts
|
|
766
989
|
var SitemapParser = class {
|
|
990
|
+
/**
|
|
991
|
+
* Parses a sitemap XML string into an array of entries.
|
|
992
|
+
*
|
|
993
|
+
* @param xml - The raw sitemap XML content.
|
|
994
|
+
* @returns An array of `SitemapEntry` objects.
|
|
995
|
+
*/
|
|
767
996
|
static parse(xml) {
|
|
768
997
|
const entries = [];
|
|
769
998
|
const urlRegex = /<url>([\s\S]*?)<\/url>/g;
|
|
@@ -776,6 +1005,14 @@ var SitemapParser = class {
|
|
|
776
1005
|
}
|
|
777
1006
|
return entries;
|
|
778
1007
|
}
|
|
1008
|
+
/**
|
|
1009
|
+
* Parses a sitemap XML stream into an async iterable of entries.
|
|
1010
|
+
*
|
|
1011
|
+
* Useful for large sitemap files that should not be fully loaded into memory.
|
|
1012
|
+
*
|
|
1013
|
+
* @param stream - An async iterable of XML chunks.
|
|
1014
|
+
* @returns An async iterable of `SitemapEntry` objects.
|
|
1015
|
+
*/
|
|
779
1016
|
static async *parseStream(stream) {
|
|
780
1017
|
let buffer = "";
|
|
781
1018
|
const urlRegex = /<url>([\s\S]*?)<\/url>/g;
|
|
@@ -796,6 +1033,9 @@ var SitemapParser = class {
|
|
|
796
1033
|
}
|
|
797
1034
|
}
|
|
798
1035
|
}
|
|
1036
|
+
/**
|
|
1037
|
+
* Parses a single `<url>` tag content into a `SitemapEntry`.
|
|
1038
|
+
*/
|
|
799
1039
|
static parseEntry(urlContent) {
|
|
800
1040
|
const entry = { url: "" };
|
|
801
1041
|
const locMatch = /<loc>(.*?)<\/loc>/.exec(urlContent);
|
|
@@ -818,6 +1058,12 @@ var SitemapParser = class {
|
|
|
818
1058
|
}
|
|
819
1059
|
return entry;
|
|
820
1060
|
}
|
|
1061
|
+
/**
|
|
1062
|
+
* Parses a sitemap index XML string into an array of sitemap URLs.
|
|
1063
|
+
*
|
|
1064
|
+
* @param xml - The raw sitemap index XML content.
|
|
1065
|
+
* @returns An array of sub-sitemap URLs.
|
|
1066
|
+
*/
|
|
821
1067
|
static parseIndex(xml) {
|
|
822
1068
|
const urls = [];
|
|
823
1069
|
const sitemapRegex = /<sitemap>([\s\S]*?)<\/sitemap>/g;
|
|
@@ -831,6 +1077,9 @@ var SitemapParser = class {
|
|
|
831
1077
|
}
|
|
832
1078
|
return urls;
|
|
833
1079
|
}
|
|
1080
|
+
/**
|
|
1081
|
+
* Unescapes special XML entities in a string.
|
|
1082
|
+
*/
|
|
834
1083
|
static unescape(str) {
|
|
835
1084
|
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'");
|
|
836
1085
|
}
|
|
@@ -853,12 +1102,26 @@ var IncrementalGenerator = class {
|
|
|
853
1102
|
this.diffCalculator = this.options.diffCalculator || new DiffCalculator();
|
|
854
1103
|
this.generator = new SitemapGenerator(this.options);
|
|
855
1104
|
}
|
|
1105
|
+
/**
|
|
1106
|
+
* Performs a full sitemap generation and optionally records all entries in the change tracker.
|
|
1107
|
+
*/
|
|
856
1108
|
async generateFull() {
|
|
857
1109
|
return this.mutex.runExclusive(() => this.performFullGeneration());
|
|
858
1110
|
}
|
|
1111
|
+
/**
|
|
1112
|
+
* Performs an incremental sitemap update based on changes recorded since a specific time.
|
|
1113
|
+
*
|
|
1114
|
+
* If the number of changes exceeds a certain threshold (e.g., 30% of total URLs),
|
|
1115
|
+
* a full generation is triggered instead to ensure consistency.
|
|
1116
|
+
*
|
|
1117
|
+
* @param since - Optional start date for the incremental update.
|
|
1118
|
+
*/
|
|
859
1119
|
async generateIncremental(since) {
|
|
860
1120
|
return this.mutex.runExclusive(() => this.performIncrementalGeneration(since));
|
|
861
1121
|
}
|
|
1122
|
+
/**
|
|
1123
|
+
* Internal implementation of full sitemap generation.
|
|
1124
|
+
*/
|
|
862
1125
|
async performFullGeneration() {
|
|
863
1126
|
await this.generator.run();
|
|
864
1127
|
if (this.options.autoTrack) {
|
|
@@ -877,6 +1140,9 @@ var IncrementalGenerator = class {
|
|
|
877
1140
|
}
|
|
878
1141
|
}
|
|
879
1142
|
}
|
|
1143
|
+
/**
|
|
1144
|
+
* Internal implementation of incremental sitemap generation.
|
|
1145
|
+
*/
|
|
880
1146
|
async performIncrementalGeneration(since) {
|
|
881
1147
|
const changes = await this.changeTracker.getChanges(since);
|
|
882
1148
|
if (changes.length === 0) {
|
|
@@ -900,6 +1166,9 @@ var IncrementalGenerator = class {
|
|
|
900
1166
|
}
|
|
901
1167
|
await this.updateShards(manifest, affectedShards);
|
|
902
1168
|
}
|
|
1169
|
+
/**
|
|
1170
|
+
* Normalizes a URL to an absolute URL using the base URL.
|
|
1171
|
+
*/
|
|
903
1172
|
normalizeUrl(url) {
|
|
904
1173
|
if (url.startsWith("http")) {
|
|
905
1174
|
return url;
|
|
@@ -909,6 +1178,9 @@ var IncrementalGenerator = class {
|
|
|
909
1178
|
const normalizedPath = url.startsWith("/") ? url : `/${url}`;
|
|
910
1179
|
return normalizedBase + normalizedPath;
|
|
911
1180
|
}
|
|
1181
|
+
/**
|
|
1182
|
+
* Loads the sitemap shard manifest from storage.
|
|
1183
|
+
*/
|
|
912
1184
|
async loadManifest() {
|
|
913
1185
|
const filename = this.options.filename?.replace(/\.xml$/, "-manifest.json") || "sitemap-manifest.json";
|
|
914
1186
|
const content = await this.options.storage.read(filename);
|
|
@@ -921,6 +1193,9 @@ var IncrementalGenerator = class {
|
|
|
921
1193
|
return null;
|
|
922
1194
|
}
|
|
923
1195
|
}
|
|
1196
|
+
/**
|
|
1197
|
+
* Identifies which shards are affected by the given set of changes.
|
|
1198
|
+
*/
|
|
924
1199
|
getAffectedShards(manifest, changes) {
|
|
925
1200
|
const affected = /* @__PURE__ */ new Map();
|
|
926
1201
|
for (const change of changes) {
|
|
@@ -942,6 +1217,9 @@ var IncrementalGenerator = class {
|
|
|
942
1217
|
}
|
|
943
1218
|
return affected;
|
|
944
1219
|
}
|
|
1220
|
+
/**
|
|
1221
|
+
* Updates the affected shards in storage.
|
|
1222
|
+
*/
|
|
945
1223
|
async updateShards(manifest, affectedShards) {
|
|
946
1224
|
for (const [filename, shardChanges] of affectedShards) {
|
|
947
1225
|
const entries = [];
|
|
@@ -979,6 +1257,9 @@ var IncrementalGenerator = class {
|
|
|
979
1257
|
JSON.stringify(manifest, null, this.options.pretty ? 2 : 0)
|
|
980
1258
|
);
|
|
981
1259
|
}
|
|
1260
|
+
/**
|
|
1261
|
+
* Applies changes to a set of sitemap entries and returns the updated, sorted list.
|
|
1262
|
+
*/
|
|
982
1263
|
applyChanges(entries, changes) {
|
|
983
1264
|
const entryMap = /* @__PURE__ */ new Map();
|
|
984
1265
|
for (const entry of entries) {
|
|
@@ -1001,6 +1282,9 @@ var IncrementalGenerator = class {
|
|
|
1001
1282
|
(a, b) => this.normalizeUrl(a.url).localeCompare(this.normalizeUrl(b.url))
|
|
1002
1283
|
);
|
|
1003
1284
|
}
|
|
1285
|
+
/**
|
|
1286
|
+
* Helper to convert an async iterable into an array.
|
|
1287
|
+
*/
|
|
1004
1288
|
async toArray(iterable) {
|
|
1005
1289
|
const array = [];
|
|
1006
1290
|
for await (const item of iterable) {
|
|
@@ -1021,7 +1305,10 @@ var ProgressTracker = class {
|
|
|
1021
1305
|
this.updateInterval = options.updateInterval || 1e3;
|
|
1022
1306
|
}
|
|
1023
1307
|
/**
|
|
1024
|
-
*
|
|
1308
|
+
* Initializes progress tracking for a new job.
|
|
1309
|
+
*
|
|
1310
|
+
* @param jobId - Unique identifier for the generation job.
|
|
1311
|
+
* @param total - Total number of entries to be processed.
|
|
1025
1312
|
*/
|
|
1026
1313
|
async init(jobId, total) {
|
|
1027
1314
|
this.currentProgress = {
|
|
@@ -1035,7 +1322,13 @@ var ProgressTracker = class {
|
|
|
1035
1322
|
await this.storage.set(jobId, this.currentProgress);
|
|
1036
1323
|
}
|
|
1037
1324
|
/**
|
|
1038
|
-
*
|
|
1325
|
+
* Updates the current progress of the job.
|
|
1326
|
+
*
|
|
1327
|
+
* Updates are debounced and flushed to storage at regular intervals
|
|
1328
|
+
* specified by `updateInterval` to avoid excessive write operations.
|
|
1329
|
+
*
|
|
1330
|
+
* @param processed - Number of entries processed so far.
|
|
1331
|
+
* @param status - Optional new status for the job.
|
|
1039
1332
|
*/
|
|
1040
1333
|
async update(processed, status) {
|
|
1041
1334
|
if (!this.currentProgress) {
|
|
@@ -1057,7 +1350,7 @@ var ProgressTracker = class {
|
|
|
1057
1350
|
}
|
|
1058
1351
|
}
|
|
1059
1352
|
/**
|
|
1060
|
-
*
|
|
1353
|
+
* Marks the current job as successfully completed.
|
|
1061
1354
|
*/
|
|
1062
1355
|
async complete() {
|
|
1063
1356
|
if (!this.currentProgress) {
|
|
@@ -1070,7 +1363,9 @@ var ProgressTracker = class {
|
|
|
1070
1363
|
this.stop();
|
|
1071
1364
|
}
|
|
1072
1365
|
/**
|
|
1073
|
-
*
|
|
1366
|
+
* Marks the current job as failed with an error message.
|
|
1367
|
+
*
|
|
1368
|
+
* @param error - The error message describing why the job failed.
|
|
1074
1369
|
*/
|
|
1075
1370
|
async fail(error) {
|
|
1076
1371
|
if (!this.currentProgress) {
|
|
@@ -1083,7 +1378,7 @@ var ProgressTracker = class {
|
|
|
1083
1378
|
this.stop();
|
|
1084
1379
|
}
|
|
1085
1380
|
/**
|
|
1086
|
-
*
|
|
1381
|
+
* Flushes the current progress state to the storage backend.
|
|
1087
1382
|
*/
|
|
1088
1383
|
async flush() {
|
|
1089
1384
|
if (!this.currentProgress) {
|
|
@@ -1098,7 +1393,7 @@ var ProgressTracker = class {
|
|
|
1098
1393
|
});
|
|
1099
1394
|
}
|
|
1100
1395
|
/**
|
|
1101
|
-
*
|
|
1396
|
+
* Stops the periodic update timer.
|
|
1102
1397
|
*/
|
|
1103
1398
|
stop() {
|
|
1104
1399
|
if (this.updateTimer) {
|
|
@@ -1107,7 +1402,9 @@ var ProgressTracker = class {
|
|
|
1107
1402
|
}
|
|
1108
1403
|
}
|
|
1109
1404
|
/**
|
|
1110
|
-
*
|
|
1405
|
+
* Returns a copy of the current progress state.
|
|
1406
|
+
*
|
|
1407
|
+
* @returns The current SitemapProgress object, or null if no job is active.
|
|
1111
1408
|
*/
|
|
1112
1409
|
getCurrentProgress() {
|
|
1113
1410
|
return this.currentProgress ? { ...this.currentProgress } : null;
|
|
@@ -1145,6 +1442,12 @@ var GenerateSitemapJob = class extends Job {
|
|
|
1145
1442
|
this.options = options;
|
|
1146
1443
|
this.generator = new SitemapGenerator(options.generatorOptions);
|
|
1147
1444
|
}
|
|
1445
|
+
/**
|
|
1446
|
+
* Main entry point for the job execution.
|
|
1447
|
+
*
|
|
1448
|
+
* Orchestrates the full lifecycle of sitemap generation, including progress
|
|
1449
|
+
* initialization, generation, shadow commit, and error handling.
|
|
1450
|
+
*/
|
|
1148
1451
|
async handle() {
|
|
1149
1452
|
const { progressTracker, onComplete, onError } = this.options;
|
|
1150
1453
|
try {
|
|
@@ -1175,7 +1478,9 @@ var GenerateSitemapJob = class extends Job {
|
|
|
1175
1478
|
}
|
|
1176
1479
|
}
|
|
1177
1480
|
/**
|
|
1178
|
-
*
|
|
1481
|
+
* Calculates the total number of URL entries from all providers.
|
|
1482
|
+
*
|
|
1483
|
+
* @returns A promise resolving to the total entry count.
|
|
1179
1484
|
*/
|
|
1180
1485
|
async calculateTotal() {
|
|
1181
1486
|
let total = 0;
|
|
@@ -1193,7 +1498,7 @@ var GenerateSitemapJob = class extends Job {
|
|
|
1193
1498
|
return total;
|
|
1194
1499
|
}
|
|
1195
1500
|
/**
|
|
1196
|
-
*
|
|
1501
|
+
* Performs sitemap generation while reporting progress to the tracker and callback.
|
|
1197
1502
|
*/
|
|
1198
1503
|
async generateWithProgress() {
|
|
1199
1504
|
const { progressTracker, onProgress } = this.options;
|
|
@@ -1212,8 +1517,558 @@ var GenerateSitemapJob = class extends Job {
|
|
|
1212
1517
|
}
|
|
1213
1518
|
};
|
|
1214
1519
|
|
|
1215
|
-
// src/
|
|
1520
|
+
// src/locks/MemoryLock.ts
|
|
1521
|
+
var MemoryLock = class {
|
|
1522
|
+
/**
|
|
1523
|
+
* Internal map storing resource identifiers to their lock expiration timestamps.
|
|
1524
|
+
*
|
|
1525
|
+
* Keys represent unique resource identifiers (e.g., 'sitemap-generation').
|
|
1526
|
+
* Values are Unix timestamps in milliseconds representing when the lock expires.
|
|
1527
|
+
* Expired locks are automatically cleaned up during `acquire()` and `isLocked()` calls.
|
|
1528
|
+
*/
|
|
1529
|
+
locks = /* @__PURE__ */ new Map();
|
|
1530
|
+
/**
|
|
1531
|
+
* Attempts to acquire an exclusive lock on the specified resource.
|
|
1532
|
+
*
|
|
1533
|
+
* Uses a test-and-set approach: checks if the lock exists and is valid, then atomically
|
|
1534
|
+
* sets the lock if available. Expired locks are treated as available and automatically
|
|
1535
|
+
* replaced during acquisition.
|
|
1536
|
+
*
|
|
1537
|
+
* **Behavior:**
|
|
1538
|
+
* - Returns `true` if lock was successfully acquired
|
|
1539
|
+
* - Returns `false` if resource is already locked by another caller
|
|
1540
|
+
* - Automatically replaces expired locks (acts as self-healing mechanism)
|
|
1541
|
+
* - Lock automatically expires after TTL milliseconds
|
|
1542
|
+
*
|
|
1543
|
+
* **Race condition handling:**
|
|
1544
|
+
* Safe within a single process due to JavaScript's single-threaded event loop.
|
|
1545
|
+
* NOT safe across multiple processes or instances (use RedisLock for that).
|
|
1546
|
+
*
|
|
1547
|
+
* @param resource - Unique identifier for the resource to lock (e.g., 'sitemap-generation', 'blog-index').
|
|
1548
|
+
* Should be consistent across all callers attempting to lock the same resource.
|
|
1549
|
+
* @param ttl - Time-to-live in milliseconds. Lock automatically expires after this duration.
|
|
1550
|
+
* Recommended: 2-5x the expected operation duration to prevent premature expiration.
|
|
1551
|
+
* @returns Promise resolving to `true` if lock acquired, `false` if already locked.
|
|
1552
|
+
*
|
|
1553
|
+
* @example Preventing concurrent sitemap generation
|
|
1554
|
+
* ```typescript
|
|
1555
|
+
* const lock = new MemoryLock()
|
|
1556
|
+
* const acquired = await lock.acquire('sitemap-generation', 60000)
|
|
1557
|
+
*
|
|
1558
|
+
* if (!acquired) {
|
|
1559
|
+
* console.log('Another process is already generating the sitemap')
|
|
1560
|
+
* return new Response('Generation in progress', { status: 503 })
|
|
1561
|
+
* }
|
|
1562
|
+
*
|
|
1563
|
+
* try {
|
|
1564
|
+
* await generateSitemap()
|
|
1565
|
+
* } finally {
|
|
1566
|
+
* await lock.release('sitemap-generation')
|
|
1567
|
+
* }
|
|
1568
|
+
* ```
|
|
1569
|
+
*
|
|
1570
|
+
* @example Setting appropriate TTL
|
|
1571
|
+
* ```typescript
|
|
1572
|
+
* // For fast operations (< 1 second), use short TTL
|
|
1573
|
+
* await lock.acquire('cache-refresh', 5000)
|
|
1574
|
+
*
|
|
1575
|
+
* // For slow operations (minutes), use longer TTL
|
|
1576
|
+
* await lock.acquire('full-reindex', 300000) // 5 minutes
|
|
1577
|
+
* ```
|
|
1578
|
+
*/
|
|
1579
|
+
async acquire(resource, ttl) {
|
|
1580
|
+
const now = Date.now();
|
|
1581
|
+
const expiresAt = this.locks.get(resource);
|
|
1582
|
+
if (expiresAt && expiresAt > now) {
|
|
1583
|
+
return false;
|
|
1584
|
+
}
|
|
1585
|
+
this.locks.set(resource, now + ttl);
|
|
1586
|
+
return true;
|
|
1587
|
+
}
|
|
1588
|
+
/**
|
|
1589
|
+
* Releases the lock on the specified resource, allowing others to acquire it.
|
|
1590
|
+
*
|
|
1591
|
+
* Immediately removes the lock from memory without any ownership validation.
|
|
1592
|
+
* Unlike RedisLock, this does NOT verify that the caller is the lock owner,
|
|
1593
|
+
* so callers must ensure they only release locks they acquired.
|
|
1594
|
+
*
|
|
1595
|
+
* **Best practices:**
|
|
1596
|
+
* - Always call `release()` in a `finally` block to prevent lock leakage
|
|
1597
|
+
* - Only release locks you successfully acquired
|
|
1598
|
+
* - If operation fails, still release the lock to allow retry
|
|
1599
|
+
*
|
|
1600
|
+
* **Idempotency:**
|
|
1601
|
+
* Safe to call multiple times on the same resource. Releasing a non-existent
|
|
1602
|
+
* lock is a no-op.
|
|
1603
|
+
*
|
|
1604
|
+
* @param resource - The resource identifier to unlock. Must match the identifier
|
|
1605
|
+
* used in the corresponding `acquire()` call.
|
|
1606
|
+
*
|
|
1607
|
+
* @example Proper release pattern with try-finally
|
|
1608
|
+
* ```typescript
|
|
1609
|
+
* const acquired = await lock.acquire('sitemap-generation', 60000)
|
|
1610
|
+
* if (!acquired) return
|
|
1611
|
+
*
|
|
1612
|
+
* try {
|
|
1613
|
+
* await generateSitemap()
|
|
1614
|
+
* } finally {
|
|
1615
|
+
* // Always release, even if operation throws
|
|
1616
|
+
* await lock.release('sitemap-generation')
|
|
1617
|
+
* }
|
|
1618
|
+
* ```
|
|
1619
|
+
*
|
|
1620
|
+
* @example Handling operation failures
|
|
1621
|
+
* ```typescript
|
|
1622
|
+
* const acquired = await lock.acquire('data-import', 120000)
|
|
1623
|
+
* if (!acquired) return
|
|
1624
|
+
*
|
|
1625
|
+
* try {
|
|
1626
|
+
* await importData()
|
|
1627
|
+
* } catch (error) {
|
|
1628
|
+
* console.error('Import failed:', error)
|
|
1629
|
+
* // Lock still released in finally block
|
|
1630
|
+
* throw error
|
|
1631
|
+
* } finally {
|
|
1632
|
+
* await lock.release('data-import')
|
|
1633
|
+
* }
|
|
1634
|
+
* ```
|
|
1635
|
+
*/
|
|
1636
|
+
async release(resource) {
|
|
1637
|
+
this.locks.delete(resource);
|
|
1638
|
+
}
|
|
1639
|
+
/**
|
|
1640
|
+
* Checks whether a resource is currently locked and has not expired.
|
|
1641
|
+
*
|
|
1642
|
+
* Performs automatic cleanup by removing expired locks during the check,
|
|
1643
|
+
* ensuring the internal map doesn't accumulate stale entries over time.
|
|
1644
|
+
*
|
|
1645
|
+
* **Use cases:**
|
|
1646
|
+
* - Pre-flight checks before attempting expensive operations
|
|
1647
|
+
* - Status monitoring and health checks
|
|
1648
|
+
* - Implementing custom retry logic
|
|
1649
|
+
* - Debugging and testing
|
|
1650
|
+
*
|
|
1651
|
+
* **Side effects:**
|
|
1652
|
+
* Automatically deletes expired locks as a garbage collection mechanism.
|
|
1653
|
+
* This is intentional to prevent memory leaks from abandoned locks.
|
|
1654
|
+
*
|
|
1655
|
+
* @param resource - The resource identifier to check for lock status.
|
|
1656
|
+
* @returns Promise resolving to `true` if resource is actively locked (not expired),
|
|
1657
|
+
* `false` if unlocked or lock has expired.
|
|
1658
|
+
*
|
|
1659
|
+
* @example Pre-flight check before starting work
|
|
1660
|
+
* ```typescript
|
|
1661
|
+
* const lock = new MemoryLock()
|
|
1662
|
+
*
|
|
1663
|
+
* if (await lock.isLocked('sitemap-generation')) {
|
|
1664
|
+
* console.log('Sitemap generation already in progress')
|
|
1665
|
+
* return
|
|
1666
|
+
* }
|
|
1667
|
+
*
|
|
1668
|
+
* // Safe to proceed
|
|
1669
|
+
* await lock.acquire('sitemap-generation', 60000)
|
|
1670
|
+
* ```
|
|
1671
|
+
*
|
|
1672
|
+
* @example Health check endpoint
|
|
1673
|
+
* ```typescript
|
|
1674
|
+
* app.get('/health/locks', async (c) => {
|
|
1675
|
+
* const isGenerating = await lock.isLocked('sitemap-generation')
|
|
1676
|
+
* const isIndexing = await lock.isLocked('search-indexing')
|
|
1677
|
+
*
|
|
1678
|
+
* return c.json({
|
|
1679
|
+
* sitemapGeneration: isGenerating ? 'in-progress' : 'idle',
|
|
1680
|
+
* searchIndexing: isIndexing ? 'in-progress' : 'idle'
|
|
1681
|
+
* })
|
|
1682
|
+
* })
|
|
1683
|
+
* ```
|
|
1684
|
+
*
|
|
1685
|
+
* @example Custom retry logic
|
|
1686
|
+
* ```typescript
|
|
1687
|
+
* let attempts = 0
|
|
1688
|
+
* while (attempts < 5) {
|
|
1689
|
+
* if (!await lock.isLocked('resource')) {
|
|
1690
|
+
* const acquired = await lock.acquire('resource', 10000)
|
|
1691
|
+
* if (acquired) break
|
|
1692
|
+
* }
|
|
1693
|
+
* await sleep(1000)
|
|
1694
|
+
* attempts++
|
|
1695
|
+
* }
|
|
1696
|
+
* ```
|
|
1697
|
+
*/
|
|
1698
|
+
async isLocked(resource) {
|
|
1699
|
+
const expiresAt = this.locks.get(resource);
|
|
1700
|
+
if (!expiresAt) {
|
|
1701
|
+
return false;
|
|
1702
|
+
}
|
|
1703
|
+
const now = Date.now();
|
|
1704
|
+
if (expiresAt <= now) {
|
|
1705
|
+
this.locks.delete(resource);
|
|
1706
|
+
return false;
|
|
1707
|
+
}
|
|
1708
|
+
return true;
|
|
1709
|
+
}
|
|
1710
|
+
/**
|
|
1711
|
+
* Clears all locks from memory, including both active and expired locks.
|
|
1712
|
+
*
|
|
1713
|
+
* **Use cases:**
|
|
1714
|
+
* - Test cleanup between test cases to ensure isolation
|
|
1715
|
+
* - Application shutdown to release all resources
|
|
1716
|
+
* - Manual intervention during debugging
|
|
1717
|
+
* - Resetting state after catastrophic errors
|
|
1718
|
+
*
|
|
1719
|
+
* **Warning:**
|
|
1720
|
+
* This forcibly releases ALL locks without any ownership validation.
|
|
1721
|
+
* Should not be called during normal operation in production environments.
|
|
1722
|
+
*
|
|
1723
|
+
* @example Test cleanup with beforeEach hook
|
|
1724
|
+
* ```typescript
|
|
1725
|
+
* import { describe, beforeEach, test } from 'vitest'
|
|
1726
|
+
*
|
|
1727
|
+
* const lock = new MemoryLock()
|
|
1728
|
+
*
|
|
1729
|
+
* beforeEach(async () => {
|
|
1730
|
+
* await lock.clear() // Ensure clean state for each test
|
|
1731
|
+
* })
|
|
1732
|
+
*
|
|
1733
|
+
* test('lock acquisition', async () => {
|
|
1734
|
+
* const acquired = await lock.acquire('test-resource', 5000)
|
|
1735
|
+
* expect(acquired).toBe(true)
|
|
1736
|
+
* })
|
|
1737
|
+
* ```
|
|
1738
|
+
*
|
|
1739
|
+
* @example Graceful shutdown handler
|
|
1740
|
+
* ```typescript
|
|
1741
|
+
* process.on('SIGTERM', async () => {
|
|
1742
|
+
* console.log('Shutting down, releasing all locks...')
|
|
1743
|
+
* await lock.clear()
|
|
1744
|
+
* process.exit(0)
|
|
1745
|
+
* })
|
|
1746
|
+
* ```
|
|
1747
|
+
*/
|
|
1748
|
+
async clear() {
|
|
1749
|
+
this.locks.clear();
|
|
1750
|
+
}
|
|
1751
|
+
/**
|
|
1752
|
+
* Returns the number of lock entries currently stored in memory.
|
|
1753
|
+
*
|
|
1754
|
+
* **Important:** This includes BOTH active and expired locks. Expired locks
|
|
1755
|
+
* are only cleaned up during `acquire()` or `isLocked()` calls, so this count
|
|
1756
|
+
* may include stale entries.
|
|
1757
|
+
*
|
|
1758
|
+
* **Use cases:**
|
|
1759
|
+
* - Monitoring memory usage and lock accumulation
|
|
1760
|
+
* - Debugging lock leakage issues
|
|
1761
|
+
* - Testing lock lifecycle behavior
|
|
1762
|
+
* - Detecting abnormal lock retention patterns
|
|
1763
|
+
*
|
|
1764
|
+
* **Not suitable for:**
|
|
1765
|
+
* - Determining number of ACTIVE locks (use `isLocked()` on each resource)
|
|
1766
|
+
* - Production health checks (includes expired locks)
|
|
1767
|
+
*
|
|
1768
|
+
* @returns The total number of lock entries in the internal Map, including expired ones.
|
|
1769
|
+
*
|
|
1770
|
+
* @example Monitoring lock accumulation
|
|
1771
|
+
* ```typescript
|
|
1772
|
+
* const lock = new MemoryLock()
|
|
1773
|
+
*
|
|
1774
|
+
* setInterval(() => {
|
|
1775
|
+
* const count = lock.size()
|
|
1776
|
+
* if (count > 100) {
|
|
1777
|
+
* console.warn(`High lock count detected: ${count}`)
|
|
1778
|
+
* // May indicate lock leakage or missing release() calls
|
|
1779
|
+
* }
|
|
1780
|
+
* }, 60000)
|
|
1781
|
+
* ```
|
|
1782
|
+
*
|
|
1783
|
+
* @example Testing lock cleanup behavior
|
|
1784
|
+
* ```typescript
|
|
1785
|
+
* import { test, expect } from 'vitest'
|
|
1786
|
+
*
|
|
1787
|
+
* test('expired locks are cleaned up', async () => {
|
|
1788
|
+
* const lock = new MemoryLock()
|
|
1789
|
+
*
|
|
1790
|
+
* await lock.acquire('resource', 10)
|
|
1791
|
+
* expect(lock.size()).toBe(1)
|
|
1792
|
+
*
|
|
1793
|
+
* await sleep(20) // Wait for expiration
|
|
1794
|
+
* expect(lock.size()).toBe(1) // Still in map (not cleaned yet)
|
|
1795
|
+
*
|
|
1796
|
+
* await lock.isLocked('resource') // Triggers cleanup
|
|
1797
|
+
* expect(lock.size()).toBe(0) // Now removed
|
|
1798
|
+
* })
|
|
1799
|
+
* ```
|
|
1800
|
+
*/
|
|
1801
|
+
size() {
|
|
1802
|
+
return this.locks.size;
|
|
1803
|
+
}
|
|
1804
|
+
};
|
|
1805
|
+
|
|
1806
|
+
// src/locks/RedisLock.ts
|
|
1216
1807
|
import { randomUUID } from "crypto";
|
|
1808
|
+
var RedisLock = class {
|
|
1809
|
+
/**
|
|
1810
|
+
* Constructs a new RedisLock instance with the specified configuration.
|
|
1811
|
+
*
|
|
1812
|
+
* @param options - Configuration including Redis client and retry parameters.
|
|
1813
|
+
*
|
|
1814
|
+
* @example With custom retry strategy
|
|
1815
|
+
* ```typescript
|
|
1816
|
+
* const lock = new RedisLock({
|
|
1817
|
+
* client: redisClient,
|
|
1818
|
+
* keyPrefix: 'app:locks:',
|
|
1819
|
+
* retryCount: 10, // More retries for high-contention scenarios
|
|
1820
|
+
* retryDelay: 50 // Shorter delay for low-latency requirements
|
|
1821
|
+
* })
|
|
1822
|
+
* ```
|
|
1823
|
+
*/
|
|
1824
|
+
constructor(options) {
|
|
1825
|
+
this.options = options;
|
|
1826
|
+
this.keyPrefix = options.keyPrefix || "sitemap:lock:";
|
|
1827
|
+
this.retryCount = options.retryCount ?? 0;
|
|
1828
|
+
this.retryDelay = options.retryDelay ?? 100;
|
|
1829
|
+
}
|
|
1830
|
+
/**
|
|
1831
|
+
* Unique identifier for this lock instance.
|
|
1832
|
+
*
|
|
1833
|
+
* Generated once during construction and used for all locks acquired by this instance.
|
|
1834
|
+
* Enables ownership validation: only the instance that acquired the lock can release it.
|
|
1835
|
+
*
|
|
1836
|
+
* **Security consideration:**
|
|
1837
|
+
* UUIDs are sufficiently random to prevent lock hijacking across instances.
|
|
1838
|
+
* However, they are stored in plain text in Redis (not encrypted).
|
|
1839
|
+
*/
|
|
1840
|
+
lockId = randomUUID();
|
|
1841
|
+
/**
|
|
1842
|
+
* Redis key prefix for all locks acquired through this instance.
|
|
1843
|
+
*
|
|
1844
|
+
* Combined with resource name to form full Redis key (e.g., 'sitemap:lock:generation').
|
|
1845
|
+
* Allows namespace isolation and easier debugging in Redis CLI.
|
|
1846
|
+
*/
|
|
1847
|
+
keyPrefix;
|
|
1848
|
+
/**
|
|
1849
|
+
* Maximum number of retry attempts when lock is held by another instance.
|
|
1850
|
+
*
|
|
1851
|
+
* Set to 0 for fail-fast behavior. Higher values increase acquisition success
|
|
1852
|
+
* rate but also increase latency under contention.
|
|
1853
|
+
*/
|
|
1854
|
+
retryCount;
|
|
1855
|
+
/**
|
|
1856
|
+
* Delay in milliseconds between consecutive retry attempts.
|
|
1857
|
+
*
|
|
1858
|
+
* Should be tuned based on expected lock hold time. Typical values: 50-500ms.
|
|
1859
|
+
*/
|
|
1860
|
+
retryDelay;
|
|
1861
|
+
/**
|
|
1862
|
+
* Attempts to acquire a distributed lock using Redis SET NX EX command.
|
|
1863
|
+
*
|
|
1864
|
+
* Uses atomic Redis operations to ensure only one instance across your entire
|
|
1865
|
+
* infrastructure can hold the lock at any given time. Implements retry logic
|
|
1866
|
+
* with exponential backoff for handling transient contention.
|
|
1867
|
+
*
|
|
1868
|
+
* **Algorithm:**
|
|
1869
|
+
* 1. Convert TTL from milliseconds to seconds (Redis requirement)
|
|
1870
|
+
* 2. Attempt Redis SET key lockId EX ttl NX (atomic operation)
|
|
1871
|
+
* 3. If successful (returns 'OK'), lock acquired
|
|
1872
|
+
* 4. If failed (returns null), retry up to retryCount times with retryDelay
|
|
1873
|
+
* 5. Return true if acquired, false if all attempts exhausted
|
|
1874
|
+
*
|
|
1875
|
+
* **Atomicity guarantee:**
|
|
1876
|
+
* The combination of NX (set if Not eXists) and EX (set EXpiration) in a single
|
|
1877
|
+
* Redis command ensures no race conditions. Either the lock is acquired or it isn't.
|
|
1878
|
+
*
|
|
1879
|
+
* **Error handling:**
|
|
1880
|
+
* Redis connection errors are caught, logged to console, and treated as acquisition
|
|
1881
|
+
* failure (returns false). This fail-safe behavior prevents exceptions from bubbling
|
|
1882
|
+
* up to application code.
|
|
1883
|
+
*
|
|
1884
|
+
* **Performance:**
|
|
1885
|
+
* - Single instance: O(1) Redis operation
|
|
1886
|
+
* - With retry: O(retryCount) worst case
|
|
1887
|
+
* - Network latency: ~1-5ms per attempt (depends on Redis location)
|
|
1888
|
+
*
|
|
1889
|
+
* @param resource - Unique identifier for the resource to lock.
|
|
1890
|
+
* Combined with keyPrefix to form Redis key.
|
|
1891
|
+
* @param ttl - Time-to-live in milliseconds. Lock auto-expires after this duration.
|
|
1892
|
+
* Recommended: 2-5x expected operation time to handle slowdowns.
|
|
1893
|
+
* Minimum: 1000ms (1 second) for practical use.
|
|
1894
|
+
* @returns Promise resolving to `true` if lock acquired, `false` if held by another instance
|
|
1895
|
+
* or Redis connection failed.
|
|
1896
|
+
*
|
|
1897
|
+
* @example Basic acquisition in distributed environment
|
|
1898
|
+
* ```typescript
|
|
1899
|
+
* const lock = new RedisLock({ client: redisClient })
|
|
1900
|
+
* const acquired = await lock.acquire('sitemap-generation', 60000)
|
|
1901
|
+
*
|
|
1902
|
+
* if (!acquired) {
|
|
1903
|
+
* // Another instance (e.g., different Kubernetes pod) holds the lock
|
|
1904
|
+
* console.log('Sitemap generation in progress on another instance')
|
|
1905
|
+
* return new Response('Service busy', {
|
|
1906
|
+
* status: 503,
|
|
1907
|
+
* headers: { 'Retry-After': '30' }
|
|
1908
|
+
* })
|
|
1909
|
+
* }
|
|
1910
|
+
*
|
|
1911
|
+
* try {
|
|
1912
|
+
* await generateSitemap()
|
|
1913
|
+
* } finally {
|
|
1914
|
+
* await lock.release('sitemap-generation')
|
|
1915
|
+
* }
|
|
1916
|
+
* ```
|
|
1917
|
+
*
|
|
1918
|
+
* @example With retry logic for transient contention
|
|
1919
|
+
* ```typescript
|
|
1920
|
+
* const lock = new RedisLock({
|
|
1921
|
+
* client: redisClient,
|
|
1922
|
+
* retryCount: 5,
|
|
1923
|
+
* retryDelay: 200
|
|
1924
|
+
* })
|
|
1925
|
+
*
|
|
1926
|
+
* // Will retry 5 times with 200ms delay between attempts
|
|
1927
|
+
* const acquired = await lock.acquire('data-import', 120000)
|
|
1928
|
+
* ```
|
|
1929
|
+
*
|
|
1930
|
+
* @example Setting appropriate TTL
|
|
1931
|
+
* ```typescript
|
|
1932
|
+
* // Fast operation: short TTL
|
|
1933
|
+
* await lock.acquire('cache-rebuild', 10000) // 10 seconds
|
|
1934
|
+
*
|
|
1935
|
+
* // Slow operation: longer TTL with buffer
|
|
1936
|
+
* await lock.acquire('full-sitemap', 300000) // 5 minutes
|
|
1937
|
+
*
|
|
1938
|
+
* // Very slow operation: generous TTL
|
|
1939
|
+
* await lock.acquire('data-migration', 1800000) // 30 minutes
|
|
1940
|
+
* ```
|
|
1941
|
+
*/
|
|
1942
|
+
async acquire(resource, ttl) {
|
|
1943
|
+
const key = this.keyPrefix + resource;
|
|
1944
|
+
const ttlSeconds = Math.ceil(ttl / 1e3);
|
|
1945
|
+
let attempts = 0;
|
|
1946
|
+
while (attempts <= this.retryCount) {
|
|
1947
|
+
try {
|
|
1948
|
+
const result = await this.options.client.set(key, this.lockId, "EX", ttlSeconds, "NX");
|
|
1949
|
+
if (result === "OK") {
|
|
1950
|
+
return true;
|
|
1951
|
+
}
|
|
1952
|
+
} catch (error) {
|
|
1953
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
1954
|
+
console.error(`[RedisLock] Failed to acquire lock for ${resource}:`, err.message);
|
|
1955
|
+
}
|
|
1956
|
+
attempts++;
|
|
1957
|
+
if (attempts <= this.retryCount) {
|
|
1958
|
+
await this.sleep(this.retryDelay);
|
|
1959
|
+
}
|
|
1960
|
+
}
|
|
1961
|
+
return false;
|
|
1962
|
+
}
|
|
1963
|
+
/**
|
|
1964
|
+
* Releases the distributed lock using Lua script for atomic ownership validation.
|
|
1965
|
+
*
|
|
1966
|
+
* Uses a Lua script to atomically check if the current instance owns the lock
|
|
1967
|
+
* (by comparing lockId) and delete it if so. This prevents accidentally releasing
|
|
1968
|
+
* locks held by other instances, which could cause data corruption in distributed systems.
|
|
1969
|
+
*
|
|
1970
|
+
* **Lua script logic:**
|
|
1971
|
+
* ```lua
|
|
1972
|
+
* if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
1973
|
+
* return redis.call("del", KEYS[1]) -- Delete only if owner matches
|
|
1974
|
+
* else
|
|
1975
|
+
* return 0 -- Not owner or lock expired, do nothing
|
|
1976
|
+
* end
|
|
1977
|
+
* ```
|
|
1978
|
+
*
|
|
1979
|
+
* **Why Lua scripts?**
|
|
1980
|
+
* - Atomicity: GET + comparison + DEL execute as one atomic operation
|
|
1981
|
+
* - Prevents race conditions: Lock cannot change between check and delete
|
|
1982
|
+
* - Server-side execution: No network round trips between steps
|
|
1983
|
+
*
|
|
1984
|
+
* **Error handling:**
|
|
1985
|
+
* Errors during release (e.g., Redis connection loss) are logged but do NOT throw.
|
|
1986
|
+
* This is intentional: if instance crashed or TTL expired, lock is already released.
|
|
1987
|
+
* Silent failure here prevents cascading errors in finally blocks.
|
|
1988
|
+
*
|
|
1989
|
+
* **Idempotency:**
|
|
1990
|
+
* Safe to call multiple times. Releasing an already-released or expired lock is a no-op.
|
|
1991
|
+
*
|
|
1992
|
+
* @param resource - The resource identifier to unlock. Must match the identifier
|
|
1993
|
+
* used in the corresponding `acquire()` call.
|
|
1994
|
+
*
|
|
1995
|
+
* @example Proper release pattern with try-finally
|
|
1996
|
+
* ```typescript
|
|
1997
|
+
* const acquired = await lock.acquire('sitemap-generation', 60000)
|
|
1998
|
+
* if (!acquired) return
|
|
1999
|
+
*
|
|
2000
|
+
* try {
|
|
2001
|
+
* await generateSitemap()
|
|
2002
|
+
* } finally {
|
|
2003
|
+
* // Always release, even if operation throws
|
|
2004
|
+
* await lock.release('sitemap-generation')
|
|
2005
|
+
* }
|
|
2006
|
+
* ```
|
|
2007
|
+
*
|
|
2008
|
+
* @example Handling operation failures
|
|
2009
|
+
* ```typescript
|
|
2010
|
+
* const acquired = await lock.acquire('data-processing', 120000)
|
|
2011
|
+
* if (!acquired) {
|
|
2012
|
+
* throw new Error('Could not acquire lock')
|
|
2013
|
+
* }
|
|
2014
|
+
*
|
|
2015
|
+
* try {
|
|
2016
|
+
* await processData()
|
|
2017
|
+
* } catch (error) {
|
|
2018
|
+
* console.error('Processing failed:', error)
|
|
2019
|
+
* // Lock still released in finally block
|
|
2020
|
+
* throw error
|
|
2021
|
+
* } finally {
|
|
2022
|
+
* await lock.release('data-processing')
|
|
2023
|
+
* }
|
|
2024
|
+
* ```
|
|
2025
|
+
*
|
|
2026
|
+
* @example Why ownership validation matters
|
|
2027
|
+
* ```typescript
|
|
2028
|
+
* // Instance A acquires lock with 10-second TTL
|
|
2029
|
+
* const lockA = new RedisLock({ client: redisClientA })
|
|
2030
|
+
* await lockA.acquire('task', 10000)
|
|
2031
|
+
*
|
|
2032
|
+
* // ... 11 seconds pass, lock auto-expires ...
|
|
2033
|
+
*
|
|
2034
|
+
* // Instance B acquires the now-expired lock
|
|
2035
|
+
* const lockB = new RedisLock({ client: redisClientB })
|
|
2036
|
+
* await lockB.acquire('task', 10000)
|
|
2037
|
+
*
|
|
2038
|
+
* // Instance A tries to release (after slowdown/GC pause)
|
|
2039
|
+
* await lockA.release('task')
|
|
2040
|
+
* // ✅ Lua script detects lockId mismatch, does NOT delete B's lock
|
|
2041
|
+
* // ❌ Without Lua: Would delete B's lock, causing data corruption
|
|
2042
|
+
* ```
|
|
2043
|
+
*/
|
|
2044
|
+
async release(resource) {
|
|
2045
|
+
const key = this.keyPrefix + resource;
|
|
2046
|
+
try {
|
|
2047
|
+
const script = `
|
|
2048
|
+
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
2049
|
+
return redis.call("del", KEYS[1])
|
|
2050
|
+
else
|
|
2051
|
+
return 0
|
|
2052
|
+
end
|
|
2053
|
+
`;
|
|
2054
|
+
await this.options.client.eval(script, 1, key, this.lockId);
|
|
2055
|
+
} catch (error) {
|
|
2056
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
2057
|
+
console.error(`[RedisLock] Failed to release lock for ${resource}:`, err.message);
|
|
2058
|
+
}
|
|
2059
|
+
}
|
|
2060
|
+
/**
|
|
2061
|
+
* Internal utility for sleeping between retry attempts.
|
|
2062
|
+
*
|
|
2063
|
+
* @param ms - Duration to sleep in milliseconds
|
|
2064
|
+
*/
|
|
2065
|
+
sleep(ms) {
|
|
2066
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2067
|
+
}
|
|
2068
|
+
};
|
|
2069
|
+
|
|
2070
|
+
// src/OrbitSitemap.ts
|
|
2071
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
1217
2072
|
|
|
1218
2073
|
// src/redirect/RedirectHandler.ts
|
|
1219
2074
|
var RedirectHandler = class {
|
|
@@ -1222,7 +2077,10 @@ var RedirectHandler = class {
|
|
|
1222
2077
|
this.options = options;
|
|
1223
2078
|
}
|
|
1224
2079
|
/**
|
|
1225
|
-
*
|
|
2080
|
+
* Processes a list of sitemap entries and handles redirects according to the configured strategy.
|
|
2081
|
+
*
|
|
2082
|
+
* @param entries - The original list of sitemap entries.
|
|
2083
|
+
* @returns A promise resolving to the processed list of entries.
|
|
1226
2084
|
*/
|
|
1227
2085
|
async processEntries(entries) {
|
|
1228
2086
|
const { manager, strategy, followChains, maxChainLength } = this.options;
|
|
@@ -1253,7 +2111,7 @@ var RedirectHandler = class {
|
|
|
1253
2111
|
}
|
|
1254
2112
|
}
|
|
1255
2113
|
/**
|
|
1256
|
-
*
|
|
2114
|
+
* Strategy 1: Remove old URL and add the new destination URL.
|
|
1257
2115
|
*/
|
|
1258
2116
|
handleRemoveOldAddNew(entries, redirectMap) {
|
|
1259
2117
|
const processed = [];
|
|
@@ -1278,7 +2136,7 @@ var RedirectHandler = class {
|
|
|
1278
2136
|
return processed;
|
|
1279
2137
|
}
|
|
1280
2138
|
/**
|
|
1281
|
-
*
|
|
2139
|
+
* Strategy 2: Keep the original URL but mark the destination as canonical.
|
|
1282
2140
|
*/
|
|
1283
2141
|
handleKeepRelation(entries, redirectMap) {
|
|
1284
2142
|
const processed = [];
|
|
@@ -1301,7 +2159,7 @@ var RedirectHandler = class {
|
|
|
1301
2159
|
return processed;
|
|
1302
2160
|
}
|
|
1303
2161
|
/**
|
|
1304
|
-
*
|
|
2162
|
+
* Strategy 3: Silently update the URL to the destination.
|
|
1305
2163
|
*/
|
|
1306
2164
|
handleUpdateUrl(entries, redirectMap) {
|
|
1307
2165
|
return entries.map((entry) => {
|
|
@@ -1321,7 +2179,7 @@ var RedirectHandler = class {
|
|
|
1321
2179
|
});
|
|
1322
2180
|
}
|
|
1323
2181
|
/**
|
|
1324
|
-
*
|
|
2182
|
+
* Strategy 4: Include both the original and destination URLs.
|
|
1325
2183
|
*/
|
|
1326
2184
|
handleDualMark(entries, redirectMap) {
|
|
1327
2185
|
const processed = [];
|
|
@@ -1363,12 +2221,58 @@ var MemorySitemapStorage = class {
|
|
|
1363
2221
|
this.baseUrl = baseUrl;
|
|
1364
2222
|
}
|
|
1365
2223
|
files = /* @__PURE__ */ new Map();
|
|
2224
|
+
/**
|
|
2225
|
+
* Writes sitemap content to memory.
|
|
2226
|
+
*
|
|
2227
|
+
* @param filename - The name of the file to store.
|
|
2228
|
+
* @param content - The XML or JSON content.
|
|
2229
|
+
*/
|
|
1366
2230
|
async write(filename, content) {
|
|
1367
2231
|
this.files.set(filename, content);
|
|
1368
2232
|
}
|
|
2233
|
+
/**
|
|
2234
|
+
* 使用串流方式寫入 sitemap 至記憶體,可選擇性啟用 gzip 壓縮。
|
|
2235
|
+
* 記憶體儲存會收集串流為完整字串。
|
|
2236
|
+
*
|
|
2237
|
+
* @param filename - 檔案名稱
|
|
2238
|
+
* @param stream - XML 內容的 AsyncIterable
|
|
2239
|
+
* @param options - 寫入選項(如壓縮)
|
|
2240
|
+
*
|
|
2241
|
+
* @since 3.1.0
|
|
2242
|
+
*/
|
|
2243
|
+
async writeStream(filename, stream, options) {
|
|
2244
|
+
const chunks = [];
|
|
2245
|
+
for await (const chunk of stream) {
|
|
2246
|
+
chunks.push(chunk);
|
|
2247
|
+
}
|
|
2248
|
+
const content = chunks.join("");
|
|
2249
|
+
const key = options?.compress ? toGzipFilename(filename) : filename;
|
|
2250
|
+
if (options?.compress) {
|
|
2251
|
+
const compressed = await compressToBuffer(
|
|
2252
|
+
(async function* () {
|
|
2253
|
+
yield content;
|
|
2254
|
+
})()
|
|
2255
|
+
);
|
|
2256
|
+
this.files.set(key, compressed.toString("base64"));
|
|
2257
|
+
} else {
|
|
2258
|
+
this.files.set(key, content);
|
|
2259
|
+
}
|
|
2260
|
+
}
|
|
2261
|
+
/**
|
|
2262
|
+
* Reads sitemap content from memory.
|
|
2263
|
+
*
|
|
2264
|
+
* @param filename - The name of the file to read.
|
|
2265
|
+
* @returns A promise resolving to the file content as a string, or null if not found.
|
|
2266
|
+
*/
|
|
1369
2267
|
async read(filename) {
|
|
1370
2268
|
return this.files.get(filename) || null;
|
|
1371
2269
|
}
|
|
2270
|
+
/**
|
|
2271
|
+
* Returns a readable stream for a sitemap file in memory.
|
|
2272
|
+
*
|
|
2273
|
+
* @param filename - The name of the file to stream.
|
|
2274
|
+
* @returns A promise resolving to an async iterable of file chunks, or null if not found.
|
|
2275
|
+
*/
|
|
1372
2276
|
async readStream(filename) {
|
|
1373
2277
|
const content = this.files.get(filename);
|
|
1374
2278
|
if (content === void 0) {
|
|
@@ -1378,9 +2282,21 @@ var MemorySitemapStorage = class {
|
|
|
1378
2282
|
yield content;
|
|
1379
2283
|
})();
|
|
1380
2284
|
}
|
|
2285
|
+
/**
|
|
2286
|
+
* Checks if a sitemap file exists in memory.
|
|
2287
|
+
*
|
|
2288
|
+
* @param filename - The name of the file to check.
|
|
2289
|
+
* @returns A promise resolving to true if the file exists, false otherwise.
|
|
2290
|
+
*/
|
|
1381
2291
|
async exists(filename) {
|
|
1382
2292
|
return this.files.has(filename);
|
|
1383
2293
|
}
|
|
2294
|
+
/**
|
|
2295
|
+
* Returns the full public URL for a sitemap file.
|
|
2296
|
+
*
|
|
2297
|
+
* @param filename - The name of the sitemap file.
|
|
2298
|
+
* @returns The public URL as a string.
|
|
2299
|
+
*/
|
|
1384
2300
|
getUrl(filename) {
|
|
1385
2301
|
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
1386
2302
|
const file = filename.startsWith("/") ? filename.slice(1) : filename;
|
|
@@ -1436,7 +2352,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1436
2352
|
});
|
|
1437
2353
|
}
|
|
1438
2354
|
/**
|
|
1439
|
-
*
|
|
2355
|
+
* Installs the sitemap module into PlanetCore.
|
|
1440
2356
|
*
|
|
1441
2357
|
* @param core - The PlanetCore instance.
|
|
1442
2358
|
*/
|
|
@@ -1447,6 +2363,9 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1447
2363
|
core.logger.info("[OrbitSitemap] Static mode configured. Use generate() to build sitemaps.");
|
|
1448
2364
|
}
|
|
1449
2365
|
}
|
|
2366
|
+
/**
|
|
2367
|
+
* Internal method to set up dynamic sitemap routes.
|
|
2368
|
+
*/
|
|
1450
2369
|
installDynamic(core) {
|
|
1451
2370
|
const opts = this.options;
|
|
1452
2371
|
const storage = opts.storage ?? new MemorySitemapStorage(opts.baseUrl);
|
|
@@ -1504,7 +2423,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1504
2423
|
core.router.get(shardRoute, handler);
|
|
1505
2424
|
}
|
|
1506
2425
|
/**
|
|
1507
|
-
*
|
|
2426
|
+
* Generates the sitemap (static mode only).
|
|
1508
2427
|
*
|
|
1509
2428
|
* @returns A promise that resolves when generation is complete.
|
|
1510
2429
|
* @throws {Error} If called in dynamic mode.
|
|
@@ -1516,7 +2435,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1516
2435
|
const opts = this.options;
|
|
1517
2436
|
let storage = opts.storage;
|
|
1518
2437
|
if (!storage) {
|
|
1519
|
-
const { DiskSitemapStorage: DiskSitemapStorage2 } = await import("./DiskSitemapStorage-
|
|
2438
|
+
const { DiskSitemapStorage: DiskSitemapStorage2 } = await import("./DiskSitemapStorage-VLN5I24C.js");
|
|
1520
2439
|
storage = new DiskSitemapStorage2(opts.outDir, opts.baseUrl);
|
|
1521
2440
|
}
|
|
1522
2441
|
let providers = opts.providers;
|
|
@@ -1546,7 +2465,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1546
2465
|
console.log(`[OrbitSitemap] Generated sitemap in ${opts.outDir}`);
|
|
1547
2466
|
}
|
|
1548
2467
|
/**
|
|
1549
|
-
*
|
|
2468
|
+
* Generates incremental sitemap updates (static mode only).
|
|
1550
2469
|
*
|
|
1551
2470
|
* @param since - Only include items modified since this date.
|
|
1552
2471
|
* @returns A promise that resolves when incremental generation is complete.
|
|
@@ -1562,7 +2481,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1562
2481
|
}
|
|
1563
2482
|
let storage = opts.storage;
|
|
1564
2483
|
if (!storage) {
|
|
1565
|
-
const { DiskSitemapStorage: DiskSitemapStorage2 } = await import("./DiskSitemapStorage-
|
|
2484
|
+
const { DiskSitemapStorage: DiskSitemapStorage2 } = await import("./DiskSitemapStorage-VLN5I24C.js");
|
|
1566
2485
|
storage = new DiskSitemapStorage2(opts.outDir, opts.baseUrl);
|
|
1567
2486
|
}
|
|
1568
2487
|
const incrementalGenerator = new IncrementalGenerator({
|
|
@@ -1577,7 +2496,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1577
2496
|
console.log(`[OrbitSitemap] Generated incremental sitemap in ${opts.outDir}`);
|
|
1578
2497
|
}
|
|
1579
2498
|
/**
|
|
1580
|
-
*
|
|
2499
|
+
* Generates sitemap asynchronously in the background (static mode only).
|
|
1581
2500
|
*
|
|
1582
2501
|
* @param options - Options for the async generation job.
|
|
1583
2502
|
* @returns A promise resolving to the job ID.
|
|
@@ -1588,10 +2507,10 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1588
2507
|
throw new Error("generateAsync() can only be called in static mode");
|
|
1589
2508
|
}
|
|
1590
2509
|
const opts = this.options;
|
|
1591
|
-
const jobId =
|
|
2510
|
+
const jobId = randomUUID2();
|
|
1592
2511
|
let storage = opts.storage;
|
|
1593
2512
|
if (!storage) {
|
|
1594
|
-
const { DiskSitemapStorage: DiskSitemapStorage2 } = await import("./DiskSitemapStorage-
|
|
2513
|
+
const { DiskSitemapStorage: DiskSitemapStorage2 } = await import("./DiskSitemapStorage-VLN5I24C.js");
|
|
1595
2514
|
storage = new DiskSitemapStorage2(opts.outDir, opts.baseUrl);
|
|
1596
2515
|
}
|
|
1597
2516
|
let providers = opts.providers;
|
|
@@ -1638,7 +2557,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1638
2557
|
return jobId;
|
|
1639
2558
|
}
|
|
1640
2559
|
/**
|
|
1641
|
-
*
|
|
2560
|
+
* Installs API endpoints for triggering and monitoring sitemap generation.
|
|
1642
2561
|
*
|
|
1643
2562
|
* @param core - The PlanetCore instance.
|
|
1644
2563
|
* @param basePath - The base path for the API endpoints (default: '/admin/sitemap').
|
|
@@ -1711,9 +2630,12 @@ var RouteScanner = class {
|
|
|
1711
2630
|
};
|
|
1712
2631
|
}
|
|
1713
2632
|
/**
|
|
1714
|
-
*
|
|
2633
|
+
* Scans the router and returns discovered static GET routes as sitemap entries.
|
|
2634
|
+
*
|
|
2635
|
+
* This method iterates through all registered routes in the Gravito router,
|
|
2636
|
+
* applying inclusion/exclusion filters and defaulting metadata for matching routes.
|
|
1715
2637
|
*
|
|
1716
|
-
* @returns An array of
|
|
2638
|
+
* @returns An array of `SitemapEntry` objects.
|
|
1717
2639
|
*/
|
|
1718
2640
|
getEntries() {
|
|
1719
2641
|
const entries = [];
|
|
@@ -1775,7 +2697,10 @@ var RedirectDetector = class {
|
|
|
1775
2697
|
this.options = options;
|
|
1776
2698
|
}
|
|
1777
2699
|
/**
|
|
1778
|
-
*
|
|
2700
|
+
* Detects redirects for a single URL using multiple strategies.
|
|
2701
|
+
*
|
|
2702
|
+
* @param url - The URL path to probe for redirects.
|
|
2703
|
+
* @returns A promise resolving to a `RedirectRule` if a redirect is found, or null.
|
|
1779
2704
|
*/
|
|
1780
2705
|
async detect(url) {
|
|
1781
2706
|
if (this.options.autoDetect?.cache) {
|
|
@@ -1807,7 +2732,10 @@ var RedirectDetector = class {
|
|
|
1807
2732
|
return null;
|
|
1808
2733
|
}
|
|
1809
2734
|
/**
|
|
1810
|
-
*
|
|
2735
|
+
* Batch detects redirects for multiple URLs with concurrency control.
|
|
2736
|
+
*
|
|
2737
|
+
* @param urls - An array of URL paths to probe.
|
|
2738
|
+
* @returns A promise resolving to a Map of URLs to their respective `RedirectRule` or null.
|
|
1811
2739
|
*/
|
|
1812
2740
|
async detectBatch(urls) {
|
|
1813
2741
|
const results = /* @__PURE__ */ new Map();
|
|
@@ -1827,7 +2755,7 @@ var RedirectDetector = class {
|
|
|
1827
2755
|
return results;
|
|
1828
2756
|
}
|
|
1829
2757
|
/**
|
|
1830
|
-
*
|
|
2758
|
+
* Detects a redirect from the configured database table.
|
|
1831
2759
|
*/
|
|
1832
2760
|
async detectFromDatabase(url) {
|
|
1833
2761
|
const { database } = this.options;
|
|
@@ -1845,14 +2773,14 @@ var RedirectDetector = class {
|
|
|
1845
2773
|
return {
|
|
1846
2774
|
from: row[columns.from],
|
|
1847
2775
|
to: row[columns.to],
|
|
1848
|
-
type: parseInt(row[columns.type], 10)
|
|
2776
|
+
type: Number.parseInt(row[columns.type], 10)
|
|
1849
2777
|
};
|
|
1850
2778
|
} catch {
|
|
1851
2779
|
return null;
|
|
1852
2780
|
}
|
|
1853
2781
|
}
|
|
1854
2782
|
/**
|
|
1855
|
-
*
|
|
2783
|
+
* Detects a redirect from a static JSON configuration file.
|
|
1856
2784
|
*/
|
|
1857
2785
|
async detectFromConfig(url) {
|
|
1858
2786
|
const { config } = this.options;
|
|
@@ -1870,7 +2798,7 @@ var RedirectDetector = class {
|
|
|
1870
2798
|
}
|
|
1871
2799
|
}
|
|
1872
2800
|
/**
|
|
1873
|
-
*
|
|
2801
|
+
* Auto-detects a redirect by sending an HTTP HEAD request.
|
|
1874
2802
|
*/
|
|
1875
2803
|
async detectAuto(url) {
|
|
1876
2804
|
const { autoDetect, baseUrl } = this.options;
|
|
@@ -1887,7 +2815,7 @@ var RedirectDetector = class {
|
|
|
1887
2815
|
method: "HEAD",
|
|
1888
2816
|
signal: controller.signal,
|
|
1889
2817
|
redirect: "manual"
|
|
1890
|
-
//
|
|
2818
|
+
// Handle redirects manually
|
|
1891
2819
|
});
|
|
1892
2820
|
clearTimeout(timeoutId);
|
|
1893
2821
|
if (response.status === 301 || response.status === 302) {
|
|
@@ -1911,7 +2839,7 @@ var RedirectDetector = class {
|
|
|
1911
2839
|
return null;
|
|
1912
2840
|
}
|
|
1913
2841
|
/**
|
|
1914
|
-
*
|
|
2842
|
+
* Caches the detection result for a URL.
|
|
1915
2843
|
*/
|
|
1916
2844
|
cacheResult(url, rule) {
|
|
1917
2845
|
if (!this.options.autoDetect?.cache) {
|
|
@@ -1932,6 +2860,11 @@ var MemoryRedirectManager = class {
|
|
|
1932
2860
|
constructor(options = {}) {
|
|
1933
2861
|
this.maxRules = options.maxRules || 1e5;
|
|
1934
2862
|
}
|
|
2863
|
+
/**
|
|
2864
|
+
* Registers a single redirect rule in memory.
|
|
2865
|
+
*
|
|
2866
|
+
* @param redirect - The redirect rule to add.
|
|
2867
|
+
*/
|
|
1935
2868
|
async register(redirect) {
|
|
1936
2869
|
this.rules.set(redirect.from, redirect);
|
|
1937
2870
|
if (this.rules.size > this.maxRules) {
|
|
@@ -1941,17 +2874,41 @@ var MemoryRedirectManager = class {
|
|
|
1941
2874
|
}
|
|
1942
2875
|
}
|
|
1943
2876
|
}
|
|
2877
|
+
/**
|
|
2878
|
+
* Registers multiple redirect rules in memory.
|
|
2879
|
+
*
|
|
2880
|
+
* @param redirects - An array of redirect rules.
|
|
2881
|
+
*/
|
|
1944
2882
|
async registerBatch(redirects) {
|
|
1945
2883
|
for (const redirect of redirects) {
|
|
1946
2884
|
await this.register(redirect);
|
|
1947
2885
|
}
|
|
1948
2886
|
}
|
|
2887
|
+
/**
|
|
2888
|
+
* Retrieves a specific redirect rule by its source path from memory.
|
|
2889
|
+
*
|
|
2890
|
+
* @param from - The source path.
|
|
2891
|
+
* @returns A promise resolving to the redirect rule, or null if not found.
|
|
2892
|
+
*/
|
|
1949
2893
|
async get(from) {
|
|
1950
2894
|
return this.rules.get(from) || null;
|
|
1951
2895
|
}
|
|
2896
|
+
/**
|
|
2897
|
+
* Retrieves all registered redirect rules from memory.
|
|
2898
|
+
*
|
|
2899
|
+
* @returns A promise resolving to an array of all redirect rules.
|
|
2900
|
+
*/
|
|
1952
2901
|
async getAll() {
|
|
1953
2902
|
return Array.from(this.rules.values());
|
|
1954
2903
|
}
|
|
2904
|
+
/**
|
|
2905
|
+
* Resolves a URL to its final destination through the redirect table.
|
|
2906
|
+
*
|
|
2907
|
+
* @param url - The URL to resolve.
|
|
2908
|
+
* @param followChains - Whether to recursively resolve chained redirects.
|
|
2909
|
+
* @param maxChainLength - Maximum depth for chain resolution.
|
|
2910
|
+
* @returns A promise resolving to the final destination URL.
|
|
2911
|
+
*/
|
|
1955
2912
|
async resolve(url, followChains = false, maxChainLength = 5) {
|
|
1956
2913
|
let current = url;
|
|
1957
2914
|
let chainLength = 0;
|
|
@@ -1984,6 +2941,11 @@ var RedisRedirectManager = class {
|
|
|
1984
2941
|
getListKey() {
|
|
1985
2942
|
return `${this.keyPrefix}list`;
|
|
1986
2943
|
}
|
|
2944
|
+
/**
|
|
2945
|
+
* Registers a single redirect rule in Redis.
|
|
2946
|
+
*
|
|
2947
|
+
* @param redirect - The redirect rule to add.
|
|
2948
|
+
*/
|
|
1987
2949
|
async register(redirect) {
|
|
1988
2950
|
const key = this.getKey(redirect.from);
|
|
1989
2951
|
const listKey = this.getListKey();
|
|
@@ -1995,11 +2957,22 @@ var RedisRedirectManager = class {
|
|
|
1995
2957
|
}
|
|
1996
2958
|
await this.client.sadd(listKey, redirect.from);
|
|
1997
2959
|
}
|
|
2960
|
+
/**
|
|
2961
|
+
* Registers multiple redirect rules in Redis.
|
|
2962
|
+
*
|
|
2963
|
+
* @param redirects - An array of redirect rules.
|
|
2964
|
+
*/
|
|
1998
2965
|
async registerBatch(redirects) {
|
|
1999
2966
|
for (const redirect of redirects) {
|
|
2000
2967
|
await this.register(redirect);
|
|
2001
2968
|
}
|
|
2002
2969
|
}
|
|
2970
|
+
/**
|
|
2971
|
+
* Retrieves a specific redirect rule by its source path from Redis.
|
|
2972
|
+
*
|
|
2973
|
+
* @param from - The source path.
|
|
2974
|
+
* @returns A promise resolving to the redirect rule, or null if not found.
|
|
2975
|
+
*/
|
|
2003
2976
|
async get(from) {
|
|
2004
2977
|
try {
|
|
2005
2978
|
const key = this.getKey(from);
|
|
@@ -2016,6 +2989,11 @@ var RedisRedirectManager = class {
|
|
|
2016
2989
|
return null;
|
|
2017
2990
|
}
|
|
2018
2991
|
}
|
|
2992
|
+
/**
|
|
2993
|
+
* Retrieves all registered redirect rules from Redis.
|
|
2994
|
+
*
|
|
2995
|
+
* @returns A promise resolving to an array of all redirect rules.
|
|
2996
|
+
*/
|
|
2019
2997
|
async getAll() {
|
|
2020
2998
|
try {
|
|
2021
2999
|
const listKey = this.getListKey();
|
|
@@ -2032,6 +3010,14 @@ var RedisRedirectManager = class {
|
|
|
2032
3010
|
return [];
|
|
2033
3011
|
}
|
|
2034
3012
|
}
|
|
3013
|
+
/**
|
|
3014
|
+
* Resolves a URL to its final destination through the Redis redirect table.
|
|
3015
|
+
*
|
|
3016
|
+
* @param url - The URL to resolve.
|
|
3017
|
+
* @param followChains - Whether to recursively resolve chained redirects.
|
|
3018
|
+
* @param maxChainLength - Maximum depth for chain resolution.
|
|
3019
|
+
* @returns A promise resolving to the final destination URL.
|
|
3020
|
+
*/
|
|
2035
3021
|
async resolve(url, followChains = false, maxChainLength = 5) {
|
|
2036
3022
|
let current = url;
|
|
2037
3023
|
let chainLength = 0;
|
|
@@ -2051,6 +3037,8 @@ var RedisRedirectManager = class {
|
|
|
2051
3037
|
};
|
|
2052
3038
|
|
|
2053
3039
|
// src/storage/GCPSitemapStorage.ts
|
|
3040
|
+
import { Readable } from "stream";
|
|
3041
|
+
import { pipeline } from "stream/promises";
|
|
2054
3042
|
var GCPSitemapStorage = class {
|
|
2055
3043
|
bucket;
|
|
2056
3044
|
prefix;
|
|
@@ -2089,6 +3077,12 @@ var GCPSitemapStorage = class {
|
|
|
2089
3077
|
const cleanPrefix = this.prefix.endsWith("/") ? this.prefix.slice(0, -1) : this.prefix;
|
|
2090
3078
|
return cleanPrefix ? `${cleanPrefix}/${filename}` : filename;
|
|
2091
3079
|
}
|
|
3080
|
+
/**
|
|
3081
|
+
* Writes sitemap content to a Google Cloud Storage object.
|
|
3082
|
+
*
|
|
3083
|
+
* @param filename - The name of the file to write.
|
|
3084
|
+
* @param content - The XML or JSON content.
|
|
3085
|
+
*/
|
|
2092
3086
|
async write(filename, content) {
|
|
2093
3087
|
const { bucket } = await this.getStorageClient();
|
|
2094
3088
|
const key = this.getKey(filename);
|
|
@@ -2100,6 +3094,41 @@ var GCPSitemapStorage = class {
|
|
|
2100
3094
|
}
|
|
2101
3095
|
});
|
|
2102
3096
|
}
|
|
3097
|
+
/**
|
|
3098
|
+
* 使用串流方式寫入 sitemap 至 GCP Cloud Storage,可選擇性啟用 gzip 壓縮。
|
|
3099
|
+
*
|
|
3100
|
+
* @param filename - 檔案名稱
|
|
3101
|
+
* @param stream - XML 內容的 AsyncIterable
|
|
3102
|
+
* @param options - 寫入選項(如壓縮、content type)
|
|
3103
|
+
*
|
|
3104
|
+
* @since 3.1.0
|
|
3105
|
+
*/
|
|
3106
|
+
async writeStream(filename, stream, options) {
|
|
3107
|
+
const { bucket } = await this.getStorageClient();
|
|
3108
|
+
const key = this.getKey(options?.compress ? toGzipFilename(filename) : filename);
|
|
3109
|
+
const file = bucket.file(key);
|
|
3110
|
+
const writeStream = file.createWriteStream({
|
|
3111
|
+
resumable: false,
|
|
3112
|
+
contentType: "application/xml",
|
|
3113
|
+
metadata: {
|
|
3114
|
+
contentEncoding: options?.compress ? "gzip" : void 0,
|
|
3115
|
+
cacheControl: "public, max-age=3600"
|
|
3116
|
+
}
|
|
3117
|
+
});
|
|
3118
|
+
const readable = Readable.from(stream, { encoding: "utf-8" });
|
|
3119
|
+
if (options?.compress) {
|
|
3120
|
+
const gzip = createCompressionStream();
|
|
3121
|
+
await pipeline(readable, gzip, writeStream);
|
|
3122
|
+
} else {
|
|
3123
|
+
await pipeline(readable, writeStream);
|
|
3124
|
+
}
|
|
3125
|
+
}
|
|
3126
|
+
/**
|
|
3127
|
+
* Reads sitemap content from a Google Cloud Storage object.
|
|
3128
|
+
*
|
|
3129
|
+
* @param filename - The name of the file to read.
|
|
3130
|
+
* @returns A promise resolving to the file content as a string, or null if not found.
|
|
3131
|
+
*/
|
|
2103
3132
|
async read(filename) {
|
|
2104
3133
|
try {
|
|
2105
3134
|
const { bucket } = await this.getStorageClient();
|
|
@@ -2118,6 +3147,12 @@ var GCPSitemapStorage = class {
|
|
|
2118
3147
|
throw error;
|
|
2119
3148
|
}
|
|
2120
3149
|
}
|
|
3150
|
+
/**
|
|
3151
|
+
* Returns a readable stream for a Google Cloud Storage object.
|
|
3152
|
+
*
|
|
3153
|
+
* @param filename - The name of the file to stream.
|
|
3154
|
+
* @returns A promise resolving to an async iterable of file chunks, or null if not found.
|
|
3155
|
+
*/
|
|
2121
3156
|
async readStream(filename) {
|
|
2122
3157
|
try {
|
|
2123
3158
|
const { bucket } = await this.getStorageClient();
|
|
@@ -2142,6 +3177,12 @@ var GCPSitemapStorage = class {
|
|
|
2142
3177
|
throw error;
|
|
2143
3178
|
}
|
|
2144
3179
|
}
|
|
3180
|
+
/**
|
|
3181
|
+
* Checks if a Google Cloud Storage object exists.
|
|
3182
|
+
*
|
|
3183
|
+
* @param filename - The name of the file to check.
|
|
3184
|
+
* @returns A promise resolving to true if the file exists, false otherwise.
|
|
3185
|
+
*/
|
|
2145
3186
|
async exists(filename) {
|
|
2146
3187
|
try {
|
|
2147
3188
|
const { bucket } = await this.getStorageClient();
|
|
@@ -2153,12 +3194,24 @@ var GCPSitemapStorage = class {
|
|
|
2153
3194
|
return false;
|
|
2154
3195
|
}
|
|
2155
3196
|
}
|
|
3197
|
+
/**
|
|
3198
|
+
* Returns the full public URL for a Google Cloud Storage object.
|
|
3199
|
+
*
|
|
3200
|
+
* @param filename - The name of the sitemap file.
|
|
3201
|
+
* @returns The public URL as a string.
|
|
3202
|
+
*/
|
|
2156
3203
|
getUrl(filename) {
|
|
2157
3204
|
const key = this.getKey(filename);
|
|
2158
3205
|
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
2159
3206
|
return `${base}/${key}`;
|
|
2160
3207
|
}
|
|
2161
|
-
|
|
3208
|
+
/**
|
|
3209
|
+
* Writes content to a shadow (staged) location in Google Cloud Storage.
|
|
3210
|
+
*
|
|
3211
|
+
* @param filename - The name of the file to write.
|
|
3212
|
+
* @param content - The XML or JSON content.
|
|
3213
|
+
* @param shadowId - Optional unique session identifier.
|
|
3214
|
+
*/
|
|
2162
3215
|
async writeShadow(filename, content, shadowId) {
|
|
2163
3216
|
if (!this.shadowEnabled) {
|
|
2164
3217
|
return this.write(filename, content);
|
|
@@ -2174,6 +3227,11 @@ var GCPSitemapStorage = class {
|
|
|
2174
3227
|
}
|
|
2175
3228
|
});
|
|
2176
3229
|
}
|
|
3230
|
+
/**
|
|
3231
|
+
* Commits all staged shadow objects in a session to production in Google Cloud Storage.
|
|
3232
|
+
*
|
|
3233
|
+
* @param shadowId - The identifier of the session to commit.
|
|
3234
|
+
*/
|
|
2177
3235
|
async commitShadow(shadowId) {
|
|
2178
3236
|
if (!this.shadowEnabled) {
|
|
2179
3237
|
return;
|
|
@@ -2200,6 +3258,12 @@ var GCPSitemapStorage = class {
|
|
|
2200
3258
|
}
|
|
2201
3259
|
}
|
|
2202
3260
|
}
|
|
3261
|
+
/**
|
|
3262
|
+
* Lists all archived versions of a specific sitemap in Google Cloud Storage.
|
|
3263
|
+
*
|
|
3264
|
+
* @param filename - The sitemap filename.
|
|
3265
|
+
* @returns A promise resolving to an array of version identifiers.
|
|
3266
|
+
*/
|
|
2203
3267
|
async listVersions(filename) {
|
|
2204
3268
|
if (this.shadowMode !== "versioned") {
|
|
2205
3269
|
return [];
|
|
@@ -2221,6 +3285,12 @@ var GCPSitemapStorage = class {
|
|
|
2221
3285
|
return [];
|
|
2222
3286
|
}
|
|
2223
3287
|
}
|
|
3288
|
+
/**
|
|
3289
|
+
* Reverts a sitemap to a previously archived version in Google Cloud Storage.
|
|
3290
|
+
*
|
|
3291
|
+
* @param filename - The sitemap filename.
|
|
3292
|
+
* @param version - The version identifier to switch to.
|
|
3293
|
+
*/
|
|
2224
3294
|
async switchVersion(filename, version) {
|
|
2225
3295
|
if (this.shadowMode !== "versioned") {
|
|
2226
3296
|
throw new Error("Version switching is only available in versioned mode");
|
|
@@ -2240,22 +3310,51 @@ var GCPSitemapStorage = class {
|
|
|
2240
3310
|
// src/storage/MemoryProgressStorage.ts
|
|
2241
3311
|
var MemoryProgressStorage = class {
|
|
2242
3312
|
storage = /* @__PURE__ */ new Map();
|
|
3313
|
+
/**
|
|
3314
|
+
* Retrieves the progress of a specific generation job from memory.
|
|
3315
|
+
*
|
|
3316
|
+
* @param jobId - Unique identifier for the job.
|
|
3317
|
+
* @returns A promise resolving to the `SitemapProgress` object, or null if not found.
|
|
3318
|
+
*/
|
|
2243
3319
|
async get(jobId) {
|
|
2244
3320
|
const progress = this.storage.get(jobId);
|
|
2245
3321
|
return progress ? { ...progress } : null;
|
|
2246
3322
|
}
|
|
3323
|
+
/**
|
|
3324
|
+
* Initializes or overwrites a progress record in memory.
|
|
3325
|
+
*
|
|
3326
|
+
* @param jobId - Unique identifier for the job.
|
|
3327
|
+
* @param progress - The initial or current state of the job progress.
|
|
3328
|
+
*/
|
|
2247
3329
|
async set(jobId, progress) {
|
|
2248
3330
|
this.storage.set(jobId, { ...progress });
|
|
2249
3331
|
}
|
|
3332
|
+
/**
|
|
3333
|
+
* Updates specific fields of an existing progress record in memory.
|
|
3334
|
+
*
|
|
3335
|
+
* @param jobId - Unique identifier for the job.
|
|
3336
|
+
* @param updates - Object containing the fields to update.
|
|
3337
|
+
*/
|
|
2250
3338
|
async update(jobId, updates) {
|
|
2251
3339
|
const existing = this.storage.get(jobId);
|
|
2252
3340
|
if (existing) {
|
|
2253
3341
|
this.storage.set(jobId, { ...existing, ...updates });
|
|
2254
3342
|
}
|
|
2255
3343
|
}
|
|
3344
|
+
/**
|
|
3345
|
+
* Deletes a progress record from memory.
|
|
3346
|
+
*
|
|
3347
|
+
* @param jobId - Unique identifier for the job to remove.
|
|
3348
|
+
*/
|
|
2256
3349
|
async delete(jobId) {
|
|
2257
3350
|
this.storage.delete(jobId);
|
|
2258
3351
|
}
|
|
3352
|
+
/**
|
|
3353
|
+
* Lists the most recent sitemap generation jobs from memory.
|
|
3354
|
+
*
|
|
3355
|
+
* @param limit - Maximum number of records to return.
|
|
3356
|
+
* @returns A promise resolving to an array of `SitemapProgress` objects, sorted by start time.
|
|
3357
|
+
*/
|
|
2259
3358
|
async list(limit) {
|
|
2260
3359
|
const all = Array.from(this.storage.values());
|
|
2261
3360
|
const sorted = all.sort((a, b) => {
|
|
@@ -2283,6 +3382,12 @@ var RedisProgressStorage = class {
|
|
|
2283
3382
|
getListKey() {
|
|
2284
3383
|
return `${this.keyPrefix}list`;
|
|
2285
3384
|
}
|
|
3385
|
+
/**
|
|
3386
|
+
* Retrieves the progress of a specific generation job from Redis.
|
|
3387
|
+
*
|
|
3388
|
+
* @param jobId - Unique identifier for the job.
|
|
3389
|
+
* @returns A promise resolving to the `SitemapProgress` object, or null if not found.
|
|
3390
|
+
*/
|
|
2286
3391
|
async get(jobId) {
|
|
2287
3392
|
try {
|
|
2288
3393
|
const key = this.getKey(jobId);
|
|
@@ -2302,6 +3407,12 @@ var RedisProgressStorage = class {
|
|
|
2302
3407
|
return null;
|
|
2303
3408
|
}
|
|
2304
3409
|
}
|
|
3410
|
+
/**
|
|
3411
|
+
* Initializes or overwrites a progress record in Redis.
|
|
3412
|
+
*
|
|
3413
|
+
* @param jobId - Unique identifier for the job.
|
|
3414
|
+
* @param progress - The initial or current state of the job progress.
|
|
3415
|
+
*/
|
|
2305
3416
|
async set(jobId, progress) {
|
|
2306
3417
|
const key = this.getKey(jobId);
|
|
2307
3418
|
const listKey = this.getListKey();
|
|
@@ -2310,18 +3421,35 @@ var RedisProgressStorage = class {
|
|
|
2310
3421
|
await this.client.zadd(listKey, Date.now(), jobId);
|
|
2311
3422
|
await this.client.expire(listKey, this.ttl);
|
|
2312
3423
|
}
|
|
3424
|
+
/**
|
|
3425
|
+
* Updates specific fields of an existing progress record in Redis.
|
|
3426
|
+
*
|
|
3427
|
+
* @param jobId - Unique identifier for the job.
|
|
3428
|
+
* @param updates - Object containing the fields to update.
|
|
3429
|
+
*/
|
|
2313
3430
|
async update(jobId, updates) {
|
|
2314
3431
|
const existing = await this.get(jobId);
|
|
2315
3432
|
if (existing) {
|
|
2316
3433
|
await this.set(jobId, { ...existing, ...updates });
|
|
2317
3434
|
}
|
|
2318
3435
|
}
|
|
3436
|
+
/**
|
|
3437
|
+
* Deletes a progress record from Redis.
|
|
3438
|
+
*
|
|
3439
|
+
* @param jobId - Unique identifier for the job to remove.
|
|
3440
|
+
*/
|
|
2319
3441
|
async delete(jobId) {
|
|
2320
3442
|
const key = this.getKey(jobId);
|
|
2321
3443
|
const listKey = this.getListKey();
|
|
2322
3444
|
await this.client.del(key);
|
|
2323
3445
|
await this.client.zrem(listKey, jobId);
|
|
2324
3446
|
}
|
|
3447
|
+
/**
|
|
3448
|
+
* Lists the most recent sitemap generation jobs from Redis.
|
|
3449
|
+
*
|
|
3450
|
+
* @param limit - Maximum number of records to return.
|
|
3451
|
+
* @returns A promise resolving to an array of `SitemapProgress` objects, sorted by start time.
|
|
3452
|
+
*/
|
|
2325
3453
|
async list(limit) {
|
|
2326
3454
|
try {
|
|
2327
3455
|
const listKey = this.getListKey();
|
|
@@ -2398,6 +3526,12 @@ var S3SitemapStorage = class {
|
|
|
2398
3526
|
const cleanPrefix = this.prefix.endsWith("/") ? this.prefix.slice(0, -1) : this.prefix;
|
|
2399
3527
|
return cleanPrefix ? `${cleanPrefix}/${filename}` : filename;
|
|
2400
3528
|
}
|
|
3529
|
+
/**
|
|
3530
|
+
* Writes sitemap content to an S3 object.
|
|
3531
|
+
*
|
|
3532
|
+
* @param filename - The name of the file to write.
|
|
3533
|
+
* @param content - The XML or JSON content.
|
|
3534
|
+
*/
|
|
2401
3535
|
async write(filename, content) {
|
|
2402
3536
|
const s3 = await this.getS3Client();
|
|
2403
3537
|
const key = this.getKey(filename);
|
|
@@ -2410,6 +3544,48 @@ var S3SitemapStorage = class {
|
|
|
2410
3544
|
})
|
|
2411
3545
|
);
|
|
2412
3546
|
}
|
|
3547
|
+
/**
|
|
3548
|
+
* 使用串流方式寫入 sitemap 至 S3,可選擇性啟用 gzip 壓縮。
|
|
3549
|
+
* S3 需要知道 Content-Length,因此會先收集串流為 Buffer。
|
|
3550
|
+
*
|
|
3551
|
+
* @param filename - 檔案名稱
|
|
3552
|
+
* @param stream - XML 內容的 AsyncIterable
|
|
3553
|
+
* @param options - 寫入選項(如壓縮、content type)
|
|
3554
|
+
*
|
|
3555
|
+
* @since 3.1.0
|
|
3556
|
+
*/
|
|
3557
|
+
async writeStream(filename, stream, options) {
|
|
3558
|
+
const s3 = await this.getS3Client();
|
|
3559
|
+
const key = this.getKey(options?.compress ? toGzipFilename(filename) : filename);
|
|
3560
|
+
let body;
|
|
3561
|
+
const contentType = options?.contentType || "application/xml";
|
|
3562
|
+
let contentEncoding;
|
|
3563
|
+
if (options?.compress) {
|
|
3564
|
+
body = await compressToBuffer(stream);
|
|
3565
|
+
contentEncoding = "gzip";
|
|
3566
|
+
} else {
|
|
3567
|
+
const chunks = [];
|
|
3568
|
+
for await (const chunk of stream) {
|
|
3569
|
+
chunks.push(chunk);
|
|
3570
|
+
}
|
|
3571
|
+
body = chunks.join("");
|
|
3572
|
+
}
|
|
3573
|
+
await s3.client.send(
|
|
3574
|
+
new s3.PutObjectCommand({
|
|
3575
|
+
Bucket: this.bucket,
|
|
3576
|
+
Key: key,
|
|
3577
|
+
Body: body,
|
|
3578
|
+
ContentType: contentType,
|
|
3579
|
+
ContentEncoding: contentEncoding
|
|
3580
|
+
})
|
|
3581
|
+
);
|
|
3582
|
+
}
|
|
3583
|
+
/**
|
|
3584
|
+
* Reads sitemap content from an S3 object.
|
|
3585
|
+
*
|
|
3586
|
+
* @param filename - The name of the file to read.
|
|
3587
|
+
* @returns A promise resolving to the file content as a string, or null if not found.
|
|
3588
|
+
*/
|
|
2413
3589
|
async read(filename) {
|
|
2414
3590
|
try {
|
|
2415
3591
|
const s3 = await this.getS3Client();
|
|
@@ -2436,6 +3612,12 @@ var S3SitemapStorage = class {
|
|
|
2436
3612
|
throw error;
|
|
2437
3613
|
}
|
|
2438
3614
|
}
|
|
3615
|
+
/**
|
|
3616
|
+
* Returns a readable stream for an S3 object.
|
|
3617
|
+
*
|
|
3618
|
+
* @param filename - The name of the file to stream.
|
|
3619
|
+
* @returns A promise resolving to an async iterable of file chunks, or null if not found.
|
|
3620
|
+
*/
|
|
2439
3621
|
async readStream(filename) {
|
|
2440
3622
|
try {
|
|
2441
3623
|
const s3 = await this.getS3Client();
|
|
@@ -2464,6 +3646,12 @@ var S3SitemapStorage = class {
|
|
|
2464
3646
|
throw error;
|
|
2465
3647
|
}
|
|
2466
3648
|
}
|
|
3649
|
+
/**
|
|
3650
|
+
* Checks if an S3 object exists.
|
|
3651
|
+
*
|
|
3652
|
+
* @param filename - The name of the file to check.
|
|
3653
|
+
* @returns A promise resolving to true if the file exists, false otherwise.
|
|
3654
|
+
*/
|
|
2467
3655
|
async exists(filename) {
|
|
2468
3656
|
try {
|
|
2469
3657
|
const s3 = await this.getS3Client();
|
|
@@ -2482,12 +3670,24 @@ var S3SitemapStorage = class {
|
|
|
2482
3670
|
throw error;
|
|
2483
3671
|
}
|
|
2484
3672
|
}
|
|
3673
|
+
/**
|
|
3674
|
+
* Returns the full public URL for an S3 object.
|
|
3675
|
+
*
|
|
3676
|
+
* @param filename - The name of the sitemap file.
|
|
3677
|
+
* @returns The public URL as a string.
|
|
3678
|
+
*/
|
|
2485
3679
|
getUrl(filename) {
|
|
2486
3680
|
const key = this.getKey(filename);
|
|
2487
3681
|
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
2488
3682
|
return `${base}/${key}`;
|
|
2489
3683
|
}
|
|
2490
|
-
|
|
3684
|
+
/**
|
|
3685
|
+
* Writes content to a shadow (staged) location in S3.
|
|
3686
|
+
*
|
|
3687
|
+
* @param filename - The name of the file to write.
|
|
3688
|
+
* @param content - The XML or JSON content.
|
|
3689
|
+
* @param shadowId - Optional unique session identifier.
|
|
3690
|
+
*/
|
|
2491
3691
|
async writeShadow(filename, content, shadowId) {
|
|
2492
3692
|
if (!this.shadowEnabled) {
|
|
2493
3693
|
return this.write(filename, content);
|
|
@@ -2504,6 +3704,11 @@ var S3SitemapStorage = class {
|
|
|
2504
3704
|
})
|
|
2505
3705
|
);
|
|
2506
3706
|
}
|
|
3707
|
+
/**
|
|
3708
|
+
* Commits all staged shadow objects in a session to production.
|
|
3709
|
+
*
|
|
3710
|
+
* @param shadowId - The identifier of the session to commit.
|
|
3711
|
+
*/
|
|
2507
3712
|
async commitShadow(shadowId) {
|
|
2508
3713
|
if (!this.shadowEnabled) {
|
|
2509
3714
|
return;
|
|
@@ -2571,6 +3776,12 @@ var S3SitemapStorage = class {
|
|
|
2571
3776
|
}
|
|
2572
3777
|
}
|
|
2573
3778
|
}
|
|
3779
|
+
/**
|
|
3780
|
+
* Lists all archived versions of a specific sitemap in S3.
|
|
3781
|
+
*
|
|
3782
|
+
* @param filename - The sitemap filename.
|
|
3783
|
+
* @returns A promise resolving to an array of version identifiers.
|
|
3784
|
+
*/
|
|
2574
3785
|
async listVersions(filename) {
|
|
2575
3786
|
if (this.shadowMode !== "versioned") {
|
|
2576
3787
|
return [];
|
|
@@ -2603,6 +3814,12 @@ var S3SitemapStorage = class {
|
|
|
2603
3814
|
return [];
|
|
2604
3815
|
}
|
|
2605
3816
|
}
|
|
3817
|
+
/**
|
|
3818
|
+
* Reverts a sitemap to a previously archived version in S3.
|
|
3819
|
+
*
|
|
3820
|
+
* @param filename - The sitemap filename.
|
|
3821
|
+
* @param version - The version identifier to switch to.
|
|
3822
|
+
*/
|
|
2606
3823
|
async switchVersion(filename, version) {
|
|
2607
3824
|
if (this.shadowMode !== "versioned") {
|
|
2608
3825
|
throw new Error("Version switching is only available in versioned mode");
|
|
@@ -2631,6 +3848,7 @@ export {
|
|
|
2631
3848
|
GenerateSitemapJob,
|
|
2632
3849
|
IncrementalGenerator,
|
|
2633
3850
|
MemoryChangeTracker,
|
|
3851
|
+
MemoryLock,
|
|
2634
3852
|
MemoryProgressStorage,
|
|
2635
3853
|
MemoryRedirectManager,
|
|
2636
3854
|
MemorySitemapStorage,
|
|
@@ -2639,6 +3857,7 @@ export {
|
|
|
2639
3857
|
RedirectDetector,
|
|
2640
3858
|
RedirectHandler,
|
|
2641
3859
|
RedisChangeTracker,
|
|
3860
|
+
RedisLock,
|
|
2642
3861
|
RedisProgressStorage,
|
|
2643
3862
|
RedisRedirectManager,
|
|
2644
3863
|
RouteScanner,
|
|
@@ -2647,6 +3866,10 @@ export {
|
|
|
2647
3866
|
SitemapGenerator,
|
|
2648
3867
|
SitemapIndex,
|
|
2649
3868
|
SitemapStream,
|
|
3869
|
+
compressToBuffer,
|
|
3870
|
+
createCompressionStream,
|
|
3871
|
+
fromGzipFilename,
|
|
2650
3872
|
generateI18nEntries,
|
|
2651
|
-
routeScanner
|
|
3873
|
+
routeScanner,
|
|
3874
|
+
toGzipFilename
|
|
2652
3875
|
};
|