@gravito/constellation 3.0.2 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -251
- package/README.zh-TW.md +130 -13
- package/dist/{DiskSitemapStorage-WP6RITUN.js → DiskSitemapStorage-VLN5I24C.js} +1 -1
- package/dist/chunk-3IZTXYU7.js +166 -0
- package/dist/index.cjs +1400 -67
- package/dist/index.d.cts +1579 -148
- package/dist/index.d.ts +1579 -148
- package/dist/index.js +1293 -70
- package/package.json +11 -9
- package/dist/chunk-IS2H7U6M.js +0 -68
package/dist/index.cjs
CHANGED
|
@@ -30,6 +30,49 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
30
30
|
));
|
|
31
31
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
32
32
|
|
|
33
|
+
// src/utils/Compression.ts
|
|
34
|
+
async function compressToBuffer(source, config) {
|
|
35
|
+
const level = config?.level ?? 6;
|
|
36
|
+
if (level < 1 || level > 9) {
|
|
37
|
+
throw new Error(`Invalid compression level: ${level}. Must be between 1 and 9.`);
|
|
38
|
+
}
|
|
39
|
+
const chunks = [];
|
|
40
|
+
const gzip = (0, import_node_zlib.createGzip)({ level });
|
|
41
|
+
const readable = import_node_stream.Readable.from(source, { encoding: "utf-8" });
|
|
42
|
+
try {
|
|
43
|
+
readable.pipe(gzip);
|
|
44
|
+
for await (const chunk of gzip) {
|
|
45
|
+
chunks.push(chunk);
|
|
46
|
+
}
|
|
47
|
+
return Buffer.concat(chunks);
|
|
48
|
+
} catch (error) {
|
|
49
|
+
readable.destroy();
|
|
50
|
+
gzip.destroy();
|
|
51
|
+
throw error;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
function createCompressionStream(config) {
|
|
55
|
+
const level = config?.level ?? 6;
|
|
56
|
+
if (level < 1 || level > 9) {
|
|
57
|
+
throw new Error(`Invalid compression level: ${level}. Must be between 1 and 9.`);
|
|
58
|
+
}
|
|
59
|
+
return (0, import_node_zlib.createGzip)({ level });
|
|
60
|
+
}
|
|
61
|
+
function toGzipFilename(filename) {
|
|
62
|
+
return filename.endsWith(".gz") ? filename : `${filename}.gz`;
|
|
63
|
+
}
|
|
64
|
+
function fromGzipFilename(filename) {
|
|
65
|
+
return filename.endsWith(".gz") ? filename.slice(0, -3) : filename;
|
|
66
|
+
}
|
|
67
|
+
var import_node_stream, import_node_zlib;
|
|
68
|
+
var init_Compression = __esm({
|
|
69
|
+
"src/utils/Compression.ts"() {
|
|
70
|
+
"use strict";
|
|
71
|
+
import_node_stream = require("stream");
|
|
72
|
+
import_node_zlib = require("zlib");
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
|
|
33
76
|
// src/storage/DiskSitemapStorage.ts
|
|
34
77
|
var DiskSitemapStorage_exports = {};
|
|
35
78
|
__export(DiskSitemapStorage_exports, {
|
|
@@ -50,23 +93,61 @@ function sanitizeFilename(filename) {
|
|
|
50
93
|
}
|
|
51
94
|
return filename;
|
|
52
95
|
}
|
|
53
|
-
var import_node_fs, import_promises, import_node_path, DiskSitemapStorage;
|
|
96
|
+
var import_node_fs, import_promises, import_node_path, import_node_stream2, import_promises2, DiskSitemapStorage;
|
|
54
97
|
var init_DiskSitemapStorage = __esm({
|
|
55
98
|
"src/storage/DiskSitemapStorage.ts"() {
|
|
56
99
|
"use strict";
|
|
57
100
|
import_node_fs = require("fs");
|
|
58
101
|
import_promises = __toESM(require("fs/promises"), 1);
|
|
59
102
|
import_node_path = __toESM(require("path"), 1);
|
|
103
|
+
import_node_stream2 = require("stream");
|
|
104
|
+
import_promises2 = require("stream/promises");
|
|
105
|
+
init_Compression();
|
|
60
106
|
DiskSitemapStorage = class {
|
|
61
107
|
constructor(outDir, baseUrl) {
|
|
62
108
|
this.outDir = outDir;
|
|
63
109
|
this.baseUrl = baseUrl;
|
|
64
110
|
}
|
|
111
|
+
/**
|
|
112
|
+
* Writes sitemap content to a file on the local disk.
|
|
113
|
+
*
|
|
114
|
+
* @param filename - The name of the file to write.
|
|
115
|
+
* @param content - The XML or JSON content.
|
|
116
|
+
*/
|
|
65
117
|
async write(filename, content) {
|
|
66
118
|
const safeName = sanitizeFilename(filename);
|
|
67
119
|
await import_promises.default.mkdir(this.outDir, { recursive: true });
|
|
68
120
|
await import_promises.default.writeFile(import_node_path.default.join(this.outDir, safeName), content);
|
|
69
121
|
}
|
|
122
|
+
/**
|
|
123
|
+
* 使用串流方式寫入 sitemap 檔案,可選擇性啟用 gzip 壓縮。
|
|
124
|
+
* 此方法可大幅降低大型 sitemap 的記憶體峰值。
|
|
125
|
+
*
|
|
126
|
+
* @param filename - 檔案名稱
|
|
127
|
+
* @param stream - XML 內容的 AsyncIterable
|
|
128
|
+
* @param options - 寫入選項(如壓縮、content type)
|
|
129
|
+
*
|
|
130
|
+
* @since 3.1.0
|
|
131
|
+
*/
|
|
132
|
+
async writeStream(filename, stream, options) {
|
|
133
|
+
const safeName = sanitizeFilename(options?.compress ? toGzipFilename(filename) : filename);
|
|
134
|
+
await import_promises.default.mkdir(this.outDir, { recursive: true });
|
|
135
|
+
const filePath = import_node_path.default.join(this.outDir, safeName);
|
|
136
|
+
const writeStream = (0, import_node_fs.createWriteStream)(filePath);
|
|
137
|
+
const readable = import_node_stream2.Readable.from(stream, { encoding: "utf-8" });
|
|
138
|
+
if (options?.compress) {
|
|
139
|
+
const gzip = createCompressionStream();
|
|
140
|
+
await (0, import_promises2.pipeline)(readable, gzip, writeStream);
|
|
141
|
+
} else {
|
|
142
|
+
await (0, import_promises2.pipeline)(readable, writeStream);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Reads sitemap content from a file on the local disk.
|
|
147
|
+
*
|
|
148
|
+
* @param filename - The name of the file to read.
|
|
149
|
+
* @returns A promise resolving to the file content as a string, or null if not found.
|
|
150
|
+
*/
|
|
70
151
|
async read(filename) {
|
|
71
152
|
try {
|
|
72
153
|
const safeName = sanitizeFilename(filename);
|
|
@@ -75,6 +156,12 @@ var init_DiskSitemapStorage = __esm({
|
|
|
75
156
|
return null;
|
|
76
157
|
}
|
|
77
158
|
}
|
|
159
|
+
/**
|
|
160
|
+
* Returns a readable stream for a sitemap file on the local disk.
|
|
161
|
+
*
|
|
162
|
+
* @param filename - The name of the file to stream.
|
|
163
|
+
* @returns A promise resolving to an async iterable of file chunks, or null if not found.
|
|
164
|
+
*/
|
|
78
165
|
async readStream(filename) {
|
|
79
166
|
try {
|
|
80
167
|
const safeName = sanitizeFilename(filename);
|
|
@@ -86,6 +173,12 @@ var init_DiskSitemapStorage = __esm({
|
|
|
86
173
|
return null;
|
|
87
174
|
}
|
|
88
175
|
}
|
|
176
|
+
/**
|
|
177
|
+
* Checks if a sitemap file exists on the local disk.
|
|
178
|
+
*
|
|
179
|
+
* @param filename - The name of the file to check.
|
|
180
|
+
* @returns A promise resolving to true if the file exists, false otherwise.
|
|
181
|
+
*/
|
|
89
182
|
async exists(filename) {
|
|
90
183
|
try {
|
|
91
184
|
const safeName = sanitizeFilename(filename);
|
|
@@ -95,6 +188,12 @@ var init_DiskSitemapStorage = __esm({
|
|
|
95
188
|
return false;
|
|
96
189
|
}
|
|
97
190
|
}
|
|
191
|
+
/**
|
|
192
|
+
* Returns the full public URL for a sitemap file.
|
|
193
|
+
*
|
|
194
|
+
* @param filename - The name of the sitemap file.
|
|
195
|
+
* @returns The public URL as a string.
|
|
196
|
+
*/
|
|
98
197
|
getUrl(filename) {
|
|
99
198
|
const safeName = sanitizeFilename(filename);
|
|
100
199
|
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
@@ -114,6 +213,7 @@ __export(index_exports, {
|
|
|
114
213
|
GenerateSitemapJob: () => GenerateSitemapJob,
|
|
115
214
|
IncrementalGenerator: () => IncrementalGenerator,
|
|
116
215
|
MemoryChangeTracker: () => MemoryChangeTracker,
|
|
216
|
+
MemoryLock: () => MemoryLock,
|
|
117
217
|
MemoryProgressStorage: () => MemoryProgressStorage,
|
|
118
218
|
MemoryRedirectManager: () => MemoryRedirectManager,
|
|
119
219
|
MemorySitemapStorage: () => MemorySitemapStorage,
|
|
@@ -122,6 +222,7 @@ __export(index_exports, {
|
|
|
122
222
|
RedirectDetector: () => RedirectDetector,
|
|
123
223
|
RedirectHandler: () => RedirectHandler,
|
|
124
224
|
RedisChangeTracker: () => RedisChangeTracker,
|
|
225
|
+
RedisLock: () => RedisLock,
|
|
125
226
|
RedisProgressStorage: () => RedisProgressStorage,
|
|
126
227
|
RedisRedirectManager: () => RedisRedirectManager,
|
|
127
228
|
RouteScanner: () => RouteScanner,
|
|
@@ -130,8 +231,12 @@ __export(index_exports, {
|
|
|
130
231
|
SitemapGenerator: () => SitemapGenerator,
|
|
131
232
|
SitemapIndex: () => SitemapIndex,
|
|
132
233
|
SitemapStream: () => SitemapStream,
|
|
234
|
+
compressToBuffer: () => compressToBuffer,
|
|
235
|
+
createCompressionStream: () => createCompressionStream,
|
|
236
|
+
fromGzipFilename: () => fromGzipFilename,
|
|
133
237
|
generateI18nEntries: () => generateI18nEntries,
|
|
134
|
-
routeScanner: () => routeScanner
|
|
238
|
+
routeScanner: () => routeScanner,
|
|
239
|
+
toGzipFilename: () => toGzipFilename
|
|
135
240
|
});
|
|
136
241
|
module.exports = __toCommonJS(index_exports);
|
|
137
242
|
|
|
@@ -143,6 +248,11 @@ var MemoryChangeTracker = class {
|
|
|
143
248
|
constructor(options = {}) {
|
|
144
249
|
this.maxChanges = options.maxChanges || 1e5;
|
|
145
250
|
}
|
|
251
|
+
/**
|
|
252
|
+
* Record a new site structure change in memory.
|
|
253
|
+
*
|
|
254
|
+
* @param change - The change event to record.
|
|
255
|
+
*/
|
|
146
256
|
async track(change) {
|
|
147
257
|
this.changes.push(change);
|
|
148
258
|
const urlChanges = this.urlIndex.get(change.url) || [];
|
|
@@ -164,15 +274,32 @@ var MemoryChangeTracker = class {
|
|
|
164
274
|
}
|
|
165
275
|
}
|
|
166
276
|
}
|
|
277
|
+
/**
|
|
278
|
+
* Retrieve all changes recorded in memory since a specific time.
|
|
279
|
+
*
|
|
280
|
+
* @param since - Optional start date for the query.
|
|
281
|
+
* @returns An array of change events.
|
|
282
|
+
*/
|
|
167
283
|
async getChanges(since) {
|
|
168
284
|
if (!since) {
|
|
169
285
|
return [...this.changes];
|
|
170
286
|
}
|
|
171
287
|
return this.changes.filter((change) => change.timestamp >= since);
|
|
172
288
|
}
|
|
289
|
+
/**
|
|
290
|
+
* Retrieve the full change history for a specific URL from memory.
|
|
291
|
+
*
|
|
292
|
+
* @param url - The URL to query history for.
|
|
293
|
+
* @returns An array of change events.
|
|
294
|
+
*/
|
|
173
295
|
async getChangesByUrl(url) {
|
|
174
296
|
return this.urlIndex.get(url) || [];
|
|
175
297
|
}
|
|
298
|
+
/**
|
|
299
|
+
* Purge old change records from memory storage.
|
|
300
|
+
*
|
|
301
|
+
* @param since - If provided, only records older than this date will be cleared.
|
|
302
|
+
*/
|
|
176
303
|
async clear(since) {
|
|
177
304
|
if (!since) {
|
|
178
305
|
this.changes = [];
|
|
@@ -203,6 +330,11 @@ var RedisChangeTracker = class {
|
|
|
203
330
|
getListKey() {
|
|
204
331
|
return `${this.keyPrefix}list`;
|
|
205
332
|
}
|
|
333
|
+
/**
|
|
334
|
+
* Record a new site structure change in Redis.
|
|
335
|
+
*
|
|
336
|
+
* @param change - The change event to record.
|
|
337
|
+
*/
|
|
206
338
|
async track(change) {
|
|
207
339
|
const key = this.getKey(change.url);
|
|
208
340
|
const listKey = this.getListKey();
|
|
@@ -212,6 +344,12 @@ var RedisChangeTracker = class {
|
|
|
212
344
|
await this.client.zadd(listKey, score, change.url);
|
|
213
345
|
await this.client.expire(listKey, this.ttl);
|
|
214
346
|
}
|
|
347
|
+
/**
|
|
348
|
+
* Retrieve all changes recorded in Redis since a specific time.
|
|
349
|
+
*
|
|
350
|
+
* @param since - Optional start date for the query.
|
|
351
|
+
* @returns An array of change events.
|
|
352
|
+
*/
|
|
215
353
|
async getChanges(since) {
|
|
216
354
|
try {
|
|
217
355
|
const listKey = this.getListKey();
|
|
@@ -232,6 +370,12 @@ var RedisChangeTracker = class {
|
|
|
232
370
|
return [];
|
|
233
371
|
}
|
|
234
372
|
}
|
|
373
|
+
/**
|
|
374
|
+
* Retrieve the full change history for a specific URL from Redis.
|
|
375
|
+
*
|
|
376
|
+
* @param url - The URL to query history for.
|
|
377
|
+
* @returns An array of change events.
|
|
378
|
+
*/
|
|
235
379
|
async getChangesByUrl(url) {
|
|
236
380
|
try {
|
|
237
381
|
const key = this.getKey(url);
|
|
@@ -246,6 +390,11 @@ var RedisChangeTracker = class {
|
|
|
246
390
|
return [];
|
|
247
391
|
}
|
|
248
392
|
}
|
|
393
|
+
/**
|
|
394
|
+
* Purge old change records from Redis storage.
|
|
395
|
+
*
|
|
396
|
+
* @param since - If provided, only records older than this date will be cleared.
|
|
397
|
+
*/
|
|
249
398
|
async clear(since) {
|
|
250
399
|
try {
|
|
251
400
|
const listKey = this.getListKey();
|
|
@@ -275,7 +424,11 @@ var DiffCalculator = class {
|
|
|
275
424
|
this.batchSize = options.batchSize || 1e4;
|
|
276
425
|
}
|
|
277
426
|
/**
|
|
278
|
-
*
|
|
427
|
+
* Calculates the difference between two sets of sitemap entries.
|
|
428
|
+
*
|
|
429
|
+
* @param oldEntries - The previous set of sitemap entries.
|
|
430
|
+
* @param newEntries - The current set of sitemap entries.
|
|
431
|
+
* @returns A DiffResult containing added, updated, and removed entries.
|
|
279
432
|
*/
|
|
280
433
|
calculate(oldEntries, newEntries) {
|
|
281
434
|
const oldMap = /* @__PURE__ */ new Map();
|
|
@@ -305,7 +458,11 @@ var DiffCalculator = class {
|
|
|
305
458
|
return { added, updated, removed };
|
|
306
459
|
}
|
|
307
460
|
/**
|
|
308
|
-
*
|
|
461
|
+
* Batch calculates differences for large datasets using async iterables.
|
|
462
|
+
*
|
|
463
|
+
* @param oldEntries - An async iterable of the previous sitemap entries.
|
|
464
|
+
* @param newEntries - An async iterable of the current sitemap entries.
|
|
465
|
+
* @returns A promise resolving to the DiffResult.
|
|
309
466
|
*/
|
|
310
467
|
async calculateBatch(oldEntries, newEntries) {
|
|
311
468
|
const oldMap = /* @__PURE__ */ new Map();
|
|
@@ -319,7 +476,11 @@ var DiffCalculator = class {
|
|
|
319
476
|
return this.calculate(Array.from(oldMap.values()), Array.from(newMap.values()));
|
|
320
477
|
}
|
|
321
478
|
/**
|
|
322
|
-
*
|
|
479
|
+
* Calculates differences based on a sequence of change records.
|
|
480
|
+
*
|
|
481
|
+
* @param baseEntries - The base set of sitemap entries.
|
|
482
|
+
* @param changes - An array of change records to apply to the base set.
|
|
483
|
+
* @returns A DiffResult comparing the base set with the applied changes.
|
|
323
484
|
*/
|
|
324
485
|
calculateFromChanges(baseEntries, changes) {
|
|
325
486
|
const entryMap = /* @__PURE__ */ new Map();
|
|
@@ -339,7 +500,11 @@ var DiffCalculator = class {
|
|
|
339
500
|
return this.calculate(baseEntries, newEntries);
|
|
340
501
|
}
|
|
341
502
|
/**
|
|
342
|
-
*
|
|
503
|
+
* Checks if a sitemap entry has changed by comparing its key properties.
|
|
504
|
+
*
|
|
505
|
+
* @param oldEntry - The previous sitemap entry.
|
|
506
|
+
* @param newEntry - The current sitemap entry.
|
|
507
|
+
* @returns True if the entry has changed, false otherwise.
|
|
343
508
|
*/
|
|
344
509
|
hasChanged(oldEntry, newEntry) {
|
|
345
510
|
if (oldEntry.lastmod !== newEntry.lastmod) {
|
|
@@ -363,6 +528,12 @@ var DiffCalculator = class {
|
|
|
363
528
|
// src/utils/Mutex.ts
|
|
364
529
|
var Mutex = class {
|
|
365
530
|
queue = Promise.resolve();
|
|
531
|
+
/**
|
|
532
|
+
* Executes a function exclusively, ensuring no other task can run it concurrently.
|
|
533
|
+
*
|
|
534
|
+
* @param fn - The async function to execute.
|
|
535
|
+
* @returns A promise resolving to the result of the function.
|
|
536
|
+
*/
|
|
366
537
|
async runExclusive(fn) {
|
|
367
538
|
const next = this.queue.then(() => fn());
|
|
368
539
|
this.queue = next.then(
|
|
@@ -375,6 +546,9 @@ var Mutex = class {
|
|
|
375
546
|
}
|
|
376
547
|
};
|
|
377
548
|
|
|
549
|
+
// src/core/SitemapGenerator.ts
|
|
550
|
+
init_Compression();
|
|
551
|
+
|
|
378
552
|
// src/core/ShadowProcessor.ts
|
|
379
553
|
var ShadowProcessor = class {
|
|
380
554
|
options;
|
|
@@ -386,7 +560,12 @@ var ShadowProcessor = class {
|
|
|
386
560
|
this.shadowId = `shadow-${Date.now()}-${crypto.randomUUID()}`;
|
|
387
561
|
}
|
|
388
562
|
/**
|
|
389
|
-
*
|
|
563
|
+
* Adds a single file write operation to the current shadow session.
|
|
564
|
+
*
|
|
565
|
+
* If shadow processing is disabled, the file is written directly to the
|
|
566
|
+
* final destination in storage. Otherwise, it is written to the shadow staging area.
|
|
567
|
+
*
|
|
568
|
+
* @param operation - The shadow write operation details.
|
|
390
569
|
*/
|
|
391
570
|
async addOperation(operation) {
|
|
392
571
|
return this.mutex.runExclusive(async () => {
|
|
@@ -406,7 +585,10 @@ var ShadowProcessor = class {
|
|
|
406
585
|
});
|
|
407
586
|
}
|
|
408
587
|
/**
|
|
409
|
-
*
|
|
588
|
+
* Commits all staged shadow operations to the final production location.
|
|
589
|
+
*
|
|
590
|
+
* Depending on the `mode`, this will either perform an atomic swap of all files
|
|
591
|
+
* or commit each file individually (potentially creating new versions).
|
|
410
592
|
*/
|
|
411
593
|
async commit() {
|
|
412
594
|
return this.mutex.runExclusive(async () => {
|
|
@@ -428,7 +610,7 @@ var ShadowProcessor = class {
|
|
|
428
610
|
});
|
|
429
611
|
}
|
|
430
612
|
/**
|
|
431
|
-
*
|
|
613
|
+
* Cancels all staged shadow operations without committing them.
|
|
432
614
|
*/
|
|
433
615
|
async rollback() {
|
|
434
616
|
if (!this.options.enabled) {
|
|
@@ -437,13 +619,13 @@ var ShadowProcessor = class {
|
|
|
437
619
|
this.operations = [];
|
|
438
620
|
}
|
|
439
621
|
/**
|
|
440
|
-
*
|
|
622
|
+
* Returns the unique identifier for the current shadow session.
|
|
441
623
|
*/
|
|
442
624
|
getShadowId() {
|
|
443
625
|
return this.shadowId;
|
|
444
626
|
}
|
|
445
627
|
/**
|
|
446
|
-
*
|
|
628
|
+
* Returns an array of all staged shadow operations.
|
|
447
629
|
*/
|
|
448
630
|
getOperations() {
|
|
449
631
|
return [...this.operations];
|
|
@@ -460,6 +642,12 @@ var SitemapIndex = class {
|
|
|
460
642
|
this.options.baseUrl = this.options.baseUrl.slice(0, -1);
|
|
461
643
|
}
|
|
462
644
|
}
|
|
645
|
+
/**
|
|
646
|
+
* Adds a single entry to the sitemap index.
|
|
647
|
+
*
|
|
648
|
+
* @param entry - A sitemap filename or a `SitemapIndexEntry` object.
|
|
649
|
+
* @returns The `SitemapIndex` instance for chaining.
|
|
650
|
+
*/
|
|
463
651
|
add(entry) {
|
|
464
652
|
if (typeof entry === "string") {
|
|
465
653
|
this.entries.push({ url: entry });
|
|
@@ -468,12 +656,23 @@ var SitemapIndex = class {
|
|
|
468
656
|
}
|
|
469
657
|
return this;
|
|
470
658
|
}
|
|
659
|
+
/**
|
|
660
|
+
* Adds multiple entries to the sitemap index.
|
|
661
|
+
*
|
|
662
|
+
* @param entries - An array of sitemap filenames or `SitemapIndexEntry` objects.
|
|
663
|
+
* @returns The `SitemapIndex` instance for chaining.
|
|
664
|
+
*/
|
|
471
665
|
addAll(entries) {
|
|
472
666
|
for (const entry of entries) {
|
|
473
667
|
this.add(entry);
|
|
474
668
|
}
|
|
475
669
|
return this;
|
|
476
670
|
}
|
|
671
|
+
/**
|
|
672
|
+
* Generates the sitemap index XML content.
|
|
673
|
+
*
|
|
674
|
+
* @returns The complete XML string for the sitemap index.
|
|
675
|
+
*/
|
|
477
676
|
toXML() {
|
|
478
677
|
const { baseUrl, pretty } = this.options;
|
|
479
678
|
let xml = `<?xml version="1.0" encoding="UTF-8"?>
|
|
@@ -502,6 +701,9 @@ var SitemapIndex = class {
|
|
|
502
701
|
xml += `</sitemapindex>`;
|
|
503
702
|
return xml;
|
|
504
703
|
}
|
|
704
|
+
/**
|
|
705
|
+
* Escapes special XML characters in a string.
|
|
706
|
+
*/
|
|
505
707
|
escape(str) {
|
|
506
708
|
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
507
709
|
}
|
|
@@ -523,6 +725,12 @@ var SitemapStream = class {
|
|
|
523
725
|
this.options.baseUrl = this.options.baseUrl.slice(0, -1);
|
|
524
726
|
}
|
|
525
727
|
}
|
|
728
|
+
/**
|
|
729
|
+
* Adds a single entry to the sitemap stream.
|
|
730
|
+
*
|
|
731
|
+
* @param entry - A URL string or a complete `SitemapEntry` object.
|
|
732
|
+
* @returns The `SitemapStream` instance for chaining.
|
|
733
|
+
*/
|
|
526
734
|
add(entry) {
|
|
527
735
|
const e = typeof entry === "string" ? { url: entry } : entry;
|
|
528
736
|
this.entries.push(e);
|
|
@@ -540,16 +748,78 @@ var SitemapStream = class {
|
|
|
540
748
|
}
|
|
541
749
|
return this;
|
|
542
750
|
}
|
|
751
|
+
/**
|
|
752
|
+
* Adds multiple entries to the sitemap stream.
|
|
753
|
+
*
|
|
754
|
+
* @param entries - An array of URL strings or `SitemapEntry` objects.
|
|
755
|
+
* @returns The `SitemapStream` instance for chaining.
|
|
756
|
+
*/
|
|
543
757
|
addAll(entries) {
|
|
544
758
|
for (const entry of entries) {
|
|
545
759
|
this.add(entry);
|
|
546
760
|
}
|
|
547
761
|
return this;
|
|
548
762
|
}
|
|
763
|
+
/**
|
|
764
|
+
* Generates the sitemap XML content.
|
|
765
|
+
*
|
|
766
|
+
* Automatically includes the necessary XML namespaces for images, videos, news,
|
|
767
|
+
* and internationalization if the entries contain such metadata.
|
|
768
|
+
*
|
|
769
|
+
* @returns The complete XML string for the sitemap.
|
|
770
|
+
*/
|
|
549
771
|
toXML() {
|
|
772
|
+
const parts = [];
|
|
773
|
+
for (const chunk of this.toSyncIterable()) {
|
|
774
|
+
parts.push(chunk);
|
|
775
|
+
}
|
|
776
|
+
return parts.join("");
|
|
777
|
+
}
|
|
778
|
+
/**
|
|
779
|
+
* 以 AsyncGenerator 方式產生 XML 內容。
|
|
780
|
+
* 每次 yield 一個邏輯區塊,適合串流寫入場景,可減少記憶體峰值。
|
|
781
|
+
*
|
|
782
|
+
* @returns AsyncGenerator 產生 XML 字串片段
|
|
783
|
+
*
|
|
784
|
+
* @example
|
|
785
|
+
* ```typescript
|
|
786
|
+
* const stream = new SitemapStream({ baseUrl: 'https://example.com' })
|
|
787
|
+
* stream.add({ url: '/page1' })
|
|
788
|
+
* stream.add({ url: '/page2' })
|
|
789
|
+
*
|
|
790
|
+
* for await (const chunk of stream.toAsyncIterable()) {
|
|
791
|
+
* process.stdout.write(chunk)
|
|
792
|
+
* }
|
|
793
|
+
* ```
|
|
794
|
+
*
|
|
795
|
+
* @since 3.1.0
|
|
796
|
+
*/
|
|
797
|
+
async *toAsyncIterable() {
|
|
798
|
+
yield '<?xml version="1.0" encoding="UTF-8"?>\n';
|
|
799
|
+
yield this.buildUrlsetOpenTag();
|
|
800
|
+
const { baseUrl, pretty } = this.options;
|
|
801
|
+
for (const entry of this.entries) {
|
|
802
|
+
yield this.renderUrl(entry, baseUrl, pretty);
|
|
803
|
+
}
|
|
804
|
+
yield "</urlset>";
|
|
805
|
+
}
|
|
806
|
+
/**
|
|
807
|
+
* 同步版本的 iterable,供 toXML() 使用。
|
|
808
|
+
*/
|
|
809
|
+
*toSyncIterable() {
|
|
810
|
+
yield '<?xml version="1.0" encoding="UTF-8"?>\n';
|
|
811
|
+
yield this.buildUrlsetOpenTag();
|
|
550
812
|
const { baseUrl, pretty } = this.options;
|
|
813
|
+
for (const entry of this.entries) {
|
|
814
|
+
yield this.renderUrl(entry, baseUrl, pretty);
|
|
815
|
+
}
|
|
816
|
+
yield "</urlset>";
|
|
817
|
+
}
|
|
818
|
+
/**
|
|
819
|
+
* 建立 urlset 開標籤與所有必要的 XML 命名空間。
|
|
820
|
+
*/
|
|
821
|
+
buildUrlsetOpenTag() {
|
|
551
822
|
const parts = [];
|
|
552
|
-
parts.push('<?xml version="1.0" encoding="UTF-8"?>\n');
|
|
553
823
|
parts.push('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"');
|
|
554
824
|
if (this.flags.hasImages) {
|
|
555
825
|
parts.push(' xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"');
|
|
@@ -564,12 +834,11 @@ var SitemapStream = class {
|
|
|
564
834
|
parts.push(' xmlns:xhtml="http://www.w3.org/1999/xhtml"');
|
|
565
835
|
}
|
|
566
836
|
parts.push(">\n");
|
|
567
|
-
for (const entry of this.entries) {
|
|
568
|
-
parts.push(this.renderUrl(entry, baseUrl, pretty));
|
|
569
|
-
}
|
|
570
|
-
parts.push("</urlset>");
|
|
571
837
|
return parts.join("");
|
|
572
838
|
}
|
|
839
|
+
/**
|
|
840
|
+
* Renders a single sitemap entry into its XML representation.
|
|
841
|
+
*/
|
|
573
842
|
renderUrl(entry, baseUrl, pretty) {
|
|
574
843
|
const indent = pretty ? " " : "";
|
|
575
844
|
const subIndent = pretty ? " " : "";
|
|
@@ -732,9 +1001,17 @@ var SitemapStream = class {
|
|
|
732
1001
|
parts.push(`${indent}</url>${nl}`);
|
|
733
1002
|
return parts.join("");
|
|
734
1003
|
}
|
|
1004
|
+
/**
|
|
1005
|
+
* Escapes special XML characters in a string.
|
|
1006
|
+
*/
|
|
735
1007
|
escape(str) {
|
|
736
1008
|
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
737
1009
|
}
|
|
1010
|
+
/**
|
|
1011
|
+
* Returns all entries currently in the stream.
|
|
1012
|
+
*
|
|
1013
|
+
* @returns An array of `SitemapEntry` objects.
|
|
1014
|
+
*/
|
|
738
1015
|
getEntries() {
|
|
739
1016
|
return this.entries;
|
|
740
1017
|
}
|
|
@@ -758,6 +1035,12 @@ var SitemapGenerator = class {
|
|
|
758
1035
|
});
|
|
759
1036
|
}
|
|
760
1037
|
}
|
|
1038
|
+
/**
|
|
1039
|
+
* Orchestrates the sitemap generation process.
|
|
1040
|
+
*
|
|
1041
|
+
* This method scans all providers, handles sharding, generates the XML files,
|
|
1042
|
+
* and optionally creates a sitemap index and manifest.
|
|
1043
|
+
*/
|
|
761
1044
|
async run() {
|
|
762
1045
|
let shardIndex = 1;
|
|
763
1046
|
let currentCount = 0;
|
|
@@ -775,21 +1058,23 @@ var SitemapGenerator = class {
|
|
|
775
1058
|
isMultiFile = true;
|
|
776
1059
|
const baseName = this.options.filename?.replace(/\.xml$/, "");
|
|
777
1060
|
const filename = `${baseName}-${shardIndex}.xml`;
|
|
778
|
-
const xml = currentStream.toXML();
|
|
779
1061
|
const entries = currentStream.getEntries();
|
|
1062
|
+
let actualFilename;
|
|
1063
|
+
if (this.shadowProcessor) {
|
|
1064
|
+
actualFilename = this.options.compression?.enabled ? toGzipFilename(filename) : filename;
|
|
1065
|
+
const xml = currentStream.toXML();
|
|
1066
|
+
await this.shadowProcessor.addOperation({ filename: actualFilename, content: xml });
|
|
1067
|
+
} else {
|
|
1068
|
+
actualFilename = await this.writeSitemap(currentStream, filename);
|
|
1069
|
+
}
|
|
780
1070
|
shards.push({
|
|
781
|
-
filename,
|
|
1071
|
+
filename: actualFilename,
|
|
782
1072
|
from: this.normalizeUrl(entries[0].url),
|
|
783
1073
|
to: this.normalizeUrl(entries[entries.length - 1].url),
|
|
784
1074
|
count: entries.length,
|
|
785
1075
|
lastmod: /* @__PURE__ */ new Date()
|
|
786
1076
|
});
|
|
787
|
-
|
|
788
|
-
await this.shadowProcessor.addOperation({ filename, content: xml });
|
|
789
|
-
} else {
|
|
790
|
-
await this.options.storage.write(filename, xml);
|
|
791
|
-
}
|
|
792
|
-
const url = this.options.storage.getUrl(filename);
|
|
1077
|
+
const url = this.options.storage.getUrl(actualFilename);
|
|
793
1078
|
index.add({
|
|
794
1079
|
url,
|
|
795
1080
|
lastmod: /* @__PURE__ */ new Date()
|
|
@@ -822,7 +1107,9 @@ var SitemapGenerator = class {
|
|
|
822
1107
|
}
|
|
823
1108
|
}
|
|
824
1109
|
const writeManifest = async () => {
|
|
825
|
-
if (!this.options.generateManifest)
|
|
1110
|
+
if (!this.options.generateManifest) {
|
|
1111
|
+
return;
|
|
1112
|
+
}
|
|
826
1113
|
const manifest = {
|
|
827
1114
|
version: 1,
|
|
828
1115
|
generatedAt: /* @__PURE__ */ new Date(),
|
|
@@ -840,24 +1127,29 @@ var SitemapGenerator = class {
|
|
|
840
1127
|
}
|
|
841
1128
|
};
|
|
842
1129
|
if (!isMultiFile) {
|
|
843
|
-
const xml = currentStream.toXML();
|
|
844
1130
|
const entries = currentStream.getEntries();
|
|
1131
|
+
let actualFilename;
|
|
1132
|
+
if (this.shadowProcessor) {
|
|
1133
|
+
actualFilename = this.options.compression?.enabled ? toGzipFilename(this.options.filename) : this.options.filename;
|
|
1134
|
+
const xml = currentStream.toXML();
|
|
1135
|
+
await this.shadowProcessor.addOperation({
|
|
1136
|
+
filename: actualFilename,
|
|
1137
|
+
content: xml
|
|
1138
|
+
});
|
|
1139
|
+
} else {
|
|
1140
|
+
actualFilename = await this.writeSitemap(currentStream, this.options.filename);
|
|
1141
|
+
}
|
|
845
1142
|
shards.push({
|
|
846
|
-
filename:
|
|
1143
|
+
filename: actualFilename,
|
|
847
1144
|
from: entries[0] ? this.normalizeUrl(entries[0].url) : "",
|
|
848
1145
|
to: entries[entries.length - 1] ? this.normalizeUrl(entries[entries.length - 1].url) : "",
|
|
849
1146
|
count: entries.length,
|
|
850
1147
|
lastmod: /* @__PURE__ */ new Date()
|
|
851
1148
|
});
|
|
852
1149
|
if (this.shadowProcessor) {
|
|
853
|
-
await this.shadowProcessor.addOperation({
|
|
854
|
-
filename: this.options.filename,
|
|
855
|
-
content: xml
|
|
856
|
-
});
|
|
857
1150
|
await writeManifest();
|
|
858
1151
|
await this.shadowProcessor.commit();
|
|
859
1152
|
} else {
|
|
860
|
-
await this.options.storage.write(this.options.filename, xml);
|
|
861
1153
|
await writeManifest();
|
|
862
1154
|
}
|
|
863
1155
|
return;
|
|
@@ -878,6 +1170,41 @@ var SitemapGenerator = class {
|
|
|
878
1170
|
await writeManifest();
|
|
879
1171
|
}
|
|
880
1172
|
}
|
|
1173
|
+
/**
|
|
1174
|
+
* 統一的 sitemap 寫入方法,優先使用串流寫入以降低記憶體使用。
|
|
1175
|
+
*
|
|
1176
|
+
* @param stream - SitemapStream 實例
|
|
1177
|
+
* @param filename - 檔案名稱(不含 .gz)
|
|
1178
|
+
* @returns 實際寫入的檔名(可能包含 .gz)
|
|
1179
|
+
* @since 3.1.0
|
|
1180
|
+
*/
|
|
1181
|
+
async writeSitemap(stream, filename) {
|
|
1182
|
+
const { storage, compression } = this.options;
|
|
1183
|
+
const compress = compression?.enabled ?? false;
|
|
1184
|
+
if (storage.writeStream) {
|
|
1185
|
+
await storage.writeStream(filename, stream.toAsyncIterable(), {
|
|
1186
|
+
compress,
|
|
1187
|
+
contentType: "application/xml"
|
|
1188
|
+
});
|
|
1189
|
+
} else {
|
|
1190
|
+
const xml = stream.toXML();
|
|
1191
|
+
if (compress) {
|
|
1192
|
+
const buffer = await compressToBuffer(
|
|
1193
|
+
(async function* () {
|
|
1194
|
+
yield xml;
|
|
1195
|
+
})(),
|
|
1196
|
+
{ level: compression?.level }
|
|
1197
|
+
);
|
|
1198
|
+
await storage.write(toGzipFilename(filename), buffer.toString("base64"));
|
|
1199
|
+
} else {
|
|
1200
|
+
await storage.write(filename, xml);
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
return compress ? toGzipFilename(filename) : filename;
|
|
1204
|
+
}
|
|
1205
|
+
/**
|
|
1206
|
+
* Normalizes a URL to an absolute URL using the base URL.
|
|
1207
|
+
*/
|
|
881
1208
|
normalizeUrl(url) {
|
|
882
1209
|
if (url.startsWith("http")) {
|
|
883
1210
|
return url;
|
|
@@ -888,7 +1215,7 @@ var SitemapGenerator = class {
|
|
|
888
1215
|
return normalizedBase + normalizedPath;
|
|
889
1216
|
}
|
|
890
1217
|
/**
|
|
891
|
-
*
|
|
1218
|
+
* Returns the shadow processor instance if enabled.
|
|
892
1219
|
*/
|
|
893
1220
|
getShadowProcessor() {
|
|
894
1221
|
return this.shadowProcessor;
|
|
@@ -897,6 +1224,12 @@ var SitemapGenerator = class {
|
|
|
897
1224
|
|
|
898
1225
|
// src/core/SitemapParser.ts
|
|
899
1226
|
var SitemapParser = class {
|
|
1227
|
+
/**
|
|
1228
|
+
* Parses a sitemap XML string into an array of entries.
|
|
1229
|
+
*
|
|
1230
|
+
* @param xml - The raw sitemap XML content.
|
|
1231
|
+
* @returns An array of `SitemapEntry` objects.
|
|
1232
|
+
*/
|
|
900
1233
|
static parse(xml) {
|
|
901
1234
|
const entries = [];
|
|
902
1235
|
const urlRegex = /<url>([\s\S]*?)<\/url>/g;
|
|
@@ -909,6 +1242,14 @@ var SitemapParser = class {
|
|
|
909
1242
|
}
|
|
910
1243
|
return entries;
|
|
911
1244
|
}
|
|
1245
|
+
/**
|
|
1246
|
+
* Parses a sitemap XML stream into an async iterable of entries.
|
|
1247
|
+
*
|
|
1248
|
+
* Useful for large sitemap files that should not be fully loaded into memory.
|
|
1249
|
+
*
|
|
1250
|
+
* @param stream - An async iterable of XML chunks.
|
|
1251
|
+
* @returns An async iterable of `SitemapEntry` objects.
|
|
1252
|
+
*/
|
|
912
1253
|
static async *parseStream(stream) {
|
|
913
1254
|
let buffer = "";
|
|
914
1255
|
const urlRegex = /<url>([\s\S]*?)<\/url>/g;
|
|
@@ -929,6 +1270,9 @@ var SitemapParser = class {
|
|
|
929
1270
|
}
|
|
930
1271
|
}
|
|
931
1272
|
}
|
|
1273
|
+
/**
|
|
1274
|
+
* Parses a single `<url>` tag content into a `SitemapEntry`.
|
|
1275
|
+
*/
|
|
932
1276
|
static parseEntry(urlContent) {
|
|
933
1277
|
const entry = { url: "" };
|
|
934
1278
|
const locMatch = /<loc>(.*?)<\/loc>/.exec(urlContent);
|
|
@@ -951,6 +1295,12 @@ var SitemapParser = class {
|
|
|
951
1295
|
}
|
|
952
1296
|
return entry;
|
|
953
1297
|
}
|
|
1298
|
+
/**
|
|
1299
|
+
* Parses a sitemap index XML string into an array of sitemap URLs.
|
|
1300
|
+
*
|
|
1301
|
+
* @param xml - The raw sitemap index XML content.
|
|
1302
|
+
* @returns An array of sub-sitemap URLs.
|
|
1303
|
+
*/
|
|
954
1304
|
static parseIndex(xml) {
|
|
955
1305
|
const urls = [];
|
|
956
1306
|
const sitemapRegex = /<sitemap>([\s\S]*?)<\/sitemap>/g;
|
|
@@ -964,6 +1314,9 @@ var SitemapParser = class {
|
|
|
964
1314
|
}
|
|
965
1315
|
return urls;
|
|
966
1316
|
}
|
|
1317
|
+
/**
|
|
1318
|
+
* Unescapes special XML entities in a string.
|
|
1319
|
+
*/
|
|
967
1320
|
static unescape(str) {
|
|
968
1321
|
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'");
|
|
969
1322
|
}
|
|
@@ -986,12 +1339,26 @@ var IncrementalGenerator = class {
|
|
|
986
1339
|
this.diffCalculator = this.options.diffCalculator || new DiffCalculator();
|
|
987
1340
|
this.generator = new SitemapGenerator(this.options);
|
|
988
1341
|
}
|
|
1342
|
+
/**
|
|
1343
|
+
* Performs a full sitemap generation and optionally records all entries in the change tracker.
|
|
1344
|
+
*/
|
|
989
1345
|
async generateFull() {
|
|
990
1346
|
return this.mutex.runExclusive(() => this.performFullGeneration());
|
|
991
1347
|
}
|
|
1348
|
+
/**
|
|
1349
|
+
* Performs an incremental sitemap update based on changes recorded since a specific time.
|
|
1350
|
+
*
|
|
1351
|
+
* If the number of changes exceeds a certain threshold (e.g., 30% of total URLs),
|
|
1352
|
+
* a full generation is triggered instead to ensure consistency.
|
|
1353
|
+
*
|
|
1354
|
+
* @param since - Optional start date for the incremental update.
|
|
1355
|
+
*/
|
|
992
1356
|
async generateIncremental(since) {
|
|
993
1357
|
return this.mutex.runExclusive(() => this.performIncrementalGeneration(since));
|
|
994
1358
|
}
|
|
1359
|
+
/**
|
|
1360
|
+
* Internal implementation of full sitemap generation.
|
|
1361
|
+
*/
|
|
995
1362
|
async performFullGeneration() {
|
|
996
1363
|
await this.generator.run();
|
|
997
1364
|
if (this.options.autoTrack) {
|
|
@@ -1010,6 +1377,9 @@ var IncrementalGenerator = class {
|
|
|
1010
1377
|
}
|
|
1011
1378
|
}
|
|
1012
1379
|
}
|
|
1380
|
+
/**
|
|
1381
|
+
* Internal implementation of incremental sitemap generation.
|
|
1382
|
+
*/
|
|
1013
1383
|
async performIncrementalGeneration(since) {
|
|
1014
1384
|
const changes = await this.changeTracker.getChanges(since);
|
|
1015
1385
|
if (changes.length === 0) {
|
|
@@ -1033,6 +1403,9 @@ var IncrementalGenerator = class {
|
|
|
1033
1403
|
}
|
|
1034
1404
|
await this.updateShards(manifest, affectedShards);
|
|
1035
1405
|
}
|
|
1406
|
+
/**
|
|
1407
|
+
* Normalizes a URL to an absolute URL using the base URL.
|
|
1408
|
+
*/
|
|
1036
1409
|
normalizeUrl(url) {
|
|
1037
1410
|
if (url.startsWith("http")) {
|
|
1038
1411
|
return url;
|
|
@@ -1042,6 +1415,9 @@ var IncrementalGenerator = class {
|
|
|
1042
1415
|
const normalizedPath = url.startsWith("/") ? url : `/${url}`;
|
|
1043
1416
|
return normalizedBase + normalizedPath;
|
|
1044
1417
|
}
|
|
1418
|
+
/**
|
|
1419
|
+
* Loads the sitemap shard manifest from storage.
|
|
1420
|
+
*/
|
|
1045
1421
|
async loadManifest() {
|
|
1046
1422
|
const filename = this.options.filename?.replace(/\.xml$/, "-manifest.json") || "sitemap-manifest.json";
|
|
1047
1423
|
const content = await this.options.storage.read(filename);
|
|
@@ -1054,6 +1430,9 @@ var IncrementalGenerator = class {
|
|
|
1054
1430
|
return null;
|
|
1055
1431
|
}
|
|
1056
1432
|
}
|
|
1433
|
+
/**
|
|
1434
|
+
* Identifies which shards are affected by the given set of changes.
|
|
1435
|
+
*/
|
|
1057
1436
|
getAffectedShards(manifest, changes) {
|
|
1058
1437
|
const affected = /* @__PURE__ */ new Map();
|
|
1059
1438
|
for (const change of changes) {
|
|
@@ -1075,6 +1454,9 @@ var IncrementalGenerator = class {
|
|
|
1075
1454
|
}
|
|
1076
1455
|
return affected;
|
|
1077
1456
|
}
|
|
1457
|
+
/**
|
|
1458
|
+
* Updates the affected shards in storage.
|
|
1459
|
+
*/
|
|
1078
1460
|
async updateShards(manifest, affectedShards) {
|
|
1079
1461
|
for (const [filename, shardChanges] of affectedShards) {
|
|
1080
1462
|
const entries = [];
|
|
@@ -1112,6 +1494,9 @@ var IncrementalGenerator = class {
|
|
|
1112
1494
|
JSON.stringify(manifest, null, this.options.pretty ? 2 : 0)
|
|
1113
1495
|
);
|
|
1114
1496
|
}
|
|
1497
|
+
/**
|
|
1498
|
+
* Applies changes to a set of sitemap entries and returns the updated, sorted list.
|
|
1499
|
+
*/
|
|
1115
1500
|
applyChanges(entries, changes) {
|
|
1116
1501
|
const entryMap = /* @__PURE__ */ new Map();
|
|
1117
1502
|
for (const entry of entries) {
|
|
@@ -1134,6 +1519,9 @@ var IncrementalGenerator = class {
|
|
|
1134
1519
|
(a, b) => this.normalizeUrl(a.url).localeCompare(this.normalizeUrl(b.url))
|
|
1135
1520
|
);
|
|
1136
1521
|
}
|
|
1522
|
+
/**
|
|
1523
|
+
* Helper to convert an async iterable into an array.
|
|
1524
|
+
*/
|
|
1137
1525
|
async toArray(iterable) {
|
|
1138
1526
|
const array = [];
|
|
1139
1527
|
for await (const item of iterable) {
|
|
@@ -1154,7 +1542,10 @@ var ProgressTracker = class {
|
|
|
1154
1542
|
this.updateInterval = options.updateInterval || 1e3;
|
|
1155
1543
|
}
|
|
1156
1544
|
/**
|
|
1157
|
-
*
|
|
1545
|
+
* Initializes progress tracking for a new job.
|
|
1546
|
+
*
|
|
1547
|
+
* @param jobId - Unique identifier for the generation job.
|
|
1548
|
+
* @param total - Total number of entries to be processed.
|
|
1158
1549
|
*/
|
|
1159
1550
|
async init(jobId, total) {
|
|
1160
1551
|
this.currentProgress = {
|
|
@@ -1168,7 +1559,13 @@ var ProgressTracker = class {
|
|
|
1168
1559
|
await this.storage.set(jobId, this.currentProgress);
|
|
1169
1560
|
}
|
|
1170
1561
|
/**
|
|
1171
|
-
*
|
|
1562
|
+
* Updates the current progress of the job.
|
|
1563
|
+
*
|
|
1564
|
+
* Updates are debounced and flushed to storage at regular intervals
|
|
1565
|
+
* specified by `updateInterval` to avoid excessive write operations.
|
|
1566
|
+
*
|
|
1567
|
+
* @param processed - Number of entries processed so far.
|
|
1568
|
+
* @param status - Optional new status for the job.
|
|
1172
1569
|
*/
|
|
1173
1570
|
async update(processed, status) {
|
|
1174
1571
|
if (!this.currentProgress) {
|
|
@@ -1190,7 +1587,7 @@ var ProgressTracker = class {
|
|
|
1190
1587
|
}
|
|
1191
1588
|
}
|
|
1192
1589
|
/**
|
|
1193
|
-
*
|
|
1590
|
+
* Marks the current job as successfully completed.
|
|
1194
1591
|
*/
|
|
1195
1592
|
async complete() {
|
|
1196
1593
|
if (!this.currentProgress) {
|
|
@@ -1203,7 +1600,9 @@ var ProgressTracker = class {
|
|
|
1203
1600
|
this.stop();
|
|
1204
1601
|
}
|
|
1205
1602
|
/**
|
|
1206
|
-
*
|
|
1603
|
+
* Marks the current job as failed with an error message.
|
|
1604
|
+
*
|
|
1605
|
+
* @param error - The error message describing why the job failed.
|
|
1207
1606
|
*/
|
|
1208
1607
|
async fail(error) {
|
|
1209
1608
|
if (!this.currentProgress) {
|
|
@@ -1216,7 +1615,7 @@ var ProgressTracker = class {
|
|
|
1216
1615
|
this.stop();
|
|
1217
1616
|
}
|
|
1218
1617
|
/**
|
|
1219
|
-
*
|
|
1618
|
+
* Flushes the current progress state to the storage backend.
|
|
1220
1619
|
*/
|
|
1221
1620
|
async flush() {
|
|
1222
1621
|
if (!this.currentProgress) {
|
|
@@ -1231,7 +1630,7 @@ var ProgressTracker = class {
|
|
|
1231
1630
|
});
|
|
1232
1631
|
}
|
|
1233
1632
|
/**
|
|
1234
|
-
*
|
|
1633
|
+
* Stops the periodic update timer.
|
|
1235
1634
|
*/
|
|
1236
1635
|
stop() {
|
|
1237
1636
|
if (this.updateTimer) {
|
|
@@ -1240,7 +1639,9 @@ var ProgressTracker = class {
|
|
|
1240
1639
|
}
|
|
1241
1640
|
}
|
|
1242
1641
|
/**
|
|
1243
|
-
*
|
|
1642
|
+
* Returns a copy of the current progress state.
|
|
1643
|
+
*
|
|
1644
|
+
* @returns The current SitemapProgress object, or null if no job is active.
|
|
1244
1645
|
*/
|
|
1245
1646
|
getCurrentProgress() {
|
|
1246
1647
|
return this.currentProgress ? { ...this.currentProgress } : null;
|
|
@@ -1278,6 +1679,12 @@ var GenerateSitemapJob = class extends import_stream.Job {
|
|
|
1278
1679
|
this.options = options;
|
|
1279
1680
|
this.generator = new SitemapGenerator(options.generatorOptions);
|
|
1280
1681
|
}
|
|
1682
|
+
/**
|
|
1683
|
+
* Main entry point for the job execution.
|
|
1684
|
+
*
|
|
1685
|
+
* Orchestrates the full lifecycle of sitemap generation, including progress
|
|
1686
|
+
* initialization, generation, shadow commit, and error handling.
|
|
1687
|
+
*/
|
|
1281
1688
|
async handle() {
|
|
1282
1689
|
const { progressTracker, onComplete, onError } = this.options;
|
|
1283
1690
|
try {
|
|
@@ -1308,7 +1715,9 @@ var GenerateSitemapJob = class extends import_stream.Job {
|
|
|
1308
1715
|
}
|
|
1309
1716
|
}
|
|
1310
1717
|
/**
|
|
1311
|
-
*
|
|
1718
|
+
* Calculates the total number of URL entries from all providers.
|
|
1719
|
+
*
|
|
1720
|
+
* @returns A promise resolving to the total entry count.
|
|
1312
1721
|
*/
|
|
1313
1722
|
async calculateTotal() {
|
|
1314
1723
|
let total = 0;
|
|
@@ -1326,7 +1735,7 @@ var GenerateSitemapJob = class extends import_stream.Job {
|
|
|
1326
1735
|
return total;
|
|
1327
1736
|
}
|
|
1328
1737
|
/**
|
|
1329
|
-
*
|
|
1738
|
+
* Performs sitemap generation while reporting progress to the tracker and callback.
|
|
1330
1739
|
*/
|
|
1331
1740
|
async generateWithProgress() {
|
|
1332
1741
|
const { progressTracker, onProgress } = this.options;
|
|
@@ -1345,8 +1754,558 @@ var GenerateSitemapJob = class extends import_stream.Job {
|
|
|
1345
1754
|
}
|
|
1346
1755
|
};
|
|
1347
1756
|
|
|
1348
|
-
// src/
|
|
1757
|
+
// src/locks/MemoryLock.ts
|
|
1758
|
+
var MemoryLock = class {
|
|
1759
|
+
/**
|
|
1760
|
+
* Internal map storing resource identifiers to their lock expiration timestamps.
|
|
1761
|
+
*
|
|
1762
|
+
* Keys represent unique resource identifiers (e.g., 'sitemap-generation').
|
|
1763
|
+
* Values are Unix timestamps in milliseconds representing when the lock expires.
|
|
1764
|
+
* Expired locks are automatically cleaned up during `acquire()` and `isLocked()` calls.
|
|
1765
|
+
*/
|
|
1766
|
+
locks = /* @__PURE__ */ new Map();
|
|
1767
|
+
/**
|
|
1768
|
+
* Attempts to acquire an exclusive lock on the specified resource.
|
|
1769
|
+
*
|
|
1770
|
+
* Uses a test-and-set approach: checks if the lock exists and is valid, then atomically
|
|
1771
|
+
* sets the lock if available. Expired locks are treated as available and automatically
|
|
1772
|
+
* replaced during acquisition.
|
|
1773
|
+
*
|
|
1774
|
+
* **Behavior:**
|
|
1775
|
+
* - Returns `true` if lock was successfully acquired
|
|
1776
|
+
* - Returns `false` if resource is already locked by another caller
|
|
1777
|
+
* - Automatically replaces expired locks (acts as self-healing mechanism)
|
|
1778
|
+
* - Lock automatically expires after TTL milliseconds
|
|
1779
|
+
*
|
|
1780
|
+
* **Race condition handling:**
|
|
1781
|
+
* Safe within a single process due to JavaScript's single-threaded event loop.
|
|
1782
|
+
* NOT safe across multiple processes or instances (use RedisLock for that).
|
|
1783
|
+
*
|
|
1784
|
+
* @param resource - Unique identifier for the resource to lock (e.g., 'sitemap-generation', 'blog-index').
|
|
1785
|
+
* Should be consistent across all callers attempting to lock the same resource.
|
|
1786
|
+
* @param ttl - Time-to-live in milliseconds. Lock automatically expires after this duration.
|
|
1787
|
+
* Recommended: 2-5x the expected operation duration to prevent premature expiration.
|
|
1788
|
+
* @returns Promise resolving to `true` if lock acquired, `false` if already locked.
|
|
1789
|
+
*
|
|
1790
|
+
* @example Preventing concurrent sitemap generation
|
|
1791
|
+
* ```typescript
|
|
1792
|
+
* const lock = new MemoryLock()
|
|
1793
|
+
* const acquired = await lock.acquire('sitemap-generation', 60000)
|
|
1794
|
+
*
|
|
1795
|
+
* if (!acquired) {
|
|
1796
|
+
* console.log('Another process is already generating the sitemap')
|
|
1797
|
+
* return new Response('Generation in progress', { status: 503 })
|
|
1798
|
+
* }
|
|
1799
|
+
*
|
|
1800
|
+
* try {
|
|
1801
|
+
* await generateSitemap()
|
|
1802
|
+
* } finally {
|
|
1803
|
+
* await lock.release('sitemap-generation')
|
|
1804
|
+
* }
|
|
1805
|
+
* ```
|
|
1806
|
+
*
|
|
1807
|
+
* @example Setting appropriate TTL
|
|
1808
|
+
* ```typescript
|
|
1809
|
+
* // For fast operations (< 1 second), use short TTL
|
|
1810
|
+
* await lock.acquire('cache-refresh', 5000)
|
|
1811
|
+
*
|
|
1812
|
+
* // For slow operations (minutes), use longer TTL
|
|
1813
|
+
* await lock.acquire('full-reindex', 300000) // 5 minutes
|
|
1814
|
+
* ```
|
|
1815
|
+
*/
|
|
1816
|
+
async acquire(resource, ttl) {
|
|
1817
|
+
const now = Date.now();
|
|
1818
|
+
const expiresAt = this.locks.get(resource);
|
|
1819
|
+
if (expiresAt && expiresAt > now) {
|
|
1820
|
+
return false;
|
|
1821
|
+
}
|
|
1822
|
+
this.locks.set(resource, now + ttl);
|
|
1823
|
+
return true;
|
|
1824
|
+
}
|
|
1825
|
+
/**
|
|
1826
|
+
* Releases the lock on the specified resource, allowing others to acquire it.
|
|
1827
|
+
*
|
|
1828
|
+
* Immediately removes the lock from memory without any ownership validation.
|
|
1829
|
+
* Unlike RedisLock, this does NOT verify that the caller is the lock owner,
|
|
1830
|
+
* so callers must ensure they only release locks they acquired.
|
|
1831
|
+
*
|
|
1832
|
+
* **Best practices:**
|
|
1833
|
+
* - Always call `release()` in a `finally` block to prevent lock leakage
|
|
1834
|
+
* - Only release locks you successfully acquired
|
|
1835
|
+
* - If operation fails, still release the lock to allow retry
|
|
1836
|
+
*
|
|
1837
|
+
* **Idempotency:**
|
|
1838
|
+
* Safe to call multiple times on the same resource. Releasing a non-existent
|
|
1839
|
+
* lock is a no-op.
|
|
1840
|
+
*
|
|
1841
|
+
* @param resource - The resource identifier to unlock. Must match the identifier
|
|
1842
|
+
* used in the corresponding `acquire()` call.
|
|
1843
|
+
*
|
|
1844
|
+
* @example Proper release pattern with try-finally
|
|
1845
|
+
* ```typescript
|
|
1846
|
+
* const acquired = await lock.acquire('sitemap-generation', 60000)
|
|
1847
|
+
* if (!acquired) return
|
|
1848
|
+
*
|
|
1849
|
+
* try {
|
|
1850
|
+
* await generateSitemap()
|
|
1851
|
+
* } finally {
|
|
1852
|
+
* // Always release, even if operation throws
|
|
1853
|
+
* await lock.release('sitemap-generation')
|
|
1854
|
+
* }
|
|
1855
|
+
* ```
|
|
1856
|
+
*
|
|
1857
|
+
* @example Handling operation failures
|
|
1858
|
+
* ```typescript
|
|
1859
|
+
* const acquired = await lock.acquire('data-import', 120000)
|
|
1860
|
+
* if (!acquired) return
|
|
1861
|
+
*
|
|
1862
|
+
* try {
|
|
1863
|
+
* await importData()
|
|
1864
|
+
* } catch (error) {
|
|
1865
|
+
* console.error('Import failed:', error)
|
|
1866
|
+
* // Lock still released in finally block
|
|
1867
|
+
* throw error
|
|
1868
|
+
* } finally {
|
|
1869
|
+
* await lock.release('data-import')
|
|
1870
|
+
* }
|
|
1871
|
+
* ```
|
|
1872
|
+
*/
|
|
1873
|
+
async release(resource) {
|
|
1874
|
+
this.locks.delete(resource);
|
|
1875
|
+
}
|
|
1876
|
+
/**
|
|
1877
|
+
* Checks whether a resource is currently locked and has not expired.
|
|
1878
|
+
*
|
|
1879
|
+
* Performs automatic cleanup by removing expired locks during the check,
|
|
1880
|
+
* ensuring the internal map doesn't accumulate stale entries over time.
|
|
1881
|
+
*
|
|
1882
|
+
* **Use cases:**
|
|
1883
|
+
* - Pre-flight checks before attempting expensive operations
|
|
1884
|
+
* - Status monitoring and health checks
|
|
1885
|
+
* - Implementing custom retry logic
|
|
1886
|
+
* - Debugging and testing
|
|
1887
|
+
*
|
|
1888
|
+
* **Side effects:**
|
|
1889
|
+
* Automatically deletes expired locks as a garbage collection mechanism.
|
|
1890
|
+
* This is intentional to prevent memory leaks from abandoned locks.
|
|
1891
|
+
*
|
|
1892
|
+
* @param resource - The resource identifier to check for lock status.
|
|
1893
|
+
* @returns Promise resolving to `true` if resource is actively locked (not expired),
|
|
1894
|
+
* `false` if unlocked or lock has expired.
|
|
1895
|
+
*
|
|
1896
|
+
* @example Pre-flight check before starting work
|
|
1897
|
+
* ```typescript
|
|
1898
|
+
* const lock = new MemoryLock()
|
|
1899
|
+
*
|
|
1900
|
+
* if (await lock.isLocked('sitemap-generation')) {
|
|
1901
|
+
* console.log('Sitemap generation already in progress')
|
|
1902
|
+
* return
|
|
1903
|
+
* }
|
|
1904
|
+
*
|
|
1905
|
+
* // Safe to proceed
|
|
1906
|
+
* await lock.acquire('sitemap-generation', 60000)
|
|
1907
|
+
* ```
|
|
1908
|
+
*
|
|
1909
|
+
* @example Health check endpoint
|
|
1910
|
+
* ```typescript
|
|
1911
|
+
* app.get('/health/locks', async (c) => {
|
|
1912
|
+
* const isGenerating = await lock.isLocked('sitemap-generation')
|
|
1913
|
+
* const isIndexing = await lock.isLocked('search-indexing')
|
|
1914
|
+
*
|
|
1915
|
+
* return c.json({
|
|
1916
|
+
* sitemapGeneration: isGenerating ? 'in-progress' : 'idle',
|
|
1917
|
+
* searchIndexing: isIndexing ? 'in-progress' : 'idle'
|
|
1918
|
+
* })
|
|
1919
|
+
* })
|
|
1920
|
+
* ```
|
|
1921
|
+
*
|
|
1922
|
+
* @example Custom retry logic
|
|
1923
|
+
* ```typescript
|
|
1924
|
+
* let attempts = 0
|
|
1925
|
+
* while (attempts < 5) {
|
|
1926
|
+
* if (!await lock.isLocked('resource')) {
|
|
1927
|
+
* const acquired = await lock.acquire('resource', 10000)
|
|
1928
|
+
* if (acquired) break
|
|
1929
|
+
* }
|
|
1930
|
+
* await sleep(1000)
|
|
1931
|
+
* attempts++
|
|
1932
|
+
* }
|
|
1933
|
+
* ```
|
|
1934
|
+
*/
|
|
1935
|
+
async isLocked(resource) {
|
|
1936
|
+
const expiresAt = this.locks.get(resource);
|
|
1937
|
+
if (!expiresAt) {
|
|
1938
|
+
return false;
|
|
1939
|
+
}
|
|
1940
|
+
const now = Date.now();
|
|
1941
|
+
if (expiresAt <= now) {
|
|
1942
|
+
this.locks.delete(resource);
|
|
1943
|
+
return false;
|
|
1944
|
+
}
|
|
1945
|
+
return true;
|
|
1946
|
+
}
|
|
1947
|
+
/**
|
|
1948
|
+
* Clears all locks from memory, including both active and expired locks.
|
|
1949
|
+
*
|
|
1950
|
+
* **Use cases:**
|
|
1951
|
+
* - Test cleanup between test cases to ensure isolation
|
|
1952
|
+
* - Application shutdown to release all resources
|
|
1953
|
+
* - Manual intervention during debugging
|
|
1954
|
+
* - Resetting state after catastrophic errors
|
|
1955
|
+
*
|
|
1956
|
+
* **Warning:**
|
|
1957
|
+
* This forcibly releases ALL locks without any ownership validation.
|
|
1958
|
+
* Should not be called during normal operation in production environments.
|
|
1959
|
+
*
|
|
1960
|
+
* @example Test cleanup with beforeEach hook
|
|
1961
|
+
* ```typescript
|
|
1962
|
+
* import { describe, beforeEach, test } from 'vitest'
|
|
1963
|
+
*
|
|
1964
|
+
* const lock = new MemoryLock()
|
|
1965
|
+
*
|
|
1966
|
+
* beforeEach(async () => {
|
|
1967
|
+
* await lock.clear() // Ensure clean state for each test
|
|
1968
|
+
* })
|
|
1969
|
+
*
|
|
1970
|
+
* test('lock acquisition', async () => {
|
|
1971
|
+
* const acquired = await lock.acquire('test-resource', 5000)
|
|
1972
|
+
* expect(acquired).toBe(true)
|
|
1973
|
+
* })
|
|
1974
|
+
* ```
|
|
1975
|
+
*
|
|
1976
|
+
* @example Graceful shutdown handler
|
|
1977
|
+
* ```typescript
|
|
1978
|
+
* process.on('SIGTERM', async () => {
|
|
1979
|
+
* console.log('Shutting down, releasing all locks...')
|
|
1980
|
+
* await lock.clear()
|
|
1981
|
+
* process.exit(0)
|
|
1982
|
+
* })
|
|
1983
|
+
* ```
|
|
1984
|
+
*/
|
|
1985
|
+
async clear() {
|
|
1986
|
+
this.locks.clear();
|
|
1987
|
+
}
|
|
1988
|
+
/**
|
|
1989
|
+
* Returns the number of lock entries currently stored in memory.
|
|
1990
|
+
*
|
|
1991
|
+
* **Important:** This includes BOTH active and expired locks. Expired locks
|
|
1992
|
+
* are only cleaned up during `acquire()` or `isLocked()` calls, so this count
|
|
1993
|
+
* may include stale entries.
|
|
1994
|
+
*
|
|
1995
|
+
* **Use cases:**
|
|
1996
|
+
* - Monitoring memory usage and lock accumulation
|
|
1997
|
+
* - Debugging lock leakage issues
|
|
1998
|
+
* - Testing lock lifecycle behavior
|
|
1999
|
+
* - Detecting abnormal lock retention patterns
|
|
2000
|
+
*
|
|
2001
|
+
* **Not suitable for:**
|
|
2002
|
+
* - Determining number of ACTIVE locks (use `isLocked()` on each resource)
|
|
2003
|
+
* - Production health checks (includes expired locks)
|
|
2004
|
+
*
|
|
2005
|
+
* @returns The total number of lock entries in the internal Map, including expired ones.
|
|
2006
|
+
*
|
|
2007
|
+
* @example Monitoring lock accumulation
|
|
2008
|
+
* ```typescript
|
|
2009
|
+
* const lock = new MemoryLock()
|
|
2010
|
+
*
|
|
2011
|
+
* setInterval(() => {
|
|
2012
|
+
* const count = lock.size()
|
|
2013
|
+
* if (count > 100) {
|
|
2014
|
+
* console.warn(`High lock count detected: ${count}`)
|
|
2015
|
+
* // May indicate lock leakage or missing release() calls
|
|
2016
|
+
* }
|
|
2017
|
+
* }, 60000)
|
|
2018
|
+
* ```
|
|
2019
|
+
*
|
|
2020
|
+
* @example Testing lock cleanup behavior
|
|
2021
|
+
* ```typescript
|
|
2022
|
+
* import { test, expect } from 'vitest'
|
|
2023
|
+
*
|
|
2024
|
+
* test('expired locks are cleaned up', async () => {
|
|
2025
|
+
* const lock = new MemoryLock()
|
|
2026
|
+
*
|
|
2027
|
+
* await lock.acquire('resource', 10)
|
|
2028
|
+
* expect(lock.size()).toBe(1)
|
|
2029
|
+
*
|
|
2030
|
+
* await sleep(20) // Wait for expiration
|
|
2031
|
+
* expect(lock.size()).toBe(1) // Still in map (not cleaned yet)
|
|
2032
|
+
*
|
|
2033
|
+
* await lock.isLocked('resource') // Triggers cleanup
|
|
2034
|
+
* expect(lock.size()).toBe(0) // Now removed
|
|
2035
|
+
* })
|
|
2036
|
+
* ```
|
|
2037
|
+
*/
|
|
2038
|
+
size() {
|
|
2039
|
+
return this.locks.size;
|
|
2040
|
+
}
|
|
2041
|
+
};
|
|
2042
|
+
|
|
2043
|
+
// src/locks/RedisLock.ts
|
|
1349
2044
|
var import_node_crypto = require("crypto");
|
|
2045
|
+
var RedisLock = class {
|
|
2046
|
+
/**
|
|
2047
|
+
* Constructs a new RedisLock instance with the specified configuration.
|
|
2048
|
+
*
|
|
2049
|
+
* @param options - Configuration including Redis client and retry parameters.
|
|
2050
|
+
*
|
|
2051
|
+
* @example With custom retry strategy
|
|
2052
|
+
* ```typescript
|
|
2053
|
+
* const lock = new RedisLock({
|
|
2054
|
+
* client: redisClient,
|
|
2055
|
+
* keyPrefix: 'app:locks:',
|
|
2056
|
+
* retryCount: 10, // More retries for high-contention scenarios
|
|
2057
|
+
* retryDelay: 50 // Shorter delay for low-latency requirements
|
|
2058
|
+
* })
|
|
2059
|
+
* ```
|
|
2060
|
+
*/
|
|
2061
|
+
constructor(options) {
|
|
2062
|
+
this.options = options;
|
|
2063
|
+
this.keyPrefix = options.keyPrefix || "sitemap:lock:";
|
|
2064
|
+
this.retryCount = options.retryCount ?? 0;
|
|
2065
|
+
this.retryDelay = options.retryDelay ?? 100;
|
|
2066
|
+
}
|
|
2067
|
+
/**
|
|
2068
|
+
* Unique identifier for this lock instance.
|
|
2069
|
+
*
|
|
2070
|
+
* Generated once during construction and used for all locks acquired by this instance.
|
|
2071
|
+
* Enables ownership validation: only the instance that acquired the lock can release it.
|
|
2072
|
+
*
|
|
2073
|
+
* **Security consideration:**
|
|
2074
|
+
* UUIDs are sufficiently random to prevent lock hijacking across instances.
|
|
2075
|
+
* However, they are stored in plain text in Redis (not encrypted).
|
|
2076
|
+
*/
|
|
2077
|
+
lockId = (0, import_node_crypto.randomUUID)();
|
|
2078
|
+
/**
|
|
2079
|
+
* Redis key prefix for all locks acquired through this instance.
|
|
2080
|
+
*
|
|
2081
|
+
* Combined with resource name to form full Redis key (e.g., 'sitemap:lock:generation').
|
|
2082
|
+
* Allows namespace isolation and easier debugging in Redis CLI.
|
|
2083
|
+
*/
|
|
2084
|
+
keyPrefix;
|
|
2085
|
+
/**
|
|
2086
|
+
* Maximum number of retry attempts when lock is held by another instance.
|
|
2087
|
+
*
|
|
2088
|
+
* Set to 0 for fail-fast behavior. Higher values increase acquisition success
|
|
2089
|
+
* rate but also increase latency under contention.
|
|
2090
|
+
*/
|
|
2091
|
+
retryCount;
|
|
2092
|
+
/**
|
|
2093
|
+
* Delay in milliseconds between consecutive retry attempts.
|
|
2094
|
+
*
|
|
2095
|
+
* Should be tuned based on expected lock hold time. Typical values: 50-500ms.
|
|
2096
|
+
*/
|
|
2097
|
+
retryDelay;
|
|
2098
|
+
/**
|
|
2099
|
+
* Attempts to acquire a distributed lock using Redis SET NX EX command.
|
|
2100
|
+
*
|
|
2101
|
+
* Uses atomic Redis operations to ensure only one instance across your entire
|
|
2102
|
+
* infrastructure can hold the lock at any given time. Implements retry logic
|
|
2103
|
+
* with exponential backoff for handling transient contention.
|
|
2104
|
+
*
|
|
2105
|
+
* **Algorithm:**
|
|
2106
|
+
* 1. Convert TTL from milliseconds to seconds (Redis requirement)
|
|
2107
|
+
* 2. Attempt Redis SET key lockId EX ttl NX (atomic operation)
|
|
2108
|
+
* 3. If successful (returns 'OK'), lock acquired
|
|
2109
|
+
* 4. If failed (returns null), retry up to retryCount times with retryDelay
|
|
2110
|
+
* 5. Return true if acquired, false if all attempts exhausted
|
|
2111
|
+
*
|
|
2112
|
+
* **Atomicity guarantee:**
|
|
2113
|
+
* The combination of NX (set if Not eXists) and EX (set EXpiration) in a single
|
|
2114
|
+
* Redis command ensures no race conditions. Either the lock is acquired or it isn't.
|
|
2115
|
+
*
|
|
2116
|
+
* **Error handling:**
|
|
2117
|
+
* Redis connection errors are caught, logged to console, and treated as acquisition
|
|
2118
|
+
* failure (returns false). This fail-safe behavior prevents exceptions from bubbling
|
|
2119
|
+
* up to application code.
|
|
2120
|
+
*
|
|
2121
|
+
* **Performance:**
|
|
2122
|
+
* - Single instance: O(1) Redis operation
|
|
2123
|
+
* - With retry: O(retryCount) worst case
|
|
2124
|
+
* - Network latency: ~1-5ms per attempt (depends on Redis location)
|
|
2125
|
+
*
|
|
2126
|
+
* @param resource - Unique identifier for the resource to lock.
|
|
2127
|
+
* Combined with keyPrefix to form Redis key.
|
|
2128
|
+
* @param ttl - Time-to-live in milliseconds. Lock auto-expires after this duration.
|
|
2129
|
+
* Recommended: 2-5x expected operation time to handle slowdowns.
|
|
2130
|
+
* Minimum: 1000ms (1 second) for practical use.
|
|
2131
|
+
* @returns Promise resolving to `true` if lock acquired, `false` if held by another instance
|
|
2132
|
+
* or Redis connection failed.
|
|
2133
|
+
*
|
|
2134
|
+
* @example Basic acquisition in distributed environment
|
|
2135
|
+
* ```typescript
|
|
2136
|
+
* const lock = new RedisLock({ client: redisClient })
|
|
2137
|
+
* const acquired = await lock.acquire('sitemap-generation', 60000)
|
|
2138
|
+
*
|
|
2139
|
+
* if (!acquired) {
|
|
2140
|
+
* // Another instance (e.g., different Kubernetes pod) holds the lock
|
|
2141
|
+
* console.log('Sitemap generation in progress on another instance')
|
|
2142
|
+
* return new Response('Service busy', {
|
|
2143
|
+
* status: 503,
|
|
2144
|
+
* headers: { 'Retry-After': '30' }
|
|
2145
|
+
* })
|
|
2146
|
+
* }
|
|
2147
|
+
*
|
|
2148
|
+
* try {
|
|
2149
|
+
* await generateSitemap()
|
|
2150
|
+
* } finally {
|
|
2151
|
+
* await lock.release('sitemap-generation')
|
|
2152
|
+
* }
|
|
2153
|
+
* ```
|
|
2154
|
+
*
|
|
2155
|
+
* @example With retry logic for transient contention
|
|
2156
|
+
* ```typescript
|
|
2157
|
+
* const lock = new RedisLock({
|
|
2158
|
+
* client: redisClient,
|
|
2159
|
+
* retryCount: 5,
|
|
2160
|
+
* retryDelay: 200
|
|
2161
|
+
* })
|
|
2162
|
+
*
|
|
2163
|
+
* // Will retry 5 times with 200ms delay between attempts
|
|
2164
|
+
* const acquired = await lock.acquire('data-import', 120000)
|
|
2165
|
+
* ```
|
|
2166
|
+
*
|
|
2167
|
+
* @example Setting appropriate TTL
|
|
2168
|
+
* ```typescript
|
|
2169
|
+
* // Fast operation: short TTL
|
|
2170
|
+
* await lock.acquire('cache-rebuild', 10000) // 10 seconds
|
|
2171
|
+
*
|
|
2172
|
+
* // Slow operation: longer TTL with buffer
|
|
2173
|
+
* await lock.acquire('full-sitemap', 300000) // 5 minutes
|
|
2174
|
+
*
|
|
2175
|
+
* // Very slow operation: generous TTL
|
|
2176
|
+
* await lock.acquire('data-migration', 1800000) // 30 minutes
|
|
2177
|
+
* ```
|
|
2178
|
+
*/
|
|
2179
|
+
async acquire(resource, ttl) {
|
|
2180
|
+
const key = this.keyPrefix + resource;
|
|
2181
|
+
const ttlSeconds = Math.ceil(ttl / 1e3);
|
|
2182
|
+
let attempts = 0;
|
|
2183
|
+
while (attempts <= this.retryCount) {
|
|
2184
|
+
try {
|
|
2185
|
+
const result = await this.options.client.set(key, this.lockId, "EX", ttlSeconds, "NX");
|
|
2186
|
+
if (result === "OK") {
|
|
2187
|
+
return true;
|
|
2188
|
+
}
|
|
2189
|
+
} catch (error) {
|
|
2190
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
2191
|
+
console.error(`[RedisLock] Failed to acquire lock for ${resource}:`, err.message);
|
|
2192
|
+
}
|
|
2193
|
+
attempts++;
|
|
2194
|
+
if (attempts <= this.retryCount) {
|
|
2195
|
+
await this.sleep(this.retryDelay);
|
|
2196
|
+
}
|
|
2197
|
+
}
|
|
2198
|
+
return false;
|
|
2199
|
+
}
|
|
2200
|
+
/**
|
|
2201
|
+
* Releases the distributed lock using Lua script for atomic ownership validation.
|
|
2202
|
+
*
|
|
2203
|
+
* Uses a Lua script to atomically check if the current instance owns the lock
|
|
2204
|
+
* (by comparing lockId) and delete it if so. This prevents accidentally releasing
|
|
2205
|
+
* locks held by other instances, which could cause data corruption in distributed systems.
|
|
2206
|
+
*
|
|
2207
|
+
* **Lua script logic:**
|
|
2208
|
+
* ```lua
|
|
2209
|
+
* if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
2210
|
+
* return redis.call("del", KEYS[1]) -- Delete only if owner matches
|
|
2211
|
+
* else
|
|
2212
|
+
* return 0 -- Not owner or lock expired, do nothing
|
|
2213
|
+
* end
|
|
2214
|
+
* ```
|
|
2215
|
+
*
|
|
2216
|
+
* **Why Lua scripts?**
|
|
2217
|
+
* - Atomicity: GET + comparison + DEL execute as one atomic operation
|
|
2218
|
+
* - Prevents race conditions: Lock cannot change between check and delete
|
|
2219
|
+
* - Server-side execution: No network round trips between steps
|
|
2220
|
+
*
|
|
2221
|
+
* **Error handling:**
|
|
2222
|
+
* Errors during release (e.g., Redis connection loss) are logged but do NOT throw.
|
|
2223
|
+
* This is intentional: if instance crashed or TTL expired, lock is already released.
|
|
2224
|
+
* Silent failure here prevents cascading errors in finally blocks.
|
|
2225
|
+
*
|
|
2226
|
+
* **Idempotency:**
|
|
2227
|
+
* Safe to call multiple times. Releasing an already-released or expired lock is a no-op.
|
|
2228
|
+
*
|
|
2229
|
+
* @param resource - The resource identifier to unlock. Must match the identifier
|
|
2230
|
+
* used in the corresponding `acquire()` call.
|
|
2231
|
+
*
|
|
2232
|
+
* @example Proper release pattern with try-finally
|
|
2233
|
+
* ```typescript
|
|
2234
|
+
* const acquired = await lock.acquire('sitemap-generation', 60000)
|
|
2235
|
+
* if (!acquired) return
|
|
2236
|
+
*
|
|
2237
|
+
* try {
|
|
2238
|
+
* await generateSitemap()
|
|
2239
|
+
* } finally {
|
|
2240
|
+
* // Always release, even if operation throws
|
|
2241
|
+
* await lock.release('sitemap-generation')
|
|
2242
|
+
* }
|
|
2243
|
+
* ```
|
|
2244
|
+
*
|
|
2245
|
+
* @example Handling operation failures
|
|
2246
|
+
* ```typescript
|
|
2247
|
+
* const acquired = await lock.acquire('data-processing', 120000)
|
|
2248
|
+
* if (!acquired) {
|
|
2249
|
+
* throw new Error('Could not acquire lock')
|
|
2250
|
+
* }
|
|
2251
|
+
*
|
|
2252
|
+
* try {
|
|
2253
|
+
* await processData()
|
|
2254
|
+
* } catch (error) {
|
|
2255
|
+
* console.error('Processing failed:', error)
|
|
2256
|
+
* // Lock still released in finally block
|
|
2257
|
+
* throw error
|
|
2258
|
+
* } finally {
|
|
2259
|
+
* await lock.release('data-processing')
|
|
2260
|
+
* }
|
|
2261
|
+
* ```
|
|
2262
|
+
*
|
|
2263
|
+
* @example Why ownership validation matters
|
|
2264
|
+
* ```typescript
|
|
2265
|
+
* // Instance A acquires lock with 10-second TTL
|
|
2266
|
+
* const lockA = new RedisLock({ client: redisClientA })
|
|
2267
|
+
* await lockA.acquire('task', 10000)
|
|
2268
|
+
*
|
|
2269
|
+
* // ... 11 seconds pass, lock auto-expires ...
|
|
2270
|
+
*
|
|
2271
|
+
* // Instance B acquires the now-expired lock
|
|
2272
|
+
* const lockB = new RedisLock({ client: redisClientB })
|
|
2273
|
+
* await lockB.acquire('task', 10000)
|
|
2274
|
+
*
|
|
2275
|
+
* // Instance A tries to release (after slowdown/GC pause)
|
|
2276
|
+
* await lockA.release('task')
|
|
2277
|
+
* // ✅ Lua script detects lockId mismatch, does NOT delete B's lock
|
|
2278
|
+
* // ❌ Without Lua: Would delete B's lock, causing data corruption
|
|
2279
|
+
* ```
|
|
2280
|
+
*/
|
|
2281
|
+
async release(resource) {
|
|
2282
|
+
const key = this.keyPrefix + resource;
|
|
2283
|
+
try {
|
|
2284
|
+
const script = `
|
|
2285
|
+
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
2286
|
+
return redis.call("del", KEYS[1])
|
|
2287
|
+
else
|
|
2288
|
+
return 0
|
|
2289
|
+
end
|
|
2290
|
+
`;
|
|
2291
|
+
await this.options.client.eval(script, 1, key, this.lockId);
|
|
2292
|
+
} catch (error) {
|
|
2293
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
2294
|
+
console.error(`[RedisLock] Failed to release lock for ${resource}:`, err.message);
|
|
2295
|
+
}
|
|
2296
|
+
}
|
|
2297
|
+
/**
|
|
2298
|
+
* Internal utility for sleeping between retry attempts.
|
|
2299
|
+
*
|
|
2300
|
+
* @param ms - Duration to sleep in milliseconds
|
|
2301
|
+
*/
|
|
2302
|
+
sleep(ms) {
|
|
2303
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2304
|
+
}
|
|
2305
|
+
};
|
|
2306
|
+
|
|
2307
|
+
// src/OrbitSitemap.ts
|
|
2308
|
+
var import_node_crypto2 = require("crypto");
|
|
1350
2309
|
|
|
1351
2310
|
// src/redirect/RedirectHandler.ts
|
|
1352
2311
|
var RedirectHandler = class {
|
|
@@ -1355,7 +2314,10 @@ var RedirectHandler = class {
|
|
|
1355
2314
|
this.options = options;
|
|
1356
2315
|
}
|
|
1357
2316
|
/**
|
|
1358
|
-
*
|
|
2317
|
+
* Processes a list of sitemap entries and handles redirects according to the configured strategy.
|
|
2318
|
+
*
|
|
2319
|
+
* @param entries - The original list of sitemap entries.
|
|
2320
|
+
* @returns A promise resolving to the processed list of entries.
|
|
1359
2321
|
*/
|
|
1360
2322
|
async processEntries(entries) {
|
|
1361
2323
|
const { manager, strategy, followChains, maxChainLength } = this.options;
|
|
@@ -1386,7 +2348,7 @@ var RedirectHandler = class {
|
|
|
1386
2348
|
}
|
|
1387
2349
|
}
|
|
1388
2350
|
/**
|
|
1389
|
-
*
|
|
2351
|
+
* Strategy 1: Remove old URL and add the new destination URL.
|
|
1390
2352
|
*/
|
|
1391
2353
|
handleRemoveOldAddNew(entries, redirectMap) {
|
|
1392
2354
|
const processed = [];
|
|
@@ -1411,7 +2373,7 @@ var RedirectHandler = class {
|
|
|
1411
2373
|
return processed;
|
|
1412
2374
|
}
|
|
1413
2375
|
/**
|
|
1414
|
-
*
|
|
2376
|
+
* Strategy 2: Keep the original URL but mark the destination as canonical.
|
|
1415
2377
|
*/
|
|
1416
2378
|
handleKeepRelation(entries, redirectMap) {
|
|
1417
2379
|
const processed = [];
|
|
@@ -1434,7 +2396,7 @@ var RedirectHandler = class {
|
|
|
1434
2396
|
return processed;
|
|
1435
2397
|
}
|
|
1436
2398
|
/**
|
|
1437
|
-
*
|
|
2399
|
+
* Strategy 3: Silently update the URL to the destination.
|
|
1438
2400
|
*/
|
|
1439
2401
|
handleUpdateUrl(entries, redirectMap) {
|
|
1440
2402
|
return entries.map((entry) => {
|
|
@@ -1454,7 +2416,7 @@ var RedirectHandler = class {
|
|
|
1454
2416
|
});
|
|
1455
2417
|
}
|
|
1456
2418
|
/**
|
|
1457
|
-
*
|
|
2419
|
+
* Strategy 4: Include both the original and destination URLs.
|
|
1458
2420
|
*/
|
|
1459
2421
|
handleDualMark(entries, redirectMap) {
|
|
1460
2422
|
const processed = [];
|
|
@@ -1491,17 +2453,64 @@ var RedirectHandler = class {
|
|
|
1491
2453
|
};
|
|
1492
2454
|
|
|
1493
2455
|
// src/storage/MemorySitemapStorage.ts
|
|
2456
|
+
init_Compression();
|
|
1494
2457
|
var MemorySitemapStorage = class {
|
|
1495
2458
|
constructor(baseUrl) {
|
|
1496
2459
|
this.baseUrl = baseUrl;
|
|
1497
2460
|
}
|
|
1498
2461
|
files = /* @__PURE__ */ new Map();
|
|
2462
|
+
/**
|
|
2463
|
+
* Writes sitemap content to memory.
|
|
2464
|
+
*
|
|
2465
|
+
* @param filename - The name of the file to store.
|
|
2466
|
+
* @param content - The XML or JSON content.
|
|
2467
|
+
*/
|
|
1499
2468
|
async write(filename, content) {
|
|
1500
2469
|
this.files.set(filename, content);
|
|
1501
2470
|
}
|
|
2471
|
+
/**
|
|
2472
|
+
* 使用串流方式寫入 sitemap 至記憶體,可選擇性啟用 gzip 壓縮。
|
|
2473
|
+
* 記憶體儲存會收集串流為完整字串。
|
|
2474
|
+
*
|
|
2475
|
+
* @param filename - 檔案名稱
|
|
2476
|
+
* @param stream - XML 內容的 AsyncIterable
|
|
2477
|
+
* @param options - 寫入選項(如壓縮)
|
|
2478
|
+
*
|
|
2479
|
+
* @since 3.1.0
|
|
2480
|
+
*/
|
|
2481
|
+
async writeStream(filename, stream, options) {
|
|
2482
|
+
const chunks = [];
|
|
2483
|
+
for await (const chunk of stream) {
|
|
2484
|
+
chunks.push(chunk);
|
|
2485
|
+
}
|
|
2486
|
+
const content = chunks.join("");
|
|
2487
|
+
const key = options?.compress ? toGzipFilename(filename) : filename;
|
|
2488
|
+
if (options?.compress) {
|
|
2489
|
+
const compressed = await compressToBuffer(
|
|
2490
|
+
(async function* () {
|
|
2491
|
+
yield content;
|
|
2492
|
+
})()
|
|
2493
|
+
);
|
|
2494
|
+
this.files.set(key, compressed.toString("base64"));
|
|
2495
|
+
} else {
|
|
2496
|
+
this.files.set(key, content);
|
|
2497
|
+
}
|
|
2498
|
+
}
|
|
2499
|
+
/**
|
|
2500
|
+
* Reads sitemap content from memory.
|
|
2501
|
+
*
|
|
2502
|
+
* @param filename - The name of the file to read.
|
|
2503
|
+
* @returns A promise resolving to the file content as a string, or null if not found.
|
|
2504
|
+
*/
|
|
1502
2505
|
async read(filename) {
|
|
1503
2506
|
return this.files.get(filename) || null;
|
|
1504
2507
|
}
|
|
2508
|
+
/**
|
|
2509
|
+
* Returns a readable stream for a sitemap file in memory.
|
|
2510
|
+
*
|
|
2511
|
+
* @param filename - The name of the file to stream.
|
|
2512
|
+
* @returns A promise resolving to an async iterable of file chunks, or null if not found.
|
|
2513
|
+
*/
|
|
1505
2514
|
async readStream(filename) {
|
|
1506
2515
|
const content = this.files.get(filename);
|
|
1507
2516
|
if (content === void 0) {
|
|
@@ -1511,9 +2520,21 @@ var MemorySitemapStorage = class {
|
|
|
1511
2520
|
yield content;
|
|
1512
2521
|
})();
|
|
1513
2522
|
}
|
|
2523
|
+
/**
|
|
2524
|
+
* Checks if a sitemap file exists in memory.
|
|
2525
|
+
*
|
|
2526
|
+
* @param filename - The name of the file to check.
|
|
2527
|
+
* @returns A promise resolving to true if the file exists, false otherwise.
|
|
2528
|
+
*/
|
|
1514
2529
|
async exists(filename) {
|
|
1515
2530
|
return this.files.has(filename);
|
|
1516
2531
|
}
|
|
2532
|
+
/**
|
|
2533
|
+
* Returns the full public URL for a sitemap file.
|
|
2534
|
+
*
|
|
2535
|
+
* @param filename - The name of the sitemap file.
|
|
2536
|
+
* @returns The public URL as a string.
|
|
2537
|
+
*/
|
|
1517
2538
|
getUrl(filename) {
|
|
1518
2539
|
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
1519
2540
|
const file = filename.startsWith("/") ? filename.slice(1) : filename;
|
|
@@ -1569,7 +2590,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1569
2590
|
});
|
|
1570
2591
|
}
|
|
1571
2592
|
/**
|
|
1572
|
-
*
|
|
2593
|
+
* Installs the sitemap module into PlanetCore.
|
|
1573
2594
|
*
|
|
1574
2595
|
* @param core - The PlanetCore instance.
|
|
1575
2596
|
*/
|
|
@@ -1580,6 +2601,9 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1580
2601
|
core.logger.info("[OrbitSitemap] Static mode configured. Use generate() to build sitemaps.");
|
|
1581
2602
|
}
|
|
1582
2603
|
}
|
|
2604
|
+
/**
|
|
2605
|
+
* Internal method to set up dynamic sitemap routes.
|
|
2606
|
+
*/
|
|
1583
2607
|
installDynamic(core) {
|
|
1584
2608
|
const opts = this.options;
|
|
1585
2609
|
const storage = opts.storage ?? new MemorySitemapStorage(opts.baseUrl);
|
|
@@ -1637,7 +2661,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1637
2661
|
core.router.get(shardRoute, handler);
|
|
1638
2662
|
}
|
|
1639
2663
|
/**
|
|
1640
|
-
*
|
|
2664
|
+
* Generates the sitemap (static mode only).
|
|
1641
2665
|
*
|
|
1642
2666
|
* @returns A promise that resolves when generation is complete.
|
|
1643
2667
|
* @throws {Error} If called in dynamic mode.
|
|
@@ -1679,7 +2703,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1679
2703
|
console.log(`[OrbitSitemap] Generated sitemap in ${opts.outDir}`);
|
|
1680
2704
|
}
|
|
1681
2705
|
/**
|
|
1682
|
-
*
|
|
2706
|
+
* Generates incremental sitemap updates (static mode only).
|
|
1683
2707
|
*
|
|
1684
2708
|
* @param since - Only include items modified since this date.
|
|
1685
2709
|
* @returns A promise that resolves when incremental generation is complete.
|
|
@@ -1710,7 +2734,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1710
2734
|
console.log(`[OrbitSitemap] Generated incremental sitemap in ${opts.outDir}`);
|
|
1711
2735
|
}
|
|
1712
2736
|
/**
|
|
1713
|
-
*
|
|
2737
|
+
* Generates sitemap asynchronously in the background (static mode only).
|
|
1714
2738
|
*
|
|
1715
2739
|
* @param options - Options for the async generation job.
|
|
1716
2740
|
* @returns A promise resolving to the job ID.
|
|
@@ -1721,7 +2745,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1721
2745
|
throw new Error("generateAsync() can only be called in static mode");
|
|
1722
2746
|
}
|
|
1723
2747
|
const opts = this.options;
|
|
1724
|
-
const jobId = (0,
|
|
2748
|
+
const jobId = (0, import_node_crypto2.randomUUID)();
|
|
1725
2749
|
let storage = opts.storage;
|
|
1726
2750
|
if (!storage) {
|
|
1727
2751
|
const { DiskSitemapStorage: DiskSitemapStorage2 } = await Promise.resolve().then(() => (init_DiskSitemapStorage(), DiskSitemapStorage_exports));
|
|
@@ -1771,7 +2795,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1771
2795
|
return jobId;
|
|
1772
2796
|
}
|
|
1773
2797
|
/**
|
|
1774
|
-
*
|
|
2798
|
+
* Installs API endpoints for triggering and monitoring sitemap generation.
|
|
1775
2799
|
*
|
|
1776
2800
|
* @param core - The PlanetCore instance.
|
|
1777
2801
|
* @param basePath - The base path for the API endpoints (default: '/admin/sitemap').
|
|
@@ -1844,9 +2868,12 @@ var RouteScanner = class {
|
|
|
1844
2868
|
};
|
|
1845
2869
|
}
|
|
1846
2870
|
/**
|
|
1847
|
-
*
|
|
2871
|
+
* Scans the router and returns discovered static GET routes as sitemap entries.
|
|
1848
2872
|
*
|
|
1849
|
-
*
|
|
2873
|
+
* This method iterates through all registered routes in the Gravito router,
|
|
2874
|
+
* applying inclusion/exclusion filters and defaulting metadata for matching routes.
|
|
2875
|
+
*
|
|
2876
|
+
* @returns An array of `SitemapEntry` objects.
|
|
1850
2877
|
*/
|
|
1851
2878
|
getEntries() {
|
|
1852
2879
|
const entries = [];
|
|
@@ -1908,7 +2935,10 @@ var RedirectDetector = class {
|
|
|
1908
2935
|
this.options = options;
|
|
1909
2936
|
}
|
|
1910
2937
|
/**
|
|
1911
|
-
*
|
|
2938
|
+
* Detects redirects for a single URL using multiple strategies.
|
|
2939
|
+
*
|
|
2940
|
+
* @param url - The URL path to probe for redirects.
|
|
2941
|
+
* @returns A promise resolving to a `RedirectRule` if a redirect is found, or null.
|
|
1912
2942
|
*/
|
|
1913
2943
|
async detect(url) {
|
|
1914
2944
|
if (this.options.autoDetect?.cache) {
|
|
@@ -1940,7 +2970,10 @@ var RedirectDetector = class {
|
|
|
1940
2970
|
return null;
|
|
1941
2971
|
}
|
|
1942
2972
|
/**
|
|
1943
|
-
*
|
|
2973
|
+
* Batch detects redirects for multiple URLs with concurrency control.
|
|
2974
|
+
*
|
|
2975
|
+
* @param urls - An array of URL paths to probe.
|
|
2976
|
+
* @returns A promise resolving to a Map of URLs to their respective `RedirectRule` or null.
|
|
1944
2977
|
*/
|
|
1945
2978
|
async detectBatch(urls) {
|
|
1946
2979
|
const results = /* @__PURE__ */ new Map();
|
|
@@ -1960,7 +2993,7 @@ var RedirectDetector = class {
|
|
|
1960
2993
|
return results;
|
|
1961
2994
|
}
|
|
1962
2995
|
/**
|
|
1963
|
-
*
|
|
2996
|
+
* Detects a redirect from the configured database table.
|
|
1964
2997
|
*/
|
|
1965
2998
|
async detectFromDatabase(url) {
|
|
1966
2999
|
const { database } = this.options;
|
|
@@ -1978,14 +3011,14 @@ var RedirectDetector = class {
|
|
|
1978
3011
|
return {
|
|
1979
3012
|
from: row[columns.from],
|
|
1980
3013
|
to: row[columns.to],
|
|
1981
|
-
type: parseInt(row[columns.type], 10)
|
|
3014
|
+
type: Number.parseInt(row[columns.type], 10)
|
|
1982
3015
|
};
|
|
1983
3016
|
} catch {
|
|
1984
3017
|
return null;
|
|
1985
3018
|
}
|
|
1986
3019
|
}
|
|
1987
3020
|
/**
|
|
1988
|
-
*
|
|
3021
|
+
* Detects a redirect from a static JSON configuration file.
|
|
1989
3022
|
*/
|
|
1990
3023
|
async detectFromConfig(url) {
|
|
1991
3024
|
const { config } = this.options;
|
|
@@ -2003,7 +3036,7 @@ var RedirectDetector = class {
|
|
|
2003
3036
|
}
|
|
2004
3037
|
}
|
|
2005
3038
|
/**
|
|
2006
|
-
*
|
|
3039
|
+
* Auto-detects a redirect by sending an HTTP HEAD request.
|
|
2007
3040
|
*/
|
|
2008
3041
|
async detectAuto(url) {
|
|
2009
3042
|
const { autoDetect, baseUrl } = this.options;
|
|
@@ -2020,7 +3053,7 @@ var RedirectDetector = class {
|
|
|
2020
3053
|
method: "HEAD",
|
|
2021
3054
|
signal: controller.signal,
|
|
2022
3055
|
redirect: "manual"
|
|
2023
|
-
//
|
|
3056
|
+
// Handle redirects manually
|
|
2024
3057
|
});
|
|
2025
3058
|
clearTimeout(timeoutId);
|
|
2026
3059
|
if (response.status === 301 || response.status === 302) {
|
|
@@ -2044,7 +3077,7 @@ var RedirectDetector = class {
|
|
|
2044
3077
|
return null;
|
|
2045
3078
|
}
|
|
2046
3079
|
/**
|
|
2047
|
-
*
|
|
3080
|
+
* Caches the detection result for a URL.
|
|
2048
3081
|
*/
|
|
2049
3082
|
cacheResult(url, rule) {
|
|
2050
3083
|
if (!this.options.autoDetect?.cache) {
|
|
@@ -2065,6 +3098,11 @@ var MemoryRedirectManager = class {
|
|
|
2065
3098
|
constructor(options = {}) {
|
|
2066
3099
|
this.maxRules = options.maxRules || 1e5;
|
|
2067
3100
|
}
|
|
3101
|
+
/**
|
|
3102
|
+
* Registers a single redirect rule in memory.
|
|
3103
|
+
*
|
|
3104
|
+
* @param redirect - The redirect rule to add.
|
|
3105
|
+
*/
|
|
2068
3106
|
async register(redirect) {
|
|
2069
3107
|
this.rules.set(redirect.from, redirect);
|
|
2070
3108
|
if (this.rules.size > this.maxRules) {
|
|
@@ -2074,17 +3112,41 @@ var MemoryRedirectManager = class {
|
|
|
2074
3112
|
}
|
|
2075
3113
|
}
|
|
2076
3114
|
}
|
|
3115
|
+
/**
|
|
3116
|
+
* Registers multiple redirect rules in memory.
|
|
3117
|
+
*
|
|
3118
|
+
* @param redirects - An array of redirect rules.
|
|
3119
|
+
*/
|
|
2077
3120
|
async registerBatch(redirects) {
|
|
2078
3121
|
for (const redirect of redirects) {
|
|
2079
3122
|
await this.register(redirect);
|
|
2080
3123
|
}
|
|
2081
3124
|
}
|
|
3125
|
+
/**
|
|
3126
|
+
* Retrieves a specific redirect rule by its source path from memory.
|
|
3127
|
+
*
|
|
3128
|
+
* @param from - The source path.
|
|
3129
|
+
* @returns A promise resolving to the redirect rule, or null if not found.
|
|
3130
|
+
*/
|
|
2082
3131
|
async get(from) {
|
|
2083
3132
|
return this.rules.get(from) || null;
|
|
2084
3133
|
}
|
|
3134
|
+
/**
|
|
3135
|
+
* Retrieves all registered redirect rules from memory.
|
|
3136
|
+
*
|
|
3137
|
+
* @returns A promise resolving to an array of all redirect rules.
|
|
3138
|
+
*/
|
|
2085
3139
|
async getAll() {
|
|
2086
3140
|
return Array.from(this.rules.values());
|
|
2087
3141
|
}
|
|
3142
|
+
/**
|
|
3143
|
+
* Resolves a URL to its final destination through the redirect table.
|
|
3144
|
+
*
|
|
3145
|
+
* @param url - The URL to resolve.
|
|
3146
|
+
* @param followChains - Whether to recursively resolve chained redirects.
|
|
3147
|
+
* @param maxChainLength - Maximum depth for chain resolution.
|
|
3148
|
+
* @returns A promise resolving to the final destination URL.
|
|
3149
|
+
*/
|
|
2088
3150
|
async resolve(url, followChains = false, maxChainLength = 5) {
|
|
2089
3151
|
let current = url;
|
|
2090
3152
|
let chainLength = 0;
|
|
@@ -2117,6 +3179,11 @@ var RedisRedirectManager = class {
|
|
|
2117
3179
|
getListKey() {
|
|
2118
3180
|
return `${this.keyPrefix}list`;
|
|
2119
3181
|
}
|
|
3182
|
+
/**
|
|
3183
|
+
* Registers a single redirect rule in Redis.
|
|
3184
|
+
*
|
|
3185
|
+
* @param redirect - The redirect rule to add.
|
|
3186
|
+
*/
|
|
2120
3187
|
async register(redirect) {
|
|
2121
3188
|
const key = this.getKey(redirect.from);
|
|
2122
3189
|
const listKey = this.getListKey();
|
|
@@ -2128,11 +3195,22 @@ var RedisRedirectManager = class {
|
|
|
2128
3195
|
}
|
|
2129
3196
|
await this.client.sadd(listKey, redirect.from);
|
|
2130
3197
|
}
|
|
3198
|
+
/**
|
|
3199
|
+
* Registers multiple redirect rules in Redis.
|
|
3200
|
+
*
|
|
3201
|
+
* @param redirects - An array of redirect rules.
|
|
3202
|
+
*/
|
|
2131
3203
|
async registerBatch(redirects) {
|
|
2132
3204
|
for (const redirect of redirects) {
|
|
2133
3205
|
await this.register(redirect);
|
|
2134
3206
|
}
|
|
2135
3207
|
}
|
|
3208
|
+
/**
|
|
3209
|
+
* Retrieves a specific redirect rule by its source path from Redis.
|
|
3210
|
+
*
|
|
3211
|
+
* @param from - The source path.
|
|
3212
|
+
* @returns A promise resolving to the redirect rule, or null if not found.
|
|
3213
|
+
*/
|
|
2136
3214
|
async get(from) {
|
|
2137
3215
|
try {
|
|
2138
3216
|
const key = this.getKey(from);
|
|
@@ -2149,6 +3227,11 @@ var RedisRedirectManager = class {
|
|
|
2149
3227
|
return null;
|
|
2150
3228
|
}
|
|
2151
3229
|
}
|
|
3230
|
+
/**
|
|
3231
|
+
* Retrieves all registered redirect rules from Redis.
|
|
3232
|
+
*
|
|
3233
|
+
* @returns A promise resolving to an array of all redirect rules.
|
|
3234
|
+
*/
|
|
2152
3235
|
async getAll() {
|
|
2153
3236
|
try {
|
|
2154
3237
|
const listKey = this.getListKey();
|
|
@@ -2165,6 +3248,14 @@ var RedisRedirectManager = class {
|
|
|
2165
3248
|
return [];
|
|
2166
3249
|
}
|
|
2167
3250
|
}
|
|
3251
|
+
/**
|
|
3252
|
+
* Resolves a URL to its final destination through the Redis redirect table.
|
|
3253
|
+
*
|
|
3254
|
+
* @param url - The URL to resolve.
|
|
3255
|
+
* @param followChains - Whether to recursively resolve chained redirects.
|
|
3256
|
+
* @param maxChainLength - Maximum depth for chain resolution.
|
|
3257
|
+
* @returns A promise resolving to the final destination URL.
|
|
3258
|
+
*/
|
|
2168
3259
|
async resolve(url, followChains = false, maxChainLength = 5) {
|
|
2169
3260
|
let current = url;
|
|
2170
3261
|
let chainLength = 0;
|
|
@@ -2187,6 +3278,9 @@ var RedisRedirectManager = class {
|
|
|
2187
3278
|
init_DiskSitemapStorage();
|
|
2188
3279
|
|
|
2189
3280
|
// src/storage/GCPSitemapStorage.ts
|
|
3281
|
+
var import_node_stream3 = require("stream");
|
|
3282
|
+
var import_promises3 = require("stream/promises");
|
|
3283
|
+
init_Compression();
|
|
2190
3284
|
var GCPSitemapStorage = class {
|
|
2191
3285
|
bucket;
|
|
2192
3286
|
prefix;
|
|
@@ -2225,6 +3319,12 @@ var GCPSitemapStorage = class {
|
|
|
2225
3319
|
const cleanPrefix = this.prefix.endsWith("/") ? this.prefix.slice(0, -1) : this.prefix;
|
|
2226
3320
|
return cleanPrefix ? `${cleanPrefix}/${filename}` : filename;
|
|
2227
3321
|
}
|
|
3322
|
+
/**
|
|
3323
|
+
* Writes sitemap content to a Google Cloud Storage object.
|
|
3324
|
+
*
|
|
3325
|
+
* @param filename - The name of the file to write.
|
|
3326
|
+
* @param content - The XML or JSON content.
|
|
3327
|
+
*/
|
|
2228
3328
|
async write(filename, content) {
|
|
2229
3329
|
const { bucket } = await this.getStorageClient();
|
|
2230
3330
|
const key = this.getKey(filename);
|
|
@@ -2236,6 +3336,41 @@ var GCPSitemapStorage = class {
|
|
|
2236
3336
|
}
|
|
2237
3337
|
});
|
|
2238
3338
|
}
|
|
3339
|
+
/**
|
|
3340
|
+
* 使用串流方式寫入 sitemap 至 GCP Cloud Storage,可選擇性啟用 gzip 壓縮。
|
|
3341
|
+
*
|
|
3342
|
+
* @param filename - 檔案名稱
|
|
3343
|
+
* @param stream - XML 內容的 AsyncIterable
|
|
3344
|
+
* @param options - 寫入選項(如壓縮、content type)
|
|
3345
|
+
*
|
|
3346
|
+
* @since 3.1.0
|
|
3347
|
+
*/
|
|
3348
|
+
async writeStream(filename, stream, options) {
|
|
3349
|
+
const { bucket } = await this.getStorageClient();
|
|
3350
|
+
const key = this.getKey(options?.compress ? toGzipFilename(filename) : filename);
|
|
3351
|
+
const file = bucket.file(key);
|
|
3352
|
+
const writeStream = file.createWriteStream({
|
|
3353
|
+
resumable: false,
|
|
3354
|
+
contentType: "application/xml",
|
|
3355
|
+
metadata: {
|
|
3356
|
+
contentEncoding: options?.compress ? "gzip" : void 0,
|
|
3357
|
+
cacheControl: "public, max-age=3600"
|
|
3358
|
+
}
|
|
3359
|
+
});
|
|
3360
|
+
const readable = import_node_stream3.Readable.from(stream, { encoding: "utf-8" });
|
|
3361
|
+
if (options?.compress) {
|
|
3362
|
+
const gzip = createCompressionStream();
|
|
3363
|
+
await (0, import_promises3.pipeline)(readable, gzip, writeStream);
|
|
3364
|
+
} else {
|
|
3365
|
+
await (0, import_promises3.pipeline)(readable, writeStream);
|
|
3366
|
+
}
|
|
3367
|
+
}
|
|
3368
|
+
/**
|
|
3369
|
+
* Reads sitemap content from a Google Cloud Storage object.
|
|
3370
|
+
*
|
|
3371
|
+
* @param filename - The name of the file to read.
|
|
3372
|
+
* @returns A promise resolving to the file content as a string, or null if not found.
|
|
3373
|
+
*/
|
|
2239
3374
|
async read(filename) {
|
|
2240
3375
|
try {
|
|
2241
3376
|
const { bucket } = await this.getStorageClient();
|
|
@@ -2254,6 +3389,12 @@ var GCPSitemapStorage = class {
|
|
|
2254
3389
|
throw error;
|
|
2255
3390
|
}
|
|
2256
3391
|
}
|
|
3392
|
+
/**
|
|
3393
|
+
* Returns a readable stream for a Google Cloud Storage object.
|
|
3394
|
+
*
|
|
3395
|
+
* @param filename - The name of the file to stream.
|
|
3396
|
+
* @returns A promise resolving to an async iterable of file chunks, or null if not found.
|
|
3397
|
+
*/
|
|
2257
3398
|
async readStream(filename) {
|
|
2258
3399
|
try {
|
|
2259
3400
|
const { bucket } = await this.getStorageClient();
|
|
@@ -2278,6 +3419,12 @@ var GCPSitemapStorage = class {
|
|
|
2278
3419
|
throw error;
|
|
2279
3420
|
}
|
|
2280
3421
|
}
|
|
3422
|
+
/**
|
|
3423
|
+
* Checks if a Google Cloud Storage object exists.
|
|
3424
|
+
*
|
|
3425
|
+
* @param filename - The name of the file to check.
|
|
3426
|
+
* @returns A promise resolving to true if the file exists, false otherwise.
|
|
3427
|
+
*/
|
|
2281
3428
|
async exists(filename) {
|
|
2282
3429
|
try {
|
|
2283
3430
|
const { bucket } = await this.getStorageClient();
|
|
@@ -2289,12 +3436,24 @@ var GCPSitemapStorage = class {
|
|
|
2289
3436
|
return false;
|
|
2290
3437
|
}
|
|
2291
3438
|
}
|
|
3439
|
+
/**
|
|
3440
|
+
* Returns the full public URL for a Google Cloud Storage object.
|
|
3441
|
+
*
|
|
3442
|
+
* @param filename - The name of the sitemap file.
|
|
3443
|
+
* @returns The public URL as a string.
|
|
3444
|
+
*/
|
|
2292
3445
|
getUrl(filename) {
|
|
2293
3446
|
const key = this.getKey(filename);
|
|
2294
3447
|
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
2295
3448
|
return `${base}/${key}`;
|
|
2296
3449
|
}
|
|
2297
|
-
|
|
3450
|
+
/**
|
|
3451
|
+
* Writes content to a shadow (staged) location in Google Cloud Storage.
|
|
3452
|
+
*
|
|
3453
|
+
* @param filename - The name of the file to write.
|
|
3454
|
+
* @param content - The XML or JSON content.
|
|
3455
|
+
* @param shadowId - Optional unique session identifier.
|
|
3456
|
+
*/
|
|
2298
3457
|
async writeShadow(filename, content, shadowId) {
|
|
2299
3458
|
if (!this.shadowEnabled) {
|
|
2300
3459
|
return this.write(filename, content);
|
|
@@ -2310,6 +3469,11 @@ var GCPSitemapStorage = class {
|
|
|
2310
3469
|
}
|
|
2311
3470
|
});
|
|
2312
3471
|
}
|
|
3472
|
+
/**
|
|
3473
|
+
* Commits all staged shadow objects in a session to production in Google Cloud Storage.
|
|
3474
|
+
*
|
|
3475
|
+
* @param shadowId - The identifier of the session to commit.
|
|
3476
|
+
*/
|
|
2313
3477
|
async commitShadow(shadowId) {
|
|
2314
3478
|
if (!this.shadowEnabled) {
|
|
2315
3479
|
return;
|
|
@@ -2336,6 +3500,12 @@ var GCPSitemapStorage = class {
|
|
|
2336
3500
|
}
|
|
2337
3501
|
}
|
|
2338
3502
|
}
|
|
3503
|
+
/**
|
|
3504
|
+
* Lists all archived versions of a specific sitemap in Google Cloud Storage.
|
|
3505
|
+
*
|
|
3506
|
+
* @param filename - The sitemap filename.
|
|
3507
|
+
* @returns A promise resolving to an array of version identifiers.
|
|
3508
|
+
*/
|
|
2339
3509
|
async listVersions(filename) {
|
|
2340
3510
|
if (this.shadowMode !== "versioned") {
|
|
2341
3511
|
return [];
|
|
@@ -2357,6 +3527,12 @@ var GCPSitemapStorage = class {
|
|
|
2357
3527
|
return [];
|
|
2358
3528
|
}
|
|
2359
3529
|
}
|
|
3530
|
+
/**
|
|
3531
|
+
* Reverts a sitemap to a previously archived version in Google Cloud Storage.
|
|
3532
|
+
*
|
|
3533
|
+
* @param filename - The sitemap filename.
|
|
3534
|
+
* @param version - The version identifier to switch to.
|
|
3535
|
+
*/
|
|
2360
3536
|
async switchVersion(filename, version) {
|
|
2361
3537
|
if (this.shadowMode !== "versioned") {
|
|
2362
3538
|
throw new Error("Version switching is only available in versioned mode");
|
|
@@ -2376,22 +3552,51 @@ var GCPSitemapStorage = class {
|
|
|
2376
3552
|
// src/storage/MemoryProgressStorage.ts
|
|
2377
3553
|
var MemoryProgressStorage = class {
|
|
2378
3554
|
storage = /* @__PURE__ */ new Map();
|
|
3555
|
+
/**
|
|
3556
|
+
* Retrieves the progress of a specific generation job from memory.
|
|
3557
|
+
*
|
|
3558
|
+
* @param jobId - Unique identifier for the job.
|
|
3559
|
+
* @returns A promise resolving to the `SitemapProgress` object, or null if not found.
|
|
3560
|
+
*/
|
|
2379
3561
|
async get(jobId) {
|
|
2380
3562
|
const progress = this.storage.get(jobId);
|
|
2381
3563
|
return progress ? { ...progress } : null;
|
|
2382
3564
|
}
|
|
3565
|
+
/**
|
|
3566
|
+
* Initializes or overwrites a progress record in memory.
|
|
3567
|
+
*
|
|
3568
|
+
* @param jobId - Unique identifier for the job.
|
|
3569
|
+
* @param progress - The initial or current state of the job progress.
|
|
3570
|
+
*/
|
|
2383
3571
|
async set(jobId, progress) {
|
|
2384
3572
|
this.storage.set(jobId, { ...progress });
|
|
2385
3573
|
}
|
|
3574
|
+
/**
|
|
3575
|
+
* Updates specific fields of an existing progress record in memory.
|
|
3576
|
+
*
|
|
3577
|
+
* @param jobId - Unique identifier for the job.
|
|
3578
|
+
* @param updates - Object containing the fields to update.
|
|
3579
|
+
*/
|
|
2386
3580
|
async update(jobId, updates) {
|
|
2387
3581
|
const existing = this.storage.get(jobId);
|
|
2388
3582
|
if (existing) {
|
|
2389
3583
|
this.storage.set(jobId, { ...existing, ...updates });
|
|
2390
3584
|
}
|
|
2391
3585
|
}
|
|
3586
|
+
/**
|
|
3587
|
+
* Deletes a progress record from memory.
|
|
3588
|
+
*
|
|
3589
|
+
* @param jobId - Unique identifier for the job to remove.
|
|
3590
|
+
*/
|
|
2392
3591
|
async delete(jobId) {
|
|
2393
3592
|
this.storage.delete(jobId);
|
|
2394
3593
|
}
|
|
3594
|
+
/**
|
|
3595
|
+
* Lists the most recent sitemap generation jobs from memory.
|
|
3596
|
+
*
|
|
3597
|
+
* @param limit - Maximum number of records to return.
|
|
3598
|
+
* @returns A promise resolving to an array of `SitemapProgress` objects, sorted by start time.
|
|
3599
|
+
*/
|
|
2395
3600
|
async list(limit) {
|
|
2396
3601
|
const all = Array.from(this.storage.values());
|
|
2397
3602
|
const sorted = all.sort((a, b) => {
|
|
@@ -2419,6 +3624,12 @@ var RedisProgressStorage = class {
|
|
|
2419
3624
|
getListKey() {
|
|
2420
3625
|
return `${this.keyPrefix}list`;
|
|
2421
3626
|
}
|
|
3627
|
+
/**
|
|
3628
|
+
* Retrieves the progress of a specific generation job from Redis.
|
|
3629
|
+
*
|
|
3630
|
+
* @param jobId - Unique identifier for the job.
|
|
3631
|
+
* @returns A promise resolving to the `SitemapProgress` object, or null if not found.
|
|
3632
|
+
*/
|
|
2422
3633
|
async get(jobId) {
|
|
2423
3634
|
try {
|
|
2424
3635
|
const key = this.getKey(jobId);
|
|
@@ -2438,6 +3649,12 @@ var RedisProgressStorage = class {
|
|
|
2438
3649
|
return null;
|
|
2439
3650
|
}
|
|
2440
3651
|
}
|
|
3652
|
+
/**
|
|
3653
|
+
* Initializes or overwrites a progress record in Redis.
|
|
3654
|
+
*
|
|
3655
|
+
* @param jobId - Unique identifier for the job.
|
|
3656
|
+
* @param progress - The initial or current state of the job progress.
|
|
3657
|
+
*/
|
|
2441
3658
|
async set(jobId, progress) {
|
|
2442
3659
|
const key = this.getKey(jobId);
|
|
2443
3660
|
const listKey = this.getListKey();
|
|
@@ -2446,18 +3663,35 @@ var RedisProgressStorage = class {
|
|
|
2446
3663
|
await this.client.zadd(listKey, Date.now(), jobId);
|
|
2447
3664
|
await this.client.expire(listKey, this.ttl);
|
|
2448
3665
|
}
|
|
3666
|
+
/**
|
|
3667
|
+
* Updates specific fields of an existing progress record in Redis.
|
|
3668
|
+
*
|
|
3669
|
+
* @param jobId - Unique identifier for the job.
|
|
3670
|
+
* @param updates - Object containing the fields to update.
|
|
3671
|
+
*/
|
|
2449
3672
|
async update(jobId, updates) {
|
|
2450
3673
|
const existing = await this.get(jobId);
|
|
2451
3674
|
if (existing) {
|
|
2452
3675
|
await this.set(jobId, { ...existing, ...updates });
|
|
2453
3676
|
}
|
|
2454
3677
|
}
|
|
3678
|
+
/**
|
|
3679
|
+
* Deletes a progress record from Redis.
|
|
3680
|
+
*
|
|
3681
|
+
* @param jobId - Unique identifier for the job to remove.
|
|
3682
|
+
*/
|
|
2455
3683
|
async delete(jobId) {
|
|
2456
3684
|
const key = this.getKey(jobId);
|
|
2457
3685
|
const listKey = this.getListKey();
|
|
2458
3686
|
await this.client.del(key);
|
|
2459
3687
|
await this.client.zrem(listKey, jobId);
|
|
2460
3688
|
}
|
|
3689
|
+
/**
|
|
3690
|
+
* Lists the most recent sitemap generation jobs from Redis.
|
|
3691
|
+
*
|
|
3692
|
+
* @param limit - Maximum number of records to return.
|
|
3693
|
+
* @returns A promise resolving to an array of `SitemapProgress` objects, sorted by start time.
|
|
3694
|
+
*/
|
|
2461
3695
|
async list(limit) {
|
|
2462
3696
|
try {
|
|
2463
3697
|
const listKey = this.getListKey();
|
|
@@ -2477,6 +3711,7 @@ var RedisProgressStorage = class {
|
|
|
2477
3711
|
};
|
|
2478
3712
|
|
|
2479
3713
|
// src/storage/S3SitemapStorage.ts
|
|
3714
|
+
init_Compression();
|
|
2480
3715
|
var S3SitemapStorage = class {
|
|
2481
3716
|
bucket;
|
|
2482
3717
|
region;
|
|
@@ -2534,6 +3769,12 @@ var S3SitemapStorage = class {
|
|
|
2534
3769
|
const cleanPrefix = this.prefix.endsWith("/") ? this.prefix.slice(0, -1) : this.prefix;
|
|
2535
3770
|
return cleanPrefix ? `${cleanPrefix}/${filename}` : filename;
|
|
2536
3771
|
}
|
|
3772
|
+
/**
|
|
3773
|
+
* Writes sitemap content to an S3 object.
|
|
3774
|
+
*
|
|
3775
|
+
* @param filename - The name of the file to write.
|
|
3776
|
+
* @param content - The XML or JSON content.
|
|
3777
|
+
*/
|
|
2537
3778
|
async write(filename, content) {
|
|
2538
3779
|
const s3 = await this.getS3Client();
|
|
2539
3780
|
const key = this.getKey(filename);
|
|
@@ -2546,6 +3787,48 @@ var S3SitemapStorage = class {
|
|
|
2546
3787
|
})
|
|
2547
3788
|
);
|
|
2548
3789
|
}
|
|
3790
|
+
/**
|
|
3791
|
+
* 使用串流方式寫入 sitemap 至 S3,可選擇性啟用 gzip 壓縮。
|
|
3792
|
+
* S3 需要知道 Content-Length,因此會先收集串流為 Buffer。
|
|
3793
|
+
*
|
|
3794
|
+
* @param filename - 檔案名稱
|
|
3795
|
+
* @param stream - XML 內容的 AsyncIterable
|
|
3796
|
+
* @param options - 寫入選項(如壓縮、content type)
|
|
3797
|
+
*
|
|
3798
|
+
* @since 3.1.0
|
|
3799
|
+
*/
|
|
3800
|
+
async writeStream(filename, stream, options) {
|
|
3801
|
+
const s3 = await this.getS3Client();
|
|
3802
|
+
const key = this.getKey(options?.compress ? toGzipFilename(filename) : filename);
|
|
3803
|
+
let body;
|
|
3804
|
+
const contentType = options?.contentType || "application/xml";
|
|
3805
|
+
let contentEncoding;
|
|
3806
|
+
if (options?.compress) {
|
|
3807
|
+
body = await compressToBuffer(stream);
|
|
3808
|
+
contentEncoding = "gzip";
|
|
3809
|
+
} else {
|
|
3810
|
+
const chunks = [];
|
|
3811
|
+
for await (const chunk of stream) {
|
|
3812
|
+
chunks.push(chunk);
|
|
3813
|
+
}
|
|
3814
|
+
body = chunks.join("");
|
|
3815
|
+
}
|
|
3816
|
+
await s3.client.send(
|
|
3817
|
+
new s3.PutObjectCommand({
|
|
3818
|
+
Bucket: this.bucket,
|
|
3819
|
+
Key: key,
|
|
3820
|
+
Body: body,
|
|
3821
|
+
ContentType: contentType,
|
|
3822
|
+
ContentEncoding: contentEncoding
|
|
3823
|
+
})
|
|
3824
|
+
);
|
|
3825
|
+
}
|
|
3826
|
+
/**
|
|
3827
|
+
* Reads sitemap content from an S3 object.
|
|
3828
|
+
*
|
|
3829
|
+
* @param filename - The name of the file to read.
|
|
3830
|
+
* @returns A promise resolving to the file content as a string, or null if not found.
|
|
3831
|
+
*/
|
|
2549
3832
|
async read(filename) {
|
|
2550
3833
|
try {
|
|
2551
3834
|
const s3 = await this.getS3Client();
|
|
@@ -2572,6 +3855,12 @@ var S3SitemapStorage = class {
|
|
|
2572
3855
|
throw error;
|
|
2573
3856
|
}
|
|
2574
3857
|
}
|
|
3858
|
+
/**
|
|
3859
|
+
* Returns a readable stream for an S3 object.
|
|
3860
|
+
*
|
|
3861
|
+
* @param filename - The name of the file to stream.
|
|
3862
|
+
* @returns A promise resolving to an async iterable of file chunks, or null if not found.
|
|
3863
|
+
*/
|
|
2575
3864
|
async readStream(filename) {
|
|
2576
3865
|
try {
|
|
2577
3866
|
const s3 = await this.getS3Client();
|
|
@@ -2600,6 +3889,12 @@ var S3SitemapStorage = class {
|
|
|
2600
3889
|
throw error;
|
|
2601
3890
|
}
|
|
2602
3891
|
}
|
|
3892
|
+
/**
|
|
3893
|
+
* Checks if an S3 object exists.
|
|
3894
|
+
*
|
|
3895
|
+
* @param filename - The name of the file to check.
|
|
3896
|
+
* @returns A promise resolving to true if the file exists, false otherwise.
|
|
3897
|
+
*/
|
|
2603
3898
|
async exists(filename) {
|
|
2604
3899
|
try {
|
|
2605
3900
|
const s3 = await this.getS3Client();
|
|
@@ -2618,12 +3913,24 @@ var S3SitemapStorage = class {
|
|
|
2618
3913
|
throw error;
|
|
2619
3914
|
}
|
|
2620
3915
|
}
|
|
3916
|
+
/**
|
|
3917
|
+
* Returns the full public URL for an S3 object.
|
|
3918
|
+
*
|
|
3919
|
+
* @param filename - The name of the sitemap file.
|
|
3920
|
+
* @returns The public URL as a string.
|
|
3921
|
+
*/
|
|
2621
3922
|
getUrl(filename) {
|
|
2622
3923
|
const key = this.getKey(filename);
|
|
2623
3924
|
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
2624
3925
|
return `${base}/${key}`;
|
|
2625
3926
|
}
|
|
2626
|
-
|
|
3927
|
+
/**
|
|
3928
|
+
* Writes content to a shadow (staged) location in S3.
|
|
3929
|
+
*
|
|
3930
|
+
* @param filename - The name of the file to write.
|
|
3931
|
+
* @param content - The XML or JSON content.
|
|
3932
|
+
* @param shadowId - Optional unique session identifier.
|
|
3933
|
+
*/
|
|
2627
3934
|
async writeShadow(filename, content, shadowId) {
|
|
2628
3935
|
if (!this.shadowEnabled) {
|
|
2629
3936
|
return this.write(filename, content);
|
|
@@ -2640,6 +3947,11 @@ var S3SitemapStorage = class {
|
|
|
2640
3947
|
})
|
|
2641
3948
|
);
|
|
2642
3949
|
}
|
|
3950
|
+
/**
|
|
3951
|
+
* Commits all staged shadow objects in a session to production.
|
|
3952
|
+
*
|
|
3953
|
+
* @param shadowId - The identifier of the session to commit.
|
|
3954
|
+
*/
|
|
2643
3955
|
async commitShadow(shadowId) {
|
|
2644
3956
|
if (!this.shadowEnabled) {
|
|
2645
3957
|
return;
|
|
@@ -2707,6 +4019,12 @@ var S3SitemapStorage = class {
|
|
|
2707
4019
|
}
|
|
2708
4020
|
}
|
|
2709
4021
|
}
|
|
4022
|
+
/**
|
|
4023
|
+
* Lists all archived versions of a specific sitemap in S3.
|
|
4024
|
+
*
|
|
4025
|
+
* @param filename - The sitemap filename.
|
|
4026
|
+
* @returns A promise resolving to an array of version identifiers.
|
|
4027
|
+
*/
|
|
2710
4028
|
async listVersions(filename) {
|
|
2711
4029
|
if (this.shadowMode !== "versioned") {
|
|
2712
4030
|
return [];
|
|
@@ -2739,6 +4057,12 @@ var S3SitemapStorage = class {
|
|
|
2739
4057
|
return [];
|
|
2740
4058
|
}
|
|
2741
4059
|
}
|
|
4060
|
+
/**
|
|
4061
|
+
* Reverts a sitemap to a previously archived version in S3.
|
|
4062
|
+
*
|
|
4063
|
+
* @param filename - The sitemap filename.
|
|
4064
|
+
* @param version - The version identifier to switch to.
|
|
4065
|
+
*/
|
|
2742
4066
|
async switchVersion(filename, version) {
|
|
2743
4067
|
if (this.shadowMode !== "versioned") {
|
|
2744
4068
|
throw new Error("Version switching is only available in versioned mode");
|
|
@@ -2760,6 +4084,9 @@ var S3SitemapStorage = class {
|
|
|
2760
4084
|
);
|
|
2761
4085
|
}
|
|
2762
4086
|
};
|
|
4087
|
+
|
|
4088
|
+
// src/index.ts
|
|
4089
|
+
init_Compression();
|
|
2763
4090
|
// Annotate the CommonJS export names for ESM import in node:
|
|
2764
4091
|
0 && (module.exports = {
|
|
2765
4092
|
DiffCalculator,
|
|
@@ -2768,6 +4095,7 @@ var S3SitemapStorage = class {
|
|
|
2768
4095
|
GenerateSitemapJob,
|
|
2769
4096
|
IncrementalGenerator,
|
|
2770
4097
|
MemoryChangeTracker,
|
|
4098
|
+
MemoryLock,
|
|
2771
4099
|
MemoryProgressStorage,
|
|
2772
4100
|
MemoryRedirectManager,
|
|
2773
4101
|
MemorySitemapStorage,
|
|
@@ -2776,6 +4104,7 @@ var S3SitemapStorage = class {
|
|
|
2776
4104
|
RedirectDetector,
|
|
2777
4105
|
RedirectHandler,
|
|
2778
4106
|
RedisChangeTracker,
|
|
4107
|
+
RedisLock,
|
|
2779
4108
|
RedisProgressStorage,
|
|
2780
4109
|
RedisRedirectManager,
|
|
2781
4110
|
RouteScanner,
|
|
@@ -2784,6 +4113,10 @@ var S3SitemapStorage = class {
|
|
|
2784
4113
|
SitemapGenerator,
|
|
2785
4114
|
SitemapIndex,
|
|
2786
4115
|
SitemapStream,
|
|
4116
|
+
compressToBuffer,
|
|
4117
|
+
createCompressionStream,
|
|
4118
|
+
fromGzipFilename,
|
|
2787
4119
|
generateI18nEntries,
|
|
2788
|
-
routeScanner
|
|
4120
|
+
routeScanner,
|
|
4121
|
+
toGzipFilename
|
|
2789
4122
|
});
|