@gravito/constellation 3.0.0 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{DiskSitemapStorage-7ZZMGC4K.js → DiskSitemapStorage-WP6RITUN.js} +1 -1
- package/dist/{chunk-7WHLC3OJ.js → chunk-IS2H7U6M.js} +12 -0
- package/dist/index.cjs +517 -156
- package/dist/index.d.cts +934 -61
- package/dist/index.d.ts +934 -61
- package/dist/index.js +508 -159
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,18 +1,34 @@
|
|
|
1
1
|
import {
|
|
2
2
|
DiskSitemapStorage
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-IS2H7U6M.js";
|
|
4
4
|
|
|
5
5
|
// src/core/ChangeTracker.ts
|
|
6
6
|
var MemoryChangeTracker = class {
|
|
7
7
|
changes = [];
|
|
8
|
+
urlIndex = /* @__PURE__ */ new Map();
|
|
8
9
|
maxChanges;
|
|
9
10
|
constructor(options = {}) {
|
|
10
11
|
this.maxChanges = options.maxChanges || 1e5;
|
|
11
12
|
}
|
|
12
13
|
async track(change) {
|
|
13
14
|
this.changes.push(change);
|
|
15
|
+
const urlChanges = this.urlIndex.get(change.url) || [];
|
|
16
|
+
urlChanges.push(change);
|
|
17
|
+
this.urlIndex.set(change.url, urlChanges);
|
|
14
18
|
if (this.changes.length > this.maxChanges) {
|
|
15
|
-
|
|
19
|
+
const removed = this.changes.shift();
|
|
20
|
+
if (removed) {
|
|
21
|
+
const changes = this.urlIndex.get(removed.url);
|
|
22
|
+
if (changes) {
|
|
23
|
+
const index = changes.indexOf(removed);
|
|
24
|
+
if (index > -1) {
|
|
25
|
+
changes.splice(index, 1);
|
|
26
|
+
}
|
|
27
|
+
if (changes.length === 0) {
|
|
28
|
+
this.urlIndex.delete(removed.url);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
16
32
|
}
|
|
17
33
|
}
|
|
18
34
|
async getChanges(since) {
|
|
@@ -22,14 +38,21 @@ var MemoryChangeTracker = class {
|
|
|
22
38
|
return this.changes.filter((change) => change.timestamp >= since);
|
|
23
39
|
}
|
|
24
40
|
async getChangesByUrl(url) {
|
|
25
|
-
return this.
|
|
41
|
+
return this.urlIndex.get(url) || [];
|
|
26
42
|
}
|
|
27
43
|
async clear(since) {
|
|
28
44
|
if (!since) {
|
|
29
45
|
this.changes = [];
|
|
46
|
+
this.urlIndex.clear();
|
|
30
47
|
return;
|
|
31
48
|
}
|
|
32
49
|
this.changes = this.changes.filter((change) => change.timestamp < since);
|
|
50
|
+
this.urlIndex.clear();
|
|
51
|
+
for (const change of this.changes) {
|
|
52
|
+
const urlChanges = this.urlIndex.get(change.url) || [];
|
|
53
|
+
urlChanges.push(change);
|
|
54
|
+
this.urlIndex.set(change.url, urlChanges);
|
|
55
|
+
}
|
|
33
56
|
}
|
|
34
57
|
};
|
|
35
58
|
var RedisChangeTracker = class {
|
|
@@ -204,52 +227,72 @@ var DiffCalculator = class {
|
|
|
204
227
|
}
|
|
205
228
|
};
|
|
206
229
|
|
|
230
|
+
// src/utils/Mutex.ts
|
|
231
|
+
var Mutex = class {
|
|
232
|
+
queue = Promise.resolve();
|
|
233
|
+
async runExclusive(fn) {
|
|
234
|
+
const next = this.queue.then(() => fn());
|
|
235
|
+
this.queue = next.then(
|
|
236
|
+
() => {
|
|
237
|
+
},
|
|
238
|
+
() => {
|
|
239
|
+
}
|
|
240
|
+
);
|
|
241
|
+
return next;
|
|
242
|
+
}
|
|
243
|
+
};
|
|
244
|
+
|
|
207
245
|
// src/core/ShadowProcessor.ts
|
|
208
246
|
var ShadowProcessor = class {
|
|
209
247
|
options;
|
|
210
248
|
shadowId;
|
|
211
249
|
operations = [];
|
|
250
|
+
mutex = new Mutex();
|
|
212
251
|
constructor(options) {
|
|
213
252
|
this.options = options;
|
|
214
|
-
this.shadowId = `shadow-${Date.now()}-${
|
|
253
|
+
this.shadowId = `shadow-${Date.now()}-${crypto.randomUUID()}`;
|
|
215
254
|
}
|
|
216
255
|
/**
|
|
217
256
|
* 添加一個影子操作
|
|
218
257
|
*/
|
|
219
258
|
async addOperation(operation) {
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
259
|
+
return this.mutex.runExclusive(async () => {
|
|
260
|
+
if (!this.options.enabled) {
|
|
261
|
+
await this.options.storage.write(operation.filename, operation.content);
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
this.operations.push({
|
|
265
|
+
...operation,
|
|
266
|
+
shadowId: operation.shadowId || this.shadowId
|
|
267
|
+
});
|
|
268
|
+
if (this.options.storage.writeShadow) {
|
|
269
|
+
await this.options.storage.writeShadow(operation.filename, operation.content, this.shadowId);
|
|
270
|
+
} else {
|
|
271
|
+
await this.options.storage.write(operation.filename, operation.content);
|
|
272
|
+
}
|
|
227
273
|
});
|
|
228
|
-
if (this.options.storage.writeShadow) {
|
|
229
|
-
await this.options.storage.writeShadow(operation.filename, operation.content, this.shadowId);
|
|
230
|
-
} else {
|
|
231
|
-
await this.options.storage.write(operation.filename, operation.content);
|
|
232
|
-
}
|
|
233
274
|
}
|
|
234
275
|
/**
|
|
235
276
|
* 提交所有影子操作
|
|
236
277
|
*/
|
|
237
278
|
async commit() {
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
if (this.options.mode === "atomic") {
|
|
242
|
-
if (this.options.storage.commitShadow) {
|
|
243
|
-
await this.options.storage.commitShadow(this.shadowId);
|
|
279
|
+
return this.mutex.runExclusive(async () => {
|
|
280
|
+
if (!this.options.enabled) {
|
|
281
|
+
return;
|
|
244
282
|
}
|
|
245
|
-
|
|
246
|
-
for (const operation of this.operations) {
|
|
283
|
+
if (this.options.mode === "atomic") {
|
|
247
284
|
if (this.options.storage.commitShadow) {
|
|
248
|
-
await this.options.storage.commitShadow(
|
|
285
|
+
await this.options.storage.commitShadow(this.shadowId);
|
|
286
|
+
}
|
|
287
|
+
} else {
|
|
288
|
+
for (const operation of this.operations) {
|
|
289
|
+
if (this.options.storage.commitShadow) {
|
|
290
|
+
await this.options.storage.commitShadow(operation.shadowId || this.shadowId);
|
|
291
|
+
}
|
|
249
292
|
}
|
|
250
293
|
}
|
|
251
|
-
|
|
252
|
-
|
|
294
|
+
this.operations = [];
|
|
295
|
+
});
|
|
253
296
|
}
|
|
254
297
|
/**
|
|
255
298
|
* 取消所有影子操作
|
|
@@ -335,6 +378,12 @@ var SitemapIndex = class {
|
|
|
335
378
|
var SitemapStream = class {
|
|
336
379
|
options;
|
|
337
380
|
entries = [];
|
|
381
|
+
flags = {
|
|
382
|
+
hasImages: false,
|
|
383
|
+
hasVideos: false,
|
|
384
|
+
hasNews: false,
|
|
385
|
+
hasAlternates: false
|
|
386
|
+
};
|
|
338
387
|
constructor(options) {
|
|
339
388
|
this.options = { ...options };
|
|
340
389
|
if (this.options.baseUrl.endsWith("/")) {
|
|
@@ -342,10 +391,19 @@ var SitemapStream = class {
|
|
|
342
391
|
}
|
|
343
392
|
}
|
|
344
393
|
add(entry) {
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
this.
|
|
394
|
+
const e = typeof entry === "string" ? { url: entry } : entry;
|
|
395
|
+
this.entries.push(e);
|
|
396
|
+
if (e.images && e.images.length > 0) {
|
|
397
|
+
this.flags.hasImages = true;
|
|
398
|
+
}
|
|
399
|
+
if (e.videos && e.videos.length > 0) {
|
|
400
|
+
this.flags.hasVideos = true;
|
|
401
|
+
}
|
|
402
|
+
if (e.news) {
|
|
403
|
+
this.flags.hasNews = true;
|
|
404
|
+
}
|
|
405
|
+
if (e.alternates && e.alternates.length > 0) {
|
|
406
|
+
this.flags.hasAlternates = true;
|
|
349
407
|
}
|
|
350
408
|
return this;
|
|
351
409
|
}
|
|
@@ -357,33 +415,33 @@ var SitemapStream = class {
|
|
|
357
415
|
}
|
|
358
416
|
toXML() {
|
|
359
417
|
const { baseUrl, pretty } = this.options;
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
if (this.hasImages
|
|
364
|
-
|
|
418
|
+
const parts = [];
|
|
419
|
+
parts.push('<?xml version="1.0" encoding="UTF-8"?>\n');
|
|
420
|
+
parts.push('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"');
|
|
421
|
+
if (this.flags.hasImages) {
|
|
422
|
+
parts.push(' xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"');
|
|
365
423
|
}
|
|
366
|
-
if (this.hasVideos
|
|
367
|
-
|
|
424
|
+
if (this.flags.hasVideos) {
|
|
425
|
+
parts.push(' xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"');
|
|
368
426
|
}
|
|
369
|
-
if (this.hasNews
|
|
370
|
-
|
|
427
|
+
if (this.flags.hasNews) {
|
|
428
|
+
parts.push(' xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"');
|
|
371
429
|
}
|
|
372
|
-
if (this.hasAlternates
|
|
373
|
-
|
|
430
|
+
if (this.flags.hasAlternates) {
|
|
431
|
+
parts.push(' xmlns:xhtml="http://www.w3.org/1999/xhtml"');
|
|
374
432
|
}
|
|
375
|
-
|
|
376
|
-
`;
|
|
433
|
+
parts.push(">\n");
|
|
377
434
|
for (const entry of this.entries) {
|
|
378
|
-
|
|
435
|
+
parts.push(this.renderUrl(entry, baseUrl, pretty));
|
|
379
436
|
}
|
|
380
|
-
|
|
381
|
-
return
|
|
437
|
+
parts.push("</urlset>");
|
|
438
|
+
return parts.join("");
|
|
382
439
|
}
|
|
383
440
|
renderUrl(entry, baseUrl, pretty) {
|
|
384
441
|
const indent = pretty ? " " : "";
|
|
385
442
|
const subIndent = pretty ? " " : "";
|
|
386
443
|
const nl = pretty ? "\n" : "";
|
|
444
|
+
const parts = [];
|
|
387
445
|
let loc = entry.url;
|
|
388
446
|
if (!loc.startsWith("http")) {
|
|
389
447
|
if (!loc.startsWith("/")) {
|
|
@@ -391,17 +449,17 @@ var SitemapStream = class {
|
|
|
391
449
|
}
|
|
392
450
|
loc = baseUrl + loc;
|
|
393
451
|
}
|
|
394
|
-
|
|
395
|
-
|
|
452
|
+
parts.push(`${indent}<url>${nl}`);
|
|
453
|
+
parts.push(`${subIndent}<loc>${this.escape(loc)}</loc>${nl}`);
|
|
396
454
|
if (entry.lastmod) {
|
|
397
455
|
const date = entry.lastmod instanceof Date ? entry.lastmod : new Date(entry.lastmod);
|
|
398
|
-
|
|
456
|
+
parts.push(`${subIndent}<lastmod>${date.toISOString().split("T")[0]}</lastmod>${nl}`);
|
|
399
457
|
}
|
|
400
458
|
if (entry.changefreq) {
|
|
401
|
-
|
|
459
|
+
parts.push(`${subIndent}<changefreq>${entry.changefreq}</changefreq>${nl}`);
|
|
402
460
|
}
|
|
403
461
|
if (entry.priority !== void 0) {
|
|
404
|
-
|
|
462
|
+
parts.push(`${subIndent}<priority>${entry.priority.toFixed(1)}</priority>${nl}`);
|
|
405
463
|
}
|
|
406
464
|
if (entry.alternates) {
|
|
407
465
|
for (const alt of entry.alternates) {
|
|
@@ -412,7 +470,9 @@ var SitemapStream = class {
|
|
|
412
470
|
}
|
|
413
471
|
altLoc = baseUrl + altLoc;
|
|
414
472
|
}
|
|
415
|
-
|
|
473
|
+
parts.push(
|
|
474
|
+
`${subIndent}<xhtml:link rel="alternate" hreflang="${alt.lang}" href="${this.escape(altLoc)}"/>${nl}`
|
|
475
|
+
);
|
|
416
476
|
}
|
|
417
477
|
}
|
|
418
478
|
if (entry.redirect?.canonical) {
|
|
@@ -423,10 +483,14 @@ var SitemapStream = class {
|
|
|
423
483
|
}
|
|
424
484
|
canonicalUrl = baseUrl + canonicalUrl;
|
|
425
485
|
}
|
|
426
|
-
|
|
486
|
+
parts.push(
|
|
487
|
+
`${subIndent}<xhtml:link rel="canonical" href="${this.escape(canonicalUrl)}"/>${nl}`
|
|
488
|
+
);
|
|
427
489
|
}
|
|
428
490
|
if (entry.redirect && !entry.redirect.canonical) {
|
|
429
|
-
|
|
491
|
+
parts.push(
|
|
492
|
+
`${subIndent}<!-- Redirect: ${entry.redirect.from} \u2192 ${entry.redirect.to} (${entry.redirect.type}) -->${nl}`
|
|
493
|
+
);
|
|
430
494
|
}
|
|
431
495
|
if (entry.images) {
|
|
432
496
|
for (const img of entry.images) {
|
|
@@ -437,91 +501,110 @@ var SitemapStream = class {
|
|
|
437
501
|
}
|
|
438
502
|
loc2 = baseUrl + loc2;
|
|
439
503
|
}
|
|
440
|
-
|
|
441
|
-
|
|
504
|
+
parts.push(`${subIndent}<image:image>${nl}`);
|
|
505
|
+
parts.push(`${subIndent} <image:loc>${this.escape(loc2)}</image:loc>${nl}`);
|
|
442
506
|
if (img.title) {
|
|
443
|
-
|
|
507
|
+
parts.push(`${subIndent} <image:title>${this.escape(img.title)}</image:title>${nl}`);
|
|
444
508
|
}
|
|
445
509
|
if (img.caption) {
|
|
446
|
-
|
|
510
|
+
parts.push(
|
|
511
|
+
`${subIndent} <image:caption>${this.escape(img.caption)}</image:caption>${nl}`
|
|
512
|
+
);
|
|
447
513
|
}
|
|
448
514
|
if (img.geo_location) {
|
|
449
|
-
|
|
515
|
+
parts.push(
|
|
516
|
+
`${subIndent} <image:geo_location>${this.escape(img.geo_location)}</image:geo_location>${nl}`
|
|
517
|
+
);
|
|
450
518
|
}
|
|
451
519
|
if (img.license) {
|
|
452
|
-
|
|
520
|
+
parts.push(
|
|
521
|
+
`${subIndent} <image:license>${this.escape(img.license)}</image:license>${nl}`
|
|
522
|
+
);
|
|
453
523
|
}
|
|
454
|
-
|
|
524
|
+
parts.push(`${subIndent}</image:image>${nl}`);
|
|
455
525
|
}
|
|
456
526
|
}
|
|
457
527
|
if (entry.videos) {
|
|
458
528
|
for (const video of entry.videos) {
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
529
|
+
parts.push(`${subIndent}<video:video>${nl}`);
|
|
530
|
+
parts.push(
|
|
531
|
+
`${subIndent} <video:thumbnail_loc>${this.escape(video.thumbnail_loc)}</video:thumbnail_loc>${nl}`
|
|
532
|
+
);
|
|
533
|
+
parts.push(`${subIndent} <video:title>${this.escape(video.title)}</video:title>${nl}`);
|
|
534
|
+
parts.push(
|
|
535
|
+
`${subIndent} <video:description>${this.escape(video.description)}</video:description>${nl}`
|
|
536
|
+
);
|
|
463
537
|
if (video.content_loc) {
|
|
464
|
-
|
|
538
|
+
parts.push(
|
|
539
|
+
`${subIndent} <video:content_loc>${this.escape(video.content_loc)}</video:content_loc>${nl}`
|
|
540
|
+
);
|
|
465
541
|
}
|
|
466
542
|
if (video.player_loc) {
|
|
467
|
-
|
|
543
|
+
parts.push(
|
|
544
|
+
`${subIndent} <video:player_loc>${this.escape(video.player_loc)}</video:player_loc>${nl}`
|
|
545
|
+
);
|
|
468
546
|
}
|
|
469
547
|
if (video.duration) {
|
|
470
|
-
|
|
548
|
+
parts.push(`${subIndent} <video:duration>${video.duration}</video:duration>${nl}`);
|
|
471
549
|
}
|
|
472
550
|
if (video.view_count) {
|
|
473
|
-
|
|
551
|
+
parts.push(`${subIndent} <video:view_count>${video.view_count}</video:view_count>${nl}`);
|
|
474
552
|
}
|
|
475
553
|
if (video.publication_date) {
|
|
476
554
|
const pubDate = video.publication_date instanceof Date ? video.publication_date : new Date(video.publication_date);
|
|
477
|
-
|
|
555
|
+
parts.push(
|
|
556
|
+
`${subIndent} <video:publication_date>${pubDate.toISOString()}</video:publication_date>${nl}`
|
|
557
|
+
);
|
|
478
558
|
}
|
|
479
559
|
if (video.family_friendly) {
|
|
480
|
-
|
|
560
|
+
parts.push(
|
|
561
|
+
`${subIndent} <video:family_friendly>${video.family_friendly}</video:family_friendly>${nl}`
|
|
562
|
+
);
|
|
481
563
|
}
|
|
482
564
|
if (video.tag) {
|
|
483
565
|
for (const tag of video.tag) {
|
|
484
|
-
|
|
566
|
+
parts.push(`${subIndent} <video:tag>${this.escape(tag)}</video:tag>${nl}`);
|
|
485
567
|
}
|
|
486
568
|
}
|
|
487
|
-
|
|
569
|
+
parts.push(`${subIndent}</video:video>${nl}`);
|
|
488
570
|
}
|
|
489
571
|
}
|
|
490
572
|
if (entry.news) {
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
573
|
+
parts.push(`${subIndent}<news:news>${nl}`);
|
|
574
|
+
parts.push(`${subIndent} <news:publication>${nl}`);
|
|
575
|
+
parts.push(
|
|
576
|
+
`${subIndent} <news:name>${this.escape(entry.news.publication.name)}</news:name>${nl}`
|
|
577
|
+
);
|
|
578
|
+
parts.push(
|
|
579
|
+
`${subIndent} <news:language>${this.escape(entry.news.publication.language)}</news:language>${nl}`
|
|
580
|
+
);
|
|
581
|
+
parts.push(`${subIndent} </news:publication>${nl}`);
|
|
496
582
|
const pubDate = entry.news.publication_date instanceof Date ? entry.news.publication_date : new Date(entry.news.publication_date);
|
|
497
|
-
|
|
498
|
-
|
|
583
|
+
parts.push(
|
|
584
|
+
`${subIndent} <news:publication_date>${pubDate.toISOString()}</news:publication_date>${nl}`
|
|
585
|
+
);
|
|
586
|
+
parts.push(`${subIndent} <news:title>${this.escape(entry.news.title)}</news:title>${nl}`);
|
|
499
587
|
if (entry.news.genres) {
|
|
500
|
-
|
|
588
|
+
parts.push(
|
|
589
|
+
`${subIndent} <news:genres>${this.escape(entry.news.genres)}</news:genres>${nl}`
|
|
590
|
+
);
|
|
501
591
|
}
|
|
502
592
|
if (entry.news.keywords) {
|
|
503
|
-
|
|
593
|
+
parts.push(
|
|
594
|
+
`${subIndent} <news:keywords>${entry.news.keywords.map((k) => this.escape(k)).join(", ")}</news:keywords>${nl}`
|
|
595
|
+
);
|
|
504
596
|
}
|
|
505
|
-
|
|
597
|
+
parts.push(`${subIndent}</news:news>${nl}`);
|
|
506
598
|
}
|
|
507
|
-
|
|
508
|
-
return
|
|
509
|
-
}
|
|
510
|
-
hasImages() {
|
|
511
|
-
return this.entries.some((e) => e.images && e.images.length > 0);
|
|
512
|
-
}
|
|
513
|
-
hasVideos() {
|
|
514
|
-
return this.entries.some((e) => e.videos && e.videos.length > 0);
|
|
515
|
-
}
|
|
516
|
-
hasNews() {
|
|
517
|
-
return this.entries.some((e) => !!e.news);
|
|
518
|
-
}
|
|
519
|
-
hasAlternates() {
|
|
520
|
-
return this.entries.some((e) => e.alternates && e.alternates.length > 0);
|
|
599
|
+
parts.push(`${indent}</url>${nl}`);
|
|
600
|
+
return parts.join("");
|
|
521
601
|
}
|
|
522
602
|
escape(str) {
|
|
523
603
|
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
524
604
|
}
|
|
605
|
+
getEntries() {
|
|
606
|
+
return this.entries;
|
|
607
|
+
}
|
|
525
608
|
};
|
|
526
609
|
|
|
527
610
|
// src/core/SitemapGenerator.ts
|
|
@@ -553,12 +636,21 @@ var SitemapGenerator = class {
|
|
|
553
636
|
baseUrl: this.options.baseUrl,
|
|
554
637
|
pretty: this.options.pretty
|
|
555
638
|
});
|
|
639
|
+
const shards = [];
|
|
556
640
|
let isMultiFile = false;
|
|
557
641
|
const flushShard = async () => {
|
|
558
642
|
isMultiFile = true;
|
|
559
643
|
const baseName = this.options.filename?.replace(/\.xml$/, "");
|
|
560
644
|
const filename = `${baseName}-${shardIndex}.xml`;
|
|
561
645
|
const xml = currentStream.toXML();
|
|
646
|
+
const entries = currentStream.getEntries();
|
|
647
|
+
shards.push({
|
|
648
|
+
filename,
|
|
649
|
+
from: this.normalizeUrl(entries[0].url),
|
|
650
|
+
to: this.normalizeUrl(entries[entries.length - 1].url),
|
|
651
|
+
count: entries.length,
|
|
652
|
+
lastmod: /* @__PURE__ */ new Date()
|
|
653
|
+
});
|
|
562
654
|
if (this.shadowProcessor) {
|
|
563
655
|
await this.shadowProcessor.addOperation({ filename, content: xml });
|
|
564
656
|
} else {
|
|
@@ -596,16 +688,44 @@ var SitemapGenerator = class {
|
|
|
596
688
|
}
|
|
597
689
|
}
|
|
598
690
|
}
|
|
691
|
+
const writeManifest = async () => {
|
|
692
|
+
if (!this.options.generateManifest) return;
|
|
693
|
+
const manifest = {
|
|
694
|
+
version: 1,
|
|
695
|
+
generatedAt: /* @__PURE__ */ new Date(),
|
|
696
|
+
baseUrl: this.options.baseUrl,
|
|
697
|
+
maxEntriesPerShard: this.options.maxEntriesPerFile,
|
|
698
|
+
sort: "url-lex",
|
|
699
|
+
shards
|
|
700
|
+
};
|
|
701
|
+
const manifestFilename = this.options.filename?.replace(/\.xml$/, "-manifest.json") || "sitemap-manifest.json";
|
|
702
|
+
const content = JSON.stringify(manifest, null, this.options.pretty ? 2 : 0);
|
|
703
|
+
if (this.shadowProcessor) {
|
|
704
|
+
await this.shadowProcessor.addOperation({ filename: manifestFilename, content });
|
|
705
|
+
} else {
|
|
706
|
+
await this.options.storage.write(manifestFilename, content);
|
|
707
|
+
}
|
|
708
|
+
};
|
|
599
709
|
if (!isMultiFile) {
|
|
600
710
|
const xml = currentStream.toXML();
|
|
711
|
+
const entries = currentStream.getEntries();
|
|
712
|
+
shards.push({
|
|
713
|
+
filename: this.options.filename,
|
|
714
|
+
from: entries[0] ? this.normalizeUrl(entries[0].url) : "",
|
|
715
|
+
to: entries[entries.length - 1] ? this.normalizeUrl(entries[entries.length - 1].url) : "",
|
|
716
|
+
count: entries.length,
|
|
717
|
+
lastmod: /* @__PURE__ */ new Date()
|
|
718
|
+
});
|
|
601
719
|
if (this.shadowProcessor) {
|
|
602
720
|
await this.shadowProcessor.addOperation({
|
|
603
721
|
filename: this.options.filename,
|
|
604
722
|
content: xml
|
|
605
723
|
});
|
|
724
|
+
await writeManifest();
|
|
606
725
|
await this.shadowProcessor.commit();
|
|
607
726
|
} else {
|
|
608
727
|
await this.options.storage.write(this.options.filename, xml);
|
|
728
|
+
await writeManifest();
|
|
609
729
|
}
|
|
610
730
|
return;
|
|
611
731
|
}
|
|
@@ -618,11 +738,22 @@ var SitemapGenerator = class {
|
|
|
618
738
|
filename: this.options.filename,
|
|
619
739
|
content: indexXml
|
|
620
740
|
});
|
|
741
|
+
await writeManifest();
|
|
621
742
|
await this.shadowProcessor.commit();
|
|
622
743
|
} else {
|
|
623
744
|
await this.options.storage.write(this.options.filename, indexXml);
|
|
745
|
+
await writeManifest();
|
|
624
746
|
}
|
|
625
747
|
}
|
|
748
|
+
normalizeUrl(url) {
|
|
749
|
+
if (url.startsWith("http")) {
|
|
750
|
+
return url;
|
|
751
|
+
}
|
|
752
|
+
const { baseUrl } = this.options;
|
|
753
|
+
const normalizedBase = baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
|
|
754
|
+
const normalizedPath = url.startsWith("/") ? url : `/${url}`;
|
|
755
|
+
return normalizedBase + normalizedPath;
|
|
756
|
+
}
|
|
626
757
|
/**
|
|
627
758
|
* 獲取影子處理器(如果啟用)
|
|
628
759
|
*/
|
|
@@ -631,22 +762,104 @@ var SitemapGenerator = class {
|
|
|
631
762
|
}
|
|
632
763
|
};
|
|
633
764
|
|
|
765
|
+
// src/core/SitemapParser.ts
|
|
766
|
+
var SitemapParser = class {
|
|
767
|
+
static parse(xml) {
|
|
768
|
+
const entries = [];
|
|
769
|
+
const urlRegex = /<url>([\s\S]*?)<\/url>/g;
|
|
770
|
+
let match;
|
|
771
|
+
while ((match = urlRegex.exec(xml)) !== null) {
|
|
772
|
+
const entry = this.parseEntry(match[1]);
|
|
773
|
+
if (entry) {
|
|
774
|
+
entries.push(entry);
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
return entries;
|
|
778
|
+
}
|
|
779
|
+
static async *parseStream(stream) {
|
|
780
|
+
let buffer = "";
|
|
781
|
+
const urlRegex = /<url>([\s\S]*?)<\/url>/g;
|
|
782
|
+
for await (const chunk of stream) {
|
|
783
|
+
buffer += chunk;
|
|
784
|
+
let match;
|
|
785
|
+
while ((match = urlRegex.exec(buffer)) !== null) {
|
|
786
|
+
const entry = this.parseEntry(match[1]);
|
|
787
|
+
if (entry) {
|
|
788
|
+
yield entry;
|
|
789
|
+
}
|
|
790
|
+
buffer = buffer.slice(match.index + match[0].length);
|
|
791
|
+
urlRegex.lastIndex = 0;
|
|
792
|
+
}
|
|
793
|
+
if (buffer.length > 1024 * 1024) {
|
|
794
|
+
const lastUrlStart = buffer.lastIndexOf("<url>");
|
|
795
|
+
buffer = lastUrlStart !== -1 ? buffer.slice(lastUrlStart) : "";
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
static parseEntry(urlContent) {
|
|
800
|
+
const entry = { url: "" };
|
|
801
|
+
const locMatch = /<loc>(.*?)<\/loc>/.exec(urlContent);
|
|
802
|
+
if (locMatch) {
|
|
803
|
+
entry.url = this.unescape(locMatch[1]);
|
|
804
|
+
} else {
|
|
805
|
+
return null;
|
|
806
|
+
}
|
|
807
|
+
const lastmodMatch = /<lastmod>(.*?)<\/lastmod>/.exec(urlContent);
|
|
808
|
+
if (lastmodMatch) {
|
|
809
|
+
entry.lastmod = new Date(lastmodMatch[1]);
|
|
810
|
+
}
|
|
811
|
+
const priorityMatch = /<priority>(.*?)<\/priority>/.exec(urlContent);
|
|
812
|
+
if (priorityMatch) {
|
|
813
|
+
entry.priority = parseFloat(priorityMatch[1]);
|
|
814
|
+
}
|
|
815
|
+
const changefreqMatch = /<changefreq>(.*?)<\/changefreq>/.exec(urlContent);
|
|
816
|
+
if (changefreqMatch) {
|
|
817
|
+
entry.changefreq = changefreqMatch[1];
|
|
818
|
+
}
|
|
819
|
+
return entry;
|
|
820
|
+
}
|
|
821
|
+
static parseIndex(xml) {
|
|
822
|
+
const urls = [];
|
|
823
|
+
const sitemapRegex = /<sitemap>([\s\S]*?)<\/sitemap>/g;
|
|
824
|
+
let match;
|
|
825
|
+
while ((match = sitemapRegex.exec(xml)) !== null) {
|
|
826
|
+
const content = match[1];
|
|
827
|
+
const locMatch = /<loc>(.*?)<\/loc>/.exec(content);
|
|
828
|
+
if (locMatch) {
|
|
829
|
+
urls.push(this.unescape(locMatch[1]));
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
return urls;
|
|
833
|
+
}
|
|
834
|
+
static unescape(str) {
|
|
835
|
+
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'");
|
|
836
|
+
}
|
|
837
|
+
};
|
|
838
|
+
|
|
634
839
|
// src/core/IncrementalGenerator.ts
|
|
635
840
|
var IncrementalGenerator = class {
|
|
636
841
|
options;
|
|
637
842
|
changeTracker;
|
|
638
843
|
diffCalculator;
|
|
639
844
|
generator;
|
|
845
|
+
mutex = new Mutex();
|
|
640
846
|
constructor(options) {
|
|
641
|
-
this.options =
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
847
|
+
this.options = {
|
|
848
|
+
autoTrack: true,
|
|
849
|
+
generateManifest: true,
|
|
850
|
+
...options
|
|
851
|
+
};
|
|
852
|
+
this.changeTracker = this.options.changeTracker;
|
|
853
|
+
this.diffCalculator = this.options.diffCalculator || new DiffCalculator();
|
|
854
|
+
this.generator = new SitemapGenerator(this.options);
|
|
645
855
|
}
|
|
646
|
-
/**
|
|
647
|
-
* 生成完整的 sitemap(首次生成)
|
|
648
|
-
*/
|
|
649
856
|
async generateFull() {
|
|
857
|
+
return this.mutex.runExclusive(() => this.performFullGeneration());
|
|
858
|
+
}
|
|
859
|
+
async generateIncremental(since) {
|
|
860
|
+
return this.mutex.runExclusive(() => this.performIncrementalGeneration(since));
|
|
861
|
+
}
|
|
862
|
+
async performFullGeneration() {
|
|
650
863
|
await this.generator.run();
|
|
651
864
|
if (this.options.autoTrack) {
|
|
652
865
|
const { providers } = this.options;
|
|
@@ -664,52 +877,130 @@ var IncrementalGenerator = class {
|
|
|
664
877
|
}
|
|
665
878
|
}
|
|
666
879
|
}
|
|
667
|
-
|
|
668
|
-
* 增量生成(只更新變更的部分)
|
|
669
|
-
*/
|
|
670
|
-
async generateIncremental(since) {
|
|
880
|
+
async performIncrementalGeneration(since) {
|
|
671
881
|
const changes = await this.changeTracker.getChanges(since);
|
|
672
882
|
if (changes.length === 0) {
|
|
673
883
|
return;
|
|
674
884
|
}
|
|
675
|
-
const
|
|
676
|
-
|
|
677
|
-
|
|
885
|
+
const manifest = await this.loadManifest();
|
|
886
|
+
if (!manifest) {
|
|
887
|
+
await this.performFullGeneration();
|
|
888
|
+
return;
|
|
889
|
+
}
|
|
890
|
+
const totalCount = manifest.shards.reduce((acc, s) => acc + s.count, 0);
|
|
891
|
+
const changeRatio = totalCount > 0 ? changes.length / totalCount : 1;
|
|
892
|
+
if (changeRatio > 0.3) {
|
|
893
|
+
await this.performFullGeneration();
|
|
894
|
+
return;
|
|
895
|
+
}
|
|
896
|
+
const affectedShards = this.getAffectedShards(manifest, changes);
|
|
897
|
+
if (affectedShards.size / manifest.shards.length > 0.5) {
|
|
898
|
+
await this.performFullGeneration();
|
|
899
|
+
return;
|
|
900
|
+
}
|
|
901
|
+
await this.updateShards(manifest, affectedShards);
|
|
678
902
|
}
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
903
|
+
normalizeUrl(url) {
|
|
904
|
+
if (url.startsWith("http")) {
|
|
905
|
+
return url;
|
|
906
|
+
}
|
|
907
|
+
const { baseUrl } = this.options;
|
|
908
|
+
const normalizedBase = baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
|
|
909
|
+
const normalizedPath = url.startsWith("/") ? url : `/${url}`;
|
|
910
|
+
return normalizedBase + normalizedPath;
|
|
684
911
|
}
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
912
|
+
async loadManifest() {
|
|
913
|
+
const filename = this.options.filename?.replace(/\.xml$/, "-manifest.json") || "sitemap-manifest.json";
|
|
914
|
+
const content = await this.options.storage.read(filename);
|
|
915
|
+
if (!content) {
|
|
916
|
+
return null;
|
|
917
|
+
}
|
|
918
|
+
try {
|
|
919
|
+
return JSON.parse(content);
|
|
920
|
+
} catch {
|
|
921
|
+
return null;
|
|
922
|
+
}
|
|
690
923
|
}
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
924
|
+
getAffectedShards(manifest, changes) {
|
|
925
|
+
const affected = /* @__PURE__ */ new Map();
|
|
926
|
+
for (const change of changes) {
|
|
927
|
+
const normalizedUrl = this.normalizeUrl(change.url);
|
|
928
|
+
let shard = manifest.shards.find((s) => {
|
|
929
|
+
return normalizedUrl >= s.from && normalizedUrl <= s.to;
|
|
930
|
+
});
|
|
931
|
+
if (!shard) {
|
|
932
|
+
shard = manifest.shards.find((s) => normalizedUrl <= s.to);
|
|
933
|
+
if (!shard) {
|
|
934
|
+
shard = manifest.shards[manifest.shards.length - 1];
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
if (shard) {
|
|
938
|
+
const shardChanges = affected.get(shard.filename) || [];
|
|
939
|
+
shardChanges.push(change);
|
|
940
|
+
affected.set(shard.filename, shardChanges);
|
|
941
|
+
}
|
|
701
942
|
}
|
|
702
|
-
return
|
|
943
|
+
return affected;
|
|
703
944
|
}
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
945
|
+
async updateShards(manifest, affectedShards) {
|
|
946
|
+
for (const [filename, shardChanges] of affectedShards) {
|
|
947
|
+
const entries = [];
|
|
948
|
+
const stream = await this.options.storage.readStream?.(filename);
|
|
949
|
+
if (stream) {
|
|
950
|
+
for await (const entry of SitemapParser.parseStream(stream)) {
|
|
951
|
+
entries.push(entry);
|
|
952
|
+
}
|
|
953
|
+
} else {
|
|
954
|
+
const xml = await this.options.storage.read(filename);
|
|
955
|
+
if (!xml) {
|
|
956
|
+
continue;
|
|
957
|
+
}
|
|
958
|
+
entries.push(...SitemapParser.parse(xml));
|
|
959
|
+
}
|
|
960
|
+
const updatedEntries = this.applyChanges(entries, shardChanges);
|
|
961
|
+
const outStream = new SitemapStream({
|
|
962
|
+
baseUrl: this.options.baseUrl,
|
|
963
|
+
pretty: this.options.pretty
|
|
964
|
+
});
|
|
965
|
+
outStream.addAll(updatedEntries);
|
|
966
|
+
const newXml = outStream.toXML();
|
|
967
|
+
await this.options.storage.write(filename, newXml);
|
|
968
|
+
const shardInfo = manifest.shards.find((s) => s.filename === filename);
|
|
969
|
+
if (shardInfo) {
|
|
970
|
+
shardInfo.count = updatedEntries.length;
|
|
971
|
+
shardInfo.lastmod = /* @__PURE__ */ new Date();
|
|
972
|
+
shardInfo.from = this.normalizeUrl(updatedEntries[0].url);
|
|
973
|
+
shardInfo.to = this.normalizeUrl(updatedEntries[updatedEntries.length - 1].url);
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
const manifestFilename = this.options.filename?.replace(/\.xml$/, "-manifest.json") || "sitemap-manifest.json";
|
|
977
|
+
await this.options.storage.write(
|
|
978
|
+
manifestFilename,
|
|
979
|
+
JSON.stringify(manifest, null, this.options.pretty ? 2 : 0)
|
|
980
|
+
);
|
|
981
|
+
}
|
|
982
|
+
applyChanges(entries, changes) {
|
|
983
|
+
const entryMap = /* @__PURE__ */ new Map();
|
|
984
|
+
for (const entry of entries) {
|
|
985
|
+
entryMap.set(this.normalizeUrl(entry.url), entry);
|
|
986
|
+
}
|
|
987
|
+
for (const change of changes) {
|
|
988
|
+
const normalizedUrl = this.normalizeUrl(change.url);
|
|
989
|
+
if (change.type === "add" || change.type === "update") {
|
|
990
|
+
if (change.entry) {
|
|
991
|
+
entryMap.set(normalizedUrl, {
|
|
992
|
+
...change.entry,
|
|
993
|
+
url: normalizedUrl
|
|
994
|
+
});
|
|
995
|
+
}
|
|
996
|
+
} else if (change.type === "remove") {
|
|
997
|
+
entryMap.delete(normalizedUrl);
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
return Array.from(entryMap.values()).sort(
|
|
1001
|
+
(a, b) => this.normalizeUrl(a.url).localeCompare(this.normalizeUrl(b.url))
|
|
1002
|
+
);
|
|
709
1003
|
}
|
|
710
|
-
/**
|
|
711
|
-
* 將 AsyncIterable 轉換為陣列
|
|
712
|
-
*/
|
|
713
1004
|
async toArray(iterable) {
|
|
714
1005
|
const array = [];
|
|
715
1006
|
for await (const item of iterable) {
|
|
@@ -855,7 +1146,7 @@ var GenerateSitemapJob = class extends Job {
|
|
|
855
1146
|
this.generator = new SitemapGenerator(options.generatorOptions);
|
|
856
1147
|
}
|
|
857
1148
|
async handle() {
|
|
858
|
-
const { progressTracker,
|
|
1149
|
+
const { progressTracker, onComplete, onError } = this.options;
|
|
859
1150
|
try {
|
|
860
1151
|
if (progressTracker) {
|
|
861
1152
|
const total = await this.calculateTotal();
|
|
@@ -863,8 +1154,8 @@ var GenerateSitemapJob = class extends Job {
|
|
|
863
1154
|
this.totalEntries = total;
|
|
864
1155
|
}
|
|
865
1156
|
await this.generateWithProgress();
|
|
866
|
-
if (shadowProcessor) {
|
|
867
|
-
await shadowProcessor.commit();
|
|
1157
|
+
if (this.options.shadowProcessor) {
|
|
1158
|
+
await this.options.shadowProcessor.commit();
|
|
868
1159
|
}
|
|
869
1160
|
if (progressTracker) {
|
|
870
1161
|
await progressTracker.complete();
|
|
@@ -905,15 +1196,7 @@ var GenerateSitemapJob = class extends Job {
|
|
|
905
1196
|
* 帶進度追蹤的生成
|
|
906
1197
|
*/
|
|
907
1198
|
async generateWithProgress() {
|
|
908
|
-
const { progressTracker,
|
|
909
|
-
const {
|
|
910
|
-
providers,
|
|
911
|
-
maxEntriesPerFile = 5e4,
|
|
912
|
-
storage,
|
|
913
|
-
baseUrl,
|
|
914
|
-
pretty,
|
|
915
|
-
filename
|
|
916
|
-
} = this.options.generatorOptions;
|
|
1199
|
+
const { progressTracker, onProgress } = this.options;
|
|
917
1200
|
await this.generator.run();
|
|
918
1201
|
this.processedEntries = this.totalEntries;
|
|
919
1202
|
if (progressTracker) {
|
|
@@ -1086,6 +1369,15 @@ var MemorySitemapStorage = class {
|
|
|
1086
1369
|
async read(filename) {
|
|
1087
1370
|
return this.files.get(filename) || null;
|
|
1088
1371
|
}
|
|
1372
|
+
async readStream(filename) {
|
|
1373
|
+
const content = this.files.get(filename);
|
|
1374
|
+
if (content === void 0) {
|
|
1375
|
+
return null;
|
|
1376
|
+
}
|
|
1377
|
+
return (async function* () {
|
|
1378
|
+
yield content;
|
|
1379
|
+
})();
|
|
1380
|
+
}
|
|
1089
1381
|
async exists(filename) {
|
|
1090
1382
|
return this.files.has(filename);
|
|
1091
1383
|
}
|
|
@@ -1152,7 +1444,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1152
1444
|
if (this.mode === "dynamic") {
|
|
1153
1445
|
this.installDynamic(core);
|
|
1154
1446
|
} else {
|
|
1155
|
-
|
|
1447
|
+
core.logger.info("[OrbitSitemap] Static mode configured. Use generate() to build sitemaps.");
|
|
1156
1448
|
}
|
|
1157
1449
|
}
|
|
1158
1450
|
installDynamic(core) {
|
|
@@ -1224,7 +1516,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1224
1516
|
const opts = this.options;
|
|
1225
1517
|
let storage = opts.storage;
|
|
1226
1518
|
if (!storage) {
|
|
1227
|
-
const { DiskSitemapStorage: DiskSitemapStorage2 } = await import("./DiskSitemapStorage-
|
|
1519
|
+
const { DiskSitemapStorage: DiskSitemapStorage2 } = await import("./DiskSitemapStorage-WP6RITUN.js");
|
|
1228
1520
|
storage = new DiskSitemapStorage2(opts.outDir, opts.baseUrl);
|
|
1229
1521
|
}
|
|
1230
1522
|
let providers = opts.providers;
|
|
@@ -1270,7 +1562,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1270
1562
|
}
|
|
1271
1563
|
let storage = opts.storage;
|
|
1272
1564
|
if (!storage) {
|
|
1273
|
-
const { DiskSitemapStorage: DiskSitemapStorage2 } = await import("./DiskSitemapStorage-
|
|
1565
|
+
const { DiskSitemapStorage: DiskSitemapStorage2 } = await import("./DiskSitemapStorage-WP6RITUN.js");
|
|
1274
1566
|
storage = new DiskSitemapStorage2(opts.outDir, opts.baseUrl);
|
|
1275
1567
|
}
|
|
1276
1568
|
const incrementalGenerator = new IncrementalGenerator({
|
|
@@ -1299,7 +1591,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1299
1591
|
const jobId = randomUUID();
|
|
1300
1592
|
let storage = opts.storage;
|
|
1301
1593
|
if (!storage) {
|
|
1302
|
-
const { DiskSitemapStorage: DiskSitemapStorage2 } = await import("./DiskSitemapStorage-
|
|
1594
|
+
const { DiskSitemapStorage: DiskSitemapStorage2 } = await import("./DiskSitemapStorage-WP6RITUN.js");
|
|
1303
1595
|
storage = new DiskSitemapStorage2(opts.outDir, opts.baseUrl);
|
|
1304
1596
|
}
|
|
1305
1597
|
let providers = opts.providers;
|
|
@@ -1418,6 +1710,11 @@ var RouteScanner = class {
|
|
|
1418
1710
|
...options
|
|
1419
1711
|
};
|
|
1420
1712
|
}
|
|
1713
|
+
/**
|
|
1714
|
+
* Scan the router and return discovered entries.
|
|
1715
|
+
*
|
|
1716
|
+
* @returns An array of sitemap entries.
|
|
1717
|
+
*/
|
|
1421
1718
|
getEntries() {
|
|
1422
1719
|
const entries = [];
|
|
1423
1720
|
const routes = this.extractRoutes(this.router);
|
|
@@ -1821,6 +2118,30 @@ var GCPSitemapStorage = class {
|
|
|
1821
2118
|
throw error;
|
|
1822
2119
|
}
|
|
1823
2120
|
}
|
|
2121
|
+
async readStream(filename) {
|
|
2122
|
+
try {
|
|
2123
|
+
const { bucket } = await this.getStorageClient();
|
|
2124
|
+
const key = this.getKey(filename);
|
|
2125
|
+
const file = bucket.file(key);
|
|
2126
|
+
const [exists] = await file.exists();
|
|
2127
|
+
if (!exists) {
|
|
2128
|
+
return null;
|
|
2129
|
+
}
|
|
2130
|
+
const stream = file.createReadStream();
|
|
2131
|
+
return (async function* () {
|
|
2132
|
+
const decoder = new TextDecoder();
|
|
2133
|
+
for await (const chunk of stream) {
|
|
2134
|
+
yield decoder.decode(chunk, { stream: true });
|
|
2135
|
+
}
|
|
2136
|
+
yield decoder.decode();
|
|
2137
|
+
})();
|
|
2138
|
+
} catch (error) {
|
|
2139
|
+
if (error.code === 404) {
|
|
2140
|
+
return null;
|
|
2141
|
+
}
|
|
2142
|
+
throw error;
|
|
2143
|
+
}
|
|
2144
|
+
}
|
|
1824
2145
|
async exists(filename) {
|
|
1825
2146
|
try {
|
|
1826
2147
|
const { bucket } = await this.getStorageClient();
|
|
@@ -1843,7 +2164,7 @@ var GCPSitemapStorage = class {
|
|
|
1843
2164
|
return this.write(filename, content);
|
|
1844
2165
|
}
|
|
1845
2166
|
const { bucket } = await this.getStorageClient();
|
|
1846
|
-
const id = shadowId || `shadow-${Date.now()}-${
|
|
2167
|
+
const id = shadowId || `shadow-${Date.now()}-${crypto.randomUUID()}`;
|
|
1847
2168
|
const shadowKey = this.getKey(`${filename}.shadow.${id}`);
|
|
1848
2169
|
const file = bucket.file(shadowKey);
|
|
1849
2170
|
await file.save(content, {
|
|
@@ -2115,6 +2436,34 @@ var S3SitemapStorage = class {
|
|
|
2115
2436
|
throw error;
|
|
2116
2437
|
}
|
|
2117
2438
|
}
|
|
2439
|
+
async readStream(filename) {
|
|
2440
|
+
try {
|
|
2441
|
+
const s3 = await this.getS3Client();
|
|
2442
|
+
const key = this.getKey(filename);
|
|
2443
|
+
const response = await s3.client.send(
|
|
2444
|
+
new s3.GetObjectCommand({
|
|
2445
|
+
Bucket: this.bucket,
|
|
2446
|
+
Key: key
|
|
2447
|
+
})
|
|
2448
|
+
);
|
|
2449
|
+
if (!response.Body) {
|
|
2450
|
+
return null;
|
|
2451
|
+
}
|
|
2452
|
+
const body = response.Body;
|
|
2453
|
+
return (async function* () {
|
|
2454
|
+
const decoder = new TextDecoder();
|
|
2455
|
+
for await (const chunk of body) {
|
|
2456
|
+
yield decoder.decode(chunk, { stream: true });
|
|
2457
|
+
}
|
|
2458
|
+
yield decoder.decode();
|
|
2459
|
+
})();
|
|
2460
|
+
} catch (error) {
|
|
2461
|
+
if (error.name === "NoSuchKey" || error.$metadata?.httpStatusCode === 404) {
|
|
2462
|
+
return null;
|
|
2463
|
+
}
|
|
2464
|
+
throw error;
|
|
2465
|
+
}
|
|
2466
|
+
}
|
|
2118
2467
|
async exists(filename) {
|
|
2119
2468
|
try {
|
|
2120
2469
|
const s3 = await this.getS3Client();
|
|
@@ -2144,7 +2493,7 @@ var S3SitemapStorage = class {
|
|
|
2144
2493
|
return this.write(filename, content);
|
|
2145
2494
|
}
|
|
2146
2495
|
const s3 = await this.getS3Client();
|
|
2147
|
-
const id = shadowId || `shadow-${Date.now()}-${
|
|
2496
|
+
const id = shadowId || `shadow-${Date.now()}-${crypto.randomUUID()}`;
|
|
2148
2497
|
const shadowKey = this.getKey(`${filename}.shadow.${id}`);
|
|
2149
2498
|
await s3.client.send(
|
|
2150
2499
|
new s3.PutObjectCommand({
|