@gravito/constellation 3.0.0 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{DiskSitemapStorage-7ZZMGC4K.js → DiskSitemapStorage-WP6RITUN.js} +1 -1
- package/dist/{chunk-7WHLC3OJ.js → chunk-IS2H7U6M.js} +12 -0
- package/dist/index.cjs +517 -156
- package/dist/index.d.cts +934 -61
- package/dist/index.d.ts +934 -61
- package/dist/index.js +508 -159
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -50,10 +50,11 @@ function sanitizeFilename(filename) {
|
|
|
50
50
|
}
|
|
51
51
|
return filename;
|
|
52
52
|
}
|
|
53
|
-
var import_promises, import_node_path, DiskSitemapStorage;
|
|
53
|
+
var import_node_fs, import_promises, import_node_path, DiskSitemapStorage;
|
|
54
54
|
var init_DiskSitemapStorage = __esm({
|
|
55
55
|
"src/storage/DiskSitemapStorage.ts"() {
|
|
56
56
|
"use strict";
|
|
57
|
+
import_node_fs = require("fs");
|
|
57
58
|
import_promises = __toESM(require("fs/promises"), 1);
|
|
58
59
|
import_node_path = __toESM(require("path"), 1);
|
|
59
60
|
DiskSitemapStorage = class {
|
|
@@ -74,6 +75,17 @@ var init_DiskSitemapStorage = __esm({
|
|
|
74
75
|
return null;
|
|
75
76
|
}
|
|
76
77
|
}
|
|
78
|
+
async readStream(filename) {
|
|
79
|
+
try {
|
|
80
|
+
const safeName = sanitizeFilename(filename);
|
|
81
|
+
const fullPath = import_node_path.default.join(this.outDir, safeName);
|
|
82
|
+
await import_promises.default.access(fullPath);
|
|
83
|
+
const stream = (0, import_node_fs.createReadStream)(fullPath, { encoding: "utf-8" });
|
|
84
|
+
return stream;
|
|
85
|
+
} catch {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
77
89
|
async exists(filename) {
|
|
78
90
|
try {
|
|
79
91
|
const safeName = sanitizeFilename(filename);
|
|
@@ -126,14 +138,30 @@ module.exports = __toCommonJS(index_exports);
|
|
|
126
138
|
// src/core/ChangeTracker.ts
|
|
127
139
|
var MemoryChangeTracker = class {
|
|
128
140
|
changes = [];
|
|
141
|
+
urlIndex = /* @__PURE__ */ new Map();
|
|
129
142
|
maxChanges;
|
|
130
143
|
constructor(options = {}) {
|
|
131
144
|
this.maxChanges = options.maxChanges || 1e5;
|
|
132
145
|
}
|
|
133
146
|
async track(change) {
|
|
134
147
|
this.changes.push(change);
|
|
148
|
+
const urlChanges = this.urlIndex.get(change.url) || [];
|
|
149
|
+
urlChanges.push(change);
|
|
150
|
+
this.urlIndex.set(change.url, urlChanges);
|
|
135
151
|
if (this.changes.length > this.maxChanges) {
|
|
136
|
-
|
|
152
|
+
const removed = this.changes.shift();
|
|
153
|
+
if (removed) {
|
|
154
|
+
const changes = this.urlIndex.get(removed.url);
|
|
155
|
+
if (changes) {
|
|
156
|
+
const index = changes.indexOf(removed);
|
|
157
|
+
if (index > -1) {
|
|
158
|
+
changes.splice(index, 1);
|
|
159
|
+
}
|
|
160
|
+
if (changes.length === 0) {
|
|
161
|
+
this.urlIndex.delete(removed.url);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
137
165
|
}
|
|
138
166
|
}
|
|
139
167
|
async getChanges(since) {
|
|
@@ -143,14 +171,21 @@ var MemoryChangeTracker = class {
|
|
|
143
171
|
return this.changes.filter((change) => change.timestamp >= since);
|
|
144
172
|
}
|
|
145
173
|
async getChangesByUrl(url) {
|
|
146
|
-
return this.
|
|
174
|
+
return this.urlIndex.get(url) || [];
|
|
147
175
|
}
|
|
148
176
|
async clear(since) {
|
|
149
177
|
if (!since) {
|
|
150
178
|
this.changes = [];
|
|
179
|
+
this.urlIndex.clear();
|
|
151
180
|
return;
|
|
152
181
|
}
|
|
153
182
|
this.changes = this.changes.filter((change) => change.timestamp < since);
|
|
183
|
+
this.urlIndex.clear();
|
|
184
|
+
for (const change of this.changes) {
|
|
185
|
+
const urlChanges = this.urlIndex.get(change.url) || [];
|
|
186
|
+
urlChanges.push(change);
|
|
187
|
+
this.urlIndex.set(change.url, urlChanges);
|
|
188
|
+
}
|
|
154
189
|
}
|
|
155
190
|
};
|
|
156
191
|
var RedisChangeTracker = class {
|
|
@@ -325,52 +360,72 @@ var DiffCalculator = class {
|
|
|
325
360
|
}
|
|
326
361
|
};
|
|
327
362
|
|
|
363
|
+
// src/utils/Mutex.ts
|
|
364
|
+
var Mutex = class {
|
|
365
|
+
queue = Promise.resolve();
|
|
366
|
+
async runExclusive(fn) {
|
|
367
|
+
const next = this.queue.then(() => fn());
|
|
368
|
+
this.queue = next.then(
|
|
369
|
+
() => {
|
|
370
|
+
},
|
|
371
|
+
() => {
|
|
372
|
+
}
|
|
373
|
+
);
|
|
374
|
+
return next;
|
|
375
|
+
}
|
|
376
|
+
};
|
|
377
|
+
|
|
328
378
|
// src/core/ShadowProcessor.ts
|
|
329
379
|
var ShadowProcessor = class {
|
|
330
380
|
options;
|
|
331
381
|
shadowId;
|
|
332
382
|
operations = [];
|
|
383
|
+
mutex = new Mutex();
|
|
333
384
|
constructor(options) {
|
|
334
385
|
this.options = options;
|
|
335
|
-
this.shadowId = `shadow-${Date.now()}-${
|
|
386
|
+
this.shadowId = `shadow-${Date.now()}-${crypto.randomUUID()}`;
|
|
336
387
|
}
|
|
337
388
|
/**
|
|
338
389
|
* 添加一個影子操作
|
|
339
390
|
*/
|
|
340
391
|
async addOperation(operation) {
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
392
|
+
return this.mutex.runExclusive(async () => {
|
|
393
|
+
if (!this.options.enabled) {
|
|
394
|
+
await this.options.storage.write(operation.filename, operation.content);
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
this.operations.push({
|
|
398
|
+
...operation,
|
|
399
|
+
shadowId: operation.shadowId || this.shadowId
|
|
400
|
+
});
|
|
401
|
+
if (this.options.storage.writeShadow) {
|
|
402
|
+
await this.options.storage.writeShadow(operation.filename, operation.content, this.shadowId);
|
|
403
|
+
} else {
|
|
404
|
+
await this.options.storage.write(operation.filename, operation.content);
|
|
405
|
+
}
|
|
348
406
|
});
|
|
349
|
-
if (this.options.storage.writeShadow) {
|
|
350
|
-
await this.options.storage.writeShadow(operation.filename, operation.content, this.shadowId);
|
|
351
|
-
} else {
|
|
352
|
-
await this.options.storage.write(operation.filename, operation.content);
|
|
353
|
-
}
|
|
354
407
|
}
|
|
355
408
|
/**
|
|
356
409
|
* 提交所有影子操作
|
|
357
410
|
*/
|
|
358
411
|
async commit() {
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
if (this.options.mode === "atomic") {
|
|
363
|
-
if (this.options.storage.commitShadow) {
|
|
364
|
-
await this.options.storage.commitShadow(this.shadowId);
|
|
412
|
+
return this.mutex.runExclusive(async () => {
|
|
413
|
+
if (!this.options.enabled) {
|
|
414
|
+
return;
|
|
365
415
|
}
|
|
366
|
-
|
|
367
|
-
for (const operation of this.operations) {
|
|
416
|
+
if (this.options.mode === "atomic") {
|
|
368
417
|
if (this.options.storage.commitShadow) {
|
|
369
|
-
await this.options.storage.commitShadow(
|
|
418
|
+
await this.options.storage.commitShadow(this.shadowId);
|
|
419
|
+
}
|
|
420
|
+
} else {
|
|
421
|
+
for (const operation of this.operations) {
|
|
422
|
+
if (this.options.storage.commitShadow) {
|
|
423
|
+
await this.options.storage.commitShadow(operation.shadowId || this.shadowId);
|
|
424
|
+
}
|
|
370
425
|
}
|
|
371
426
|
}
|
|
372
|
-
|
|
373
|
-
|
|
427
|
+
this.operations = [];
|
|
428
|
+
});
|
|
374
429
|
}
|
|
375
430
|
/**
|
|
376
431
|
* 取消所有影子操作
|
|
@@ -456,6 +511,12 @@ var SitemapIndex = class {
|
|
|
456
511
|
var SitemapStream = class {
|
|
457
512
|
options;
|
|
458
513
|
entries = [];
|
|
514
|
+
flags = {
|
|
515
|
+
hasImages: false,
|
|
516
|
+
hasVideos: false,
|
|
517
|
+
hasNews: false,
|
|
518
|
+
hasAlternates: false
|
|
519
|
+
};
|
|
459
520
|
constructor(options) {
|
|
460
521
|
this.options = { ...options };
|
|
461
522
|
if (this.options.baseUrl.endsWith("/")) {
|
|
@@ -463,10 +524,19 @@ var SitemapStream = class {
|
|
|
463
524
|
}
|
|
464
525
|
}
|
|
465
526
|
add(entry) {
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
this.
|
|
527
|
+
const e = typeof entry === "string" ? { url: entry } : entry;
|
|
528
|
+
this.entries.push(e);
|
|
529
|
+
if (e.images && e.images.length > 0) {
|
|
530
|
+
this.flags.hasImages = true;
|
|
531
|
+
}
|
|
532
|
+
if (e.videos && e.videos.length > 0) {
|
|
533
|
+
this.flags.hasVideos = true;
|
|
534
|
+
}
|
|
535
|
+
if (e.news) {
|
|
536
|
+
this.flags.hasNews = true;
|
|
537
|
+
}
|
|
538
|
+
if (e.alternates && e.alternates.length > 0) {
|
|
539
|
+
this.flags.hasAlternates = true;
|
|
470
540
|
}
|
|
471
541
|
return this;
|
|
472
542
|
}
|
|
@@ -478,33 +548,33 @@ var SitemapStream = class {
|
|
|
478
548
|
}
|
|
479
549
|
toXML() {
|
|
480
550
|
const { baseUrl, pretty } = this.options;
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
if (this.hasImages
|
|
485
|
-
|
|
551
|
+
const parts = [];
|
|
552
|
+
parts.push('<?xml version="1.0" encoding="UTF-8"?>\n');
|
|
553
|
+
parts.push('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"');
|
|
554
|
+
if (this.flags.hasImages) {
|
|
555
|
+
parts.push(' xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"');
|
|
486
556
|
}
|
|
487
|
-
if (this.hasVideos
|
|
488
|
-
|
|
557
|
+
if (this.flags.hasVideos) {
|
|
558
|
+
parts.push(' xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"');
|
|
489
559
|
}
|
|
490
|
-
if (this.hasNews
|
|
491
|
-
|
|
560
|
+
if (this.flags.hasNews) {
|
|
561
|
+
parts.push(' xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"');
|
|
492
562
|
}
|
|
493
|
-
if (this.hasAlternates
|
|
494
|
-
|
|
563
|
+
if (this.flags.hasAlternates) {
|
|
564
|
+
parts.push(' xmlns:xhtml="http://www.w3.org/1999/xhtml"');
|
|
495
565
|
}
|
|
496
|
-
|
|
497
|
-
`;
|
|
566
|
+
parts.push(">\n");
|
|
498
567
|
for (const entry of this.entries) {
|
|
499
|
-
|
|
568
|
+
parts.push(this.renderUrl(entry, baseUrl, pretty));
|
|
500
569
|
}
|
|
501
|
-
|
|
502
|
-
return
|
|
570
|
+
parts.push("</urlset>");
|
|
571
|
+
return parts.join("");
|
|
503
572
|
}
|
|
504
573
|
renderUrl(entry, baseUrl, pretty) {
|
|
505
574
|
const indent = pretty ? " " : "";
|
|
506
575
|
const subIndent = pretty ? " " : "";
|
|
507
576
|
const nl = pretty ? "\n" : "";
|
|
577
|
+
const parts = [];
|
|
508
578
|
let loc = entry.url;
|
|
509
579
|
if (!loc.startsWith("http")) {
|
|
510
580
|
if (!loc.startsWith("/")) {
|
|
@@ -512,17 +582,17 @@ var SitemapStream = class {
|
|
|
512
582
|
}
|
|
513
583
|
loc = baseUrl + loc;
|
|
514
584
|
}
|
|
515
|
-
|
|
516
|
-
|
|
585
|
+
parts.push(`${indent}<url>${nl}`);
|
|
586
|
+
parts.push(`${subIndent}<loc>${this.escape(loc)}</loc>${nl}`);
|
|
517
587
|
if (entry.lastmod) {
|
|
518
588
|
const date = entry.lastmod instanceof Date ? entry.lastmod : new Date(entry.lastmod);
|
|
519
|
-
|
|
589
|
+
parts.push(`${subIndent}<lastmod>${date.toISOString().split("T")[0]}</lastmod>${nl}`);
|
|
520
590
|
}
|
|
521
591
|
if (entry.changefreq) {
|
|
522
|
-
|
|
592
|
+
parts.push(`${subIndent}<changefreq>${entry.changefreq}</changefreq>${nl}`);
|
|
523
593
|
}
|
|
524
594
|
if (entry.priority !== void 0) {
|
|
525
|
-
|
|
595
|
+
parts.push(`${subIndent}<priority>${entry.priority.toFixed(1)}</priority>${nl}`);
|
|
526
596
|
}
|
|
527
597
|
if (entry.alternates) {
|
|
528
598
|
for (const alt of entry.alternates) {
|
|
@@ -533,7 +603,9 @@ var SitemapStream = class {
|
|
|
533
603
|
}
|
|
534
604
|
altLoc = baseUrl + altLoc;
|
|
535
605
|
}
|
|
536
|
-
|
|
606
|
+
parts.push(
|
|
607
|
+
`${subIndent}<xhtml:link rel="alternate" hreflang="${alt.lang}" href="${this.escape(altLoc)}"/>${nl}`
|
|
608
|
+
);
|
|
537
609
|
}
|
|
538
610
|
}
|
|
539
611
|
if (entry.redirect?.canonical) {
|
|
@@ -544,10 +616,14 @@ var SitemapStream = class {
|
|
|
544
616
|
}
|
|
545
617
|
canonicalUrl = baseUrl + canonicalUrl;
|
|
546
618
|
}
|
|
547
|
-
|
|
619
|
+
parts.push(
|
|
620
|
+
`${subIndent}<xhtml:link rel="canonical" href="${this.escape(canonicalUrl)}"/>${nl}`
|
|
621
|
+
);
|
|
548
622
|
}
|
|
549
623
|
if (entry.redirect && !entry.redirect.canonical) {
|
|
550
|
-
|
|
624
|
+
parts.push(
|
|
625
|
+
`${subIndent}<!-- Redirect: ${entry.redirect.from} \u2192 ${entry.redirect.to} (${entry.redirect.type}) -->${nl}`
|
|
626
|
+
);
|
|
551
627
|
}
|
|
552
628
|
if (entry.images) {
|
|
553
629
|
for (const img of entry.images) {
|
|
@@ -558,91 +634,110 @@ var SitemapStream = class {
|
|
|
558
634
|
}
|
|
559
635
|
loc2 = baseUrl + loc2;
|
|
560
636
|
}
|
|
561
|
-
|
|
562
|
-
|
|
637
|
+
parts.push(`${subIndent}<image:image>${nl}`);
|
|
638
|
+
parts.push(`${subIndent} <image:loc>${this.escape(loc2)}</image:loc>${nl}`);
|
|
563
639
|
if (img.title) {
|
|
564
|
-
|
|
640
|
+
parts.push(`${subIndent} <image:title>${this.escape(img.title)}</image:title>${nl}`);
|
|
565
641
|
}
|
|
566
642
|
if (img.caption) {
|
|
567
|
-
|
|
643
|
+
parts.push(
|
|
644
|
+
`${subIndent} <image:caption>${this.escape(img.caption)}</image:caption>${nl}`
|
|
645
|
+
);
|
|
568
646
|
}
|
|
569
647
|
if (img.geo_location) {
|
|
570
|
-
|
|
648
|
+
parts.push(
|
|
649
|
+
`${subIndent} <image:geo_location>${this.escape(img.geo_location)}</image:geo_location>${nl}`
|
|
650
|
+
);
|
|
571
651
|
}
|
|
572
652
|
if (img.license) {
|
|
573
|
-
|
|
653
|
+
parts.push(
|
|
654
|
+
`${subIndent} <image:license>${this.escape(img.license)}</image:license>${nl}`
|
|
655
|
+
);
|
|
574
656
|
}
|
|
575
|
-
|
|
657
|
+
parts.push(`${subIndent}</image:image>${nl}`);
|
|
576
658
|
}
|
|
577
659
|
}
|
|
578
660
|
if (entry.videos) {
|
|
579
661
|
for (const video of entry.videos) {
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
662
|
+
parts.push(`${subIndent}<video:video>${nl}`);
|
|
663
|
+
parts.push(
|
|
664
|
+
`${subIndent} <video:thumbnail_loc>${this.escape(video.thumbnail_loc)}</video:thumbnail_loc>${nl}`
|
|
665
|
+
);
|
|
666
|
+
parts.push(`${subIndent} <video:title>${this.escape(video.title)}</video:title>${nl}`);
|
|
667
|
+
parts.push(
|
|
668
|
+
`${subIndent} <video:description>${this.escape(video.description)}</video:description>${nl}`
|
|
669
|
+
);
|
|
584
670
|
if (video.content_loc) {
|
|
585
|
-
|
|
671
|
+
parts.push(
|
|
672
|
+
`${subIndent} <video:content_loc>${this.escape(video.content_loc)}</video:content_loc>${nl}`
|
|
673
|
+
);
|
|
586
674
|
}
|
|
587
675
|
if (video.player_loc) {
|
|
588
|
-
|
|
676
|
+
parts.push(
|
|
677
|
+
`${subIndent} <video:player_loc>${this.escape(video.player_loc)}</video:player_loc>${nl}`
|
|
678
|
+
);
|
|
589
679
|
}
|
|
590
680
|
if (video.duration) {
|
|
591
|
-
|
|
681
|
+
parts.push(`${subIndent} <video:duration>${video.duration}</video:duration>${nl}`);
|
|
592
682
|
}
|
|
593
683
|
if (video.view_count) {
|
|
594
|
-
|
|
684
|
+
parts.push(`${subIndent} <video:view_count>${video.view_count}</video:view_count>${nl}`);
|
|
595
685
|
}
|
|
596
686
|
if (video.publication_date) {
|
|
597
687
|
const pubDate = video.publication_date instanceof Date ? video.publication_date : new Date(video.publication_date);
|
|
598
|
-
|
|
688
|
+
parts.push(
|
|
689
|
+
`${subIndent} <video:publication_date>${pubDate.toISOString()}</video:publication_date>${nl}`
|
|
690
|
+
);
|
|
599
691
|
}
|
|
600
692
|
if (video.family_friendly) {
|
|
601
|
-
|
|
693
|
+
parts.push(
|
|
694
|
+
`${subIndent} <video:family_friendly>${video.family_friendly}</video:family_friendly>${nl}`
|
|
695
|
+
);
|
|
602
696
|
}
|
|
603
697
|
if (video.tag) {
|
|
604
698
|
for (const tag of video.tag) {
|
|
605
|
-
|
|
699
|
+
parts.push(`${subIndent} <video:tag>${this.escape(tag)}</video:tag>${nl}`);
|
|
606
700
|
}
|
|
607
701
|
}
|
|
608
|
-
|
|
702
|
+
parts.push(`${subIndent}</video:video>${nl}`);
|
|
609
703
|
}
|
|
610
704
|
}
|
|
611
705
|
if (entry.news) {
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
706
|
+
parts.push(`${subIndent}<news:news>${nl}`);
|
|
707
|
+
parts.push(`${subIndent} <news:publication>${nl}`);
|
|
708
|
+
parts.push(
|
|
709
|
+
`${subIndent} <news:name>${this.escape(entry.news.publication.name)}</news:name>${nl}`
|
|
710
|
+
);
|
|
711
|
+
parts.push(
|
|
712
|
+
`${subIndent} <news:language>${this.escape(entry.news.publication.language)}</news:language>${nl}`
|
|
713
|
+
);
|
|
714
|
+
parts.push(`${subIndent} </news:publication>${nl}`);
|
|
617
715
|
const pubDate = entry.news.publication_date instanceof Date ? entry.news.publication_date : new Date(entry.news.publication_date);
|
|
618
|
-
|
|
619
|
-
|
|
716
|
+
parts.push(
|
|
717
|
+
`${subIndent} <news:publication_date>${pubDate.toISOString()}</news:publication_date>${nl}`
|
|
718
|
+
);
|
|
719
|
+
parts.push(`${subIndent} <news:title>${this.escape(entry.news.title)}</news:title>${nl}`);
|
|
620
720
|
if (entry.news.genres) {
|
|
621
|
-
|
|
721
|
+
parts.push(
|
|
722
|
+
`${subIndent} <news:genres>${this.escape(entry.news.genres)}</news:genres>${nl}`
|
|
723
|
+
);
|
|
622
724
|
}
|
|
623
725
|
if (entry.news.keywords) {
|
|
624
|
-
|
|
726
|
+
parts.push(
|
|
727
|
+
`${subIndent} <news:keywords>${entry.news.keywords.map((k) => this.escape(k)).join(", ")}</news:keywords>${nl}`
|
|
728
|
+
);
|
|
625
729
|
}
|
|
626
|
-
|
|
730
|
+
parts.push(`${subIndent}</news:news>${nl}`);
|
|
627
731
|
}
|
|
628
|
-
|
|
629
|
-
return
|
|
630
|
-
}
|
|
631
|
-
hasImages() {
|
|
632
|
-
return this.entries.some((e) => e.images && e.images.length > 0);
|
|
633
|
-
}
|
|
634
|
-
hasVideos() {
|
|
635
|
-
return this.entries.some((e) => e.videos && e.videos.length > 0);
|
|
636
|
-
}
|
|
637
|
-
hasNews() {
|
|
638
|
-
return this.entries.some((e) => !!e.news);
|
|
639
|
-
}
|
|
640
|
-
hasAlternates() {
|
|
641
|
-
return this.entries.some((e) => e.alternates && e.alternates.length > 0);
|
|
732
|
+
parts.push(`${indent}</url>${nl}`);
|
|
733
|
+
return parts.join("");
|
|
642
734
|
}
|
|
643
735
|
escape(str) {
|
|
644
736
|
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
645
737
|
}
|
|
738
|
+
getEntries() {
|
|
739
|
+
return this.entries;
|
|
740
|
+
}
|
|
646
741
|
};
|
|
647
742
|
|
|
648
743
|
// src/core/SitemapGenerator.ts
|
|
@@ -674,12 +769,21 @@ var SitemapGenerator = class {
|
|
|
674
769
|
baseUrl: this.options.baseUrl,
|
|
675
770
|
pretty: this.options.pretty
|
|
676
771
|
});
|
|
772
|
+
const shards = [];
|
|
677
773
|
let isMultiFile = false;
|
|
678
774
|
const flushShard = async () => {
|
|
679
775
|
isMultiFile = true;
|
|
680
776
|
const baseName = this.options.filename?.replace(/\.xml$/, "");
|
|
681
777
|
const filename = `${baseName}-${shardIndex}.xml`;
|
|
682
778
|
const xml = currentStream.toXML();
|
|
779
|
+
const entries = currentStream.getEntries();
|
|
780
|
+
shards.push({
|
|
781
|
+
filename,
|
|
782
|
+
from: this.normalizeUrl(entries[0].url),
|
|
783
|
+
to: this.normalizeUrl(entries[entries.length - 1].url),
|
|
784
|
+
count: entries.length,
|
|
785
|
+
lastmod: /* @__PURE__ */ new Date()
|
|
786
|
+
});
|
|
683
787
|
if (this.shadowProcessor) {
|
|
684
788
|
await this.shadowProcessor.addOperation({ filename, content: xml });
|
|
685
789
|
} else {
|
|
@@ -717,16 +821,44 @@ var SitemapGenerator = class {
|
|
|
717
821
|
}
|
|
718
822
|
}
|
|
719
823
|
}
|
|
824
|
+
const writeManifest = async () => {
|
|
825
|
+
if (!this.options.generateManifest) return;
|
|
826
|
+
const manifest = {
|
|
827
|
+
version: 1,
|
|
828
|
+
generatedAt: /* @__PURE__ */ new Date(),
|
|
829
|
+
baseUrl: this.options.baseUrl,
|
|
830
|
+
maxEntriesPerShard: this.options.maxEntriesPerFile,
|
|
831
|
+
sort: "url-lex",
|
|
832
|
+
shards
|
|
833
|
+
};
|
|
834
|
+
const manifestFilename = this.options.filename?.replace(/\.xml$/, "-manifest.json") || "sitemap-manifest.json";
|
|
835
|
+
const content = JSON.stringify(manifest, null, this.options.pretty ? 2 : 0);
|
|
836
|
+
if (this.shadowProcessor) {
|
|
837
|
+
await this.shadowProcessor.addOperation({ filename: manifestFilename, content });
|
|
838
|
+
} else {
|
|
839
|
+
await this.options.storage.write(manifestFilename, content);
|
|
840
|
+
}
|
|
841
|
+
};
|
|
720
842
|
if (!isMultiFile) {
|
|
721
843
|
const xml = currentStream.toXML();
|
|
844
|
+
const entries = currentStream.getEntries();
|
|
845
|
+
shards.push({
|
|
846
|
+
filename: this.options.filename,
|
|
847
|
+
from: entries[0] ? this.normalizeUrl(entries[0].url) : "",
|
|
848
|
+
to: entries[entries.length - 1] ? this.normalizeUrl(entries[entries.length - 1].url) : "",
|
|
849
|
+
count: entries.length,
|
|
850
|
+
lastmod: /* @__PURE__ */ new Date()
|
|
851
|
+
});
|
|
722
852
|
if (this.shadowProcessor) {
|
|
723
853
|
await this.shadowProcessor.addOperation({
|
|
724
854
|
filename: this.options.filename,
|
|
725
855
|
content: xml
|
|
726
856
|
});
|
|
857
|
+
await writeManifest();
|
|
727
858
|
await this.shadowProcessor.commit();
|
|
728
859
|
} else {
|
|
729
860
|
await this.options.storage.write(this.options.filename, xml);
|
|
861
|
+
await writeManifest();
|
|
730
862
|
}
|
|
731
863
|
return;
|
|
732
864
|
}
|
|
@@ -739,10 +871,21 @@ var SitemapGenerator = class {
|
|
|
739
871
|
filename: this.options.filename,
|
|
740
872
|
content: indexXml
|
|
741
873
|
});
|
|
874
|
+
await writeManifest();
|
|
742
875
|
await this.shadowProcessor.commit();
|
|
743
876
|
} else {
|
|
744
877
|
await this.options.storage.write(this.options.filename, indexXml);
|
|
878
|
+
await writeManifest();
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
normalizeUrl(url) {
|
|
882
|
+
if (url.startsWith("http")) {
|
|
883
|
+
return url;
|
|
745
884
|
}
|
|
885
|
+
const { baseUrl } = this.options;
|
|
886
|
+
const normalizedBase = baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
|
|
887
|
+
const normalizedPath = url.startsWith("/") ? url : `/${url}`;
|
|
888
|
+
return normalizedBase + normalizedPath;
|
|
746
889
|
}
|
|
747
890
|
/**
|
|
748
891
|
* 獲取影子處理器(如果啟用)
|
|
@@ -752,22 +895,104 @@ var SitemapGenerator = class {
|
|
|
752
895
|
}
|
|
753
896
|
};
|
|
754
897
|
|
|
898
|
+
// src/core/SitemapParser.ts
|
|
899
|
+
var SitemapParser = class {
|
|
900
|
+
static parse(xml) {
|
|
901
|
+
const entries = [];
|
|
902
|
+
const urlRegex = /<url>([\s\S]*?)<\/url>/g;
|
|
903
|
+
let match;
|
|
904
|
+
while ((match = urlRegex.exec(xml)) !== null) {
|
|
905
|
+
const entry = this.parseEntry(match[1]);
|
|
906
|
+
if (entry) {
|
|
907
|
+
entries.push(entry);
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
return entries;
|
|
911
|
+
}
|
|
912
|
+
static async *parseStream(stream) {
|
|
913
|
+
let buffer = "";
|
|
914
|
+
const urlRegex = /<url>([\s\S]*?)<\/url>/g;
|
|
915
|
+
for await (const chunk of stream) {
|
|
916
|
+
buffer += chunk;
|
|
917
|
+
let match;
|
|
918
|
+
while ((match = urlRegex.exec(buffer)) !== null) {
|
|
919
|
+
const entry = this.parseEntry(match[1]);
|
|
920
|
+
if (entry) {
|
|
921
|
+
yield entry;
|
|
922
|
+
}
|
|
923
|
+
buffer = buffer.slice(match.index + match[0].length);
|
|
924
|
+
urlRegex.lastIndex = 0;
|
|
925
|
+
}
|
|
926
|
+
if (buffer.length > 1024 * 1024) {
|
|
927
|
+
const lastUrlStart = buffer.lastIndexOf("<url>");
|
|
928
|
+
buffer = lastUrlStart !== -1 ? buffer.slice(lastUrlStart) : "";
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
static parseEntry(urlContent) {
|
|
933
|
+
const entry = { url: "" };
|
|
934
|
+
const locMatch = /<loc>(.*?)<\/loc>/.exec(urlContent);
|
|
935
|
+
if (locMatch) {
|
|
936
|
+
entry.url = this.unescape(locMatch[1]);
|
|
937
|
+
} else {
|
|
938
|
+
return null;
|
|
939
|
+
}
|
|
940
|
+
const lastmodMatch = /<lastmod>(.*?)<\/lastmod>/.exec(urlContent);
|
|
941
|
+
if (lastmodMatch) {
|
|
942
|
+
entry.lastmod = new Date(lastmodMatch[1]);
|
|
943
|
+
}
|
|
944
|
+
const priorityMatch = /<priority>(.*?)<\/priority>/.exec(urlContent);
|
|
945
|
+
if (priorityMatch) {
|
|
946
|
+
entry.priority = parseFloat(priorityMatch[1]);
|
|
947
|
+
}
|
|
948
|
+
const changefreqMatch = /<changefreq>(.*?)<\/changefreq>/.exec(urlContent);
|
|
949
|
+
if (changefreqMatch) {
|
|
950
|
+
entry.changefreq = changefreqMatch[1];
|
|
951
|
+
}
|
|
952
|
+
return entry;
|
|
953
|
+
}
|
|
954
|
+
static parseIndex(xml) {
|
|
955
|
+
const urls = [];
|
|
956
|
+
const sitemapRegex = /<sitemap>([\s\S]*?)<\/sitemap>/g;
|
|
957
|
+
let match;
|
|
958
|
+
while ((match = sitemapRegex.exec(xml)) !== null) {
|
|
959
|
+
const content = match[1];
|
|
960
|
+
const locMatch = /<loc>(.*?)<\/loc>/.exec(content);
|
|
961
|
+
if (locMatch) {
|
|
962
|
+
urls.push(this.unescape(locMatch[1]));
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
return urls;
|
|
966
|
+
}
|
|
967
|
+
static unescape(str) {
|
|
968
|
+
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'");
|
|
969
|
+
}
|
|
970
|
+
};
|
|
971
|
+
|
|
755
972
|
// src/core/IncrementalGenerator.ts
|
|
756
973
|
var IncrementalGenerator = class {
|
|
757
974
|
options;
|
|
758
975
|
changeTracker;
|
|
759
976
|
diffCalculator;
|
|
760
977
|
generator;
|
|
978
|
+
mutex = new Mutex();
|
|
761
979
|
constructor(options) {
|
|
762
|
-
this.options =
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
980
|
+
this.options = {
|
|
981
|
+
autoTrack: true,
|
|
982
|
+
generateManifest: true,
|
|
983
|
+
...options
|
|
984
|
+
};
|
|
985
|
+
this.changeTracker = this.options.changeTracker;
|
|
986
|
+
this.diffCalculator = this.options.diffCalculator || new DiffCalculator();
|
|
987
|
+
this.generator = new SitemapGenerator(this.options);
|
|
766
988
|
}
|
|
767
|
-
/**
|
|
768
|
-
* 生成完整的 sitemap(首次生成)
|
|
769
|
-
*/
|
|
770
989
|
async generateFull() {
|
|
990
|
+
return this.mutex.runExclusive(() => this.performFullGeneration());
|
|
991
|
+
}
|
|
992
|
+
async generateIncremental(since) {
|
|
993
|
+
return this.mutex.runExclusive(() => this.performIncrementalGeneration(since));
|
|
994
|
+
}
|
|
995
|
+
async performFullGeneration() {
|
|
771
996
|
await this.generator.run();
|
|
772
997
|
if (this.options.autoTrack) {
|
|
773
998
|
const { providers } = this.options;
|
|
@@ -785,52 +1010,130 @@ var IncrementalGenerator = class {
|
|
|
785
1010
|
}
|
|
786
1011
|
}
|
|
787
1012
|
}
|
|
788
|
-
|
|
789
|
-
* 增量生成(只更新變更的部分)
|
|
790
|
-
*/
|
|
791
|
-
async generateIncremental(since) {
|
|
1013
|
+
async performIncrementalGeneration(since) {
|
|
792
1014
|
const changes = await this.changeTracker.getChanges(since);
|
|
793
1015
|
if (changes.length === 0) {
|
|
794
1016
|
return;
|
|
795
1017
|
}
|
|
796
|
-
const
|
|
797
|
-
|
|
798
|
-
|
|
1018
|
+
const manifest = await this.loadManifest();
|
|
1019
|
+
if (!manifest) {
|
|
1020
|
+
await this.performFullGeneration();
|
|
1021
|
+
return;
|
|
1022
|
+
}
|
|
1023
|
+
const totalCount = manifest.shards.reduce((acc, s) => acc + s.count, 0);
|
|
1024
|
+
const changeRatio = totalCount > 0 ? changes.length / totalCount : 1;
|
|
1025
|
+
if (changeRatio > 0.3) {
|
|
1026
|
+
await this.performFullGeneration();
|
|
1027
|
+
return;
|
|
1028
|
+
}
|
|
1029
|
+
const affectedShards = this.getAffectedShards(manifest, changes);
|
|
1030
|
+
if (affectedShards.size / manifest.shards.length > 0.5) {
|
|
1031
|
+
await this.performFullGeneration();
|
|
1032
|
+
return;
|
|
1033
|
+
}
|
|
1034
|
+
await this.updateShards(manifest, affectedShards);
|
|
799
1035
|
}
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
1036
|
+
normalizeUrl(url) {
|
|
1037
|
+
if (url.startsWith("http")) {
|
|
1038
|
+
return url;
|
|
1039
|
+
}
|
|
1040
|
+
const { baseUrl } = this.options;
|
|
1041
|
+
const normalizedBase = baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
|
|
1042
|
+
const normalizedPath = url.startsWith("/") ? url : `/${url}`;
|
|
1043
|
+
return normalizedBase + normalizedPath;
|
|
805
1044
|
}
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
1045
|
+
async loadManifest() {
|
|
1046
|
+
const filename = this.options.filename?.replace(/\.xml$/, "-manifest.json") || "sitemap-manifest.json";
|
|
1047
|
+
const content = await this.options.storage.read(filename);
|
|
1048
|
+
if (!content) {
|
|
1049
|
+
return null;
|
|
1050
|
+
}
|
|
1051
|
+
try {
|
|
1052
|
+
return JSON.parse(content);
|
|
1053
|
+
} catch {
|
|
1054
|
+
return null;
|
|
1055
|
+
}
|
|
811
1056
|
}
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
1057
|
+
getAffectedShards(manifest, changes) {
|
|
1058
|
+
const affected = /* @__PURE__ */ new Map();
|
|
1059
|
+
for (const change of changes) {
|
|
1060
|
+
const normalizedUrl = this.normalizeUrl(change.url);
|
|
1061
|
+
let shard = manifest.shards.find((s) => {
|
|
1062
|
+
return normalizedUrl >= s.from && normalizedUrl <= s.to;
|
|
1063
|
+
});
|
|
1064
|
+
if (!shard) {
|
|
1065
|
+
shard = manifest.shards.find((s) => normalizedUrl <= s.to);
|
|
1066
|
+
if (!shard) {
|
|
1067
|
+
shard = manifest.shards[manifest.shards.length - 1];
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
if (shard) {
|
|
1071
|
+
const shardChanges = affected.get(shard.filename) || [];
|
|
1072
|
+
shardChanges.push(change);
|
|
1073
|
+
affected.set(shard.filename, shardChanges);
|
|
1074
|
+
}
|
|
822
1075
|
}
|
|
823
|
-
return
|
|
1076
|
+
return affected;
|
|
824
1077
|
}
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
1078
|
+
async updateShards(manifest, affectedShards) {
|
|
1079
|
+
for (const [filename, shardChanges] of affectedShards) {
|
|
1080
|
+
const entries = [];
|
|
1081
|
+
const stream = await this.options.storage.readStream?.(filename);
|
|
1082
|
+
if (stream) {
|
|
1083
|
+
for await (const entry of SitemapParser.parseStream(stream)) {
|
|
1084
|
+
entries.push(entry);
|
|
1085
|
+
}
|
|
1086
|
+
} else {
|
|
1087
|
+
const xml = await this.options.storage.read(filename);
|
|
1088
|
+
if (!xml) {
|
|
1089
|
+
continue;
|
|
1090
|
+
}
|
|
1091
|
+
entries.push(...SitemapParser.parse(xml));
|
|
1092
|
+
}
|
|
1093
|
+
const updatedEntries = this.applyChanges(entries, shardChanges);
|
|
1094
|
+
const outStream = new SitemapStream({
|
|
1095
|
+
baseUrl: this.options.baseUrl,
|
|
1096
|
+
pretty: this.options.pretty
|
|
1097
|
+
});
|
|
1098
|
+
outStream.addAll(updatedEntries);
|
|
1099
|
+
const newXml = outStream.toXML();
|
|
1100
|
+
await this.options.storage.write(filename, newXml);
|
|
1101
|
+
const shardInfo = manifest.shards.find((s) => s.filename === filename);
|
|
1102
|
+
if (shardInfo) {
|
|
1103
|
+
shardInfo.count = updatedEntries.length;
|
|
1104
|
+
shardInfo.lastmod = /* @__PURE__ */ new Date();
|
|
1105
|
+
shardInfo.from = this.normalizeUrl(updatedEntries[0].url);
|
|
1106
|
+
shardInfo.to = this.normalizeUrl(updatedEntries[updatedEntries.length - 1].url);
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
const manifestFilename = this.options.filename?.replace(/\.xml$/, "-manifest.json") || "sitemap-manifest.json";
|
|
1110
|
+
await this.options.storage.write(
|
|
1111
|
+
manifestFilename,
|
|
1112
|
+
JSON.stringify(manifest, null, this.options.pretty ? 2 : 0)
|
|
1113
|
+
);
|
|
1114
|
+
}
|
|
1115
|
+
applyChanges(entries, changes) {
|
|
1116
|
+
const entryMap = /* @__PURE__ */ new Map();
|
|
1117
|
+
for (const entry of entries) {
|
|
1118
|
+
entryMap.set(this.normalizeUrl(entry.url), entry);
|
|
1119
|
+
}
|
|
1120
|
+
for (const change of changes) {
|
|
1121
|
+
const normalizedUrl = this.normalizeUrl(change.url);
|
|
1122
|
+
if (change.type === "add" || change.type === "update") {
|
|
1123
|
+
if (change.entry) {
|
|
1124
|
+
entryMap.set(normalizedUrl, {
|
|
1125
|
+
...change.entry,
|
|
1126
|
+
url: normalizedUrl
|
|
1127
|
+
});
|
|
1128
|
+
}
|
|
1129
|
+
} else if (change.type === "remove") {
|
|
1130
|
+
entryMap.delete(normalizedUrl);
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
return Array.from(entryMap.values()).sort(
|
|
1134
|
+
(a, b) => this.normalizeUrl(a.url).localeCompare(this.normalizeUrl(b.url))
|
|
1135
|
+
);
|
|
830
1136
|
}
|
|
831
|
-
/**
|
|
832
|
-
* 將 AsyncIterable 轉換為陣列
|
|
833
|
-
*/
|
|
834
1137
|
async toArray(iterable) {
|
|
835
1138
|
const array = [];
|
|
836
1139
|
for await (const item of iterable) {
|
|
@@ -976,7 +1279,7 @@ var GenerateSitemapJob = class extends import_stream.Job {
|
|
|
976
1279
|
this.generator = new SitemapGenerator(options.generatorOptions);
|
|
977
1280
|
}
|
|
978
1281
|
async handle() {
|
|
979
|
-
const { progressTracker,
|
|
1282
|
+
const { progressTracker, onComplete, onError } = this.options;
|
|
980
1283
|
try {
|
|
981
1284
|
if (progressTracker) {
|
|
982
1285
|
const total = await this.calculateTotal();
|
|
@@ -984,8 +1287,8 @@ var GenerateSitemapJob = class extends import_stream.Job {
|
|
|
984
1287
|
this.totalEntries = total;
|
|
985
1288
|
}
|
|
986
1289
|
await this.generateWithProgress();
|
|
987
|
-
if (shadowProcessor) {
|
|
988
|
-
await shadowProcessor.commit();
|
|
1290
|
+
if (this.options.shadowProcessor) {
|
|
1291
|
+
await this.options.shadowProcessor.commit();
|
|
989
1292
|
}
|
|
990
1293
|
if (progressTracker) {
|
|
991
1294
|
await progressTracker.complete();
|
|
@@ -1026,15 +1329,7 @@ var GenerateSitemapJob = class extends import_stream.Job {
|
|
|
1026
1329
|
* 帶進度追蹤的生成
|
|
1027
1330
|
*/
|
|
1028
1331
|
async generateWithProgress() {
|
|
1029
|
-
const { progressTracker,
|
|
1030
|
-
const {
|
|
1031
|
-
providers,
|
|
1032
|
-
maxEntriesPerFile = 5e4,
|
|
1033
|
-
storage,
|
|
1034
|
-
baseUrl,
|
|
1035
|
-
pretty,
|
|
1036
|
-
filename
|
|
1037
|
-
} = this.options.generatorOptions;
|
|
1332
|
+
const { progressTracker, onProgress } = this.options;
|
|
1038
1333
|
await this.generator.run();
|
|
1039
1334
|
this.processedEntries = this.totalEntries;
|
|
1040
1335
|
if (progressTracker) {
|
|
@@ -1207,6 +1502,15 @@ var MemorySitemapStorage = class {
|
|
|
1207
1502
|
async read(filename) {
|
|
1208
1503
|
return this.files.get(filename) || null;
|
|
1209
1504
|
}
|
|
1505
|
+
async readStream(filename) {
|
|
1506
|
+
const content = this.files.get(filename);
|
|
1507
|
+
if (content === void 0) {
|
|
1508
|
+
return null;
|
|
1509
|
+
}
|
|
1510
|
+
return (async function* () {
|
|
1511
|
+
yield content;
|
|
1512
|
+
})();
|
|
1513
|
+
}
|
|
1210
1514
|
async exists(filename) {
|
|
1211
1515
|
return this.files.has(filename);
|
|
1212
1516
|
}
|
|
@@ -1273,7 +1577,7 @@ var OrbitSitemap = class _OrbitSitemap {
|
|
|
1273
1577
|
if (this.mode === "dynamic") {
|
|
1274
1578
|
this.installDynamic(core);
|
|
1275
1579
|
} else {
|
|
1276
|
-
|
|
1580
|
+
core.logger.info("[OrbitSitemap] Static mode configured. Use generate() to build sitemaps.");
|
|
1277
1581
|
}
|
|
1278
1582
|
}
|
|
1279
1583
|
installDynamic(core) {
|
|
@@ -1539,6 +1843,11 @@ var RouteScanner = class {
|
|
|
1539
1843
|
...options
|
|
1540
1844
|
};
|
|
1541
1845
|
}
|
|
1846
|
+
/**
|
|
1847
|
+
* Scan the router and return discovered entries.
|
|
1848
|
+
*
|
|
1849
|
+
* @returns An array of sitemap entries.
|
|
1850
|
+
*/
|
|
1542
1851
|
getEntries() {
|
|
1543
1852
|
const entries = [];
|
|
1544
1853
|
const routes = this.extractRoutes(this.router);
|
|
@@ -1945,6 +2254,30 @@ var GCPSitemapStorage = class {
|
|
|
1945
2254
|
throw error;
|
|
1946
2255
|
}
|
|
1947
2256
|
}
|
|
2257
|
+
async readStream(filename) {
|
|
2258
|
+
try {
|
|
2259
|
+
const { bucket } = await this.getStorageClient();
|
|
2260
|
+
const key = this.getKey(filename);
|
|
2261
|
+
const file = bucket.file(key);
|
|
2262
|
+
const [exists] = await file.exists();
|
|
2263
|
+
if (!exists) {
|
|
2264
|
+
return null;
|
|
2265
|
+
}
|
|
2266
|
+
const stream = file.createReadStream();
|
|
2267
|
+
return (async function* () {
|
|
2268
|
+
const decoder = new TextDecoder();
|
|
2269
|
+
for await (const chunk of stream) {
|
|
2270
|
+
yield decoder.decode(chunk, { stream: true });
|
|
2271
|
+
}
|
|
2272
|
+
yield decoder.decode();
|
|
2273
|
+
})();
|
|
2274
|
+
} catch (error) {
|
|
2275
|
+
if (error.code === 404) {
|
|
2276
|
+
return null;
|
|
2277
|
+
}
|
|
2278
|
+
throw error;
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
1948
2281
|
async exists(filename) {
|
|
1949
2282
|
try {
|
|
1950
2283
|
const { bucket } = await this.getStorageClient();
|
|
@@ -1967,7 +2300,7 @@ var GCPSitemapStorage = class {
|
|
|
1967
2300
|
return this.write(filename, content);
|
|
1968
2301
|
}
|
|
1969
2302
|
const { bucket } = await this.getStorageClient();
|
|
1970
|
-
const id = shadowId || `shadow-${Date.now()}-${
|
|
2303
|
+
const id = shadowId || `shadow-${Date.now()}-${crypto.randomUUID()}`;
|
|
1971
2304
|
const shadowKey = this.getKey(`${filename}.shadow.${id}`);
|
|
1972
2305
|
const file = bucket.file(shadowKey);
|
|
1973
2306
|
await file.save(content, {
|
|
@@ -2239,6 +2572,34 @@ var S3SitemapStorage = class {
|
|
|
2239
2572
|
throw error;
|
|
2240
2573
|
}
|
|
2241
2574
|
}
|
|
2575
|
+
async readStream(filename) {
|
|
2576
|
+
try {
|
|
2577
|
+
const s3 = await this.getS3Client();
|
|
2578
|
+
const key = this.getKey(filename);
|
|
2579
|
+
const response = await s3.client.send(
|
|
2580
|
+
new s3.GetObjectCommand({
|
|
2581
|
+
Bucket: this.bucket,
|
|
2582
|
+
Key: key
|
|
2583
|
+
})
|
|
2584
|
+
);
|
|
2585
|
+
if (!response.Body) {
|
|
2586
|
+
return null;
|
|
2587
|
+
}
|
|
2588
|
+
const body = response.Body;
|
|
2589
|
+
return (async function* () {
|
|
2590
|
+
const decoder = new TextDecoder();
|
|
2591
|
+
for await (const chunk of body) {
|
|
2592
|
+
yield decoder.decode(chunk, { stream: true });
|
|
2593
|
+
}
|
|
2594
|
+
yield decoder.decode();
|
|
2595
|
+
})();
|
|
2596
|
+
} catch (error) {
|
|
2597
|
+
if (error.name === "NoSuchKey" || error.$metadata?.httpStatusCode === 404) {
|
|
2598
|
+
return null;
|
|
2599
|
+
}
|
|
2600
|
+
throw error;
|
|
2601
|
+
}
|
|
2602
|
+
}
|
|
2242
2603
|
async exists(filename) {
|
|
2243
2604
|
try {
|
|
2244
2605
|
const s3 = await this.getS3Client();
|
|
@@ -2268,7 +2629,7 @@ var S3SitemapStorage = class {
|
|
|
2268
2629
|
return this.write(filename, content);
|
|
2269
2630
|
}
|
|
2270
2631
|
const s3 = await this.getS3Client();
|
|
2271
|
-
const id = shadowId || `shadow-${Date.now()}-${
|
|
2632
|
+
const id = shadowId || `shadow-${Date.now()}-${crypto.randomUUID()}`;
|
|
2272
2633
|
const shadowKey = this.getKey(`${filename}.shadow.${id}`);
|
|
2273
2634
|
await s3.client.send(
|
|
2274
2635
|
new s3.PutObjectCommand({
|