@gravito/constellation 1.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +334 -0
- package/dist/index.cjs +2142 -0
- package/dist/index.mjs +2123 -0
- package/package.json +49 -0
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,2142 @@
|
|
|
1
|
+
var __create = Object.create;
|
|
2
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
+
var __toESM = (mod, isNodeMode, target) => {
|
|
8
|
+
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
9
|
+
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
10
|
+
for (let key of __getOwnPropNames(mod))
|
|
11
|
+
if (!__hasOwnProp.call(to, key))
|
|
12
|
+
__defProp(to, key, {
|
|
13
|
+
get: () => mod[key],
|
|
14
|
+
enumerable: true
|
|
15
|
+
});
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __moduleCache = /* @__PURE__ */ new WeakMap;
|
|
19
|
+
var __toCommonJS = (from) => {
|
|
20
|
+
var entry = __moduleCache.get(from), desc;
|
|
21
|
+
if (entry)
|
|
22
|
+
return entry;
|
|
23
|
+
entry = __defProp({}, "__esModule", { value: true });
|
|
24
|
+
if (from && typeof from === "object" || typeof from === "function")
|
|
25
|
+
__getOwnPropNames(from).map((key) => !__hasOwnProp.call(entry, key) && __defProp(entry, key, {
|
|
26
|
+
get: () => from[key],
|
|
27
|
+
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
|
|
28
|
+
}));
|
|
29
|
+
__moduleCache.set(from, entry);
|
|
30
|
+
return entry;
|
|
31
|
+
};
|
|
32
|
+
var __export = (target, all) => {
|
|
33
|
+
for (var name in all)
|
|
34
|
+
__defProp(target, name, {
|
|
35
|
+
get: all[name],
|
|
36
|
+
enumerable: true,
|
|
37
|
+
configurable: true,
|
|
38
|
+
set: (newValue) => all[name] = () => newValue
|
|
39
|
+
});
|
|
40
|
+
};
|
|
41
|
+
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
42
|
+
|
|
43
|
+
// src/storage/DiskSitemapStorage.ts
|
|
44
|
+
var exports_DiskSitemapStorage = {};
|
|
45
|
+
__export(exports_DiskSitemapStorage, {
|
|
46
|
+
DiskSitemapStorage: () => DiskSitemapStorage
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
class DiskSitemapStorage {
|
|
50
|
+
outDir;
|
|
51
|
+
baseUrl;
|
|
52
|
+
constructor(outDir, baseUrl) {
|
|
53
|
+
this.outDir = outDir;
|
|
54
|
+
this.baseUrl = baseUrl;
|
|
55
|
+
}
|
|
56
|
+
async write(filename, content) {
|
|
57
|
+
await import_promises.default.mkdir(this.outDir, { recursive: true });
|
|
58
|
+
await import_promises.default.writeFile(import_node_path.default.join(this.outDir, filename), content);
|
|
59
|
+
}
|
|
60
|
+
async read(filename) {
|
|
61
|
+
try {
|
|
62
|
+
return await import_promises.default.readFile(import_node_path.default.join(this.outDir, filename), "utf-8");
|
|
63
|
+
} catch {
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
async exists(filename) {
|
|
68
|
+
try {
|
|
69
|
+
await import_promises.default.access(import_node_path.default.join(this.outDir, filename));
|
|
70
|
+
return true;
|
|
71
|
+
} catch {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
getUrl(filename) {
|
|
76
|
+
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
77
|
+
const file = filename.startsWith("/") ? filename.slice(1) : filename;
|
|
78
|
+
return `${base}/${file}`;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
var import_promises, import_node_path;
|
|
82
|
+
var init_DiskSitemapStorage = __esm(() => {
|
|
83
|
+
import_promises = __toESM(require("node:fs/promises"));
|
|
84
|
+
import_node_path = __toESM(require("node:path"));
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
// src/index.ts
|
|
88
|
+
var exports_src = {};
|
|
89
|
+
__export(exports_src, {
|
|
90
|
+
routeScanner: () => routeScanner,
|
|
91
|
+
generateI18nEntries: () => generateI18nEntries,
|
|
92
|
+
SitemapStream: () => SitemapStream,
|
|
93
|
+
SitemapIndex: () => SitemapIndex,
|
|
94
|
+
SitemapGenerator: () => SitemapGenerator,
|
|
95
|
+
ShadowProcessor: () => ShadowProcessor,
|
|
96
|
+
S3SitemapStorage: () => S3SitemapStorage,
|
|
97
|
+
RouteScanner: () => RouteScanner,
|
|
98
|
+
RedisRedirectManager: () => RedisRedirectManager,
|
|
99
|
+
RedisProgressStorage: () => RedisProgressStorage,
|
|
100
|
+
RedisChangeTracker: () => RedisChangeTracker,
|
|
101
|
+
RedirectHandler: () => RedirectHandler,
|
|
102
|
+
RedirectDetector: () => RedirectDetector,
|
|
103
|
+
ProgressTracker: () => ProgressTracker,
|
|
104
|
+
OrbitSitemap: () => OrbitSitemap,
|
|
105
|
+
MemorySitemapStorage: () => MemorySitemapStorage,
|
|
106
|
+
MemoryRedirectManager: () => MemoryRedirectManager,
|
|
107
|
+
MemoryProgressStorage: () => MemoryProgressStorage,
|
|
108
|
+
MemoryChangeTracker: () => MemoryChangeTracker,
|
|
109
|
+
IncrementalGenerator: () => IncrementalGenerator,
|
|
110
|
+
GenerateSitemapJob: () => GenerateSitemapJob,
|
|
111
|
+
GCPSitemapStorage: () => GCPSitemapStorage,
|
|
112
|
+
DiskSitemapStorage: () => DiskSitemapStorage,
|
|
113
|
+
DiffCalculator: () => DiffCalculator
|
|
114
|
+
});
|
|
115
|
+
module.exports = __toCommonJS(exports_src);
|
|
116
|
+
|
|
117
|
+
// src/core/ChangeTracker.ts
|
|
118
|
+
class MemoryChangeTracker {
|
|
119
|
+
changes = [];
|
|
120
|
+
maxChanges;
|
|
121
|
+
constructor(options = {}) {
|
|
122
|
+
this.maxChanges = options.maxChanges || 1e5;
|
|
123
|
+
}
|
|
124
|
+
async track(change) {
|
|
125
|
+
this.changes.push(change);
|
|
126
|
+
if (this.changes.length > this.maxChanges) {
|
|
127
|
+
this.changes = this.changes.slice(-this.maxChanges);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
async getChanges(since) {
|
|
131
|
+
if (!since) {
|
|
132
|
+
return [...this.changes];
|
|
133
|
+
}
|
|
134
|
+
return this.changes.filter((change) => change.timestamp >= since);
|
|
135
|
+
}
|
|
136
|
+
async getChangesByUrl(url) {
|
|
137
|
+
return this.changes.filter((change) => change.url === url);
|
|
138
|
+
}
|
|
139
|
+
async clear(since) {
|
|
140
|
+
if (!since) {
|
|
141
|
+
this.changes = [];
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
this.changes = this.changes.filter((change) => change.timestamp < since);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
class RedisChangeTracker {
|
|
149
|
+
client;
|
|
150
|
+
keyPrefix;
|
|
151
|
+
ttl;
|
|
152
|
+
constructor(options) {
|
|
153
|
+
this.client = options.client;
|
|
154
|
+
this.keyPrefix = options.keyPrefix || "sitemap:changes:";
|
|
155
|
+
this.ttl = options.ttl || 604800;
|
|
156
|
+
}
|
|
157
|
+
getKey(url) {
|
|
158
|
+
return `${this.keyPrefix}${url}`;
|
|
159
|
+
}
|
|
160
|
+
getListKey() {
|
|
161
|
+
return `${this.keyPrefix}list`;
|
|
162
|
+
}
|
|
163
|
+
async track(change) {
|
|
164
|
+
const key = this.getKey(change.url);
|
|
165
|
+
const listKey = this.getListKey();
|
|
166
|
+
const data = JSON.stringify(change);
|
|
167
|
+
await this.client.set(key, data, "EX", this.ttl);
|
|
168
|
+
const score = change.timestamp.getTime();
|
|
169
|
+
await this.client.zadd(listKey, score, change.url);
|
|
170
|
+
await this.client.expire(listKey, this.ttl);
|
|
171
|
+
}
|
|
172
|
+
async getChanges(since) {
|
|
173
|
+
try {
|
|
174
|
+
const listKey = this.getListKey();
|
|
175
|
+
const minScore = since ? since.getTime() : 0;
|
|
176
|
+
const urls = await this.client.zrangebyscore(listKey, minScore, "+inf");
|
|
177
|
+
const changes = [];
|
|
178
|
+
for (const url of urls) {
|
|
179
|
+
const key = this.getKey(url);
|
|
180
|
+
const data = await this.client.get(key);
|
|
181
|
+
if (data) {
|
|
182
|
+
const change = JSON.parse(data);
|
|
183
|
+
change.timestamp = new Date(change.timestamp);
|
|
184
|
+
changes.push(change);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return changes.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
|
|
188
|
+
} catch {
|
|
189
|
+
return [];
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
async getChangesByUrl(url) {
|
|
193
|
+
try {
|
|
194
|
+
const key = this.getKey(url);
|
|
195
|
+
const data = await this.client.get(key);
|
|
196
|
+
if (!data) {
|
|
197
|
+
return [];
|
|
198
|
+
}
|
|
199
|
+
const change = JSON.parse(data);
|
|
200
|
+
change.timestamp = new Date(change.timestamp);
|
|
201
|
+
return [change];
|
|
202
|
+
} catch {
|
|
203
|
+
return [];
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
async clear(since) {
|
|
207
|
+
try {
|
|
208
|
+
const listKey = this.getListKey();
|
|
209
|
+
if (!since) {
|
|
210
|
+
const urls = await this.client.zrange(listKey, 0, -1);
|
|
211
|
+
for (const url of urls) {
|
|
212
|
+
await this.client.del(this.getKey(url));
|
|
213
|
+
}
|
|
214
|
+
await this.client.del(listKey);
|
|
215
|
+
} else {
|
|
216
|
+
const maxScore = since.getTime();
|
|
217
|
+
const urls = await this.client.zrangebyscore(listKey, 0, maxScore);
|
|
218
|
+
for (const url of urls) {
|
|
219
|
+
await this.client.del(this.getKey(url));
|
|
220
|
+
await this.client.zrem(listKey, url);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
} catch {}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
// src/core/DiffCalculator.ts
|
|
227
|
+
class DiffCalculator {
|
|
228
|
+
batchSize;
|
|
229
|
+
constructor(options = {}) {
|
|
230
|
+
this.batchSize = options.batchSize || 1e4;
|
|
231
|
+
}
|
|
232
|
+
calculate(oldEntries, newEntries) {
|
|
233
|
+
const oldMap = new Map;
|
|
234
|
+
const newMap = new Map;
|
|
235
|
+
for (const entry of oldEntries) {
|
|
236
|
+
oldMap.set(entry.url, entry);
|
|
237
|
+
}
|
|
238
|
+
for (const entry of newEntries) {
|
|
239
|
+
newMap.set(entry.url, entry);
|
|
240
|
+
}
|
|
241
|
+
const added = [];
|
|
242
|
+
const updated = [];
|
|
243
|
+
const removed = [];
|
|
244
|
+
for (const [url, newEntry] of newMap) {
|
|
245
|
+
const oldEntry = oldMap.get(url);
|
|
246
|
+
if (!oldEntry) {
|
|
247
|
+
added.push(newEntry);
|
|
248
|
+
} else if (this.hasChanged(oldEntry, newEntry)) {
|
|
249
|
+
updated.push(newEntry);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
for (const [url] of oldMap) {
|
|
253
|
+
if (!newMap.has(url)) {
|
|
254
|
+
removed.push(url);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
return { added, updated, removed };
|
|
258
|
+
}
|
|
259
|
+
async calculateBatch(oldEntries, newEntries) {
|
|
260
|
+
const oldMap = new Map;
|
|
261
|
+
const newMap = new Map;
|
|
262
|
+
for await (const entry of oldEntries) {
|
|
263
|
+
oldMap.set(entry.url, entry);
|
|
264
|
+
}
|
|
265
|
+
for await (const entry of newEntries) {
|
|
266
|
+
newMap.set(entry.url, entry);
|
|
267
|
+
}
|
|
268
|
+
return this.calculate(Array.from(oldMap.values()), Array.from(newMap.values()));
|
|
269
|
+
}
|
|
270
|
+
calculateFromChanges(baseEntries, changes) {
|
|
271
|
+
const entryMap = new Map;
|
|
272
|
+
for (const entry of baseEntries) {
|
|
273
|
+
entryMap.set(entry.url, entry);
|
|
274
|
+
}
|
|
275
|
+
for (const change of changes) {
|
|
276
|
+
if (change.type === "add" && change.entry) {
|
|
277
|
+
entryMap.set(change.url, change.entry);
|
|
278
|
+
} else if (change.type === "update" && change.entry) {
|
|
279
|
+
entryMap.set(change.url, change.entry);
|
|
280
|
+
} else if (change.type === "remove") {
|
|
281
|
+
entryMap.delete(change.url);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
const newEntries = Array.from(entryMap.values());
|
|
285
|
+
return this.calculate(baseEntries, newEntries);
|
|
286
|
+
}
|
|
287
|
+
hasChanged(oldEntry, newEntry) {
|
|
288
|
+
if (oldEntry.lastmod !== newEntry.lastmod) {
|
|
289
|
+
return true;
|
|
290
|
+
}
|
|
291
|
+
if (oldEntry.changefreq !== newEntry.changefreq) {
|
|
292
|
+
return true;
|
|
293
|
+
}
|
|
294
|
+
if (oldEntry.priority !== newEntry.priority) {
|
|
295
|
+
return true;
|
|
296
|
+
}
|
|
297
|
+
const oldAlternates = JSON.stringify(oldEntry.alternates || []);
|
|
298
|
+
const newAlternates = JSON.stringify(newEntry.alternates || []);
|
|
299
|
+
if (oldAlternates !== newAlternates) {
|
|
300
|
+
return true;
|
|
301
|
+
}
|
|
302
|
+
return false;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
// src/core/ShadowProcessor.ts
|
|
306
|
+
class ShadowProcessor {
|
|
307
|
+
options;
|
|
308
|
+
shadowId;
|
|
309
|
+
operations = [];
|
|
310
|
+
constructor(options) {
|
|
311
|
+
this.options = options;
|
|
312
|
+
this.shadowId = `shadow-${Date.now()}-${Math.random().toString(36).substring(7)}`;
|
|
313
|
+
}
|
|
314
|
+
async addOperation(operation) {
|
|
315
|
+
if (!this.options.enabled) {
|
|
316
|
+
await this.options.storage.write(operation.filename, operation.content);
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
this.operations.push({
|
|
320
|
+
...operation,
|
|
321
|
+
shadowId: operation.shadowId || this.shadowId
|
|
322
|
+
});
|
|
323
|
+
if (this.options.storage.writeShadow) {
|
|
324
|
+
await this.options.storage.writeShadow(operation.filename, operation.content, this.shadowId);
|
|
325
|
+
} else {
|
|
326
|
+
await this.options.storage.write(operation.filename, operation.content);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
async commit() {
|
|
330
|
+
if (!this.options.enabled) {
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
if (this.options.mode === "atomic") {
|
|
334
|
+
if (this.options.storage.commitShadow) {
|
|
335
|
+
await this.options.storage.commitShadow(this.shadowId);
|
|
336
|
+
}
|
|
337
|
+
} else {
|
|
338
|
+
for (const operation of this.operations) {
|
|
339
|
+
if (this.options.storage.commitShadow) {
|
|
340
|
+
await this.options.storage.commitShadow(operation.shadowId || this.shadowId);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
this.operations = [];
|
|
345
|
+
}
|
|
346
|
+
async rollback() {
|
|
347
|
+
if (!this.options.enabled) {
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
this.operations = [];
|
|
351
|
+
}
|
|
352
|
+
getShadowId() {
|
|
353
|
+
return this.shadowId;
|
|
354
|
+
}
|
|
355
|
+
getOperations() {
|
|
356
|
+
return [...this.operations];
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// src/core/SitemapIndex.ts
|
|
361
|
+
class SitemapIndex {
|
|
362
|
+
options;
|
|
363
|
+
entries = [];
|
|
364
|
+
constructor(options) {
|
|
365
|
+
this.options = { ...options };
|
|
366
|
+
if (this.options.baseUrl.endsWith("/")) {
|
|
367
|
+
this.options.baseUrl = this.options.baseUrl.slice(0, -1);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
add(entry) {
|
|
371
|
+
if (typeof entry === "string") {
|
|
372
|
+
this.entries.push({ url: entry });
|
|
373
|
+
} else {
|
|
374
|
+
this.entries.push(entry);
|
|
375
|
+
}
|
|
376
|
+
return this;
|
|
377
|
+
}
|
|
378
|
+
addAll(entries) {
|
|
379
|
+
for (const entry of entries) {
|
|
380
|
+
this.add(entry);
|
|
381
|
+
}
|
|
382
|
+
return this;
|
|
383
|
+
}
|
|
384
|
+
toXML() {
|
|
385
|
+
const { baseUrl, pretty } = this.options;
|
|
386
|
+
let xml = `<?xml version="1.0" encoding="UTF-8"?>
|
|
387
|
+
`;
|
|
388
|
+
xml += `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
|
389
|
+
`;
|
|
390
|
+
const indent = pretty ? " " : "";
|
|
391
|
+
const subIndent = pretty ? " " : "";
|
|
392
|
+
const nl = pretty ? `
|
|
393
|
+
` : "";
|
|
394
|
+
for (const entry of this.entries) {
|
|
395
|
+
let loc = entry.url;
|
|
396
|
+
if (!loc.startsWith("http")) {
|
|
397
|
+
if (!loc.startsWith("/")) {
|
|
398
|
+
loc = `/${loc}`;
|
|
399
|
+
}
|
|
400
|
+
loc = baseUrl + loc;
|
|
401
|
+
}
|
|
402
|
+
xml += `${indent}<sitemap>${nl}`;
|
|
403
|
+
xml += `${subIndent}<loc>${this.escape(loc)}</loc>${nl}`;
|
|
404
|
+
if (entry.lastmod) {
|
|
405
|
+
const date = entry.lastmod instanceof Date ? entry.lastmod : new Date(entry.lastmod);
|
|
406
|
+
xml += `${subIndent}<lastmod>${date.toISOString().split("T")[0]}</lastmod>${nl}`;
|
|
407
|
+
}
|
|
408
|
+
xml += `${indent}</sitemap>${nl}`;
|
|
409
|
+
}
|
|
410
|
+
xml += `</sitemapindex>`;
|
|
411
|
+
return xml;
|
|
412
|
+
}
|
|
413
|
+
escape(str) {
|
|
414
|
+
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// src/core/SitemapStream.ts
|
|
419
|
+
class SitemapStream {
|
|
420
|
+
options;
|
|
421
|
+
entries = [];
|
|
422
|
+
constructor(options) {
|
|
423
|
+
this.options = { ...options };
|
|
424
|
+
if (this.options.baseUrl.endsWith("/")) {
|
|
425
|
+
this.options.baseUrl = this.options.baseUrl.slice(0, -1);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
add(entry) {
|
|
429
|
+
if (typeof entry === "string") {
|
|
430
|
+
this.entries.push({ url: entry });
|
|
431
|
+
} else {
|
|
432
|
+
this.entries.push(entry);
|
|
433
|
+
}
|
|
434
|
+
return this;
|
|
435
|
+
}
|
|
436
|
+
addAll(entries) {
|
|
437
|
+
for (const entry of entries) {
|
|
438
|
+
this.add(entry);
|
|
439
|
+
}
|
|
440
|
+
return this;
|
|
441
|
+
}
|
|
442
|
+
toXML() {
|
|
443
|
+
const { baseUrl, pretty } = this.options;
|
|
444
|
+
let xml = `<?xml version="1.0" encoding="UTF-8"?>
|
|
445
|
+
`;
|
|
446
|
+
xml += `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
|
|
447
|
+
if (this.hasImages()) {
|
|
448
|
+
xml += ` xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"`;
|
|
449
|
+
}
|
|
450
|
+
if (this.hasVideos()) {
|
|
451
|
+
xml += ` xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"`;
|
|
452
|
+
}
|
|
453
|
+
if (this.hasNews()) {
|
|
454
|
+
xml += ` xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"`;
|
|
455
|
+
}
|
|
456
|
+
if (this.hasAlternates()) {
|
|
457
|
+
xml += ` xmlns:xhtml="http://www.w3.org/1999/xhtml"`;
|
|
458
|
+
}
|
|
459
|
+
xml += `>
|
|
460
|
+
`;
|
|
461
|
+
for (const entry of this.entries) {
|
|
462
|
+
xml += this.renderUrl(entry, baseUrl, pretty);
|
|
463
|
+
}
|
|
464
|
+
xml += `</urlset>`;
|
|
465
|
+
return xml;
|
|
466
|
+
}
|
|
467
|
+
renderUrl(entry, baseUrl, pretty) {
|
|
468
|
+
const indent = pretty ? " " : "";
|
|
469
|
+
const subIndent = pretty ? " " : "";
|
|
470
|
+
const nl = pretty ? `
|
|
471
|
+
` : "";
|
|
472
|
+
let loc = entry.url;
|
|
473
|
+
if (!loc.startsWith("http")) {
|
|
474
|
+
if (!loc.startsWith("/")) {
|
|
475
|
+
loc = `/${loc}`;
|
|
476
|
+
}
|
|
477
|
+
loc = baseUrl + loc;
|
|
478
|
+
}
|
|
479
|
+
let item = `${indent}<url>${nl}`;
|
|
480
|
+
item += `${subIndent}<loc>${this.escape(loc)}</loc>${nl}`;
|
|
481
|
+
if (entry.lastmod) {
|
|
482
|
+
const date = entry.lastmod instanceof Date ? entry.lastmod : new Date(entry.lastmod);
|
|
483
|
+
item += `${subIndent}<lastmod>${date.toISOString().split("T")[0]}</lastmod>${nl}`;
|
|
484
|
+
}
|
|
485
|
+
if (entry.changefreq) {
|
|
486
|
+
item += `${subIndent}<changefreq>${entry.changefreq}</changefreq>${nl}`;
|
|
487
|
+
}
|
|
488
|
+
if (entry.priority !== undefined) {
|
|
489
|
+
item += `${subIndent}<priority>${entry.priority.toFixed(1)}</priority>${nl}`;
|
|
490
|
+
}
|
|
491
|
+
if (entry.alternates) {
|
|
492
|
+
for (const alt of entry.alternates) {
|
|
493
|
+
let altLoc = alt.url;
|
|
494
|
+
if (!altLoc.startsWith("http")) {
|
|
495
|
+
if (!altLoc.startsWith("/")) {
|
|
496
|
+
altLoc = `/${altLoc}`;
|
|
497
|
+
}
|
|
498
|
+
altLoc = baseUrl + altLoc;
|
|
499
|
+
}
|
|
500
|
+
item += `${subIndent}<xhtml:link rel="alternate" hreflang="${alt.lang}" href="${this.escape(altLoc)}"/>${nl}`;
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
if (entry.redirect?.canonical) {
|
|
504
|
+
let canonicalUrl = entry.redirect.canonical;
|
|
505
|
+
if (!canonicalUrl.startsWith("http")) {
|
|
506
|
+
if (!canonicalUrl.startsWith("/")) {
|
|
507
|
+
canonicalUrl = `/${canonicalUrl}`;
|
|
508
|
+
}
|
|
509
|
+
canonicalUrl = baseUrl + canonicalUrl;
|
|
510
|
+
}
|
|
511
|
+
item += `${subIndent}<xhtml:link rel="canonical" href="${this.escape(canonicalUrl)}"/>${nl}`;
|
|
512
|
+
}
|
|
513
|
+
if (entry.redirect && !entry.redirect.canonical) {
|
|
514
|
+
item += `${subIndent}<!-- Redirect: ${entry.redirect.from} → ${entry.redirect.to} (${entry.redirect.type}) -->${nl}`;
|
|
515
|
+
}
|
|
516
|
+
if (entry.images) {
|
|
517
|
+
for (const img of entry.images) {
|
|
518
|
+
let loc2 = img.loc;
|
|
519
|
+
if (!loc2.startsWith("http")) {
|
|
520
|
+
if (!loc2.startsWith("/")) {
|
|
521
|
+
loc2 = `/${loc2}`;
|
|
522
|
+
}
|
|
523
|
+
loc2 = baseUrl + loc2;
|
|
524
|
+
}
|
|
525
|
+
item += `${subIndent}<image:image>${nl}`;
|
|
526
|
+
item += `${subIndent} <image:loc>${this.escape(loc2)}</image:loc>${nl}`;
|
|
527
|
+
if (img.title) {
|
|
528
|
+
item += `${subIndent} <image:title>${this.escape(img.title)}</image:title>${nl}`;
|
|
529
|
+
}
|
|
530
|
+
if (img.caption) {
|
|
531
|
+
item += `${subIndent} <image:caption>${this.escape(img.caption)}</image:caption>${nl}`;
|
|
532
|
+
}
|
|
533
|
+
if (img.geo_location) {
|
|
534
|
+
item += `${subIndent} <image:geo_location>${this.escape(img.geo_location)}</image:geo_location>${nl}`;
|
|
535
|
+
}
|
|
536
|
+
if (img.license) {
|
|
537
|
+
item += `${subIndent} <image:license>${this.escape(img.license)}</image:license>${nl}`;
|
|
538
|
+
}
|
|
539
|
+
item += `${subIndent}</image:image>${nl}`;
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
if (entry.videos) {
|
|
543
|
+
for (const video of entry.videos) {
|
|
544
|
+
item += `${subIndent}<video:video>${nl}`;
|
|
545
|
+
item += `${subIndent} <video:thumbnail_loc>${this.escape(video.thumbnail_loc)}</video:thumbnail_loc>${nl}`;
|
|
546
|
+
item += `${subIndent} <video:title>${this.escape(video.title)}</video:title>${nl}`;
|
|
547
|
+
item += `${subIndent} <video:description>${this.escape(video.description)}</video:description>${nl}`;
|
|
548
|
+
if (video.content_loc) {
|
|
549
|
+
item += `${subIndent} <video:content_loc>${this.escape(video.content_loc)}</video:content_loc>${nl}`;
|
|
550
|
+
}
|
|
551
|
+
if (video.player_loc) {
|
|
552
|
+
item += `${subIndent} <video:player_loc>${this.escape(video.player_loc)}</video:player_loc>${nl}`;
|
|
553
|
+
}
|
|
554
|
+
if (video.duration) {
|
|
555
|
+
item += `${subIndent} <video:duration>${video.duration}</video:duration>${nl}`;
|
|
556
|
+
}
|
|
557
|
+
if (video.view_count) {
|
|
558
|
+
item += `${subIndent} <video:view_count>${video.view_count}</video:view_count>${nl}`;
|
|
559
|
+
}
|
|
560
|
+
if (video.publication_date) {
|
|
561
|
+
const pubDate = video.publication_date instanceof Date ? video.publication_date : new Date(video.publication_date);
|
|
562
|
+
item += `${subIndent} <video:publication_date>${pubDate.toISOString()}</video:publication_date>${nl}`;
|
|
563
|
+
}
|
|
564
|
+
if (video.family_friendly) {
|
|
565
|
+
item += `${subIndent} <video:family_friendly>${video.family_friendly}</video:family_friendly>${nl}`;
|
|
566
|
+
}
|
|
567
|
+
if (video.tag) {
|
|
568
|
+
for (const tag of video.tag) {
|
|
569
|
+
item += `${subIndent} <video:tag>${this.escape(tag)}</video:tag>${nl}`;
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
item += `${subIndent}</video:video>${nl}`;
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
if (entry.news) {
|
|
576
|
+
item += `${subIndent}<news:news>${nl}`;
|
|
577
|
+
item += `${subIndent} <news:publication>${nl}`;
|
|
578
|
+
item += `${subIndent} <news:name>${this.escape(entry.news.publication.name)}</news:name>${nl}`;
|
|
579
|
+
item += `${subIndent} <news:language>${this.escape(entry.news.publication.language)}</news:language>${nl}`;
|
|
580
|
+
item += `${subIndent} </news:publication>${nl}`;
|
|
581
|
+
const pubDate = entry.news.publication_date instanceof Date ? entry.news.publication_date : new Date(entry.news.publication_date);
|
|
582
|
+
item += `${subIndent} <news:publication_date>${pubDate.toISOString()}</news:publication_date>${nl}`;
|
|
583
|
+
item += `${subIndent} <news:title>${this.escape(entry.news.title)}</news:title>${nl}`;
|
|
584
|
+
if (entry.news.genres) {
|
|
585
|
+
item += `${subIndent} <news:genres>${this.escape(entry.news.genres)}</news:genres>${nl}`;
|
|
586
|
+
}
|
|
587
|
+
if (entry.news.keywords) {
|
|
588
|
+
item += `${subIndent} <news:keywords>${entry.news.keywords.map((k) => this.escape(k)).join(", ")}</news:keywords>${nl}`;
|
|
589
|
+
}
|
|
590
|
+
item += `${subIndent}</news:news>${nl}`;
|
|
591
|
+
}
|
|
592
|
+
item += `${indent}</url>${nl}`;
|
|
593
|
+
return item;
|
|
594
|
+
}
|
|
595
|
+
hasImages() {
|
|
596
|
+
return this.entries.some((e) => e.images && e.images.length > 0);
|
|
597
|
+
}
|
|
598
|
+
hasVideos() {
|
|
599
|
+
return this.entries.some((e) => e.videos && e.videos.length > 0);
|
|
600
|
+
}
|
|
601
|
+
hasNews() {
|
|
602
|
+
return this.entries.some((e) => !!e.news);
|
|
603
|
+
}
|
|
604
|
+
hasAlternates() {
|
|
605
|
+
return this.entries.some((e) => e.alternates && e.alternates.length > 0);
|
|
606
|
+
}
|
|
607
|
+
escape(str) {
|
|
608
|
+
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
// src/core/SitemapGenerator.ts
|
|
613
|
+
class SitemapGenerator {
|
|
614
|
+
options;
|
|
615
|
+
shadowProcessor = null;
|
|
616
|
+
constructor(options) {
|
|
617
|
+
this.options = {
|
|
618
|
+
maxEntriesPerFile: 50000,
|
|
619
|
+
filename: "sitemap.xml",
|
|
620
|
+
...options
|
|
621
|
+
};
|
|
622
|
+
if (this.options.shadow?.enabled) {
|
|
623
|
+
this.shadowProcessor = new ShadowProcessor({
|
|
624
|
+
storage: this.options.storage,
|
|
625
|
+
mode: this.options.shadow.mode,
|
|
626
|
+
enabled: true
|
|
627
|
+
});
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
async run() {
|
|
631
|
+
let shardIndex = 1;
|
|
632
|
+
let currentCount = 0;
|
|
633
|
+
let currentStream = new SitemapStream({
|
|
634
|
+
baseUrl: this.options.baseUrl,
|
|
635
|
+
pretty: this.options.pretty
|
|
636
|
+
});
|
|
637
|
+
const index = new SitemapIndex({
|
|
638
|
+
baseUrl: this.options.baseUrl,
|
|
639
|
+
pretty: this.options.pretty
|
|
640
|
+
});
|
|
641
|
+
const flushShard = async () => {
|
|
642
|
+
if (currentCount === 0) {
|
|
643
|
+
return;
|
|
644
|
+
}
|
|
645
|
+
const baseName = this.options.filename?.replace(/\.xml$/, "");
|
|
646
|
+
const filename = `${baseName}-${shardIndex}.xml`;
|
|
647
|
+
const xml = currentStream.toXML();
|
|
648
|
+
if (this.shadowProcessor) {
|
|
649
|
+
await this.shadowProcessor.addOperation({ filename, content: xml });
|
|
650
|
+
} else {
|
|
651
|
+
await this.options.storage.write(filename, xml);
|
|
652
|
+
}
|
|
653
|
+
const url = this.options.storage.getUrl(filename);
|
|
654
|
+
index.add({
|
|
655
|
+
url,
|
|
656
|
+
lastmod: new Date
|
|
657
|
+
});
|
|
658
|
+
shardIndex++;
|
|
659
|
+
currentCount = 0;
|
|
660
|
+
currentStream = new SitemapStream({
|
|
661
|
+
baseUrl: this.options.baseUrl,
|
|
662
|
+
pretty: this.options.pretty
|
|
663
|
+
});
|
|
664
|
+
};
|
|
665
|
+
const { providers, maxEntriesPerFile } = this.options;
|
|
666
|
+
for (const provider of providers) {
|
|
667
|
+
const entries = await provider.getEntries();
|
|
668
|
+
const processEntry = async (entry) => {
|
|
669
|
+
currentStream.add(entry);
|
|
670
|
+
currentCount++;
|
|
671
|
+
if (currentCount >= maxEntriesPerFile) {
|
|
672
|
+
await flushShard();
|
|
673
|
+
}
|
|
674
|
+
};
|
|
675
|
+
if (Array.isArray(entries)) {
|
|
676
|
+
for (const entry of entries) {
|
|
677
|
+
await processEntry(entry);
|
|
678
|
+
}
|
|
679
|
+
} else if (entries && typeof entries[Symbol.asyncIterator] === "function") {
|
|
680
|
+
for await (const entry of entries) {
|
|
681
|
+
await processEntry(entry);
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
await flushShard();
|
|
686
|
+
const indexXml = index.toXML();
|
|
687
|
+
if (this.shadowProcessor) {
|
|
688
|
+
await this.shadowProcessor.addOperation({
|
|
689
|
+
filename: this.options.filename,
|
|
690
|
+
content: indexXml
|
|
691
|
+
});
|
|
692
|
+
await this.shadowProcessor.commit();
|
|
693
|
+
} else {
|
|
694
|
+
await this.options.storage.write(this.options.filename, indexXml);
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
getShadowProcessor() {
|
|
698
|
+
return this.shadowProcessor;
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
// src/core/IncrementalGenerator.ts
|
|
703
|
+
class IncrementalGenerator {
|
|
704
|
+
options;
|
|
705
|
+
changeTracker;
|
|
706
|
+
diffCalculator;
|
|
707
|
+
generator;
|
|
708
|
+
constructor(options) {
|
|
709
|
+
this.options = options;
|
|
710
|
+
this.changeTracker = options.changeTracker;
|
|
711
|
+
this.diffCalculator = options.diffCalculator || new DiffCalculator;
|
|
712
|
+
this.generator = new SitemapGenerator(options);
|
|
713
|
+
}
|
|
714
|
+
async generateFull() {
|
|
715
|
+
await this.generator.run();
|
|
716
|
+
if (this.options.autoTrack) {
|
|
717
|
+
const { providers } = this.options;
|
|
718
|
+
for (const provider of providers) {
|
|
719
|
+
const entries = await provider.getEntries();
|
|
720
|
+
const entriesArray = Array.isArray(entries) ? entries : await this.toArray(entries);
|
|
721
|
+
for (const entry of entriesArray) {
|
|
722
|
+
await this.changeTracker.track({
|
|
723
|
+
type: "add",
|
|
724
|
+
url: entry.url,
|
|
725
|
+
entry,
|
|
726
|
+
timestamp: new Date
|
|
727
|
+
});
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
async generateIncremental(since) {
|
|
733
|
+
const changes = await this.changeTracker.getChanges(since);
|
|
734
|
+
if (changes.length === 0) {
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
const baseEntries = await this.loadBaseEntries();
|
|
738
|
+
const diff = this.diffCalculator.calculateFromChanges(baseEntries, changes);
|
|
739
|
+
await this.generateDiff(diff);
|
|
740
|
+
}
|
|
741
|
+
async trackChange(change) {
|
|
742
|
+
await this.changeTracker.track(change);
|
|
743
|
+
}
|
|
744
|
+
async getChanges(since) {
|
|
745
|
+
return this.changeTracker.getChanges(since);
|
|
746
|
+
}
|
|
747
|
+
async loadBaseEntries() {
|
|
748
|
+
const entries = [];
|
|
749
|
+
const { providers } = this.options;
|
|
750
|
+
for (const provider of providers) {
|
|
751
|
+
const providerEntries = await provider.getEntries();
|
|
752
|
+
const entriesArray = Array.isArray(providerEntries) ? providerEntries : await this.toArray(providerEntries);
|
|
753
|
+
entries.push(...entriesArray);
|
|
754
|
+
}
|
|
755
|
+
return entries;
|
|
756
|
+
}
|
|
757
|
+
async generateDiff(_diff) {
|
|
758
|
+
await this.generator.run();
|
|
759
|
+
}
|
|
760
|
+
async toArray(iterable) {
|
|
761
|
+
const array = [];
|
|
762
|
+
for await (const item of iterable) {
|
|
763
|
+
array.push(item);
|
|
764
|
+
}
|
|
765
|
+
return array;
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
// src/core/ProgressTracker.ts
|
|
769
|
+
class ProgressTracker {
|
|
770
|
+
storage;
|
|
771
|
+
updateInterval;
|
|
772
|
+
currentProgress = null;
|
|
773
|
+
updateTimer = null;
|
|
774
|
+
constructor(options) {
|
|
775
|
+
this.storage = options.storage;
|
|
776
|
+
this.updateInterval = options.updateInterval || 1000;
|
|
777
|
+
}
|
|
778
|
+
async init(jobId, total) {
|
|
779
|
+
this.currentProgress = {
|
|
780
|
+
jobId,
|
|
781
|
+
status: "pending",
|
|
782
|
+
total,
|
|
783
|
+
processed: 0,
|
|
784
|
+
percentage: 0,
|
|
785
|
+
startTime: new Date
|
|
786
|
+
};
|
|
787
|
+
await this.storage.set(jobId, this.currentProgress);
|
|
788
|
+
}
|
|
789
|
+
async update(processed, status) {
|
|
790
|
+
if (!this.currentProgress) {
|
|
791
|
+
return;
|
|
792
|
+
}
|
|
793
|
+
this.currentProgress.processed = processed;
|
|
794
|
+
this.currentProgress.percentage = Math.round(processed / this.currentProgress.total * 100);
|
|
795
|
+
if (status) {
|
|
796
|
+
this.currentProgress.status = status;
|
|
797
|
+
} else if (this.currentProgress.status === "pending") {
|
|
798
|
+
this.currentProgress.status = "processing";
|
|
799
|
+
}
|
|
800
|
+
if (!this.updateTimer) {
|
|
801
|
+
this.updateTimer = setInterval(() => {
|
|
802
|
+
this.flush().catch((err) => {
|
|
803
|
+
console.error("[ProgressTracker] Failed to flush progress:", err);
|
|
804
|
+
});
|
|
805
|
+
}, this.updateInterval);
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
async complete() {
|
|
809
|
+
if (!this.currentProgress) {
|
|
810
|
+
return;
|
|
811
|
+
}
|
|
812
|
+
this.currentProgress.status = "completed";
|
|
813
|
+
this.currentProgress.endTime = new Date;
|
|
814
|
+
this.currentProgress.percentage = 100;
|
|
815
|
+
await this.flush();
|
|
816
|
+
this.stop();
|
|
817
|
+
}
|
|
818
|
+
async fail(error) {
|
|
819
|
+
if (!this.currentProgress) {
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
this.currentProgress.status = "failed";
|
|
823
|
+
this.currentProgress.endTime = new Date;
|
|
824
|
+
this.currentProgress.error = error;
|
|
825
|
+
await this.flush();
|
|
826
|
+
this.stop();
|
|
827
|
+
}
|
|
828
|
+
async flush() {
|
|
829
|
+
if (!this.currentProgress) {
|
|
830
|
+
return;
|
|
831
|
+
}
|
|
832
|
+
await this.storage.update(this.currentProgress.jobId, {
|
|
833
|
+
processed: this.currentProgress.processed,
|
|
834
|
+
percentage: this.currentProgress.percentage,
|
|
835
|
+
status: this.currentProgress.status,
|
|
836
|
+
endTime: this.currentProgress.endTime,
|
|
837
|
+
error: this.currentProgress.error
|
|
838
|
+
});
|
|
839
|
+
}
|
|
840
|
+
stop() {
|
|
841
|
+
if (this.updateTimer) {
|
|
842
|
+
clearInterval(this.updateTimer);
|
|
843
|
+
this.updateTimer = null;
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
getCurrentProgress() {
|
|
847
|
+
return this.currentProgress ? { ...this.currentProgress } : null;
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
// src/helpers/I18nSitemap.ts
|
|
851
|
+
function generateI18nEntries(path, locales, baseUrl = "", options = {}) {
|
|
852
|
+
const cleanPath = path.startsWith("/") ? path : `/${path}`;
|
|
853
|
+
const cleanBaseUrl = baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
|
|
854
|
+
const alternates = locales.map((locale) => {
|
|
855
|
+
return {
|
|
856
|
+
lang: locale,
|
|
857
|
+
url: `${cleanBaseUrl}/${locale}${cleanPath}`
|
|
858
|
+
};
|
|
859
|
+
});
|
|
860
|
+
return locales.map((locale) => {
|
|
861
|
+
return {
|
|
862
|
+
...options,
|
|
863
|
+
url: `${cleanBaseUrl}/${locale}${cleanPath}`,
|
|
864
|
+
alternates
|
|
865
|
+
};
|
|
866
|
+
});
|
|
867
|
+
}
|
|
868
|
+
// src/jobs/GenerateSitemapJob.ts
|
|
869
|
+
var import_stream = require("@gravito/stream");
|
|
870
|
+
class GenerateSitemapJob extends import_stream.Job {
|
|
871
|
+
options;
|
|
872
|
+
generator;
|
|
873
|
+
totalEntries = 0;
|
|
874
|
+
processedEntries = 0;
|
|
875
|
+
constructor(options) {
|
|
876
|
+
super();
|
|
877
|
+
this.options = options;
|
|
878
|
+
this.generator = new SitemapGenerator(options.generatorOptions);
|
|
879
|
+
}
|
|
880
|
+
async handle() {
|
|
881
|
+
const { progressTracker, shadowProcessor, onProgress, onComplete, onError } = this.options;
|
|
882
|
+
try {
|
|
883
|
+
if (progressTracker) {
|
|
884
|
+
const total = await this.calculateTotal();
|
|
885
|
+
await progressTracker.init(this.options.jobId, total);
|
|
886
|
+
this.totalEntries = total;
|
|
887
|
+
}
|
|
888
|
+
await this.generateWithProgress();
|
|
889
|
+
if (shadowProcessor) {
|
|
890
|
+
await shadowProcessor.commit();
|
|
891
|
+
}
|
|
892
|
+
if (progressTracker) {
|
|
893
|
+
await progressTracker.complete();
|
|
894
|
+
}
|
|
895
|
+
if (onComplete) {
|
|
896
|
+
onComplete();
|
|
897
|
+
}
|
|
898
|
+
} catch (error) {
|
|
899
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
900
|
+
if (progressTracker) {
|
|
901
|
+
await progressTracker.fail(err.message);
|
|
902
|
+
}
|
|
903
|
+
if (onError) {
|
|
904
|
+
onError(err);
|
|
905
|
+
}
|
|
906
|
+
throw err;
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
async calculateTotal() {
|
|
910
|
+
let total = 0;
|
|
911
|
+
const { providers } = this.options.generatorOptions;
|
|
912
|
+
for (const provider of providers) {
|
|
913
|
+
const entries = await provider.getEntries();
|
|
914
|
+
if (Array.isArray(entries)) {
|
|
915
|
+
total += entries.length;
|
|
916
|
+
} else if (entries && typeof entries[Symbol.asyncIterator] === "function") {
|
|
917
|
+
for await (const _ of entries) {
|
|
918
|
+
total++;
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
return total;
|
|
923
|
+
}
|
|
924
|
+
async generateWithProgress() {
|
|
925
|
+
const { progressTracker, shadowProcessor, onProgress } = this.options;
|
|
926
|
+
const {
|
|
927
|
+
providers,
|
|
928
|
+
maxEntriesPerFile = 50000,
|
|
929
|
+
storage,
|
|
930
|
+
baseUrl,
|
|
931
|
+
pretty,
|
|
932
|
+
filename
|
|
933
|
+
} = this.options.generatorOptions;
|
|
934
|
+
await this.generator.run();
|
|
935
|
+
this.processedEntries = this.totalEntries;
|
|
936
|
+
if (progressTracker) {
|
|
937
|
+
await progressTracker.update(this.processedEntries, "processing");
|
|
938
|
+
}
|
|
939
|
+
if (onProgress) {
|
|
940
|
+
onProgress({
|
|
941
|
+
processed: this.processedEntries,
|
|
942
|
+
total: this.totalEntries,
|
|
943
|
+
percentage: this.totalEntries > 0 ? Math.round(this.processedEntries / this.totalEntries * 100) : 100
|
|
944
|
+
});
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
// src/OrbitSitemap.ts
|
|
949
|
+
var import_node_crypto = require("node:crypto");
|
|
950
|
+
|
|
951
|
+
// src/redirect/RedirectHandler.ts
|
|
952
|
+
class RedirectHandler {
|
|
953
|
+
options;
|
|
954
|
+
constructor(options) {
|
|
955
|
+
this.options = options;
|
|
956
|
+
}
|
|
957
|
+
async processEntries(entries) {
|
|
958
|
+
const { manager, strategy, followChains, maxChainLength } = this.options;
|
|
959
|
+
const _processedEntries = [];
|
|
960
|
+
const redirectMap = new Map;
|
|
961
|
+
for (const entry of entries) {
|
|
962
|
+
const redirectTarget = await manager.resolve(entry.url, followChains, maxChainLength);
|
|
963
|
+
if (redirectTarget && entry.url !== redirectTarget) {
|
|
964
|
+
redirectMap.set(entry.url, {
|
|
965
|
+
from: entry.url,
|
|
966
|
+
to: redirectTarget,
|
|
967
|
+
type: 301
|
|
968
|
+
});
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
switch (strategy) {
|
|
972
|
+
case "remove_old_add_new":
|
|
973
|
+
return this.handleRemoveOldAddNew(entries, redirectMap);
|
|
974
|
+
case "keep_relation":
|
|
975
|
+
return this.handleKeepRelation(entries, redirectMap);
|
|
976
|
+
case "update_url":
|
|
977
|
+
return this.handleUpdateUrl(entries, redirectMap);
|
|
978
|
+
case "dual_mark":
|
|
979
|
+
return this.handleDualMark(entries, redirectMap);
|
|
980
|
+
default:
|
|
981
|
+
return entries;
|
|
982
|
+
}
|
|
983
|
+
}
|
|
984
|
+
handleRemoveOldAddNew(entries, redirectMap) {
|
|
985
|
+
const processed = [];
|
|
986
|
+
const redirectedUrls = new Set;
|
|
987
|
+
for (const entry of entries) {
|
|
988
|
+
const redirect = redirectMap.get(entry.url);
|
|
989
|
+
if (redirect) {
|
|
990
|
+
redirectedUrls.add(entry.url);
|
|
991
|
+
processed.push({
|
|
992
|
+
...entry,
|
|
993
|
+
url: redirect.to,
|
|
994
|
+
redirect: {
|
|
995
|
+
from: redirect.from,
|
|
996
|
+
to: redirect.to,
|
|
997
|
+
type: redirect.type
|
|
998
|
+
}
|
|
999
|
+
});
|
|
1000
|
+
} else if (!redirectedUrls.has(entry.url)) {
|
|
1001
|
+
processed.push(entry);
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
return processed;
|
|
1005
|
+
}
|
|
1006
|
+
handleKeepRelation(entries, redirectMap) {
|
|
1007
|
+
const processed = [];
|
|
1008
|
+
for (const entry of entries) {
|
|
1009
|
+
const redirect = redirectMap.get(entry.url);
|
|
1010
|
+
if (redirect) {
|
|
1011
|
+
processed.push({
|
|
1012
|
+
...entry,
|
|
1013
|
+
redirect: {
|
|
1014
|
+
from: redirect.from,
|
|
1015
|
+
to: redirect.to,
|
|
1016
|
+
type: redirect.type,
|
|
1017
|
+
canonical: redirect.to
|
|
1018
|
+
}
|
|
1019
|
+
});
|
|
1020
|
+
} else {
|
|
1021
|
+
processed.push(entry);
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
return processed;
|
|
1025
|
+
}
|
|
1026
|
+
handleUpdateUrl(entries, redirectMap) {
|
|
1027
|
+
return entries.map((entry) => {
|
|
1028
|
+
const redirect = redirectMap.get(entry.url);
|
|
1029
|
+
if (redirect) {
|
|
1030
|
+
return {
|
|
1031
|
+
...entry,
|
|
1032
|
+
url: redirect.to,
|
|
1033
|
+
redirect: {
|
|
1034
|
+
from: redirect.from,
|
|
1035
|
+
to: redirect.to,
|
|
1036
|
+
type: redirect.type
|
|
1037
|
+
}
|
|
1038
|
+
};
|
|
1039
|
+
}
|
|
1040
|
+
return entry;
|
|
1041
|
+
});
|
|
1042
|
+
}
|
|
1043
|
+
handleDualMark(entries, redirectMap) {
|
|
1044
|
+
const processed = [];
|
|
1045
|
+
const addedUrls = new Set;
|
|
1046
|
+
for (const entry of entries) {
|
|
1047
|
+
const redirect = redirectMap.get(entry.url);
|
|
1048
|
+
if (redirect) {
|
|
1049
|
+
processed.push({
|
|
1050
|
+
...entry,
|
|
1051
|
+
redirect: {
|
|
1052
|
+
from: redirect.from,
|
|
1053
|
+
to: redirect.to,
|
|
1054
|
+
type: redirect.type
|
|
1055
|
+
}
|
|
1056
|
+
});
|
|
1057
|
+
if (!addedUrls.has(redirect.to)) {
|
|
1058
|
+
processed.push({
|
|
1059
|
+
...entry,
|
|
1060
|
+
url: redirect.to,
|
|
1061
|
+
redirect: {
|
|
1062
|
+
from: redirect.from,
|
|
1063
|
+
to: redirect.to,
|
|
1064
|
+
type: redirect.type
|
|
1065
|
+
}
|
|
1066
|
+
});
|
|
1067
|
+
addedUrls.add(redirect.to);
|
|
1068
|
+
}
|
|
1069
|
+
} else {
|
|
1070
|
+
processed.push(entry);
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
return processed;
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
// src/storage/MemorySitemapStorage.ts
|
|
1078
|
+
class MemorySitemapStorage {
|
|
1079
|
+
baseUrl;
|
|
1080
|
+
files = new Map;
|
|
1081
|
+
constructor(baseUrl) {
|
|
1082
|
+
this.baseUrl = baseUrl;
|
|
1083
|
+
}
|
|
1084
|
+
async write(filename, content) {
|
|
1085
|
+
this.files.set(filename, content);
|
|
1086
|
+
}
|
|
1087
|
+
async read(filename) {
|
|
1088
|
+
return this.files.get(filename) || null;
|
|
1089
|
+
}
|
|
1090
|
+
async exists(filename) {
|
|
1091
|
+
return this.files.has(filename);
|
|
1092
|
+
}
|
|
1093
|
+
getUrl(filename) {
|
|
1094
|
+
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
1095
|
+
const file = filename.startsWith("/") ? filename.slice(1) : filename;
|
|
1096
|
+
return `${base}/${file}`;
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
// src/OrbitSitemap.ts
|
|
1101
|
+
class OrbitSitemap {
|
|
1102
|
+
options;
|
|
1103
|
+
mode;
|
|
1104
|
+
constructor(mode, options) {
|
|
1105
|
+
this.mode = mode;
|
|
1106
|
+
this.options = options;
|
|
1107
|
+
}
|
|
1108
|
+
static dynamic(options) {
|
|
1109
|
+
return new OrbitSitemap("dynamic", {
|
|
1110
|
+
path: "/sitemap.xml",
|
|
1111
|
+
...options
|
|
1112
|
+
});
|
|
1113
|
+
}
|
|
1114
|
+
static static(options) {
|
|
1115
|
+
return new OrbitSitemap("static", {
|
|
1116
|
+
filename: "sitemap.xml",
|
|
1117
|
+
...options
|
|
1118
|
+
});
|
|
1119
|
+
}
|
|
1120
|
+
install(core) {
|
|
1121
|
+
if (this.mode === "dynamic") {
|
|
1122
|
+
this.installDynamic(core);
|
|
1123
|
+
} else {
|
|
1124
|
+
console.log("[OrbitSitemap] Static mode configured. Use generate() to build sitemaps.");
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
installDynamic(core) {
|
|
1128
|
+
const opts = this.options;
|
|
1129
|
+
const storage = opts.storage ?? new MemorySitemapStorage(opts.baseUrl);
|
|
1130
|
+
const indexFilename = opts.path?.split("/").pop() ?? "sitemap.xml";
|
|
1131
|
+
const baseDir = opts.path ? opts.path.substring(0, opts.path.lastIndexOf("/")) : undefined;
|
|
1132
|
+
const handler = async (ctx) => {
|
|
1133
|
+
const reqPath = ctx.req.path;
|
|
1134
|
+
const filename = reqPath.split("/").pop() || indexFilename;
|
|
1135
|
+
const isIndex = filename === indexFilename;
|
|
1136
|
+
let content = await storage.read(filename);
|
|
1137
|
+
if (!content && isIndex) {
|
|
1138
|
+
if (opts.lock) {
|
|
1139
|
+
const locked = await opts.lock.acquire(filename, 60);
|
|
1140
|
+
if (!locked) {
|
|
1141
|
+
return ctx.text("Generating...", 503, { "Retry-After": "5" });
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
try {
|
|
1145
|
+
const generator = new SitemapGenerator({
|
|
1146
|
+
...opts,
|
|
1147
|
+
storage,
|
|
1148
|
+
filename: indexFilename
|
|
1149
|
+
});
|
|
1150
|
+
await generator.run();
|
|
1151
|
+
} finally {
|
|
1152
|
+
if (opts.lock) {
|
|
1153
|
+
await opts.lock.release(filename);
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
content = await storage.read(filename);
|
|
1157
|
+
}
|
|
1158
|
+
if (!content) {
|
|
1159
|
+
return ctx.text("Not Found", 404);
|
|
1160
|
+
}
|
|
1161
|
+
return ctx.body(content, 200, {
|
|
1162
|
+
"Content-Type": "application/xml",
|
|
1163
|
+
"Cache-Control": opts.cacheSeconds ? `public, max-age=${opts.cacheSeconds}` : "no-cache"
|
|
1164
|
+
});
|
|
1165
|
+
};
|
|
1166
|
+
core.router.get(opts.path, handler);
|
|
1167
|
+
const basename = indexFilename.replace(".xml", "");
|
|
1168
|
+
const shardRoute = `${baseDir}/${basename}-:shard.xml`;
|
|
1169
|
+
core.router.get(shardRoute, handler);
|
|
1170
|
+
}
|
|
1171
|
+
async generate() {
|
|
1172
|
+
if (this.mode !== "static") {
|
|
1173
|
+
throw new Error("generate() can only be called in static mode");
|
|
1174
|
+
}
|
|
1175
|
+
const opts = this.options;
|
|
1176
|
+
let storage = opts.storage;
|
|
1177
|
+
if (!storage) {
|
|
1178
|
+
const { DiskSitemapStorage: DiskSitemapStorage2 } = await Promise.resolve().then(() => (init_DiskSitemapStorage(), exports_DiskSitemapStorage));
|
|
1179
|
+
storage = new DiskSitemapStorage2(opts.outDir, opts.baseUrl);
|
|
1180
|
+
}
|
|
1181
|
+
let providers = opts.providers;
|
|
1182
|
+
if (opts.redirect?.enabled && opts.redirect.manager) {
|
|
1183
|
+
const handler = new RedirectHandler({
|
|
1184
|
+
manager: opts.redirect.manager,
|
|
1185
|
+
strategy: opts.redirect.strategy || "remove_old_add_new",
|
|
1186
|
+
followChains: opts.redirect.followChains,
|
|
1187
|
+
maxChainLength: opts.redirect.maxChainLength
|
|
1188
|
+
});
|
|
1189
|
+
providers = opts.providers.map((provider) => ({
|
|
1190
|
+
getEntries: async () => {
|
|
1191
|
+
const entries = await provider.getEntries();
|
|
1192
|
+
const entriesArray = Array.isArray(entries) ? entries : await this.toArray(entries);
|
|
1193
|
+
return handler.processEntries(entriesArray);
|
|
1194
|
+
}
|
|
1195
|
+
}));
|
|
1196
|
+
}
|
|
1197
|
+
const generator = new SitemapGenerator({
|
|
1198
|
+
...opts,
|
|
1199
|
+
providers,
|
|
1200
|
+
storage,
|
|
1201
|
+
filename: opts.filename || "sitemap.xml",
|
|
1202
|
+
shadow: opts.shadow
|
|
1203
|
+
});
|
|
1204
|
+
await generator.run();
|
|
1205
|
+
console.log(`[OrbitSitemap] Generated sitemap in ${opts.outDir}`);
|
|
1206
|
+
}
|
|
1207
|
+
async generateIncremental(since) {
|
|
1208
|
+
if (this.mode !== "static") {
|
|
1209
|
+
throw new Error("generateIncremental() can only be called in static mode");
|
|
1210
|
+
}
|
|
1211
|
+
const opts = this.options;
|
|
1212
|
+
if (!opts.incremental?.enabled || !opts.incremental.changeTracker) {
|
|
1213
|
+
throw new Error("Incremental generation is not enabled or changeTracker is not configured");
|
|
1214
|
+
}
|
|
1215
|
+
let storage = opts.storage;
|
|
1216
|
+
if (!storage) {
|
|
1217
|
+
const { DiskSitemapStorage: DiskSitemapStorage2 } = await Promise.resolve().then(() => (init_DiskSitemapStorage(), exports_DiskSitemapStorage));
|
|
1218
|
+
storage = new DiskSitemapStorage2(opts.outDir, opts.baseUrl);
|
|
1219
|
+
}
|
|
1220
|
+
const incrementalGenerator = new IncrementalGenerator({
|
|
1221
|
+
...opts,
|
|
1222
|
+
storage,
|
|
1223
|
+
filename: opts.filename || "sitemap.xml",
|
|
1224
|
+
changeTracker: opts.incremental.changeTracker,
|
|
1225
|
+
autoTrack: opts.incremental.autoTrack,
|
|
1226
|
+
shadow: opts.shadow
|
|
1227
|
+
});
|
|
1228
|
+
await incrementalGenerator.generateIncremental(since);
|
|
1229
|
+
console.log(`[OrbitSitemap] Generated incremental sitemap in ${opts.outDir}`);
|
|
1230
|
+
}
|
|
1231
|
+
async generateAsync(options) {
|
|
1232
|
+
if (this.mode !== "static") {
|
|
1233
|
+
throw new Error("generateAsync() can only be called in static mode");
|
|
1234
|
+
}
|
|
1235
|
+
const opts = this.options;
|
|
1236
|
+
const jobId = import_node_crypto.randomUUID();
|
|
1237
|
+
let storage = opts.storage;
|
|
1238
|
+
if (!storage) {
|
|
1239
|
+
const { DiskSitemapStorage: DiskSitemapStorage2 } = await Promise.resolve().then(() => (init_DiskSitemapStorage(), exports_DiskSitemapStorage));
|
|
1240
|
+
storage = new DiskSitemapStorage2(opts.outDir, opts.baseUrl);
|
|
1241
|
+
}
|
|
1242
|
+
let providers = opts.providers;
|
|
1243
|
+
if (opts.redirect?.enabled && opts.redirect.manager) {
|
|
1244
|
+
const handler = new RedirectHandler({
|
|
1245
|
+
manager: opts.redirect.manager,
|
|
1246
|
+
strategy: opts.redirect.strategy || "remove_old_add_new",
|
|
1247
|
+
followChains: opts.redirect.followChains,
|
|
1248
|
+
maxChainLength: opts.redirect.maxChainLength
|
|
1249
|
+
});
|
|
1250
|
+
providers = opts.providers.map((provider) => ({
|
|
1251
|
+
getEntries: async () => {
|
|
1252
|
+
const entries = await provider.getEntries();
|
|
1253
|
+
const entriesArray = Array.isArray(entries) ? entries : await this.toArray(entries);
|
|
1254
|
+
return handler.processEntries(entriesArray);
|
|
1255
|
+
}
|
|
1256
|
+
}));
|
|
1257
|
+
}
|
|
1258
|
+
let progressTracker;
|
|
1259
|
+
if (opts.progressStorage) {
|
|
1260
|
+
progressTracker = new ProgressTracker({
|
|
1261
|
+
storage: opts.progressStorage
|
|
1262
|
+
});
|
|
1263
|
+
}
|
|
1264
|
+
const job = new GenerateSitemapJob({
|
|
1265
|
+
jobId,
|
|
1266
|
+
generatorOptions: {
|
|
1267
|
+
...opts,
|
|
1268
|
+
providers,
|
|
1269
|
+
storage,
|
|
1270
|
+
filename: opts.filename || "sitemap.xml",
|
|
1271
|
+
shadow: opts.shadow
|
|
1272
|
+
},
|
|
1273
|
+
progressTracker,
|
|
1274
|
+
onProgress: options?.onProgress,
|
|
1275
|
+
onComplete: options?.onComplete,
|
|
1276
|
+
onError: options?.onError
|
|
1277
|
+
});
|
|
1278
|
+
job.handle().catch((error) => {
|
|
1279
|
+
if (options?.onError) {
|
|
1280
|
+
options.onError(error);
|
|
1281
|
+
}
|
|
1282
|
+
});
|
|
1283
|
+
return jobId;
|
|
1284
|
+
}
|
|
1285
|
+
installApiEndpoints(core, basePath = "/admin/sitemap") {
|
|
1286
|
+
const opts = this.options;
|
|
1287
|
+
core.router.post(`${basePath}/generate`, async (ctx) => {
|
|
1288
|
+
try {
|
|
1289
|
+
const body = await ctx.req.json().catch(() => ({}));
|
|
1290
|
+
const jobId = await this.generateAsync({
|
|
1291
|
+
incremental: body.incremental,
|
|
1292
|
+
since: body.since ? new Date(body.since) : undefined
|
|
1293
|
+
});
|
|
1294
|
+
return ctx.json({ jobId, status: "started" });
|
|
1295
|
+
} catch (error) {
|
|
1296
|
+
return ctx.json({ error: error instanceof Error ? error.message : String(error) }, 500);
|
|
1297
|
+
}
|
|
1298
|
+
});
|
|
1299
|
+
core.router.get(`${basePath}/status/:jobId`, async (ctx) => {
|
|
1300
|
+
const jobId = ctx.req.param("jobId");
|
|
1301
|
+
if (!opts.progressStorage) {
|
|
1302
|
+
return ctx.json({ error: "Progress tracking is not enabled" }, 400);
|
|
1303
|
+
}
|
|
1304
|
+
const progress = await opts.progressStorage.get(jobId);
|
|
1305
|
+
if (!progress) {
|
|
1306
|
+
return ctx.json({ error: "Job not found" }, 404);
|
|
1307
|
+
}
|
|
1308
|
+
return ctx.json(progress);
|
|
1309
|
+
});
|
|
1310
|
+
core.router.get(`${basePath}/history`, async (ctx) => {
|
|
1311
|
+
if (!opts.progressStorage) {
|
|
1312
|
+
return ctx.json({ error: "Progress tracking is not enabled" }, 400);
|
|
1313
|
+
}
|
|
1314
|
+
const limit = Number.parseInt(ctx.req.query("limit") || "10", 10);
|
|
1315
|
+
const history = await opts.progressStorage.list(limit);
|
|
1316
|
+
return ctx.json(history);
|
|
1317
|
+
});
|
|
1318
|
+
}
|
|
1319
|
+
async toArray(iterable) {
|
|
1320
|
+
const array = [];
|
|
1321
|
+
for await (const item of iterable) {
|
|
1322
|
+
array.push(item);
|
|
1323
|
+
}
|
|
1324
|
+
return array;
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
// src/providers/RouteScanner.ts
|
|
1328
|
+
function matchGlob(str, pattern) {
|
|
1329
|
+
const regexPattern = pattern.replace(/\./g, "\\.").replace(/\*/g, ".*").replace(/\?/g, ".");
|
|
1330
|
+
const regex = new RegExp(`^${regexPattern}$`);
|
|
1331
|
+
return regex.test(str);
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
class RouteScanner {
|
|
1335
|
+
router;
|
|
1336
|
+
options;
|
|
1337
|
+
constructor(router, options = {}) {
|
|
1338
|
+
this.router = router;
|
|
1339
|
+
this.options = {
|
|
1340
|
+
defaultChangefreq: "weekly",
|
|
1341
|
+
defaultPriority: 0.5,
|
|
1342
|
+
...options
|
|
1343
|
+
};
|
|
1344
|
+
}
|
|
1345
|
+
getEntries() {
|
|
1346
|
+
const entries = [];
|
|
1347
|
+
const routes = this.extractRoutes(this.router);
|
|
1348
|
+
for (const route of routes) {
|
|
1349
|
+
if (route.method !== "GET") {
|
|
1350
|
+
continue;
|
|
1351
|
+
}
|
|
1352
|
+
if (route.path.includes(":") || route.path.includes("*")) {
|
|
1353
|
+
continue;
|
|
1354
|
+
}
|
|
1355
|
+
if (this.shouldInclude(route.path)) {
|
|
1356
|
+
entries.push({
|
|
1357
|
+
url: route.path,
|
|
1358
|
+
changefreq: this.options.defaultChangefreq,
|
|
1359
|
+
priority: this.options.defaultPriority
|
|
1360
|
+
});
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
return entries;
|
|
1364
|
+
}
|
|
1365
|
+
extractRoutes(router) {
|
|
1366
|
+
const routes = [];
|
|
1367
|
+
if (router.routes) {
|
|
1368
|
+
return router.routes;
|
|
1369
|
+
}
|
|
1370
|
+
return routes;
|
|
1371
|
+
}
|
|
1372
|
+
shouldInclude(path2) {
|
|
1373
|
+
if (this.options.exclude) {
|
|
1374
|
+
for (const pattern of this.options.exclude) {
|
|
1375
|
+
if (matchGlob(path2, pattern)) {
|
|
1376
|
+
return false;
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
if (this.options.include) {
|
|
1381
|
+
let matched = false;
|
|
1382
|
+
for (const pattern of this.options.include) {
|
|
1383
|
+
if (matchGlob(path2, pattern)) {
|
|
1384
|
+
matched = true;
|
|
1385
|
+
break;
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
return matched;
|
|
1389
|
+
}
|
|
1390
|
+
return true;
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
function routeScanner(router, options) {
|
|
1394
|
+
return new RouteScanner(router, options);
|
|
1395
|
+
}
|
|
1396
|
+
// src/redirect/RedirectDetector.ts
|
|
1397
|
+
class RedirectDetector {
|
|
1398
|
+
options;
|
|
1399
|
+
cache = new Map;
|
|
1400
|
+
constructor(options) {
|
|
1401
|
+
this.options = options;
|
|
1402
|
+
}
|
|
1403
|
+
async detect(url) {
|
|
1404
|
+
if (this.options.autoDetect?.cache) {
|
|
1405
|
+
const cached = this.cache.get(url);
|
|
1406
|
+
if (cached && cached.expires > Date.now()) {
|
|
1407
|
+
return cached.rule;
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
let rule = null;
|
|
1411
|
+
if (this.options.database?.enabled) {
|
|
1412
|
+
rule = await this.detectFromDatabase(url);
|
|
1413
|
+
if (rule) {
|
|
1414
|
+
this.cacheResult(url, rule);
|
|
1415
|
+
return rule;
|
|
1416
|
+
}
|
|
1417
|
+
}
|
|
1418
|
+
if (this.options.config?.enabled) {
|
|
1419
|
+
rule = await this.detectFromConfig(url);
|
|
1420
|
+
if (rule) {
|
|
1421
|
+
this.cacheResult(url, rule);
|
|
1422
|
+
return rule;
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
if (this.options.autoDetect?.enabled) {
|
|
1426
|
+
rule = await this.detectAuto(url);
|
|
1427
|
+
this.cacheResult(url, rule);
|
|
1428
|
+
return rule;
|
|
1429
|
+
}
|
|
1430
|
+
return null;
|
|
1431
|
+
}
|
|
1432
|
+
async detectBatch(urls) {
|
|
1433
|
+
const results = new Map;
|
|
1434
|
+
const maxConcurrent = this.options.autoDetect?.maxConcurrent || 10;
|
|
1435
|
+
const batches = [];
|
|
1436
|
+
for (let i = 0;i < urls.length; i += maxConcurrent) {
|
|
1437
|
+
batches.push(urls.slice(i, i + maxConcurrent));
|
|
1438
|
+
}
|
|
1439
|
+
for (const batch of batches) {
|
|
1440
|
+
const promises = batch.map((url) => this.detect(url).then((rule) => {
|
|
1441
|
+
results.set(url, rule);
|
|
1442
|
+
}));
|
|
1443
|
+
await Promise.all(promises);
|
|
1444
|
+
}
|
|
1445
|
+
return results;
|
|
1446
|
+
}
|
|
1447
|
+
async detectFromDatabase(url) {
|
|
1448
|
+
const { database } = this.options;
|
|
1449
|
+
if (!database?.enabled) {
|
|
1450
|
+
return null;
|
|
1451
|
+
}
|
|
1452
|
+
try {
|
|
1453
|
+
const { connection, table, columns } = database;
|
|
1454
|
+
const query = `SELECT ${columns.from}, ${columns.to}, ${columns.type} FROM ${table} WHERE ${columns.from} = ? LIMIT 1`;
|
|
1455
|
+
const results = await connection.query(query, [url]);
|
|
1456
|
+
if (results.length === 0) {
|
|
1457
|
+
return null;
|
|
1458
|
+
}
|
|
1459
|
+
const row = results[0];
|
|
1460
|
+
return {
|
|
1461
|
+
from: row[columns.from],
|
|
1462
|
+
to: row[columns.to],
|
|
1463
|
+
type: parseInt(row[columns.type], 10)
|
|
1464
|
+
};
|
|
1465
|
+
} catch {
|
|
1466
|
+
return null;
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
async detectFromConfig(url) {
|
|
1470
|
+
const { config } = this.options;
|
|
1471
|
+
if (!config?.enabled) {
|
|
1472
|
+
return null;
|
|
1473
|
+
}
|
|
1474
|
+
try {
|
|
1475
|
+
const fs2 = await import("node:fs/promises");
|
|
1476
|
+
const data = await fs2.readFile(config.path, "utf-8");
|
|
1477
|
+
const redirects = JSON.parse(data);
|
|
1478
|
+
const rule = redirects.find((r) => r.from === url);
|
|
1479
|
+
return rule || null;
|
|
1480
|
+
} catch {
|
|
1481
|
+
return null;
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
async detectAuto(url) {
|
|
1485
|
+
const { autoDetect, baseUrl } = this.options;
|
|
1486
|
+
if (!autoDetect?.enabled) {
|
|
1487
|
+
return null;
|
|
1488
|
+
}
|
|
1489
|
+
try {
|
|
1490
|
+
const fullUrl = url.startsWith("http") ? url : `${baseUrl}${url}`;
|
|
1491
|
+
const timeout = autoDetect.timeout || 5000;
|
|
1492
|
+
const controller = new AbortController;
|
|
1493
|
+
const timeoutId = setTimeout(() => controller.abort(), timeout);
|
|
1494
|
+
try {
|
|
1495
|
+
const response = await fetch(fullUrl, {
|
|
1496
|
+
method: "HEAD",
|
|
1497
|
+
signal: controller.signal,
|
|
1498
|
+
redirect: "manual"
|
|
1499
|
+
});
|
|
1500
|
+
clearTimeout(timeoutId);
|
|
1501
|
+
if (response.status === 301 || response.status === 302) {
|
|
1502
|
+
const location = response.headers.get("Location");
|
|
1503
|
+
if (location) {
|
|
1504
|
+
return {
|
|
1505
|
+
from: url,
|
|
1506
|
+
to: location,
|
|
1507
|
+
type: response.status
|
|
1508
|
+
};
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
} catch (error) {
|
|
1512
|
+
clearTimeout(timeoutId);
|
|
1513
|
+
if (error.name !== "AbortError") {
|
|
1514
|
+
throw error;
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
} catch {}
|
|
1518
|
+
return null;
|
|
1519
|
+
}
|
|
1520
|
+
cacheResult(url, rule) {
|
|
1521
|
+
if (!this.options.autoDetect?.cache) {
|
|
1522
|
+
return;
|
|
1523
|
+
}
|
|
1524
|
+
const ttl = (this.options.autoDetect.cacheTtl || 3600) * 1000;
|
|
1525
|
+
this.cache.set(url, {
|
|
1526
|
+
rule,
|
|
1527
|
+
expires: Date.now() + ttl
|
|
1528
|
+
});
|
|
1529
|
+
}
|
|
1530
|
+
}
|
|
1531
|
+
// src/redirect/RedirectManager.ts
|
|
1532
|
+
class MemoryRedirectManager {
|
|
1533
|
+
rules = new Map;
|
|
1534
|
+
maxRules;
|
|
1535
|
+
constructor(options = {}) {
|
|
1536
|
+
this.maxRules = options.maxRules || 1e5;
|
|
1537
|
+
}
|
|
1538
|
+
async register(redirect) {
|
|
1539
|
+
this.rules.set(redirect.from, redirect);
|
|
1540
|
+
if (this.rules.size > this.maxRules) {
|
|
1541
|
+
const firstKey = this.rules.keys().next().value;
|
|
1542
|
+
if (firstKey) {
|
|
1543
|
+
this.rules.delete(firstKey);
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
async registerBatch(redirects) {
|
|
1548
|
+
for (const redirect of redirects) {
|
|
1549
|
+
await this.register(redirect);
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
async get(from) {
|
|
1553
|
+
return this.rules.get(from) || null;
|
|
1554
|
+
}
|
|
1555
|
+
async getAll() {
|
|
1556
|
+
return Array.from(this.rules.values());
|
|
1557
|
+
}
|
|
1558
|
+
async resolve(url, followChains = false, maxChainLength = 5) {
|
|
1559
|
+
let current = url;
|
|
1560
|
+
let chainLength = 0;
|
|
1561
|
+
while (chainLength < maxChainLength) {
|
|
1562
|
+
const rule = await this.get(current);
|
|
1563
|
+
if (!rule) {
|
|
1564
|
+
return current;
|
|
1565
|
+
}
|
|
1566
|
+
current = rule.to;
|
|
1567
|
+
chainLength++;
|
|
1568
|
+
if (!followChains) {
|
|
1569
|
+
return current;
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
return current;
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
class RedisRedirectManager {
|
|
1577
|
+
client;
|
|
1578
|
+
keyPrefix;
|
|
1579
|
+
ttl;
|
|
1580
|
+
constructor(options) {
|
|
1581
|
+
this.client = options.client;
|
|
1582
|
+
this.keyPrefix = options.keyPrefix || "sitemap:redirects:";
|
|
1583
|
+
this.ttl = options.ttl;
|
|
1584
|
+
}
|
|
1585
|
+
getKey(from) {
|
|
1586
|
+
return `${this.keyPrefix}${from}`;
|
|
1587
|
+
}
|
|
1588
|
+
getListKey() {
|
|
1589
|
+
return `${this.keyPrefix}list`;
|
|
1590
|
+
}
|
|
1591
|
+
async register(redirect) {
|
|
1592
|
+
const key = this.getKey(redirect.from);
|
|
1593
|
+
const listKey = this.getListKey();
|
|
1594
|
+
const data = JSON.stringify(redirect);
|
|
1595
|
+
if (this.ttl) {
|
|
1596
|
+
await this.client.set(key, data, "EX", this.ttl);
|
|
1597
|
+
} else {
|
|
1598
|
+
await this.client.set(key, data);
|
|
1599
|
+
}
|
|
1600
|
+
await this.client.sadd(listKey, redirect.from);
|
|
1601
|
+
}
|
|
1602
|
+
async registerBatch(redirects) {
|
|
1603
|
+
for (const redirect of redirects) {
|
|
1604
|
+
await this.register(redirect);
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
async get(from) {
|
|
1608
|
+
try {
|
|
1609
|
+
const key = this.getKey(from);
|
|
1610
|
+
const data = await this.client.get(key);
|
|
1611
|
+
if (!data) {
|
|
1612
|
+
return null;
|
|
1613
|
+
}
|
|
1614
|
+
const rule = JSON.parse(data);
|
|
1615
|
+
if (rule.createdAt) {
|
|
1616
|
+
rule.createdAt = new Date(rule.createdAt);
|
|
1617
|
+
}
|
|
1618
|
+
return rule;
|
|
1619
|
+
} catch {
|
|
1620
|
+
return null;
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
async getAll() {
|
|
1624
|
+
try {
|
|
1625
|
+
const listKey = this.getListKey();
|
|
1626
|
+
const froms = await this.client.smembers(listKey);
|
|
1627
|
+
const rules = [];
|
|
1628
|
+
for (const from of froms) {
|
|
1629
|
+
const rule = await this.get(from);
|
|
1630
|
+
if (rule) {
|
|
1631
|
+
rules.push(rule);
|
|
1632
|
+
}
|
|
1633
|
+
}
|
|
1634
|
+
return rules;
|
|
1635
|
+
} catch {
|
|
1636
|
+
return [];
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
async resolve(url, followChains = false, maxChainLength = 5) {
|
|
1640
|
+
let current = url;
|
|
1641
|
+
let chainLength = 0;
|
|
1642
|
+
while (chainLength < maxChainLength) {
|
|
1643
|
+
const rule = await this.get(current);
|
|
1644
|
+
if (!rule) {
|
|
1645
|
+
return current;
|
|
1646
|
+
}
|
|
1647
|
+
current = rule.to;
|
|
1648
|
+
chainLength++;
|
|
1649
|
+
if (!followChains) {
|
|
1650
|
+
return current;
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1653
|
+
return current;
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
// src/index.ts
|
|
1658
|
+
init_DiskSitemapStorage();
|
|
1659
|
+
|
|
1660
|
+
// src/storage/GCPSitemapStorage.ts
|
|
1661
|
+
class GCPSitemapStorage {
|
|
1662
|
+
bucket;
|
|
1663
|
+
prefix;
|
|
1664
|
+
baseUrl;
|
|
1665
|
+
shadowEnabled;
|
|
1666
|
+
shadowMode;
|
|
1667
|
+
storageClient;
|
|
1668
|
+
bucketInstance;
|
|
1669
|
+
constructor(options) {
|
|
1670
|
+
this.bucket = options.bucket;
|
|
1671
|
+
this.prefix = options.prefix || "";
|
|
1672
|
+
this.baseUrl = options.baseUrl || `https://storage.googleapis.com/${options.bucket}`;
|
|
1673
|
+
this.shadowEnabled = options.shadow?.enabled ?? false;
|
|
1674
|
+
this.shadowMode = options.shadow?.mode || "atomic";
|
|
1675
|
+
}
|
|
1676
|
+
async getStorageClient() {
|
|
1677
|
+
if (this.storageClient) {
|
|
1678
|
+
return { client: this.storageClient, bucket: this.bucketInstance };
|
|
1679
|
+
}
|
|
1680
|
+
try {
|
|
1681
|
+
const { Storage } = await import("@google-cloud/storage");
|
|
1682
|
+
const clientOptions = {};
|
|
1683
|
+
if (this.constructor.name === "GCPSitemapStorage") {}
|
|
1684
|
+
this.storageClient = new Storage(clientOptions);
|
|
1685
|
+
this.bucketInstance = this.storageClient.bucket(this.bucket);
|
|
1686
|
+
return { client: this.storageClient, bucket: this.bucketInstance };
|
|
1687
|
+
} catch (error) {
|
|
1688
|
+
throw new Error(`Failed to load Google Cloud Storage. Please install @google-cloud/storage: ${error instanceof Error ? error.message : String(error)}`);
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
getKey(filename) {
|
|
1692
|
+
const cleanPrefix = this.prefix.endsWith("/") ? this.prefix.slice(0, -1) : this.prefix;
|
|
1693
|
+
return cleanPrefix ? `${cleanPrefix}/${filename}` : filename;
|
|
1694
|
+
}
|
|
1695
|
+
async write(filename, content) {
|
|
1696
|
+
const { bucket } = await this.getStorageClient();
|
|
1697
|
+
const key = this.getKey(filename);
|
|
1698
|
+
const file = bucket.file(key);
|
|
1699
|
+
await file.save(content, {
|
|
1700
|
+
contentType: "application/xml",
|
|
1701
|
+
metadata: {
|
|
1702
|
+
cacheControl: "public, max-age=3600"
|
|
1703
|
+
}
|
|
1704
|
+
});
|
|
1705
|
+
}
|
|
1706
|
+
async read(filename) {
|
|
1707
|
+
try {
|
|
1708
|
+
const { bucket } = await this.getStorageClient();
|
|
1709
|
+
const key = this.getKey(filename);
|
|
1710
|
+
const file = bucket.file(key);
|
|
1711
|
+
const [exists] = await file.exists();
|
|
1712
|
+
if (!exists) {
|
|
1713
|
+
return null;
|
|
1714
|
+
}
|
|
1715
|
+
const [content] = await file.download();
|
|
1716
|
+
return content.toString("utf-8");
|
|
1717
|
+
} catch (error) {
|
|
1718
|
+
if (error.code === 404) {
|
|
1719
|
+
return null;
|
|
1720
|
+
}
|
|
1721
|
+
throw error;
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
async exists(filename) {
|
|
1725
|
+
try {
|
|
1726
|
+
const { bucket } = await this.getStorageClient();
|
|
1727
|
+
const key = this.getKey(filename);
|
|
1728
|
+
const file = bucket.file(key);
|
|
1729
|
+
const [exists] = await file.exists();
|
|
1730
|
+
return exists;
|
|
1731
|
+
} catch {
|
|
1732
|
+
return false;
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
getUrl(filename) {
|
|
1736
|
+
const key = this.getKey(filename);
|
|
1737
|
+
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
1738
|
+
return `${base}/${key}`;
|
|
1739
|
+
}
|
|
1740
|
+
async writeShadow(filename, content, shadowId) {
|
|
1741
|
+
if (!this.shadowEnabled) {
|
|
1742
|
+
return this.write(filename, content);
|
|
1743
|
+
}
|
|
1744
|
+
const { bucket } = await this.getStorageClient();
|
|
1745
|
+
const id = shadowId || `shadow-${Date.now()}-${Math.random().toString(36).substring(7)}`;
|
|
1746
|
+
const shadowKey = this.getKey(`${filename}.shadow.${id}`);
|
|
1747
|
+
const file = bucket.file(shadowKey);
|
|
1748
|
+
await file.save(content, {
|
|
1749
|
+
contentType: "application/xml",
|
|
1750
|
+
metadata: {
|
|
1751
|
+
cacheControl: "public, max-age=3600"
|
|
1752
|
+
}
|
|
1753
|
+
});
|
|
1754
|
+
}
|
|
1755
|
+
async commitShadow(shadowId) {
|
|
1756
|
+
if (!this.shadowEnabled) {
|
|
1757
|
+
return;
|
|
1758
|
+
}
|
|
1759
|
+
const { bucket } = await this.getStorageClient();
|
|
1760
|
+
const prefix = this.prefix ? `${this.prefix}/` : "";
|
|
1761
|
+
const [files] = await bucket.getFiles({ prefix });
|
|
1762
|
+
const shadowFiles = files.filter((file) => {
|
|
1763
|
+
const name = file.name;
|
|
1764
|
+
return name.includes(`.shadow.${shadowId}`);
|
|
1765
|
+
});
|
|
1766
|
+
for (const shadowFile of shadowFiles) {
|
|
1767
|
+
const originalKey = shadowFile.name.replace(/\.shadow\.[^/]+$/, "");
|
|
1768
|
+
const _originalFilename = originalKey.replace(prefix, "");
|
|
1769
|
+
if (this.shadowMode === "atomic") {
|
|
1770
|
+
await shadowFile.copy(bucket.file(originalKey));
|
|
1771
|
+
await shadowFile.delete();
|
|
1772
|
+
} else {
|
|
1773
|
+
const version = shadowId;
|
|
1774
|
+
const versionedKey = `${originalKey}.v${version}`;
|
|
1775
|
+
await shadowFile.copy(bucket.file(versionedKey));
|
|
1776
|
+
await shadowFile.copy(bucket.file(originalKey));
|
|
1777
|
+
await shadowFile.delete();
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
async listVersions(filename) {
|
|
1782
|
+
if (this.shadowMode !== "versioned") {
|
|
1783
|
+
return [];
|
|
1784
|
+
}
|
|
1785
|
+
try {
|
|
1786
|
+
const { bucket } = await this.getStorageClient();
|
|
1787
|
+
const key = this.getKey(filename);
|
|
1788
|
+
const prefix = key.replace(/\.xml$/, "");
|
|
1789
|
+
const [files] = await bucket.getFiles({ prefix });
|
|
1790
|
+
const versions = [];
|
|
1791
|
+
for (const file of files) {
|
|
1792
|
+
const match = file.name.match(/\.v([^/]+)$/);
|
|
1793
|
+
if (match) {
|
|
1794
|
+
versions.push(match[1]);
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
return versions.sort();
|
|
1798
|
+
} catch {
|
|
1799
|
+
return [];
|
|
1800
|
+
}
|
|
1801
|
+
}
|
|
1802
|
+
async switchVersion(filename, version) {
|
|
1803
|
+
if (this.shadowMode !== "versioned") {
|
|
1804
|
+
throw new Error("Version switching is only available in versioned mode");
|
|
1805
|
+
}
|
|
1806
|
+
const { bucket } = await this.getStorageClient();
|
|
1807
|
+
const key = this.getKey(filename);
|
|
1808
|
+
const versionedKey = `${key}.v${version}`;
|
|
1809
|
+
const versionedFile = bucket.file(versionedKey);
|
|
1810
|
+
const [exists] = await versionedFile.exists();
|
|
1811
|
+
if (!exists) {
|
|
1812
|
+
throw new Error(`Version ${version} not found for ${filename}`);
|
|
1813
|
+
}
|
|
1814
|
+
await versionedFile.copy(bucket.file(key));
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
// src/storage/MemoryProgressStorage.ts
|
|
1818
|
+
class MemoryProgressStorage {
|
|
1819
|
+
storage = new Map;
|
|
1820
|
+
async get(jobId) {
|
|
1821
|
+
const progress = this.storage.get(jobId);
|
|
1822
|
+
return progress ? { ...progress } : null;
|
|
1823
|
+
}
|
|
1824
|
+
async set(jobId, progress) {
|
|
1825
|
+
this.storage.set(jobId, { ...progress });
|
|
1826
|
+
}
|
|
1827
|
+
async update(jobId, updates) {
|
|
1828
|
+
const existing = this.storage.get(jobId);
|
|
1829
|
+
if (existing) {
|
|
1830
|
+
this.storage.set(jobId, { ...existing, ...updates });
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
async delete(jobId) {
|
|
1834
|
+
this.storage.delete(jobId);
|
|
1835
|
+
}
|
|
1836
|
+
async list(limit) {
|
|
1837
|
+
const all = Array.from(this.storage.values());
|
|
1838
|
+
const sorted = all.sort((a, b) => {
|
|
1839
|
+
const aTime = a.startTime?.getTime() || 0;
|
|
1840
|
+
const bTime = b.startTime?.getTime() || 0;
|
|
1841
|
+
return bTime - aTime;
|
|
1842
|
+
});
|
|
1843
|
+
return limit ? sorted.slice(0, limit) : sorted;
|
|
1844
|
+
}
|
|
1845
|
+
}
|
|
1846
|
+
// src/storage/RedisProgressStorage.ts
|
|
1847
|
+
class RedisProgressStorage {
|
|
1848
|
+
client;
|
|
1849
|
+
keyPrefix;
|
|
1850
|
+
ttl;
|
|
1851
|
+
constructor(options) {
|
|
1852
|
+
this.client = options.client;
|
|
1853
|
+
this.keyPrefix = options.keyPrefix || "sitemap:progress:";
|
|
1854
|
+
this.ttl = options.ttl || 86400;
|
|
1855
|
+
}
|
|
1856
|
+
getKey(jobId) {
|
|
1857
|
+
return `${this.keyPrefix}${jobId}`;
|
|
1858
|
+
}
|
|
1859
|
+
getListKey() {
|
|
1860
|
+
return `${this.keyPrefix}list`;
|
|
1861
|
+
}
|
|
1862
|
+
async get(jobId) {
|
|
1863
|
+
try {
|
|
1864
|
+
const key = this.getKey(jobId);
|
|
1865
|
+
const data = await this.client.get(key);
|
|
1866
|
+
if (!data) {
|
|
1867
|
+
return null;
|
|
1868
|
+
}
|
|
1869
|
+
const progress = JSON.parse(data);
|
|
1870
|
+
if (progress.startTime) {
|
|
1871
|
+
progress.startTime = new Date(progress.startTime);
|
|
1872
|
+
}
|
|
1873
|
+
if (progress.endTime) {
|
|
1874
|
+
progress.endTime = new Date(progress.endTime);
|
|
1875
|
+
}
|
|
1876
|
+
return progress;
|
|
1877
|
+
} catch {
|
|
1878
|
+
return null;
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
async set(jobId, progress) {
|
|
1882
|
+
const key = this.getKey(jobId);
|
|
1883
|
+
const listKey = this.getListKey();
|
|
1884
|
+
const data = JSON.stringify(progress);
|
|
1885
|
+
await this.client.set(key, data, "EX", this.ttl);
|
|
1886
|
+
await this.client.zadd(listKey, Date.now(), jobId);
|
|
1887
|
+
await this.client.expire(listKey, this.ttl);
|
|
1888
|
+
}
|
|
1889
|
+
async update(jobId, updates) {
|
|
1890
|
+
const existing = await this.get(jobId);
|
|
1891
|
+
if (existing) {
|
|
1892
|
+
await this.set(jobId, { ...existing, ...updates });
|
|
1893
|
+
}
|
|
1894
|
+
}
|
|
1895
|
+
async delete(jobId) {
|
|
1896
|
+
const key = this.getKey(jobId);
|
|
1897
|
+
const listKey = this.getListKey();
|
|
1898
|
+
await this.client.del(key);
|
|
1899
|
+
await this.client.zrem(listKey, jobId);
|
|
1900
|
+
}
|
|
1901
|
+
async list(limit) {
|
|
1902
|
+
try {
|
|
1903
|
+
const listKey = this.getListKey();
|
|
1904
|
+
const jobIds = await this.client.zrevrange(listKey, 0, (limit || 100) - 1);
|
|
1905
|
+
const results = [];
|
|
1906
|
+
for (const jobId of jobIds) {
|
|
1907
|
+
const progress = await this.get(jobId);
|
|
1908
|
+
if (progress) {
|
|
1909
|
+
results.push(progress);
|
|
1910
|
+
}
|
|
1911
|
+
}
|
|
1912
|
+
return results;
|
|
1913
|
+
} catch {
|
|
1914
|
+
return [];
|
|
1915
|
+
}
|
|
1916
|
+
}
|
|
1917
|
+
}
|
|
1918
|
+
// src/storage/S3SitemapStorage.ts
|
|
1919
|
+
class S3SitemapStorage {
|
|
1920
|
+
bucket;
|
|
1921
|
+
region;
|
|
1922
|
+
prefix;
|
|
1923
|
+
baseUrl;
|
|
1924
|
+
shadowEnabled;
|
|
1925
|
+
shadowMode;
|
|
1926
|
+
s3Client;
|
|
1927
|
+
constructor(options) {
|
|
1928
|
+
this.bucket = options.bucket;
|
|
1929
|
+
this.region = options.region || "us-east-1";
|
|
1930
|
+
this.prefix = options.prefix || "";
|
|
1931
|
+
this.baseUrl = options.baseUrl || `https://${options.bucket}.s3.${this.region}.amazonaws.com`;
|
|
1932
|
+
this.shadowEnabled = options.shadow?.enabled ?? false;
|
|
1933
|
+
this.shadowMode = options.shadow?.mode || "atomic";
|
|
1934
|
+
}
|
|
1935
|
+
async getS3Client() {
|
|
1936
|
+
if (this.s3Client) {
|
|
1937
|
+
return this.s3Client;
|
|
1938
|
+
}
|
|
1939
|
+
try {
|
|
1940
|
+
const {
|
|
1941
|
+
S3Client,
|
|
1942
|
+
PutObjectCommand,
|
|
1943
|
+
GetObjectCommand,
|
|
1944
|
+
HeadObjectCommand,
|
|
1945
|
+
ListObjectsV2Command,
|
|
1946
|
+
CopyObjectCommand,
|
|
1947
|
+
DeleteObjectCommand
|
|
1948
|
+
} = await import("@aws-sdk/client-s3");
|
|
1949
|
+
const clientOptions = {
|
|
1950
|
+
region: this.region
|
|
1951
|
+
};
|
|
1952
|
+
if (this.constructor.name === "S3SitemapStorage") {}
|
|
1953
|
+
this.s3Client = {
|
|
1954
|
+
S3Client,
|
|
1955
|
+
PutObjectCommand,
|
|
1956
|
+
GetObjectCommand,
|
|
1957
|
+
HeadObjectCommand,
|
|
1958
|
+
ListObjectsV2Command,
|
|
1959
|
+
CopyObjectCommand,
|
|
1960
|
+
DeleteObjectCommand,
|
|
1961
|
+
client: new S3Client(clientOptions)
|
|
1962
|
+
};
|
|
1963
|
+
return this.s3Client;
|
|
1964
|
+
} catch (error) {
|
|
1965
|
+
throw new Error(`Failed to load AWS SDK. Please install @aws-sdk/client-s3: ${error instanceof Error ? error.message : String(error)}`);
|
|
1966
|
+
}
|
|
1967
|
+
}
|
|
1968
|
+
getKey(filename) {
|
|
1969
|
+
const cleanPrefix = this.prefix.endsWith("/") ? this.prefix.slice(0, -1) : this.prefix;
|
|
1970
|
+
return cleanPrefix ? `${cleanPrefix}/${filename}` : filename;
|
|
1971
|
+
}
|
|
1972
|
+
async write(filename, content) {
|
|
1973
|
+
const s3 = await this.getS3Client();
|
|
1974
|
+
const key = this.getKey(filename);
|
|
1975
|
+
await s3.client.send(new s3.PutObjectCommand({
|
|
1976
|
+
Bucket: this.bucket,
|
|
1977
|
+
Key: key,
|
|
1978
|
+
Body: content,
|
|
1979
|
+
ContentType: "application/xml"
|
|
1980
|
+
}));
|
|
1981
|
+
}
|
|
1982
|
+
async read(filename) {
|
|
1983
|
+
try {
|
|
1984
|
+
const s3 = await this.getS3Client();
|
|
1985
|
+
const key = this.getKey(filename);
|
|
1986
|
+
const response = await s3.client.send(new s3.GetObjectCommand({
|
|
1987
|
+
Bucket: this.bucket,
|
|
1988
|
+
Key: key
|
|
1989
|
+
}));
|
|
1990
|
+
if (!response.Body) {
|
|
1991
|
+
return null;
|
|
1992
|
+
}
|
|
1993
|
+
const chunks = [];
|
|
1994
|
+
for await (const chunk of response.Body) {
|
|
1995
|
+
chunks.push(chunk);
|
|
1996
|
+
}
|
|
1997
|
+
const buffer = Buffer.concat(chunks);
|
|
1998
|
+
return buffer.toString("utf-8");
|
|
1999
|
+
} catch (error) {
|
|
2000
|
+
if (error.name === "NoSuchKey" || error.$metadata?.httpStatusCode === 404) {
|
|
2001
|
+
return null;
|
|
2002
|
+
}
|
|
2003
|
+
throw error;
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
async exists(filename) {
|
|
2007
|
+
try {
|
|
2008
|
+
const s3 = await this.getS3Client();
|
|
2009
|
+
const key = this.getKey(filename);
|
|
2010
|
+
await s3.client.send(new s3.HeadObjectCommand({
|
|
2011
|
+
Bucket: this.bucket,
|
|
2012
|
+
Key: key
|
|
2013
|
+
}));
|
|
2014
|
+
return true;
|
|
2015
|
+
} catch (error) {
|
|
2016
|
+
if (error.name === "NotFound" || error.$metadata?.httpStatusCode === 404) {
|
|
2017
|
+
return false;
|
|
2018
|
+
}
|
|
2019
|
+
throw error;
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
getUrl(filename) {
|
|
2023
|
+
const key = this.getKey(filename);
|
|
2024
|
+
const base = this.baseUrl.endsWith("/") ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
2025
|
+
return `${base}/${key}`;
|
|
2026
|
+
}
|
|
2027
|
+
async writeShadow(filename, content, shadowId) {
|
|
2028
|
+
if (!this.shadowEnabled) {
|
|
2029
|
+
return this.write(filename, content);
|
|
2030
|
+
}
|
|
2031
|
+
const s3 = await this.getS3Client();
|
|
2032
|
+
const id = shadowId || `shadow-${Date.now()}-${Math.random().toString(36).substring(7)}`;
|
|
2033
|
+
const shadowKey = this.getKey(`${filename}.shadow.${id}`);
|
|
2034
|
+
await s3.client.send(new s3.PutObjectCommand({
|
|
2035
|
+
Bucket: this.bucket,
|
|
2036
|
+
Key: shadowKey,
|
|
2037
|
+
Body: content,
|
|
2038
|
+
ContentType: "application/xml"
|
|
2039
|
+
}));
|
|
2040
|
+
}
|
|
2041
|
+
async commitShadow(shadowId) {
|
|
2042
|
+
if (!this.shadowEnabled) {
|
|
2043
|
+
return;
|
|
2044
|
+
}
|
|
2045
|
+
const s3 = await this.getS3Client();
|
|
2046
|
+
const prefix = this.prefix ? `${this.prefix}/` : "";
|
|
2047
|
+
const listResponse = await s3.client.send(new s3.ListObjectsV2Command({
|
|
2048
|
+
Bucket: this.bucket,
|
|
2049
|
+
Prefix: prefix
|
|
2050
|
+
}));
|
|
2051
|
+
if (!listResponse.Contents) {
|
|
2052
|
+
return;
|
|
2053
|
+
}
|
|
2054
|
+
const shadowFiles = listResponse.Contents.filter((obj) => obj.Key?.includes(`.shadow.${shadowId}`));
|
|
2055
|
+
for (const shadowFile of shadowFiles) {
|
|
2056
|
+
if (!shadowFile.Key) {
|
|
2057
|
+
continue;
|
|
2058
|
+
}
|
|
2059
|
+
const originalKey = shadowFile.Key.replace(/\.shadow\.[^/]+$/, "");
|
|
2060
|
+
const _originalFilename = originalKey.replace(prefix, "");
|
|
2061
|
+
if (this.shadowMode === "atomic") {
|
|
2062
|
+
await s3.client.send(new s3.CopyObjectCommand({
|
|
2063
|
+
Bucket: this.bucket,
|
|
2064
|
+
CopySource: `${this.bucket}/${shadowFile.Key}`,
|
|
2065
|
+
Key: originalKey,
|
|
2066
|
+
ContentType: "application/xml"
|
|
2067
|
+
}));
|
|
2068
|
+
await s3.client.send(new s3.DeleteObjectCommand({
|
|
2069
|
+
Bucket: this.bucket,
|
|
2070
|
+
Key: shadowFile.Key
|
|
2071
|
+
}));
|
|
2072
|
+
} else {
|
|
2073
|
+
const version = shadowId;
|
|
2074
|
+
const versionedKey = `${originalKey}.v${version}`;
|
|
2075
|
+
await s3.client.send(new s3.CopyObjectCommand({
|
|
2076
|
+
Bucket: this.bucket,
|
|
2077
|
+
CopySource: `${this.bucket}/${shadowFile.Key}`,
|
|
2078
|
+
Key: versionedKey,
|
|
2079
|
+
ContentType: "application/xml"
|
|
2080
|
+
}));
|
|
2081
|
+
await s3.client.send(new s3.CopyObjectCommand({
|
|
2082
|
+
Bucket: this.bucket,
|
|
2083
|
+
CopySource: `${this.bucket}/${shadowFile.Key}`,
|
|
2084
|
+
Key: originalKey,
|
|
2085
|
+
ContentType: "application/xml"
|
|
2086
|
+
}));
|
|
2087
|
+
await s3.client.send(new s3.DeleteObjectCommand({
|
|
2088
|
+
Bucket: this.bucket,
|
|
2089
|
+
Key: shadowFile.Key
|
|
2090
|
+
}));
|
|
2091
|
+
}
|
|
2092
|
+
}
|
|
2093
|
+
}
|
|
2094
|
+
async listVersions(filename) {
|
|
2095
|
+
if (this.shadowMode !== "versioned") {
|
|
2096
|
+
return [];
|
|
2097
|
+
}
|
|
2098
|
+
try {
|
|
2099
|
+
const s3 = await this.getS3Client();
|
|
2100
|
+
const key = this.getKey(filename);
|
|
2101
|
+
const prefix = key.replace(/\.xml$/, "");
|
|
2102
|
+
const listResponse = await s3.client.send(new s3.ListObjectsV2Command({
|
|
2103
|
+
Bucket: this.bucket,
|
|
2104
|
+
Prefix: prefix
|
|
2105
|
+
}));
|
|
2106
|
+
if (!listResponse.Contents) {
|
|
2107
|
+
return [];
|
|
2108
|
+
}
|
|
2109
|
+
const versions = [];
|
|
2110
|
+
for (const obj of listResponse.Contents) {
|
|
2111
|
+
if (!obj.Key) {
|
|
2112
|
+
continue;
|
|
2113
|
+
}
|
|
2114
|
+
const match = obj.Key.match(/\.v([^/]+)$/);
|
|
2115
|
+
if (match) {
|
|
2116
|
+
versions.push(match[1]);
|
|
2117
|
+
}
|
|
2118
|
+
}
|
|
2119
|
+
return versions.sort();
|
|
2120
|
+
} catch {
|
|
2121
|
+
return [];
|
|
2122
|
+
}
|
|
2123
|
+
}
|
|
2124
|
+
async switchVersion(filename, version) {
|
|
2125
|
+
if (this.shadowMode !== "versioned") {
|
|
2126
|
+
throw new Error("Version switching is only available in versioned mode");
|
|
2127
|
+
}
|
|
2128
|
+
const s3 = await this.getS3Client();
|
|
2129
|
+
const key = this.getKey(filename);
|
|
2130
|
+
const versionedKey = `${key}.v${version}`;
|
|
2131
|
+
const exists = await this.exists(versionedKey.replace(this.prefix ? `${this.prefix}/` : "", ""));
|
|
2132
|
+
if (!exists) {
|
|
2133
|
+
throw new Error(`Version ${version} not found for ${filename}`);
|
|
2134
|
+
}
|
|
2135
|
+
await s3.client.send(new s3.CopyObjectCommand({
|
|
2136
|
+
Bucket: this.bucket,
|
|
2137
|
+
CopySource: `${this.bucket}/${versionedKey}`,
|
|
2138
|
+
Key: key,
|
|
2139
|
+
ContentType: "application/xml"
|
|
2140
|
+
}));
|
|
2141
|
+
}
|
|
2142
|
+
}
|