@warlock.js/sitemap 5.15.0 → 5.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +179 -137
- package/cjs/index.cjs +677 -201
- package/cjs/index.cjs.map +1 -1
- package/esm/atomic-publish.mjs +85 -0
- package/esm/atomic-publish.mjs.map +1 -0
- package/esm/atomic-write-file.mjs +50 -0
- package/esm/atomic-write-file.mjs.map +1 -0
- package/esm/duplicate-path-tracker.mjs +36 -0
- package/esm/duplicate-path-tracker.mjs.map +1 -0
- package/esm/errors.d.mts +39 -0
- package/esm/errors.mjs +52 -0
- package/esm/errors.mjs.map +1 -0
- package/esm/index.d.mts +8 -8
- package/esm/index.mjs +6 -6
- package/esm/lastmod.mjs +23 -0
- package/esm/lastmod.mjs.map +1 -0
- package/esm/normalize-entry.mjs +59 -0
- package/esm/normalize-entry.mjs.map +1 -0
- package/esm/route-counter.mjs +19 -0
- package/esm/route-counter.mjs.map +1 -0
- package/esm/shard-name.mjs +28 -0
- package/esm/shard-name.mjs.map +1 -0
- package/esm/sitemap-index-options.mjs +31 -0
- package/esm/sitemap-index-options.mjs.map +1 -0
- package/esm/sitemap-index-types.d.mts +39 -0
- package/esm/sitemap-index-xml.mjs +20 -0
- package/esm/sitemap-index-xml.mjs.map +1 -0
- package/esm/sitemap-index.d.mts +31 -0
- package/esm/sitemap-index.mjs +96 -0
- package/esm/sitemap-index.mjs.map +1 -0
- package/esm/sitemap-shard-writer.mjs +89 -0
- package/esm/sitemap-shard-writer.mjs.map +1 -0
- package/esm/sitemap.d.mts +63 -0
- package/esm/sitemap.mjs +129 -0
- package/esm/sitemap.mjs.map +1 -0
- package/esm/types.d.mts +51 -20
- package/esm/url.d.mts +1 -20
- package/esm/url.mjs +26 -19
- package/esm/url.mjs.map +1 -1
- package/esm/xml.d.mts +15 -4
- package/esm/xml.mjs +30 -8
- package/esm/xml.mjs.map +1 -1
- package/llms-full.txt +139 -159
- package/llms.txt +2 -2
- package/package.json +2 -14
- package/skills/sitemap-overview/SKILL.md +139 -159
- package/esm/collect-entries.d.mts +0 -44
- package/esm/collect-entries.mjs +0 -73
- package/esm/collect-entries.mjs.map +0 -1
- package/esm/diagnostic.d.mts +0 -13
- package/esm/diagnostic.mjs +0 -19
- package/esm/diagnostic.mjs.map +0 -1
- package/esm/routable-page.d.mts +0 -27
- package/esm/sitemap-connector.d.mts +0 -60
- package/esm/sitemap-connector.mjs +0 -117
- package/esm/sitemap-connector.mjs.map +0 -1
package/cjs/index.cjs
CHANGED
|
@@ -1,90 +1,252 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
|
|
2
|
+
let node_fs_promises = require("node:fs/promises");
|
|
3
|
+
let node_path = require("node:path");
|
|
4
|
+
let node_zlib = require("node:zlib");
|
|
2
5
|
|
|
3
|
-
//#region ../sitemap/src/
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
6
|
+
//#region ../sitemap/src/errors.ts
|
|
7
|
+
/**
|
|
8
|
+
* The `baseUrl` given to a `Sitemap` is not an absolute http(s) URL.
|
|
9
|
+
*
|
|
10
|
+
* Thrown from the CONSTRUCTOR: a sitemap that cannot produce a valid URL
|
|
11
|
+
* should not exist, and the mistake belongs at the line that wrote the value
|
|
12
|
+
* rather than at the first request that reads it.
|
|
13
|
+
*/
|
|
14
|
+
var InvalidBaseUrlError = class extends Error {
|
|
15
|
+
constructor(value, reason) {
|
|
16
|
+
super(`Invalid sitemap baseUrl ${JSON.stringify(value)}: ${reason}.`);
|
|
17
|
+
this.name = "InvalidBaseUrlError";
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
/** An entry the sitemap protocol cannot represent. Thrown from `add()`. */
|
|
21
|
+
var InvalidSitemapEntryError = class extends Error {
|
|
22
|
+
constructor(reason) {
|
|
23
|
+
super(`Invalid sitemap entry: ${reason}.`);
|
|
24
|
+
this.name = "InvalidSitemapEntryError";
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Two `SitemapIndex` source keys collide once canonicalised (`en-US` and
|
|
29
|
+
* `en-us` would produce the same filename on a case-insensitive filesystem
|
|
30
|
+
* and silently overwrite one another). Thrown from `addSource()`, not at
|
|
31
|
+
* `saveTo()`, so the mistake is caught at the line that registered it.
|
|
32
|
+
*/
|
|
33
|
+
var DuplicateSourceKeyError = class extends Error {
|
|
34
|
+
constructor(key) {
|
|
35
|
+
super(`Duplicate sitemap source key ${JSON.stringify(key)}: keys collide case-insensitively and would overwrite one another's shard files.`);
|
|
36
|
+
this.name = "DuplicateSourceKeyError";
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* `SitemapIndex.saveTo(outDir)` swaps the ENTIRE `outDir` for a freshly
|
|
41
|
+
* written set (`atomic-publish.ts`). That is safe only when `outDir` is a
|
|
42
|
+
* directory this package already owns — marked by its own
|
|
43
|
+
* `.sitemap-set.json` from a prior publish. A non-empty directory with no
|
|
44
|
+
* marker is presumed to belong to someone else (a caller's `public/`, most
|
|
45
|
+
* dangerously) and is never swapped or deleted; this is thrown instead, from
|
|
46
|
+
* `saveTo()` before anything is written.
|
|
47
|
+
*/
|
|
48
|
+
var UnownedOutputDirectoryError = class extends Error {
|
|
49
|
+
constructor(outDir) {
|
|
50
|
+
super(`Refusing to publish a sitemap set to ${JSON.stringify(outDir)}: this directory already has content but no ".sitemap-set.json" marker from a previous @warlock.js/sitemap publish, so it is not safe to swap or delete. Point saveTo() at a dedicated, sitemap-only directory instead.`);
|
|
51
|
+
this.name = "UnownedOutputDirectoryError";
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
//#endregion
|
|
56
|
+
//#region ../sitemap/src/atomic-publish.ts
|
|
57
|
+
/**
|
|
58
|
+
* Marks a directory as one `publishAtomically` swapped into place. Its
|
|
59
|
+
* presence is the ONLY thing that lets a later publish treat the directory
|
|
60
|
+
* as safe to swap out from under itself — see {@link assertOutDirIsOwned}.
|
|
61
|
+
*/
|
|
62
|
+
const OWNERSHIP_MARKER_FILE = ".sitemap-set.json";
|
|
63
|
+
const OWNERSHIP_MARKER_VERSION = 1;
|
|
64
|
+
async function validateShards(tempDir, fileNames) {
|
|
65
|
+
for (const fileName of fileNames) {
|
|
66
|
+
const info = await (0, node_fs_promises.stat)((0, node_path.join)(tempDir, fileName)).catch(() => void 0);
|
|
67
|
+
if (!info || !info.isFile() || info.size === 0) throw new Error(`sitemap publish aborted: shard "${fileName}" is missing or empty.`);
|
|
68
|
+
}
|
|
19
69
|
}
|
|
20
70
|
/**
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* entry wins, since it was written for that exact path on purpose.
|
|
71
|
+
* `outDir` is safe to take over when it does not exist yet, is empty, or
|
|
72
|
+
* already carries {@link OWNERSHIP_MARKER_FILE} from a previous publish.
|
|
73
|
+
* Anything else — a non-empty directory this package never wrote — is
|
|
74
|
+
* refused rather than swapped or deleted (`de97020e`).
|
|
26
75
|
*/
|
|
27
|
-
function
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
76
|
+
async function assertOutDirIsOwned(outDir) {
|
|
77
|
+
const entries = await (0, node_fs_promises.readdir)(outDir).catch((error) => {
|
|
78
|
+
if (error.code === "ENOENT") return void 0;
|
|
79
|
+
throw error;
|
|
80
|
+
});
|
|
81
|
+
if (entries === void 0 || entries.length === 0) return;
|
|
82
|
+
if (!entries.includes(OWNERSHIP_MARKER_FILE)) throw new UnownedOutputDirectoryError(outDir);
|
|
32
83
|
}
|
|
33
84
|
/**
|
|
34
|
-
*
|
|
35
|
-
*
|
|
85
|
+
* Writes a complete set into a sibling temp directory, validates that every
|
|
86
|
+
* file it named actually exists and is non-empty, then swaps it into
|
|
87
|
+
* `outDir`. A crawler arriving mid-write sees the previous complete set or
|
|
88
|
+
* the new one, never a partial one, and a failed run leaves the previous set
|
|
89
|
+
* untouched and rejects.
|
|
36
90
|
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
* page's own route path.
|
|
41
|
-
* - a dynamic route with no `sitemap` export -> omitted, name collected.
|
|
42
|
-
* - everything else (static routes) -> one entry at the page's own route path.
|
|
43
|
-
*
|
|
44
|
-
* Not-found and error pages are excluded by construction: the caller is
|
|
45
|
-
* expected to hand this only `DiscoveredRoutablePage`-derived entries, and
|
|
46
|
-
* the not-found route is never one of those (`@warlock.js/web`'s discovery
|
|
47
|
-
* reports it as a routable page for the client matcher, but the runtime
|
|
48
|
-
* wiring filters it out before calling here — see the sitemap README).
|
|
91
|
+
* `write` performs the writes and returns the file names (relative to the
|
|
92
|
+
* temp dir) that must be present for the set to be considered valid — it is
|
|
93
|
+
* only known after writing, since shard count depends on what was walked.
|
|
49
94
|
*/
|
|
50
|
-
async function
|
|
51
|
-
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
95
|
+
async function publishAtomically(outDir, write) {
|
|
96
|
+
await assertOutDirIsOwned(outDir);
|
|
97
|
+
const parent = (0, node_path.dirname)(outDir);
|
|
98
|
+
await (0, node_fs_promises.mkdir)(parent, { recursive: true });
|
|
99
|
+
const tempDir = (0, node_path.join)(parent, `.${(0, node_path.basename)(outDir)}.tmp-${process.pid}-${Date.now()}`);
|
|
100
|
+
await (0, node_fs_promises.mkdir)(tempDir, { recursive: true });
|
|
101
|
+
try {
|
|
102
|
+
await validateShards(tempDir, await write(tempDir));
|
|
103
|
+
await (0, node_fs_promises.writeFile)((0, node_path.join)(tempDir, OWNERSHIP_MARKER_FILE), JSON.stringify({
|
|
104
|
+
package: "@warlock.js/sitemap",
|
|
105
|
+
version: OWNERSHIP_MARKER_VERSION
|
|
106
|
+
}), "utf8");
|
|
107
|
+
const displacedDir = (0, node_path.join)(parent, `.${(0, node_path.basename)(outDir)}.previous-${Date.now()}`);
|
|
108
|
+
let displacedPrevious = false;
|
|
109
|
+
try {
|
|
110
|
+
await (0, node_fs_promises.rename)(outDir, displacedDir);
|
|
111
|
+
displacedPrevious = true;
|
|
112
|
+
} catch (error) {
|
|
113
|
+
if (error.code !== "ENOENT") throw error;
|
|
60
114
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
115
|
+
try {
|
|
116
|
+
await (0, node_fs_promises.rename)(tempDir, outDir);
|
|
117
|
+
} catch (error) {
|
|
118
|
+
if (displacedPrevious) await (0, node_fs_promises.rename)(displacedDir, outDir);
|
|
119
|
+
throw error;
|
|
64
120
|
}
|
|
65
|
-
|
|
121
|
+
if (displacedPrevious) await (0, node_fs_promises.rm)(displacedDir, {
|
|
122
|
+
recursive: true,
|
|
123
|
+
force: true
|
|
124
|
+
});
|
|
125
|
+
} catch (error) {
|
|
126
|
+
await (0, node_fs_promises.rm)(tempDir, {
|
|
127
|
+
recursive: true,
|
|
128
|
+
force: true
|
|
129
|
+
});
|
|
130
|
+
throw error;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
//#endregion
|
|
135
|
+
//#region ../sitemap/src/atomic-write-file.ts
|
|
136
|
+
const defaultDeps = {
|
|
137
|
+
writeFile: node_fs_promises.writeFile,
|
|
138
|
+
rename: node_fs_promises.rename,
|
|
139
|
+
rm: node_fs_promises.rm
|
|
140
|
+
};
|
|
141
|
+
const RENAME_RETRY_ATTEMPTS = 5;
|
|
142
|
+
const RENAME_RETRY_DELAY_MS = 20;
|
|
143
|
+
function delay(ms) {
|
|
144
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
145
|
+
}
|
|
146
|
+
/** Windows can report a rename over an existing file as EPERM/EBUSY while something briefly holds the target (an AV scan, a reader) — retrying is correct there, not a masked bug. */
|
|
147
|
+
function isTransientRenameError(error) {
|
|
148
|
+
const code = error?.code;
|
|
149
|
+
return code === "EPERM" || code === "EBUSY";
|
|
150
|
+
}
|
|
151
|
+
async function renameWithRetry(from, to, deps) {
|
|
152
|
+
for (let attempt = 1; attempt <= RENAME_RETRY_ATTEMPTS; attempt++) try {
|
|
153
|
+
await deps.rename(from, to);
|
|
154
|
+
return;
|
|
155
|
+
} catch (error) {
|
|
156
|
+
if (attempt === RENAME_RETRY_ATTEMPTS || !isTransientRenameError(error)) throw error;
|
|
157
|
+
await delay(RENAME_RETRY_DELAY_MS * attempt);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Writes `content` to `filePath` atomically: the content lands in a unique
|
|
162
|
+
* sibling temp file first — same directory, so same filesystem, so the
|
|
163
|
+
* rename that follows is atomic — and only then is renamed over the target.
|
|
164
|
+
* An interrupted or failing write never truncates or otherwise touches the
|
|
165
|
+
* existing target; on any failure the temp file is removed and the error is
|
|
166
|
+
* rethrown.
|
|
167
|
+
*/
|
|
168
|
+
async function atomicWriteFile(filePath, content, deps = defaultDeps) {
|
|
169
|
+
const tempPath = (0, node_path.join)((0, node_path.dirname)(filePath), `.${(0, node_path.basename)(filePath)}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
|
170
|
+
try {
|
|
171
|
+
await deps.writeFile(tempPath, content, "utf8");
|
|
172
|
+
await renameWithRetry(tempPath, filePath, deps);
|
|
173
|
+
} catch (error) {
|
|
174
|
+
await deps.rm(tempPath, { force: true }).catch(() => void 0);
|
|
175
|
+
throw error;
|
|
66
176
|
}
|
|
67
|
-
return {
|
|
68
|
-
entries,
|
|
69
|
-
unresolvedDynamicRoutes
|
|
70
|
-
};
|
|
71
177
|
}
|
|
72
178
|
|
|
73
179
|
//#endregion
|
|
74
|
-
//#region ../sitemap/src/
|
|
180
|
+
//#region ../sitemap/src/lastmod.ts
|
|
75
181
|
/**
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
182
|
+
* Serialises a `lastmod` value.
|
|
183
|
+
*
|
|
184
|
+
* A `Date` becomes W3C datetime, which is what the schema wants and what
|
|
185
|
+
* `toISOString()` already produces. A string is passed through UNTOUCHED: a
|
|
186
|
+
* caller who already holds an ISO string gets it back verbatim rather than
|
|
187
|
+
* having us re-parse it and risk shifting it across a timezone.
|
|
82
188
|
*/
|
|
83
|
-
function
|
|
84
|
-
if (
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
189
|
+
function formatLastmod(value) {
|
|
190
|
+
if (value instanceof Date) {
|
|
191
|
+
if (Number.isNaN(value.getTime())) throw new InvalidSitemapEntryError("lastmod is an invalid Date");
|
|
192
|
+
return value.toISOString();
|
|
193
|
+
}
|
|
194
|
+
if (typeof value !== "string" || value.trim() === "") throw new InvalidSitemapEntryError("lastmod must be a non-empty string or a Date");
|
|
195
|
+
return value;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
//#endregion
|
|
199
|
+
//#region ../sitemap/src/normalize-entry.ts
|
|
200
|
+
const CHANGE_FREQS = [
|
|
201
|
+
"always",
|
|
202
|
+
"hourly",
|
|
203
|
+
"daily",
|
|
204
|
+
"weekly",
|
|
205
|
+
"monthly",
|
|
206
|
+
"yearly",
|
|
207
|
+
"never"
|
|
208
|
+
];
|
|
209
|
+
function assertChangeFreq(value) {
|
|
210
|
+
if (!CHANGE_FREQS.includes(value)) throw new InvalidSitemapEntryError(`changefreq ${JSON.stringify(value)} is not one of ${CHANGE_FREQS.join(", ")}`);
|
|
211
|
+
}
|
|
212
|
+
function assertPriority(value) {
|
|
213
|
+
if (typeof value !== "number" || Number.isNaN(value) || value < 0 || value > 1) throw new InvalidSitemapEntryError(`priority ${JSON.stringify(value)} is outside the protocol range 0.0–1.0`);
|
|
214
|
+
}
|
|
215
|
+
/** Every stored path carries its leading slash, so `/a` and `a` are one entry, not two. */
|
|
216
|
+
function normalizePath(path) {
|
|
217
|
+
if (typeof path !== "string" || path.trim() === "") throw new InvalidSitemapEntryError("path is required and must be a non-empty string");
|
|
218
|
+
if (/^https?:\/\//i.test(path)) return path;
|
|
219
|
+
return path.startsWith("/") ? path : `/${path}`;
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Validates one entry and folds the builder's defaults into it. Defaults are
|
|
223
|
+
* resolved HERE rather than at serialisation time so that `entries()` shows
|
|
224
|
+
* what will actually be emitted — a diagnostic that reports something other
|
|
225
|
+
* than the output is worse than none.
|
|
226
|
+
*/
|
|
227
|
+
function normalizeEntry(entry, defaults) {
|
|
228
|
+
const path = normalizePath(entry.path);
|
|
229
|
+
const changefreq = entry.changefreq ?? defaults.changefreq;
|
|
230
|
+
const priority = entry.priority ?? defaults.priority;
|
|
231
|
+
const lastmod = entry.lastmod ?? defaults.lastmod;
|
|
232
|
+
if (changefreq !== void 0) assertChangeFreq(changefreq);
|
|
233
|
+
if (priority !== void 0) assertPriority(priority);
|
|
234
|
+
const alternates = entry.alternates?.map((alternate) => {
|
|
235
|
+
if (typeof alternate?.hreflang !== "string" || alternate.hreflang.trim() === "") throw new InvalidSitemapEntryError("alternate hreflang is required");
|
|
236
|
+
return {
|
|
237
|
+
hreflang: alternate.hreflang,
|
|
238
|
+
path: normalizePath(alternate.path)
|
|
239
|
+
};
|
|
240
|
+
});
|
|
241
|
+
return {
|
|
242
|
+
path,
|
|
243
|
+
...entry.name !== void 0 ? { name: entry.name } : {},
|
|
244
|
+
...entry.route !== void 0 ? { route: entry.route } : {},
|
|
245
|
+
...lastmod !== void 0 ? { lastmod: formatLastmod(lastmod) } : {},
|
|
246
|
+
...changefreq !== void 0 ? { changefreq } : {},
|
|
247
|
+
...priority !== void 0 ? { priority } : {},
|
|
248
|
+
...alternates !== void 0 ? { alternates } : {}
|
|
249
|
+
};
|
|
88
250
|
}
|
|
89
251
|
|
|
90
252
|
//#endregion
|
|
@@ -98,26 +260,31 @@ function joinOrigin(origin, routePath) {
|
|
|
98
260
|
return `${origin.endsWith("/") ? origin.slice(0, -1) : origin}${routePath.startsWith("/") ? routePath : `/${routePath}`}`;
|
|
99
261
|
}
|
|
100
262
|
/**
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
263
|
+
* Validates a `baseUrl` and returns it without its trailing slash.
|
|
264
|
+
*
|
|
265
|
+
* `new URL()` accepts `mailto:` and `file:` happily, so the protocol is
|
|
266
|
+
* checked explicitly — a sitemap `<loc>` that is not http(s) is not a document
|
|
267
|
+
* any crawler will fetch.
|
|
105
268
|
*/
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
269
|
+
function normalizeBaseUrl(value) {
|
|
270
|
+
if (typeof value !== "string" || value.trim() === "") throw new InvalidBaseUrlError(value, "expected a non-empty string");
|
|
271
|
+
let parsed;
|
|
272
|
+
try {
|
|
273
|
+
parsed = new URL(value);
|
|
274
|
+
} catch {
|
|
275
|
+
throw new InvalidBaseUrlError(value, "not an absolute URL");
|
|
110
276
|
}
|
|
111
|
-
};
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
277
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new InvalidBaseUrlError(value, `unsupported protocol "${parsed.protocol}"`);
|
|
278
|
+
const href = parsed.href;
|
|
279
|
+
return href.endsWith("/") ? href.slice(0, -1) : href;
|
|
280
|
+
}
|
|
281
|
+
/** True for a value already usable as a `<loc>` without joining an origin. */
|
|
282
|
+
function isAbsoluteUrl(value) {
|
|
283
|
+
return /^https?:\/\//i.test(value);
|
|
284
|
+
}
|
|
285
|
+
/** Resolves an entry or alternate path against the base URL, unless it is already absolute. */
|
|
286
|
+
function resolveAgainstBase(baseUrl, path) {
|
|
287
|
+
return isAbsoluteUrl(path) ? path : joinOrigin(baseUrl, path);
|
|
121
288
|
}
|
|
122
289
|
|
|
123
290
|
//#endregion
|
|
@@ -133,147 +300,456 @@ const XML_ESCAPES = {
|
|
|
133
300
|
function escapeXml(value) {
|
|
134
301
|
return value.replace(/[&<>"']/g, (char) => XML_ESCAPES[char] ?? char);
|
|
135
302
|
}
|
|
136
|
-
|
|
137
|
-
|
|
303
|
+
const XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml";
|
|
304
|
+
/**
|
|
305
|
+
* The `xmlns:xhtml` attribute exactly as it appears on `<urlset>`, including
|
|
306
|
+
* its leading space. Exported so the shard writer can charge its byte length
|
|
307
|
+
* against the ceiling without duplicating the string it measures.
|
|
308
|
+
*/
|
|
309
|
+
const XHTML_NAMESPACE_ATTR = ` xmlns:xhtml="${XHTML_NAMESPACE}"`;
|
|
310
|
+
/**
|
|
311
|
+
* Renders one `<url>` block. Exported so the streaming writer can measure the
|
|
312
|
+
* exact bytes it is about to append before deciding whether the byte ceiling
|
|
313
|
+
* forces a new shard — a separate approximation could disagree with what is
|
|
314
|
+
* actually written.
|
|
315
|
+
*/
|
|
316
|
+
function renderUrlBlock(entry, baseUrl) {
|
|
317
|
+
const lines = [` <url>`, ` <loc>${escapeXml(resolveAgainstBase(baseUrl, entry.path))}</loc>`];
|
|
138
318
|
if (entry.lastmod !== void 0) lines.push(` <lastmod>${escapeXml(entry.lastmod)}</lastmod>`);
|
|
139
319
|
if (entry.changefreq !== void 0) lines.push(` <changefreq>${entry.changefreq}</changefreq>`);
|
|
140
320
|
if (entry.priority !== void 0) lines.push(` <priority>${entry.priority}</priority>`);
|
|
321
|
+
for (const alternate of entry.alternates ?? []) {
|
|
322
|
+
const href = escapeXml(resolveAgainstBase(baseUrl, alternate.path));
|
|
323
|
+
lines.push(` <xhtml:link rel="alternate" hreflang="${escapeXml(alternate.hreflang)}" href="${href}"/>`);
|
|
324
|
+
}
|
|
141
325
|
lines.push(` </url>`);
|
|
142
326
|
return lines.join("\n");
|
|
143
327
|
}
|
|
144
328
|
/**
|
|
145
329
|
* Serialises entries into a `urlset` sitemap document — the sitemaps.org
|
|
146
330
|
* namespace, `<url>` per entry, element order `loc` / `lastmod` / `changefreq`
|
|
147
|
-
* / `priority` (schema order; a validator that checks order rejects any other)
|
|
331
|
+
* / `priority` (schema order; a validator that checks order rejects any other),
|
|
332
|
+
* then any `xhtml:link` alternates.
|
|
333
|
+
*
|
|
334
|
+
* The xhtml namespace is declared only when some entry actually carries an
|
|
335
|
+
* alternate — an unused namespace on every single-language sitemap is noise.
|
|
148
336
|
*/
|
|
149
|
-
function buildSitemapXml(entries,
|
|
150
|
-
const
|
|
151
|
-
|
|
337
|
+
function buildSitemapXml(entries, baseUrl) {
|
|
338
|
+
const namespaces = entries.some((entry) => (entry.alternates?.length ?? 0) > 0) ? XHTML_NAMESPACE_ATTR : "";
|
|
339
|
+
const body = entries.map((entry) => renderUrlBlock(entry, baseUrl)).join("\n");
|
|
340
|
+
return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"${namespaces}>\n` + (body.length > 0 ? `${body}\n` : "") + `</urlset>\n`;
|
|
152
341
|
}
|
|
153
342
|
|
|
154
343
|
//#endregion
|
|
155
|
-
//#region ../sitemap/src/sitemap
|
|
156
|
-
/** Default path when `src/config/sitemap.ts` does not set one. */
|
|
157
|
-
const DEFAULT_SITEMAP_PATH = "/sitemap.xml";
|
|
158
|
-
/**
|
|
159
|
-
* Boots after the HTTP connector (`ConnectorPriority.HTTP` is `5`) and after
|
|
160
|
-
* web (`5.5`, `web-connector-factory.ts`) — the route it registers has to
|
|
161
|
-
* land on the same router web's pages already share, and `listRoutablePages`
|
|
162
|
-
* only has a page graph to read once web has scanned it.
|
|
163
|
-
*/
|
|
164
|
-
const SITEMAP_CONNECTOR_PRIORITY = 5.6;
|
|
344
|
+
//#region ../sitemap/src/sitemap.ts
|
|
165
345
|
/**
|
|
166
|
-
*
|
|
167
|
-
* entries
|
|
168
|
-
*
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
* that
|
|
346
|
+
* A bounded sitemap builder: it RETAINS every entry, which is what makes
|
|
347
|
+
* `entries()` and a repeatable `toXML()` possible, and is exactly right up to
|
|
348
|
+
* the sitemaps.org ceiling of 50,000 URLs / 50MB.
|
|
349
|
+
*
|
|
350
|
+
* Above that ceiling this is the wrong tool — the streaming writer retains
|
|
351
|
+
* nothing and emits shards plus an index instead. The two modes are separate
|
|
352
|
+
* on purpose: a class that promised not to retain entries and still offered
|
|
353
|
+
* `entries()` would be lying about one of them.
|
|
173
354
|
*/
|
|
174
|
-
var
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
355
|
+
var Sitemap = class {
|
|
356
|
+
baseUrl;
|
|
357
|
+
defaults;
|
|
358
|
+
/** Keyed by path: a duplicate `<loc>` makes the document invalid, so the later add wins. */
|
|
359
|
+
entriesByPath = /* @__PURE__ */ new Map();
|
|
360
|
+
/** Insertion order per path, so `duplicates()` can name every colliding source. */
|
|
361
|
+
routesByPath = /* @__PURE__ */ new Map();
|
|
362
|
+
/** Declared patterns, including ones that never contributed a URL. */
|
|
363
|
+
declaredRoutes = /* @__PURE__ */ new Set();
|
|
364
|
+
constructor(options) {
|
|
365
|
+
this.baseUrl = normalizeBaseUrl(options?.baseUrl);
|
|
366
|
+
this.defaults = {
|
|
367
|
+
changefreq: options.changefreq,
|
|
368
|
+
priority: options.priority,
|
|
369
|
+
lastmod: options.lastmod
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
add(entry) {
|
|
373
|
+
const resolved = normalizeEntry(entry, this.defaults);
|
|
374
|
+
this.entriesByPath.set(resolved.path, resolved);
|
|
375
|
+
const seen = this.routesByPath.get(resolved.path);
|
|
376
|
+
if (seen) seen.push(resolved.route);
|
|
377
|
+
else this.routesByPath.set(resolved.path, [resolved.route]);
|
|
378
|
+
if (resolved.route !== void 0) this.declaredRoutes.add(resolved.route);
|
|
379
|
+
return this;
|
|
380
|
+
}
|
|
381
|
+
addMany(entries) {
|
|
382
|
+
for (const entry of entries) this.add(entry);
|
|
383
|
+
return this;
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* Names a pattern the caller EXPECTS to contribute URLs, so that one which
|
|
387
|
+
* contributes none shows up in `routes()` as a `count: 0` row instead of as
|
|
388
|
+
* silence. A dynamic route whose supplier returned nothing is the failure
|
|
389
|
+
* this package exists to make visible.
|
|
390
|
+
*/
|
|
391
|
+
declareRoute(route) {
|
|
392
|
+
this.declaredRoutes.add(route);
|
|
393
|
+
return this;
|
|
394
|
+
}
|
|
395
|
+
get size() {
|
|
396
|
+
return this.entriesByPath.size;
|
|
397
|
+
}
|
|
398
|
+
entries() {
|
|
399
|
+
return [...this.entriesByPath.values()];
|
|
400
|
+
}
|
|
401
|
+
routes() {
|
|
402
|
+
const counts = /* @__PURE__ */ new Map();
|
|
403
|
+
for (const route of this.declaredRoutes) counts.set(route, 0);
|
|
404
|
+
for (const entry of this.entriesByPath.values()) {
|
|
405
|
+
if (entry.route === void 0) continue;
|
|
406
|
+
counts.set(entry.route, (counts.get(entry.route) ?? 0) + 1);
|
|
407
|
+
}
|
|
408
|
+
return [...counts].map(([route, count]) => ({
|
|
409
|
+
route,
|
|
410
|
+
count
|
|
411
|
+
}));
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Every path added more than once. The override is silent — it is not
|
|
415
|
+
* hidden: a collision between static discovery and a dynamic supplier is a
|
|
416
|
+
* real defect, and the CALLER decides whether it fails their build. This
|
|
417
|
+
* package reports; it never prints and never throws over a duplicate.
|
|
418
|
+
*/
|
|
419
|
+
duplicates() {
|
|
420
|
+
const reports = [];
|
|
421
|
+
for (const [path, routes] of this.routesByPath) {
|
|
422
|
+
if (routes.length < 2) continue;
|
|
423
|
+
reports.push({
|
|
424
|
+
path,
|
|
425
|
+
count: routes.length,
|
|
426
|
+
routes: [...routes]
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
return reports;
|
|
430
|
+
}
|
|
431
|
+
/** Pure and repeatable: calling it twice returns the same string and mutates nothing. */
|
|
432
|
+
toXML() {
|
|
433
|
+
return buildSitemapXml(this.entries(), this.baseUrl);
|
|
434
|
+
}
|
|
435
|
+
/**
|
|
436
|
+
* Writes the document, creating parent directories so a clean checkout
|
|
437
|
+
* works. The write is atomic: a failed or interrupted publish leaves
|
|
438
|
+
* whatever was already at `filePath` untouched instead of truncating it.
|
|
439
|
+
*/
|
|
440
|
+
async saveTo(filePath) {
|
|
441
|
+
await (0, node_fs_promises.mkdir)((0, node_path.dirname)(filePath), { recursive: true });
|
|
442
|
+
await atomicWriteFile(filePath, this.toXML());
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* Publishes the document as the whole content of `outDir`, exactly the way
|
|
446
|
+
* `SitemapIndex.saveTo` publishes a set: swapped in atomically, and marked
|
|
447
|
+
* as owned. So a site that later outgrows one file can publish an index
|
|
448
|
+
* into the same directory, and a later single file removes stale shards.
|
|
449
|
+
* Refuses a non-empty directory this package did not write
|
|
450
|
+
* (`UnownedOutputDirectoryError`). Returns the published file's path.
|
|
451
|
+
*/
|
|
452
|
+
async publishTo(outDir, fileName = "sitemap.xml") {
|
|
453
|
+
const xml = this.toXML();
|
|
454
|
+
await publishAtomically(outDir, async (tempDir) => {
|
|
455
|
+
await (0, node_fs_promises.writeFile)((0, node_path.join)(tempDir, fileName), xml, "utf8");
|
|
456
|
+
return [fileName];
|
|
457
|
+
});
|
|
458
|
+
return (0, node_path.join)(outDir, fileName);
|
|
178
459
|
}
|
|
179
460
|
};
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
461
|
+
|
|
462
|
+
//#endregion
|
|
463
|
+
//#region ../sitemap/src/sitemap-index-options.ts
|
|
464
|
+
/** The sitemaps.org limits. Never clamped to — a silently clamped option is a lie about what was written. */
|
|
465
|
+
const PROTOCOL_MAX_URLS_PER_FILE = 5e4;
|
|
466
|
+
const PROTOCOL_MAX_BYTES_PER_FILE = 50 * 1024 * 1024;
|
|
467
|
+
/** Validates and folds in defaults, once, at the constructor — the same discipline as `Sitemap`. */
|
|
468
|
+
function normalizeSitemapIndexOptions(options) {
|
|
469
|
+
const baseUrl = normalizeBaseUrl(options?.baseUrl);
|
|
470
|
+
const maxUrlsPerFile = options.maxUrlsPerFile ?? 5e4;
|
|
471
|
+
const maxBytesPerFile = options.maxBytesPerFile ?? 52428800;
|
|
472
|
+
if (!Number.isInteger(maxUrlsPerFile) || maxUrlsPerFile < 1 || maxUrlsPerFile > 5e4) throw new RangeError(`maxUrlsPerFile must be an integer between 1 and the sitemaps.org ceiling of ${PROTOCOL_MAX_URLS_PER_FILE}, got ${JSON.stringify(maxUrlsPerFile)}.`);
|
|
473
|
+
if (!Number.isFinite(maxBytesPerFile) || maxBytesPerFile < 1 || maxBytesPerFile > 52428800) throw new RangeError(`maxBytesPerFile must be between 1 and the sitemaps.org ceiling of ${PROTOCOL_MAX_BYTES_PER_FILE} bytes, got ${JSON.stringify(maxBytesPerFile)}.`);
|
|
184
474
|
return {
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
475
|
+
baseUrl,
|
|
476
|
+
filePrefix: options.filePrefix ?? "sitemap",
|
|
477
|
+
indexFileName: options.indexFileName ?? "sitemap_index.xml",
|
|
478
|
+
gzip: options.gzip ?? false,
|
|
479
|
+
maxUrlsPerFile,
|
|
480
|
+
maxBytesPerFile,
|
|
481
|
+
defaults: {
|
|
482
|
+
changefreq: options.changefreq,
|
|
483
|
+
priority: options.priority,
|
|
484
|
+
lastmod: options.lastmod
|
|
485
|
+
}
|
|
189
486
|
};
|
|
190
487
|
}
|
|
488
|
+
|
|
489
|
+
//#endregion
|
|
490
|
+
//#region ../sitemap/src/shard-name.ts
|
|
491
|
+
const SAFE_KEY_PATTERN = /^[A-Za-z0-9_-]+$/;
|
|
191
492
|
/**
|
|
192
|
-
*
|
|
193
|
-
*
|
|
194
|
-
*
|
|
195
|
-
* `true`), resolves the public origin ONCE — failing loud via
|
|
196
|
-
* {@link resolveOrigin}'s {@link MissingPublicUrlError} rather than falling
|
|
197
|
-
* back to a request-derived host — and registers `GET <config.path>`.
|
|
198
|
-
*
|
|
199
|
-
* The route itself re-reads the page graph on every request via
|
|
200
|
-
* `listRoutablePages()`, not once at boot: the registry can change under
|
|
201
|
-
* `warlock dev`, and a sitemap that only reflects the app's shape at the
|
|
202
|
-
* moment it booted is stale in exactly the way that made `2ede40cf`-class
|
|
203
|
-
* defects expensive.
|
|
493
|
+
* Canonical form used to detect a case collision before it reaches the
|
|
494
|
+
* filesystem: `en-US` and `en-us` would produce the same file on a
|
|
495
|
+
* case-insensitive filesystem and silently overwrite one another.
|
|
204
496
|
*
|
|
205
|
-
*
|
|
206
|
-
*
|
|
207
|
-
* import { sitemapConnector } from "@warlock.js/sitemap";
|
|
208
|
-
*
|
|
209
|
-
* export default defineConfig({ connectors: [sitemapConnector()] });
|
|
497
|
+
* The key reaches the filename and nothing else — it is validated as a
|
|
498
|
+
* filename fragment, not interpreted as a locale or anything else.
|
|
210
499
|
*/
|
|
211
|
-
function
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
500
|
+
function canonicalizeSourceKey(key) {
|
|
501
|
+
if (typeof key !== "string" || !SAFE_KEY_PATTERN.test(key)) throw new RangeError(`sitemap source key ${JSON.stringify(key)} must be a non-empty filename fragment (letters, digits, "-", "_").`);
|
|
502
|
+
return key.toLowerCase();
|
|
503
|
+
}
|
|
504
|
+
/**
|
|
505
|
+
* Stable, zero-padded, ordinal from shard one. A group that later crosses a
|
|
506
|
+
* ceiling GAINS a file; it never renames the first one, so a crawler that has
|
|
507
|
+
* already indexed `sitemap-en-0001.xml` never loses it because the site grew.
|
|
508
|
+
*/
|
|
509
|
+
function shardFileName(filePrefix, key, ordinal, gzip) {
|
|
510
|
+
const paddedOrdinal = String(ordinal).padStart(4, "0");
|
|
511
|
+
const base = key !== void 0 ? `${filePrefix}-${key}-${paddedOrdinal}` : `${filePrefix}-${paddedOrdinal}`;
|
|
512
|
+
return gzip ? `${base}.xml.gz` : `${base}.xml`;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
//#endregion
|
|
516
|
+
//#region ../sitemap/src/sitemap-shard-writer.ts
|
|
517
|
+
function envelopeBytes(baseUrl) {
|
|
518
|
+
return Buffer.byteLength(buildSitemapXml([], baseUrl), "utf8");
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* The extra bytes `<urlset>` gains for `xmlns:xhtml="…"` once a shard holds
|
|
522
|
+
* an entry with alternates. Charged separately from `envelopeBytes()` so the
|
|
523
|
+
* ceiling check can add it exactly once — the moment the first alternate
|
|
524
|
+
* enters the buffer — rather than missing it entirely, which would let a
|
|
525
|
+
* shard's real bytes on disk exceed the ceiling it was rolled against.
|
|
526
|
+
*/
|
|
527
|
+
const NAMESPACE_BYTES = Buffer.byteLength(XHTML_NAMESPACE_ATTR, "utf8");
|
|
528
|
+
async function writeShardFile(ctx, fileName, entries) {
|
|
529
|
+
const xml = buildSitemapXml(entries, ctx.baseUrl);
|
|
530
|
+
const filePath = (0, node_path.join)(ctx.tempDir, fileName);
|
|
531
|
+
if (ctx.gzip) {
|
|
532
|
+
const compressed = (0, node_zlib.gzipSync)(Buffer.from(xml, "utf8"));
|
|
533
|
+
await (0, node_fs_promises.writeFile)(filePath, compressed);
|
|
534
|
+
return { bytes: compressed.byteLength };
|
|
535
|
+
}
|
|
536
|
+
await (0, node_fs_promises.writeFile)(filePath, xml, "utf8");
|
|
537
|
+
return { bytes: Buffer.byteLength(xml, "utf8") };
|
|
538
|
+
}
|
|
539
|
+
/**
|
|
540
|
+
* Walks one group's factories, one at a time, rolling to a new shard on
|
|
541
|
+
* whichever ceiling — URL count or serialised bytes — is hit first. Holds
|
|
542
|
+
* only the current shard's buffer, never the whole group.
|
|
543
|
+
*/
|
|
544
|
+
async function writeShardGroup(group, ctx) {
|
|
545
|
+
const results = [];
|
|
546
|
+
let buffer = [];
|
|
547
|
+
let bufferBytes = envelopeBytes(ctx.baseUrl);
|
|
548
|
+
let bufferHasAlternates = false;
|
|
549
|
+
let ordinal = 1;
|
|
550
|
+
const flush = async () => {
|
|
551
|
+
if (buffer.length === 0) return;
|
|
552
|
+
const fileName = shardFileName(ctx.filePrefix, group.key, ordinal, ctx.gzip);
|
|
553
|
+
const { bytes } = await writeShardFile(ctx, fileName, buffer);
|
|
554
|
+
results.push({
|
|
555
|
+
fileName,
|
|
556
|
+
key: group.key,
|
|
557
|
+
urls: buffer.length,
|
|
558
|
+
bytes,
|
|
559
|
+
gzipped: ctx.gzip
|
|
560
|
+
});
|
|
561
|
+
ordinal += 1;
|
|
562
|
+
buffer = [];
|
|
563
|
+
bufferBytes = envelopeBytes(ctx.baseUrl);
|
|
564
|
+
bufferHasAlternates = false;
|
|
260
565
|
};
|
|
261
|
-
|
|
566
|
+
for (const factory of group.factories) {
|
|
567
|
+
const source = await factory();
|
|
568
|
+
for await (const rawEntry of source) {
|
|
569
|
+
const resolved = normalizeEntry(rawEntry, ctx.defaults);
|
|
570
|
+
if (!ctx.duplicates.attempt(resolved.path, resolved.route)) continue;
|
|
571
|
+
const entryHasAlternates = (resolved.alternates?.length ?? 0) > 0;
|
|
572
|
+
const blockBytes = Buffer.byteLength(renderUrlBlock(resolved, ctx.baseUrl), "utf8") + 1;
|
|
573
|
+
const addedBytes = blockBytes + (entryHasAlternates && !bufferHasAlternates ? NAMESPACE_BYTES : 0);
|
|
574
|
+
const hitsUrlCeiling = buffer.length >= ctx.maxUrlsPerFile;
|
|
575
|
+
const hitsByteCeiling = buffer.length > 0 && bufferBytes + addedBytes > ctx.maxBytesPerFile;
|
|
576
|
+
if (hitsUrlCeiling || hitsByteCeiling) await flush();
|
|
577
|
+
const addsNamespaceNow = entryHasAlternates && !bufferHasAlternates;
|
|
578
|
+
buffer.push(resolved);
|
|
579
|
+
bufferBytes += blockBytes + (addsNamespaceNow ? NAMESPACE_BYTES : 0);
|
|
580
|
+
if (addsNamespaceNow) bufferHasAlternates = true;
|
|
581
|
+
ctx.routes.record(resolved.route);
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
await flush();
|
|
585
|
+
if (results.length === 0) results.push({
|
|
586
|
+
fileName: shardFileName(ctx.filePrefix, group.key, 1, ctx.gzip),
|
|
587
|
+
key: group.key,
|
|
588
|
+
urls: 0,
|
|
589
|
+
bytes: 0,
|
|
590
|
+
gzipped: false
|
|
591
|
+
});
|
|
592
|
+
return results;
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
//#endregion
|
|
596
|
+
//#region ../sitemap/src/sitemap-index-xml.ts
|
|
597
|
+
/**
|
|
598
|
+
* Serialises the flat master `sitemapindex` document: every shard of every
|
|
599
|
+
* group, directly, in the order the groups and shards were produced — no
|
|
600
|
+
* nested per-group indexes, and never a zero-url row (that is a diagnostic
|
|
601
|
+
* for `files`, not something a crawler should be told to fetch).
|
|
602
|
+
*/
|
|
603
|
+
function buildSitemapIndexXml(files, baseUrl) {
|
|
604
|
+
const body = files.filter((file) => file.urls > 0).map((file) => {
|
|
605
|
+
return ` <sitemap>\n <loc>${escapeXml(joinOrigin(baseUrl, file.fileName))}</loc>\n </sitemap>`;
|
|
606
|
+
}).join("\n");
|
|
607
|
+
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n" + (body.length > 0 ? `${body}\n` : "") + `</sitemapindex>\n`;
|
|
262
608
|
}
|
|
263
609
|
|
|
264
610
|
//#endregion
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
611
|
+
//#region ../sitemap/src/duplicate-path-tracker.ts
|
|
612
|
+
/**
|
|
613
|
+
* The streaming equivalent of `Sitemap`'s `routesByPath` map. A streaming
|
|
614
|
+
* writer cannot compare an entry against 500K predecessors, so it keeps only
|
|
615
|
+
* the paths (and contributing routes) it has already seen. A duplicate is
|
|
616
|
+
* SKIPPED here, not last-wins — the earlier one is already on disk.
|
|
617
|
+
*/
|
|
618
|
+
var DuplicatePathTracker = class {
|
|
619
|
+
routesByPath = /* @__PURE__ */ new Map();
|
|
620
|
+
/** Records an attempt to write `path`. Returns `false` when it was already seen — skip it. */
|
|
621
|
+
attempt(path, route) {
|
|
622
|
+
const seen = this.routesByPath.get(path);
|
|
623
|
+
if (seen) {
|
|
624
|
+
seen.push(route);
|
|
625
|
+
return false;
|
|
626
|
+
}
|
|
627
|
+
this.routesByPath.set(path, [route]);
|
|
628
|
+
return true;
|
|
629
|
+
}
|
|
630
|
+
report() {
|
|
631
|
+
const reports = [];
|
|
632
|
+
for (const [path, routes] of this.routesByPath) {
|
|
633
|
+
if (routes.length < 2) continue;
|
|
634
|
+
reports.push({
|
|
635
|
+
path,
|
|
636
|
+
count: routes.length,
|
|
637
|
+
routes: [...routes]
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
return reports;
|
|
641
|
+
}
|
|
642
|
+
};
|
|
643
|
+
|
|
644
|
+
//#endregion
|
|
645
|
+
//#region ../sitemap/src/route-counter.ts
|
|
646
|
+
/** Counts URLs actually written per declared route, across every shard of every group. */
|
|
647
|
+
var RouteCounter = class {
|
|
648
|
+
counts = /* @__PURE__ */ new Map();
|
|
649
|
+
record(route) {
|
|
650
|
+
if (route === void 0) return;
|
|
651
|
+
this.counts.set(route, (this.counts.get(route) ?? 0) + 1);
|
|
652
|
+
}
|
|
653
|
+
summary() {
|
|
654
|
+
return [...this.counts].map(([route, count]) => ({
|
|
655
|
+
route,
|
|
656
|
+
count
|
|
657
|
+
}));
|
|
658
|
+
}
|
|
659
|
+
};
|
|
660
|
+
|
|
661
|
+
//#endregion
|
|
662
|
+
//#region ../sitemap/src/sitemap-index.ts
|
|
663
|
+
/**
|
|
664
|
+
* The streaming path for 300–500K URLs: shards, one flat master index, gzip,
|
|
665
|
+
* and an atomic publish. It is a SECOND mode, not a bigger `Sitemap` — it
|
|
666
|
+
* retains only the current shard's buffer and the set of paths it has seen.
|
|
667
|
+
* There is no `entries()`, `toXML()` or `size`; anyone who can afford those
|
|
668
|
+
* is in `Sitemap` and should be there instead.
|
|
669
|
+
*/
|
|
670
|
+
var SitemapIndex = class {
|
|
671
|
+
options;
|
|
672
|
+
/** Every unnamed `addSource(factory)` call merges into this one group, sharing one shard counter. */
|
|
673
|
+
unnamedFactories = [];
|
|
674
|
+
namedGroups = [];
|
|
675
|
+
/** Canonical (lowercased) keys already registered, so a case collision is caught at `addSource()`. */
|
|
676
|
+
registeredKeys = /* @__PURE__ */ new Set();
|
|
677
|
+
constructor(options) {
|
|
678
|
+
this.options = normalizeSitemapIndexOptions(options);
|
|
679
|
+
}
|
|
680
|
+
addSource(keyOrSource, maybeSource) {
|
|
681
|
+
if (typeof keyOrSource === "string") {
|
|
682
|
+
const canonicalKey = canonicalizeSourceKey(keyOrSource);
|
|
683
|
+
if (this.registeredKeys.has(canonicalKey)) throw new DuplicateSourceKeyError(keyOrSource);
|
|
684
|
+
this.registeredKeys.add(canonicalKey);
|
|
685
|
+
this.namedGroups.push({
|
|
686
|
+
key: keyOrSource,
|
|
687
|
+
factories: [maybeSource]
|
|
688
|
+
});
|
|
689
|
+
} else this.unnamedFactories.push(keyOrSource);
|
|
690
|
+
return this;
|
|
691
|
+
}
|
|
692
|
+
/**
|
|
693
|
+
* Walks every group one at a time — never all at once — into a sibling
|
|
694
|
+
* temp directory, then publishes the whole set atomically. `files` is
|
|
695
|
+
* reported in registration order: the unnamed group first (if any), then
|
|
696
|
+
* named groups key-then-ordinal.
|
|
697
|
+
*/
|
|
698
|
+
async saveTo(outDir) {
|
|
699
|
+
const groups = [...this.unnamedFactories.length > 0 ? [{
|
|
700
|
+
key: void 0,
|
|
701
|
+
factories: this.unnamedFactories
|
|
702
|
+
}] : [], ...this.namedGroups];
|
|
703
|
+
const duplicates = new DuplicatePathTracker();
|
|
704
|
+
const routes = new RouteCounter();
|
|
705
|
+
let shardFiles = [];
|
|
706
|
+
const write = async (tempDir) => {
|
|
707
|
+
shardFiles = [];
|
|
708
|
+
for (const group of groups) {
|
|
709
|
+
const files = await writeShardGroup(group, {
|
|
710
|
+
tempDir,
|
|
711
|
+
baseUrl: this.options.baseUrl,
|
|
712
|
+
filePrefix: this.options.filePrefix,
|
|
713
|
+
gzip: this.options.gzip,
|
|
714
|
+
maxUrlsPerFile: this.options.maxUrlsPerFile,
|
|
715
|
+
maxBytesPerFile: this.options.maxBytesPerFile,
|
|
716
|
+
defaults: this.options.defaults,
|
|
717
|
+
duplicates,
|
|
718
|
+
routes
|
|
719
|
+
});
|
|
720
|
+
shardFiles.push(...files);
|
|
721
|
+
}
|
|
722
|
+
const indexXml = buildSitemapIndexXml(shardFiles, this.options.baseUrl);
|
|
723
|
+
await (0, node_fs_promises.writeFile)((0, node_path.join)(tempDir, this.options.indexFileName), indexXml, "utf8");
|
|
724
|
+
return [...shardFiles.filter((file) => file.urls > 0).map((file) => file.fileName), this.options.indexFileName];
|
|
725
|
+
};
|
|
726
|
+
await publishAtomically(outDir, write);
|
|
727
|
+
const files = shardFiles.map((file) => ({
|
|
728
|
+
path: (0, node_path.join)(outDir, file.fileName),
|
|
729
|
+
...file.key !== void 0 ? { key: file.key } : {},
|
|
730
|
+
urls: file.urls,
|
|
731
|
+
bytes: file.bytes,
|
|
732
|
+
gzipped: file.gzipped
|
|
733
|
+
}));
|
|
734
|
+
return {
|
|
735
|
+
indexPath: (0, node_path.join)(outDir, this.options.indexFileName),
|
|
736
|
+
files,
|
|
737
|
+
totalUrls: files.reduce((total, file) => total + file.urls, 0),
|
|
738
|
+
duplicates: duplicates.report(),
|
|
739
|
+
routes: routes.summary()
|
|
740
|
+
};
|
|
741
|
+
}
|
|
742
|
+
};
|
|
743
|
+
|
|
744
|
+
//#endregion
|
|
745
|
+
exports.DuplicateSourceKeyError = DuplicateSourceKeyError;
|
|
746
|
+
exports.InvalidBaseUrlError = InvalidBaseUrlError;
|
|
747
|
+
exports.InvalidSitemapEntryError = InvalidSitemapEntryError;
|
|
748
|
+
exports.Sitemap = Sitemap;
|
|
749
|
+
exports.SitemapIndex = SitemapIndex;
|
|
750
|
+
exports.UnownedOutputDirectoryError = UnownedOutputDirectoryError;
|
|
269
751
|
exports.buildSitemapXml = buildSitemapXml;
|
|
270
|
-
exports.collectSitemapEntries = collectSitemapEntries;
|
|
271
|
-
exports.describeUnresolvedDynamicRoutes = describeUnresolvedDynamicRoutes;
|
|
272
752
|
exports.escapeXml = escapeXml;
|
|
273
|
-
exports.isDynamicRoutePath = isDynamicRoutePath;
|
|
274
753
|
exports.joinOrigin = joinOrigin;
|
|
275
|
-
exports.
|
|
276
|
-
exports.resolveOrigin = resolveOrigin;
|
|
277
|
-
exports.sitemapConnector = sitemapConnector;
|
|
278
|
-
exports.withDefaults = withDefaults;
|
|
754
|
+
exports.renderUrlBlock = renderUrlBlock;
|
|
279
755
|
//# sourceMappingURL=index.cjs.map
|