@warlock.js/sitemap 5.15.0 → 5.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +179 -137
- package/cjs/index.cjs +677 -201
- package/cjs/index.cjs.map +1 -1
- package/esm/atomic-publish.mjs +85 -0
- package/esm/atomic-publish.mjs.map +1 -0
- package/esm/atomic-write-file.mjs +50 -0
- package/esm/atomic-write-file.mjs.map +1 -0
- package/esm/duplicate-path-tracker.mjs +36 -0
- package/esm/duplicate-path-tracker.mjs.map +1 -0
- package/esm/errors.d.mts +39 -0
- package/esm/errors.mjs +52 -0
- package/esm/errors.mjs.map +1 -0
- package/esm/index.d.mts +8 -8
- package/esm/index.mjs +6 -6
- package/esm/lastmod.mjs +23 -0
- package/esm/lastmod.mjs.map +1 -0
- package/esm/normalize-entry.mjs +59 -0
- package/esm/normalize-entry.mjs.map +1 -0
- package/esm/route-counter.mjs +19 -0
- package/esm/route-counter.mjs.map +1 -0
- package/esm/shard-name.mjs +28 -0
- package/esm/shard-name.mjs.map +1 -0
- package/esm/sitemap-index-options.mjs +31 -0
- package/esm/sitemap-index-options.mjs.map +1 -0
- package/esm/sitemap-index-types.d.mts +39 -0
- package/esm/sitemap-index-xml.mjs +20 -0
- package/esm/sitemap-index-xml.mjs.map +1 -0
- package/esm/sitemap-index.d.mts +31 -0
- package/esm/sitemap-index.mjs +96 -0
- package/esm/sitemap-index.mjs.map +1 -0
- package/esm/sitemap-shard-writer.mjs +89 -0
- package/esm/sitemap-shard-writer.mjs.map +1 -0
- package/esm/sitemap.d.mts +63 -0
- package/esm/sitemap.mjs +129 -0
- package/esm/sitemap.mjs.map +1 -0
- package/esm/types.d.mts +51 -20
- package/esm/url.d.mts +1 -20
- package/esm/url.mjs +26 -19
- package/esm/url.mjs.map +1 -1
- package/esm/xml.d.mts +15 -4
- package/esm/xml.mjs +30 -8
- package/esm/xml.mjs.map +1 -1
- package/llms-full.txt +139 -159
- package/llms.txt +2 -2
- package/package.json +2 -14
- package/skills/sitemap-overview/SKILL.md +139 -159
- package/esm/collect-entries.d.mts +0 -44
- package/esm/collect-entries.mjs +0 -73
- package/esm/collect-entries.mjs.map +0 -1
- package/esm/diagnostic.d.mts +0 -13
- package/esm/diagnostic.mjs +0 -19
- package/esm/diagnostic.mjs.map +0 -1
- package/esm/routable-page.d.mts +0 -27
- package/esm/sitemap-connector.d.mts +0 -60
- package/esm/sitemap-connector.mjs +0 -117
- package/esm/sitemap-connector.mjs.map +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"normalize-entry.mjs","names":[],"sources":["../../../../../../sitemap/src/normalize-entry.ts"],"sourcesContent":["import { InvalidSitemapEntryError } from \"./errors\";\nimport { formatLastmod } from \"./lastmod\";\nimport type { ChangeFreq, ResolvedSitemapEntry, SitemapEntry, SitemapOptions } from \"./types\";\n\nconst CHANGE_FREQS: readonly ChangeFreq[] = [\n \"always\",\n \"hourly\",\n \"daily\",\n \"weekly\",\n \"monthly\",\n \"yearly\",\n \"never\",\n];\n\nfunction assertChangeFreq(value: unknown): asserts value is ChangeFreq {\n if (!CHANGE_FREQS.includes(value as ChangeFreq)) {\n throw new InvalidSitemapEntryError(\n `changefreq ${JSON.stringify(value)} is not one of ${CHANGE_FREQS.join(\", \")}`,\n );\n }\n}\n\nfunction assertPriority(value: unknown): asserts value is number {\n if (typeof value !== \"number\" || Number.isNaN(value) || value < 0 || value > 1) {\n throw new InvalidSitemapEntryError(\n `priority ${JSON.stringify(value)} is outside the protocol range 0.0–1.0`,\n );\n }\n}\n\n/** Every stored path carries its leading slash, so `/a` and `a` are one entry, not two. */\nexport function normalizePath(path: unknown): string {\n if (typeof path !== \"string\" || path.trim() === \"\") {\n throw new InvalidSitemapEntryError(\"path is required and must be a non-empty string\");\n }\n\n // An absolute URL is left alone: the caller is overriding the base origin\n // deliberately, which a cross-origin alternate legitimately needs.\n if (/^https?:\\/\\//i.test(path)) return path;\n\n return path.startsWith(\"/\") ? path : `/${path}`;\n}\n\n/**\n * Validates one entry and folds the builder's defaults into it. Defaults are\n * resolved HERE rather than at serialisation time so that `entries()` shows\n * what will actually be emitted — a diagnostic that reports something other\n * than the output is worse than none.\n */\nexport function normalizeEntry(\n entry: SitemapEntry,\n defaults: Pick<SitemapOptions, \"changefreq\" | \"priority\" | \"lastmod\">,\n): ResolvedSitemapEntry {\n const path = normalizePath(entry.path);\n const changefreq = entry.changefreq ?? defaults.changefreq;\n const priority = entry.priority ?? defaults.priority;\n const lastmod = entry.lastmod ?? defaults.lastmod;\n\n if (changefreq !== undefined) assertChangeFreq(changefreq);\n if (priority !== undefined) assertPriority(priority);\n\n const alternates = entry.alternates?.map((alternate) => {\n if (typeof alternate?.hreflang !== \"string\" || alternate.hreflang.trim() === \"\") {\n throw new InvalidSitemapEntryError(\"alternate hreflang is required\");\n }\n\n return { hreflang: alternate.hreflang, path: normalizePath(alternate.path) };\n });\n\n return {\n path,\n ...(entry.name !== undefined ? { name: entry.name } : {}),\n ...(entry.route !== undefined ? { route: entry.route } : {}),\n ...(lastmod !== undefined ? { lastmod: formatLastmod(lastmod) } : {}),\n ...(changefreq !== undefined ? { changefreq } : {}),\n ...(priority !== undefined ? { priority } : {}),\n ...(alternates !== undefined ? { alternates } : {}),\n };\n}\n"],"mappings":";;;;AAIA,MAAM,eAAsC;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,SAAS,iBAAiB,OAA6C;CACrE,IAAI,CAAC,aAAa,SAAS,KAAmB,GAC5C,MAAM,IAAI,yBACR,cAAc,KAAK,UAAU,KAAK,EAAE,iBAAiB,aAAa,KAAK,IAAI,GAC7E;AAEJ;AAEA,SAAS,eAAe,OAAyC;CAC/D,IAAI,OAAO,UAAU,YAAY,OAAO,MAAM,KAAK,KAAK,QAAQ,KAAK,QAAQ,GAC3E,MAAM,IAAI,yBACR,YAAY,KAAK,UAAU,KAAK,EAAE,uCACpC;AAEJ;;AAGA,SAAgB,cAAc,MAAuB;CACnD,IAAI,OAAO,SAAS,YAAY,KAAK,KAAK,MAAM,IAC9C,MAAM,IAAI,yBAAyB,iDAAiD;CAKtF,IAAI,gBAAgB,KAAK,IAAI,GAAG,OAAO;CAEvC,OAAO,KAAK,WAAW,GAAG,IAAI,OAAO,IAAI;AAC3C;;;;;;;AAQA,SAAgB,eACd,OACA,UACsB;CACtB,MAAM,OAAO,cAAc,MAAM,IAAI;CACrC,MAAM,aAAa,MAAM,cAAc,SAAS;CAChD,MAAM,WAAW,MAAM,YAAY,SAAS;CAC5C,MAAM,UAAU,MAAM,WAAW,SAAS;CAE1C,IAAI,eAAe,QAAW,iBAAiB,UAAU;CACzD,IAAI,aAAa,QAAW,eAAe,QAAQ;CAEnD,MAAM,aAAa,MAAM,YAAY,KAAK,cAAc;EACtD,IAAI,OAAO,WAAW,aAAa,YAAY,UAAU,SAAS,KAAK,MAAM,IAC3E,MAAM,IAAI,yBAAyB,gCAAgC;EAGrE,OAAO;GAAE,UAAU,UAAU;GAAU,MAAM,cAAc,UAAU,IAAI;EAAE;CAC7E,CAAC;CAED,OAAO;EACL;EACA,GAAI,MAAM,SAAS,SAAY,EAAE,MAAM,MAAM,KAAK,IAAI,CAAC;EACvD,GAAI,MAAM,UAAU,SAAY,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;EAC1D,GAAI,YAAY,SAAY,EAAE,SAAS,cAAc,OAAO,EAAE,IAAI,CAAC;EACnE,GAAI,eAAe,SAAY,EAAE,WAAW,IAAI,CAAC;EACjD,GAAI,aAAa,SAAY,EAAE,SAAS,IAAI,CAAC;EAC7C,GAAI,eAAe,SAAY,EAAE,WAAW,IAAI,CAAC;CACnD;AACF"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
//#region ../sitemap/src/route-counter.ts
|
|
2
|
+
/** Counts URLs actually written per declared route, across every shard of every group. */
|
|
3
|
+
var RouteCounter = class {
|
|
4
|
+
counts = /* @__PURE__ */ new Map();
|
|
5
|
+
record(route) {
|
|
6
|
+
if (route === void 0) return;
|
|
7
|
+
this.counts.set(route, (this.counts.get(route) ?? 0) + 1);
|
|
8
|
+
}
|
|
9
|
+
summary() {
|
|
10
|
+
return [...this.counts].map(([route, count]) => ({
|
|
11
|
+
route,
|
|
12
|
+
count
|
|
13
|
+
}));
|
|
14
|
+
}
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
//#endregion
|
|
18
|
+
export { RouteCounter };
|
|
19
|
+
//# sourceMappingURL=route-counter.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"route-counter.mjs","names":[],"sources":["../../../../../../sitemap/src/route-counter.ts"],"sourcesContent":["import type { RouteSummary } from \"./types\";\n\n/** Counts URLs actually written per declared route, across every shard of every group. */\nexport class RouteCounter {\n private readonly counts = new Map<string, number>();\n\n public record(route: string | undefined): void {\n if (route === undefined) return;\n\n this.counts.set(route, (this.counts.get(route) ?? 0) + 1);\n }\n\n public summary(): RouteSummary[] {\n return [...this.counts].map(([route, count]) => ({ route, count }));\n }\n}\n"],"mappings":";;AAGA,IAAa,eAAb,MAA0B;CACxB,AAAiB,yBAAS,IAAI,IAAoB;CAElD,AAAO,OAAO,OAAiC;EAC7C,IAAI,UAAU,QAAW;EAEzB,KAAK,OAAO,IAAI,QAAQ,KAAK,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC;CAC1D;CAEA,AAAO,UAA0B;EAC/B,OAAO,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY;GAAE;GAAO;EAAM,EAAE;CACpE;AACF"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
//#region ../sitemap/src/shard-name.ts
|
|
2
|
+
const SAFE_KEY_PATTERN = /^[A-Za-z0-9_-]+$/;
|
|
3
|
+
/**
|
|
4
|
+
* Canonical form used to detect a case collision before it reaches the
|
|
5
|
+
* filesystem: `en-US` and `en-us` would produce the same file on a
|
|
6
|
+
* case-insensitive filesystem and silently overwrite one another.
|
|
7
|
+
*
|
|
8
|
+
* The key reaches the filename and nothing else — it is validated as a
|
|
9
|
+
* filename fragment, not interpreted as a locale or anything else.
|
|
10
|
+
*/
|
|
11
|
+
function canonicalizeSourceKey(key) {
|
|
12
|
+
if (typeof key !== "string" || !SAFE_KEY_PATTERN.test(key)) throw new RangeError(`sitemap source key ${JSON.stringify(key)} must be a non-empty filename fragment (letters, digits, "-", "_").`);
|
|
13
|
+
return key.toLowerCase();
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Stable, zero-padded, ordinal from shard one. A group that later crosses a
|
|
17
|
+
* ceiling GAINS a file; it never renames the first one, so a crawler that has
|
|
18
|
+
* already indexed `sitemap-en-0001.xml` never loses it because the site grew.
|
|
19
|
+
*/
|
|
20
|
+
function shardFileName(filePrefix, key, ordinal, gzip) {
|
|
21
|
+
const paddedOrdinal = String(ordinal).padStart(4, "0");
|
|
22
|
+
const base = key !== void 0 ? `${filePrefix}-${key}-${paddedOrdinal}` : `${filePrefix}-${paddedOrdinal}`;
|
|
23
|
+
return gzip ? `${base}.xml.gz` : `${base}.xml`;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
//#endregion
|
|
27
|
+
export { canonicalizeSourceKey, shardFileName };
|
|
28
|
+
//# sourceMappingURL=shard-name.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shard-name.mjs","names":[],"sources":["../../../../../../sitemap/src/shard-name.ts"],"sourcesContent":["const SAFE_KEY_PATTERN = /^[A-Za-z0-9_-]+$/;\n\n/**\n * Canonical form used to detect a case collision before it reaches the\n * filesystem: `en-US` and `en-us` would produce the same file on a\n * case-insensitive filesystem and silently overwrite one another.\n *\n * The key reaches the filename and nothing else — it is validated as a\n * filename fragment, not interpreted as a locale or anything else.\n */\nexport function canonicalizeSourceKey(key: string): string {\n if (typeof key !== \"string\" || !SAFE_KEY_PATTERN.test(key)) {\n throw new RangeError(\n `sitemap source key ${JSON.stringify(key)} must be a non-empty filename fragment ` +\n `(letters, digits, \"-\", \"_\").`,\n );\n }\n\n return key.toLowerCase();\n}\n\n/**\n * Stable, zero-padded, ordinal from shard one. A group that later crosses a\n * ceiling GAINS a file; it never renames the first one, so a crawler that has\n * already indexed `sitemap-en-0001.xml` never loses it because the site grew.\n */\nexport function shardFileName(\n filePrefix: string,\n key: string | undefined,\n ordinal: number,\n gzip: boolean,\n): string {\n const paddedOrdinal = String(ordinal).padStart(4, \"0\");\n const base =\n key !== undefined ? `${filePrefix}-${key}-${paddedOrdinal}` : `${filePrefix}-${paddedOrdinal}`;\n\n return gzip ? `${base}.xml.gz` : `${base}.xml`;\n}\n"],"mappings":";AAAA,MAAM,mBAAmB;;;;;;;;;AAUzB,SAAgB,sBAAsB,KAAqB;CACzD,IAAI,OAAO,QAAQ,YAAY,CAAC,iBAAiB,KAAK,GAAG,GACvD,MAAM,IAAI,WACR,sBAAsB,KAAK,UAAU,GAAG,EAAE,oEAE5C;CAGF,OAAO,IAAI,YAAY;AACzB;;;;;;AAOA,SAAgB,cACd,YACA,KACA,SACA,MACQ;CACR,MAAM,gBAAgB,OAAO,OAAO,CAAC,CAAC,SAAS,GAAG,GAAG;CACrD,MAAM,OACJ,QAAQ,SAAY,GAAG,WAAW,GAAG,IAAI,GAAG,kBAAkB,GAAG,WAAW,GAAG;CAEjF,OAAO,OAAO,GAAG,KAAK,WAAW,GAAG,KAAK;AAC3C"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { normalizeBaseUrl } from "./url.mjs";
|
|
2
|
+
|
|
3
|
+
//#region ../sitemap/src/sitemap-index-options.ts
|
|
4
|
+
/** The sitemaps.org limits. Never clamped to — a silently clamped option is a lie about what was written. */
|
|
5
|
+
const PROTOCOL_MAX_URLS_PER_FILE = 5e4;
|
|
6
|
+
const PROTOCOL_MAX_BYTES_PER_FILE = 50 * 1024 * 1024;
|
|
7
|
+
/** Validates and folds in defaults, once, at the constructor — the same discipline as `Sitemap`. */
|
|
8
|
+
function normalizeSitemapIndexOptions(options) {
|
|
9
|
+
const baseUrl = normalizeBaseUrl(options?.baseUrl);
|
|
10
|
+
const maxUrlsPerFile = options.maxUrlsPerFile ?? 5e4;
|
|
11
|
+
const maxBytesPerFile = options.maxBytesPerFile ?? 52428800;
|
|
12
|
+
if (!Number.isInteger(maxUrlsPerFile) || maxUrlsPerFile < 1 || maxUrlsPerFile > 5e4) throw new RangeError(`maxUrlsPerFile must be an integer between 1 and the sitemaps.org ceiling of ${PROTOCOL_MAX_URLS_PER_FILE}, got ${JSON.stringify(maxUrlsPerFile)}.`);
|
|
13
|
+
if (!Number.isFinite(maxBytesPerFile) || maxBytesPerFile < 1 || maxBytesPerFile > 52428800) throw new RangeError(`maxBytesPerFile must be between 1 and the sitemaps.org ceiling of ${PROTOCOL_MAX_BYTES_PER_FILE} bytes, got ${JSON.stringify(maxBytesPerFile)}.`);
|
|
14
|
+
return {
|
|
15
|
+
baseUrl,
|
|
16
|
+
filePrefix: options.filePrefix ?? "sitemap",
|
|
17
|
+
indexFileName: options.indexFileName ?? "sitemap_index.xml",
|
|
18
|
+
gzip: options.gzip ?? false,
|
|
19
|
+
maxUrlsPerFile,
|
|
20
|
+
maxBytesPerFile,
|
|
21
|
+
defaults: {
|
|
22
|
+
changefreq: options.changefreq,
|
|
23
|
+
priority: options.priority,
|
|
24
|
+
lastmod: options.lastmod
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
//#endregion
|
|
30
|
+
export { normalizeSitemapIndexOptions };
|
|
31
|
+
//# sourceMappingURL=sitemap-index-options.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sitemap-index-options.mjs","names":[],"sources":["../../../../../../sitemap/src/sitemap-index-options.ts"],"sourcesContent":["import type { SitemapOptions } from \"./types\";\nimport { normalizeBaseUrl } from \"./url\";\nimport type { SitemapIndexOptions } from \"./sitemap-index-types\";\n\n/** The sitemaps.org limits. Never clamped to — a silently clamped option is a lie about what was written. */\nexport const PROTOCOL_MAX_URLS_PER_FILE = 50_000;\nexport const PROTOCOL_MAX_BYTES_PER_FILE = 50 * 1024 * 1024;\n\nexport type ResolvedSitemapIndexOptions = {\n readonly baseUrl: string;\n readonly filePrefix: string;\n readonly indexFileName: string;\n readonly gzip: boolean;\n readonly maxUrlsPerFile: number;\n readonly maxBytesPerFile: number;\n readonly defaults: Pick<SitemapOptions, \"changefreq\" | \"priority\" | \"lastmod\">;\n};\n\n/** Validates and folds in defaults, once, at the constructor — the same discipline as `Sitemap`. */\nexport function normalizeSitemapIndexOptions(\n options: SitemapIndexOptions,\n): ResolvedSitemapIndexOptions {\n const baseUrl = normalizeBaseUrl(options?.baseUrl);\n\n const maxUrlsPerFile = options.maxUrlsPerFile ?? PROTOCOL_MAX_URLS_PER_FILE;\n const maxBytesPerFile = options.maxBytesPerFile ?? PROTOCOL_MAX_BYTES_PER_FILE;\n\n if (\n !Number.isInteger(maxUrlsPerFile) ||\n maxUrlsPerFile < 1 ||\n maxUrlsPerFile > PROTOCOL_MAX_URLS_PER_FILE\n ) {\n throw new RangeError(\n `maxUrlsPerFile must be an integer between 1 and the sitemaps.org ceiling of ` +\n `${PROTOCOL_MAX_URLS_PER_FILE}, got ${JSON.stringify(maxUrlsPerFile)}.`,\n );\n }\n\n if (\n !Number.isFinite(maxBytesPerFile) ||\n maxBytesPerFile < 1 ||\n maxBytesPerFile > PROTOCOL_MAX_BYTES_PER_FILE\n ) {\n throw new RangeError(\n `maxBytesPerFile must be between 1 and the sitemaps.org ceiling of ` +\n `${PROTOCOL_MAX_BYTES_PER_FILE} bytes, got ${JSON.stringify(maxBytesPerFile)}.`,\n );\n }\n\n return {\n baseUrl,\n filePrefix: options.filePrefix ?? \"sitemap\",\n indexFileName: options.indexFileName ?? \"sitemap_index.xml\",\n gzip: options.gzip ?? false,\n maxUrlsPerFile,\n maxBytesPerFile,\n defaults: {\n changefreq: options.changefreq,\n priority: options.priority,\n lastmod: options.lastmod,\n },\n };\n}\n"],"mappings":";;;;AAKA,MAAa,6BAA6B;AAC1C,MAAa,8BAA8B,KAAK,OAAO;;AAavD,SAAgB,6BACd,SAC6B;CAC7B,MAAM,UAAU,iBAAiB,SAAS,OAAO;CAEjD,MAAM,iBAAiB,QAAQ;CAC/B,MAAM,kBAAkB,QAAQ;CAEhC,IACE,CAAC,OAAO,UAAU,cAAc,KAChC,iBAAiB,KACjB,sBAEA,MAAM,IAAI,WACR,+EACK,2BAA2B,QAAQ,KAAK,UAAU,cAAc,EAAE,EACzE;CAGF,IACE,CAAC,OAAO,SAAS,eAAe,KAChC,kBAAkB,KAClB,4BAEA,MAAM,IAAI,WACR,qEACK,4BAA4B,cAAc,KAAK,UAAU,eAAe,EAAE,EACjF;CAGF,OAAO;EACL;EACA,YAAY,QAAQ,cAAc;EAClC,eAAe,QAAQ,iBAAiB;EACxC,MAAM,QAAQ,QAAQ;EACtB;EACA;EACA,UAAU;GACR,YAAY,QAAQ;GACpB,UAAU,QAAQ;GAClB,SAAS,QAAQ;EACnB;CACF;AACF"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { ChangeFreq, DuplicateReport, RouteSummary, SitemapEntry } from "./types.mjs";
|
|
2
|
+
|
|
3
|
+
//#region ../sitemap/src/sitemap-index-types.d.ts
|
|
4
|
+
/** What a source factory produces: one walk over a group's entries. */
|
|
5
|
+
type SitemapSource = Iterable<SitemapEntry> | AsyncIterable<SitemapEntry>;
|
|
6
|
+
/**
|
|
7
|
+
* A FACTORY, not a bare iterable. An async iterable can only be walked once,
|
|
8
|
+
* so it cannot survive a retry, a second `saveTo()`, or a failed run that has
|
|
9
|
+
* to rebuild the whole set — the factory can simply be called again.
|
|
10
|
+
*/
|
|
11
|
+
type SitemapSourceFactory = () => SitemapSource | Promise<SitemapSource>;
|
|
12
|
+
type SitemapIndexOptions = {
|
|
13
|
+
/** Absolute origin, validated exactly as `Sitemap` validates it. */readonly baseUrl: string; /** Shard file name prefix. Default `sitemap`, giving `sitemap-0001.xml`. */
|
|
14
|
+
readonly filePrefix?: string; /** Index file name. Default `sitemap_index.xml`. */
|
|
15
|
+
readonly indexFileName?: string; /** Write `.xml.gz` beside each shard and point the index at it. Default false. */
|
|
16
|
+
readonly gzip?: boolean; /** Hard ceiling per shard. Default 50_000; never accepted above the protocol ceiling. */
|
|
17
|
+
readonly maxUrlsPerFile?: number; /** Hard ceiling per shard in bytes, uncompressed. Default 50 * 1024 * 1024. */
|
|
18
|
+
readonly maxBytesPerFile?: number;
|
|
19
|
+
readonly changefreq?: ChangeFreq;
|
|
20
|
+
readonly priority?: number;
|
|
21
|
+
readonly lastmod?: string | Date;
|
|
22
|
+
};
|
|
23
|
+
type SitemapSetResult = {
|
|
24
|
+
readonly indexPath: string;
|
|
25
|
+
readonly files: readonly SitemapFileResult[];
|
|
26
|
+
readonly totalUrls: number;
|
|
27
|
+
readonly duplicates: readonly DuplicateReport[];
|
|
28
|
+
readonly routes: readonly RouteSummary[];
|
|
29
|
+
};
|
|
30
|
+
type SitemapFileResult = {
|
|
31
|
+
readonly path: string; /** The source key this shard came from, when the source was named. */
|
|
32
|
+
readonly key?: string;
|
|
33
|
+
readonly urls: number;
|
|
34
|
+
readonly bytes: number;
|
|
35
|
+
readonly gzipped: boolean;
|
|
36
|
+
};
|
|
37
|
+
//#endregion
|
|
38
|
+
export { SitemapFileResult, SitemapIndexOptions, SitemapSetResult, SitemapSource, SitemapSourceFactory };
|
|
39
|
+
//# sourceMappingURL=sitemap-index-types.d.mts.map
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { joinOrigin } from "./url.mjs";
|
|
2
|
+
import { escapeXml } from "./xml.mjs";
|
|
3
|
+
|
|
4
|
+
//#region ../sitemap/src/sitemap-index-xml.ts
|
|
5
|
+
/**
|
|
6
|
+
* Serialises the flat master `sitemapindex` document: every shard of every
|
|
7
|
+
* group, directly, in the order the groups and shards were produced — no
|
|
8
|
+
* nested per-group indexes, and never a zero-url row (that is a diagnostic
|
|
9
|
+
* for `files`, not something a crawler should be told to fetch).
|
|
10
|
+
*/
|
|
11
|
+
function buildSitemapIndexXml(files, baseUrl) {
|
|
12
|
+
const body = files.filter((file) => file.urls > 0).map((file) => {
|
|
13
|
+
return ` <sitemap>\n <loc>${escapeXml(joinOrigin(baseUrl, file.fileName))}</loc>\n </sitemap>`;
|
|
14
|
+
}).join("\n");
|
|
15
|
+
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n" + (body.length > 0 ? `${body}\n` : "") + `</sitemapindex>\n`;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
//#endregion
|
|
19
|
+
export { buildSitemapIndexXml };
|
|
20
|
+
//# sourceMappingURL=sitemap-index-xml.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sitemap-index-xml.mjs","names":[],"sources":["../../../../../../sitemap/src/sitemap-index-xml.ts"],"sourcesContent":["import { escapeXml } from \"./xml\";\nimport { joinOrigin } from \"./url\";\nimport type { ShardFile } from \"./sitemap-shard-writer\";\n\n/**\n * Serialises the flat master `sitemapindex` document: every shard of every\n * group, directly, in the order the groups and shards were produced — no\n * nested per-group indexes, and never a zero-url row (that is a diagnostic\n * for `files`, not something a crawler should be told to fetch).\n */\nexport function buildSitemapIndexXml(files: readonly ShardFile[], baseUrl: string): string {\n const body = files\n .filter((file) => file.urls > 0)\n .map((file) => {\n const loc = escapeXml(joinOrigin(baseUrl, file.fileName));\n\n return ` <sitemap>\\n <loc>${loc}</loc>\\n </sitemap>`;\n })\n .join(\"\\n\");\n\n return (\n `<?xml version=\"1.0\" encoding=\"UTF-8\"?>\\n` +\n `<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\\n` +\n (body.length > 0 ? `${body}\\n` : \"\") +\n `</sitemapindex>\\n`\n );\n}\n"],"mappings":";;;;;;;;;;AAUA,SAAgB,qBAAqB,OAA6B,SAAyB;CACzF,MAAM,OAAO,MACV,QAAQ,SAAS,KAAK,OAAO,CAAC,CAAC,CAC/B,KAAK,SAAS;EAGb,OAAO,yBAFK,UAAU,WAAW,SAAS,KAAK,QAAQ,CAErB,EAAE;CACtC,CAAC,CAAC,CACD,KAAK,IAAI;CAEZ,OACE,wHAEC,KAAK,SAAS,IAAI,GAAG,KAAK,MAAM,MACjC;AAEJ"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { SitemapIndexOptions, SitemapSetResult, SitemapSourceFactory } from "./sitemap-index-types.mjs";
|
|
2
|
+
|
|
3
|
+
//#region ../sitemap/src/sitemap-index.d.ts
|
|
4
|
+
/**
|
|
5
|
+
* The streaming path for 300–500K URLs: shards, one flat master index, gzip,
|
|
6
|
+
* and an atomic publish. It is a SECOND mode, not a bigger `Sitemap` — it
|
|
7
|
+
* retains only the current shard's buffer and the set of paths it has seen.
|
|
8
|
+
* There is no `entries()`, `toXML()` or `size`; anyone who can afford those
|
|
9
|
+
* is in `Sitemap` and should be there instead.
|
|
10
|
+
*/
|
|
11
|
+
declare class SitemapIndex {
|
|
12
|
+
private readonly options;
|
|
13
|
+
/** Every unnamed `addSource(factory)` call merges into this one group, sharing one shard counter. */
|
|
14
|
+
private readonly unnamedFactories;
|
|
15
|
+
private readonly namedGroups;
|
|
16
|
+
/** Canonical (lowercased) keys already registered, so a case collision is caught at `addSource()`. */
|
|
17
|
+
private readonly registeredKeys;
|
|
18
|
+
constructor(options: SitemapIndexOptions);
|
|
19
|
+
addSource(source: SitemapSourceFactory): this;
|
|
20
|
+
addSource(key: string, source: SitemapSourceFactory): this;
|
|
21
|
+
/**
|
|
22
|
+
* Walks every group one at a time — never all at once — into a sibling
|
|
23
|
+
* temp directory, then publishes the whole set atomically. `files` is
|
|
24
|
+
* reported in registration order: the unnamed group first (if any), then
|
|
25
|
+
* named groups key-then-ordinal.
|
|
26
|
+
*/
|
|
27
|
+
saveTo(outDir: string): Promise<SitemapSetResult>;
|
|
28
|
+
}
|
|
29
|
+
//#endregion
|
|
30
|
+
export { SitemapIndex };
|
|
31
|
+
//# sourceMappingURL=sitemap-index.d.mts.map
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { DuplicateSourceKeyError } from "./errors.mjs";
|
|
2
|
+
import { publishAtomically } from "./atomic-publish.mjs";
|
|
3
|
+
import { normalizeSitemapIndexOptions } from "./sitemap-index-options.mjs";
|
|
4
|
+
import { canonicalizeSourceKey } from "./shard-name.mjs";
|
|
5
|
+
import { writeShardGroup } from "./sitemap-shard-writer.mjs";
|
|
6
|
+
import { buildSitemapIndexXml } from "./sitemap-index-xml.mjs";
|
|
7
|
+
import { DuplicatePathTracker } from "./duplicate-path-tracker.mjs";
|
|
8
|
+
import { RouteCounter } from "./route-counter.mjs";
|
|
9
|
+
import { writeFile } from "node:fs/promises";
|
|
10
|
+
import { join } from "node:path";
|
|
11
|
+
|
|
12
|
+
//#region ../sitemap/src/sitemap-index.ts
|
|
13
|
+
/**
|
|
14
|
+
* The streaming path for 300–500K URLs: shards, one flat master index, gzip,
|
|
15
|
+
* and an atomic publish. It is a SECOND mode, not a bigger `Sitemap` — it
|
|
16
|
+
* retains only the current shard's buffer and the set of paths it has seen.
|
|
17
|
+
* There is no `entries()`, `toXML()` or `size`; anyone who can afford those
|
|
18
|
+
* is in `Sitemap` and should be there instead.
|
|
19
|
+
*/
|
|
20
|
+
var SitemapIndex = class {
|
|
21
|
+
options;
|
|
22
|
+
/** Every unnamed `addSource(factory)` call merges into this one group, sharing one shard counter. */
|
|
23
|
+
unnamedFactories = [];
|
|
24
|
+
namedGroups = [];
|
|
25
|
+
/** Canonical (lowercased) keys already registered, so a case collision is caught at `addSource()`. */
|
|
26
|
+
registeredKeys = /* @__PURE__ */ new Set();
|
|
27
|
+
constructor(options) {
|
|
28
|
+
this.options = normalizeSitemapIndexOptions(options);
|
|
29
|
+
}
|
|
30
|
+
addSource(keyOrSource, maybeSource) {
|
|
31
|
+
if (typeof keyOrSource === "string") {
|
|
32
|
+
const canonicalKey = canonicalizeSourceKey(keyOrSource);
|
|
33
|
+
if (this.registeredKeys.has(canonicalKey)) throw new DuplicateSourceKeyError(keyOrSource);
|
|
34
|
+
this.registeredKeys.add(canonicalKey);
|
|
35
|
+
this.namedGroups.push({
|
|
36
|
+
key: keyOrSource,
|
|
37
|
+
factories: [maybeSource]
|
|
38
|
+
});
|
|
39
|
+
} else this.unnamedFactories.push(keyOrSource);
|
|
40
|
+
return this;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Walks every group one at a time — never all at once — into a sibling
|
|
44
|
+
* temp directory, then publishes the whole set atomically. `files` is
|
|
45
|
+
* reported in registration order: the unnamed group first (if any), then
|
|
46
|
+
* named groups key-then-ordinal.
|
|
47
|
+
*/
|
|
48
|
+
async saveTo(outDir) {
|
|
49
|
+
const groups = [...this.unnamedFactories.length > 0 ? [{
|
|
50
|
+
key: void 0,
|
|
51
|
+
factories: this.unnamedFactories
|
|
52
|
+
}] : [], ...this.namedGroups];
|
|
53
|
+
const duplicates = new DuplicatePathTracker();
|
|
54
|
+
const routes = new RouteCounter();
|
|
55
|
+
let shardFiles = [];
|
|
56
|
+
const write = async (tempDir) => {
|
|
57
|
+
shardFiles = [];
|
|
58
|
+
for (const group of groups) {
|
|
59
|
+
const files = await writeShardGroup(group, {
|
|
60
|
+
tempDir,
|
|
61
|
+
baseUrl: this.options.baseUrl,
|
|
62
|
+
filePrefix: this.options.filePrefix,
|
|
63
|
+
gzip: this.options.gzip,
|
|
64
|
+
maxUrlsPerFile: this.options.maxUrlsPerFile,
|
|
65
|
+
maxBytesPerFile: this.options.maxBytesPerFile,
|
|
66
|
+
defaults: this.options.defaults,
|
|
67
|
+
duplicates,
|
|
68
|
+
routes
|
|
69
|
+
});
|
|
70
|
+
shardFiles.push(...files);
|
|
71
|
+
}
|
|
72
|
+
const indexXml = buildSitemapIndexXml(shardFiles, this.options.baseUrl);
|
|
73
|
+
await writeFile(join(tempDir, this.options.indexFileName), indexXml, "utf8");
|
|
74
|
+
return [...shardFiles.filter((file) => file.urls > 0).map((file) => file.fileName), this.options.indexFileName];
|
|
75
|
+
};
|
|
76
|
+
await publishAtomically(outDir, write);
|
|
77
|
+
const files = shardFiles.map((file) => ({
|
|
78
|
+
path: join(outDir, file.fileName),
|
|
79
|
+
...file.key !== void 0 ? { key: file.key } : {},
|
|
80
|
+
urls: file.urls,
|
|
81
|
+
bytes: file.bytes,
|
|
82
|
+
gzipped: file.gzipped
|
|
83
|
+
}));
|
|
84
|
+
return {
|
|
85
|
+
indexPath: join(outDir, this.options.indexFileName),
|
|
86
|
+
files,
|
|
87
|
+
totalUrls: files.reduce((total, file) => total + file.urls, 0),
|
|
88
|
+
duplicates: duplicates.report(),
|
|
89
|
+
routes: routes.summary()
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
//#endregion
|
|
95
|
+
export { SitemapIndex };
|
|
96
|
+
//# sourceMappingURL=sitemap-index.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sitemap-index.mjs","names":[],"sources":["../../../../../../sitemap/src/sitemap-index.ts"],"sourcesContent":["import { join } from \"node:path\";\nimport { writeFile } from \"node:fs/promises\";\nimport { DuplicateSourceKeyError } from \"./errors\";\nimport {\n normalizeSitemapIndexOptions,\n type ResolvedSitemapIndexOptions,\n} from \"./sitemap-index-options\";\nimport { canonicalizeSourceKey } from \"./shard-name\";\nimport { publishAtomically } from \"./atomic-publish\";\nimport { writeShardGroup, type ShardFile, type ShardGroup } from \"./sitemap-shard-writer\";\nimport { buildSitemapIndexXml } from \"./sitemap-index-xml\";\nimport { DuplicatePathTracker } from \"./duplicate-path-tracker\";\nimport { RouteCounter } from \"./route-counter\";\nimport type {\n SitemapFileResult,\n SitemapIndexOptions,\n SitemapSetResult,\n SitemapSourceFactory,\n} from \"./sitemap-index-types\";\n\n/**\n * The streaming path for 300–500K URLs: shards, one flat master index, gzip,\n * and an atomic publish. It is a SECOND mode, not a bigger `Sitemap` — it\n * retains only the current shard's buffer and the set of paths it has seen.\n * There is no `entries()`, `toXML()` or `size`; anyone who can afford those\n * is in `Sitemap` and should be there instead.\n */\nexport class SitemapIndex {\n private readonly options: ResolvedSitemapIndexOptions;\n\n /** Every unnamed `addSource(factory)` call merges into this one group, sharing one shard counter. */\n private readonly unnamedFactories: SitemapSourceFactory[] = [];\n\n private readonly namedGroups: ShardGroup[] = [];\n\n /** Canonical (lowercased) keys already registered, so a case collision is caught at `addSource()`. */\n private readonly registeredKeys = new Set<string>();\n\n public constructor(options: SitemapIndexOptions) {\n this.options = normalizeSitemapIndexOptions(options);\n }\n\n public addSource(source: SitemapSourceFactory): this;\n public addSource(key: string, source: SitemapSourceFactory): this;\n public addSource(\n keyOrSource: string | SitemapSourceFactory,\n maybeSource?: SitemapSourceFactory,\n ): this {\n if (typeof keyOrSource === \"string\") {\n const canonicalKey = canonicalizeSourceKey(keyOrSource);\n\n if (this.registeredKeys.has(canonicalKey)) {\n throw new DuplicateSourceKeyError(keyOrSource);\n }\n\n this.registeredKeys.add(canonicalKey);\n this.namedGroups.push({ key: keyOrSource, factories: [maybeSource as SitemapSourceFactory] });\n } else {\n this.unnamedFactories.push(keyOrSource);\n }\n\n return this;\n }\n\n /**\n * Walks every group one at a time — never all at once — into a sibling\n * temp directory, then publishes the whole set atomically. `files` is\n * reported in registration order: the unnamed group first (if any), then\n * named groups key-then-ordinal.\n */\n public async saveTo(outDir: string): Promise<SitemapSetResult> {\n const groups: ShardGroup[] = [\n ...(this.unnamedFactories.length > 0\n ? [{ key: undefined, factories: this.unnamedFactories }]\n : []),\n ...this.namedGroups,\n ];\n\n const duplicates = new DuplicatePathTracker();\n const routes = new RouteCounter();\n let shardFiles: ShardFile[] = [];\n\n const write = async (tempDir: string): Promise<readonly string[]> => {\n shardFiles = [];\n\n for (const group of groups) {\n const files = await writeShardGroup(group, {\n tempDir,\n baseUrl: this.options.baseUrl,\n filePrefix: this.options.filePrefix,\n gzip: this.options.gzip,\n maxUrlsPerFile: this.options.maxUrlsPerFile,\n maxBytesPerFile: this.options.maxBytesPerFile,\n defaults: this.options.defaults,\n duplicates,\n routes,\n });\n\n shardFiles.push(...files);\n }\n\n const indexXml = buildSitemapIndexXml(shardFiles, this.options.baseUrl);\n\n await writeFile(join(tempDir, this.options.indexFileName), indexXml, \"utf8\");\n\n const writtenShardNames = shardFiles\n .filter((file) => file.urls > 0)\n .map((file) => file.fileName);\n\n return [...writtenShardNames, this.options.indexFileName];\n };\n\n await publishAtomically(outDir, write);\n\n const files: SitemapFileResult[] = shardFiles.map((file) => ({\n path: join(outDir, file.fileName),\n ...(file.key !== undefined ? { key: file.key } : {}),\n urls: file.urls,\n bytes: file.bytes,\n gzipped: file.gzipped,\n }));\n\n return {\n indexPath: join(outDir, this.options.indexFileName),\n files,\n totalUrls: files.reduce((total, file) => total + file.urls, 0),\n duplicates: duplicates.report(),\n routes: routes.summary(),\n };\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AA2BA,IAAa,eAAb,MAA0B;CACxB,AAAiB;;CAGjB,AAAiB,mBAA2C,CAAC;CAE7D,AAAiB,cAA4B,CAAC;;CAG9C,AAAiB,iCAAiB,IAAI,IAAY;CAElD,AAAO,YAAY,SAA8B;EAC/C,KAAK,UAAU,6BAA6B,OAAO;CACrD;CAIA,AAAO,UACL,aACA,aACM;EACN,IAAI,OAAO,gBAAgB,UAAU;GACnC,MAAM,eAAe,sBAAsB,WAAW;GAEtD,IAAI,KAAK,eAAe,IAAI,YAAY,GACtC,MAAM,IAAI,wBAAwB,WAAW;GAG/C,KAAK,eAAe,IAAI,YAAY;GACpC,KAAK,YAAY,KAAK;IAAE,KAAK;IAAa,WAAW,CAAC,WAAmC;GAAE,CAAC;EAC9F,OACE,KAAK,iBAAiB,KAAK,WAAW;EAGxC,OAAO;CACT;;;;;;;CAQA,MAAa,OAAO,QAA2C;EAC7D,MAAM,SAAuB,CAC3B,GAAI,KAAK,iBAAiB,SAAS,IAC/B,CAAC;GAAE,KAAK;GAAW,WAAW,KAAK;EAAiB,CAAC,IACrD,CAAC,GACL,GAAG,KAAK,WACV;EAEA,MAAM,aAAa,IAAI,qBAAqB;EAC5C,MAAM,SAAS,IAAI,aAAa;EAChC,IAAI,aAA0B,CAAC;EAE/B,MAAM,QAAQ,OAAO,YAAgD;GACnE,aAAa,CAAC;GAEd,KAAK,MAAM,SAAS,QAAQ;IAC1B,MAAM,QAAQ,MAAM,gBAAgB,OAAO;KACzC;KACA,SAAS,KAAK,QAAQ;KACtB,YAAY,KAAK,QAAQ;KACzB,MAAM,KAAK,QAAQ;KACnB,gBAAgB,KAAK,QAAQ;KAC7B,iBAAiB,KAAK,QAAQ;KAC9B,UAAU,KAAK,QAAQ;KACvB;KACA;IACF,CAAC;IAED,WAAW,KAAK,GAAG,KAAK;GAC1B;GAEA,MAAM,WAAW,qBAAqB,YAAY,KAAK,QAAQ,OAAO;GAEtE,MAAM,UAAU,KAAK,SAAS,KAAK,QAAQ,aAAa,GAAG,UAAU,MAAM;GAM3E,OAAO,CAAC,GAJkB,WACvB,QAAQ,SAAS,KAAK,OAAO,CAAC,CAAC,CAC/B,KAAK,SAAS,KAAK,QAEK,GAAG,KAAK,QAAQ,aAAa;EAC1D;EAEA,MAAM,kBAAkB,QAAQ,KAAK;EAErC,MAAM,QAA6B,WAAW,KAAK,UAAU;GAC3D,MAAM,KAAK,QAAQ,KAAK,QAAQ;GAChC,GAAI,KAAK,QAAQ,SAAY,EAAE,KAAK,KAAK,IAAI,IAAI,CAAC;GAClD,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,SAAS,KAAK;EAChB,EAAE;EAEF,OAAO;GACL,WAAW,KAAK,QAAQ,KAAK,QAAQ,aAAa;GAClD;GACA,WAAW,MAAM,QAAQ,OAAO,SAAS,QAAQ,KAAK,MAAM,CAAC;GAC7D,YAAY,WAAW,OAAO;GAC9B,QAAQ,OAAO,QAAQ;EACzB;CACF;AACF"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { normalizeEntry } from "./normalize-entry.mjs";
|
|
2
|
+
import { XHTML_NAMESPACE_ATTR, buildSitemapXml, renderUrlBlock } from "./xml.mjs";
|
|
3
|
+
import { shardFileName } from "./shard-name.mjs";
|
|
4
|
+
import { writeFile } from "node:fs/promises";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { gzipSync } from "node:zlib";
|
|
7
|
+
|
|
8
|
+
//#region ../sitemap/src/sitemap-shard-writer.ts
|
|
9
|
+
function envelopeBytes(baseUrl) {
|
|
10
|
+
return Buffer.byteLength(buildSitemapXml([], baseUrl), "utf8");
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* The extra bytes `<urlset>` gains for `xmlns:xhtml="…"` once a shard holds
|
|
14
|
+
* an entry with alternates. Charged separately from `envelopeBytes()` so the
|
|
15
|
+
* ceiling check can add it exactly once — the moment the first alternate
|
|
16
|
+
* enters the buffer — rather than missing it entirely, which would let a
|
|
17
|
+
* shard's real bytes on disk exceed the ceiling it was rolled against.
|
|
18
|
+
*/
|
|
19
|
+
const NAMESPACE_BYTES = Buffer.byteLength(XHTML_NAMESPACE_ATTR, "utf8");
|
|
20
|
+
async function writeShardFile(ctx, fileName, entries) {
|
|
21
|
+
const xml = buildSitemapXml(entries, ctx.baseUrl);
|
|
22
|
+
const filePath = join(ctx.tempDir, fileName);
|
|
23
|
+
if (ctx.gzip) {
|
|
24
|
+
const compressed = gzipSync(Buffer.from(xml, "utf8"));
|
|
25
|
+
await writeFile(filePath, compressed);
|
|
26
|
+
return { bytes: compressed.byteLength };
|
|
27
|
+
}
|
|
28
|
+
await writeFile(filePath, xml, "utf8");
|
|
29
|
+
return { bytes: Buffer.byteLength(xml, "utf8") };
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Walks one group's factories, one at a time, rolling to a new shard on
|
|
33
|
+
* whichever ceiling — URL count or serialised bytes — is hit first. Holds
|
|
34
|
+
* only the current shard's buffer, never the whole group.
|
|
35
|
+
*/
|
|
36
|
+
async function writeShardGroup(group, ctx) {
|
|
37
|
+
const results = [];
|
|
38
|
+
let buffer = [];
|
|
39
|
+
let bufferBytes = envelopeBytes(ctx.baseUrl);
|
|
40
|
+
let bufferHasAlternates = false;
|
|
41
|
+
let ordinal = 1;
|
|
42
|
+
const flush = async () => {
|
|
43
|
+
if (buffer.length === 0) return;
|
|
44
|
+
const fileName = shardFileName(ctx.filePrefix, group.key, ordinal, ctx.gzip);
|
|
45
|
+
const { bytes } = await writeShardFile(ctx, fileName, buffer);
|
|
46
|
+
results.push({
|
|
47
|
+
fileName,
|
|
48
|
+
key: group.key,
|
|
49
|
+
urls: buffer.length,
|
|
50
|
+
bytes,
|
|
51
|
+
gzipped: ctx.gzip
|
|
52
|
+
});
|
|
53
|
+
ordinal += 1;
|
|
54
|
+
buffer = [];
|
|
55
|
+
bufferBytes = envelopeBytes(ctx.baseUrl);
|
|
56
|
+
bufferHasAlternates = false;
|
|
57
|
+
};
|
|
58
|
+
for (const factory of group.factories) {
|
|
59
|
+
const source = await factory();
|
|
60
|
+
for await (const rawEntry of source) {
|
|
61
|
+
const resolved = normalizeEntry(rawEntry, ctx.defaults);
|
|
62
|
+
if (!ctx.duplicates.attempt(resolved.path, resolved.route)) continue;
|
|
63
|
+
const entryHasAlternates = (resolved.alternates?.length ?? 0) > 0;
|
|
64
|
+
const blockBytes = Buffer.byteLength(renderUrlBlock(resolved, ctx.baseUrl), "utf8") + 1;
|
|
65
|
+
const addedBytes = blockBytes + (entryHasAlternates && !bufferHasAlternates ? NAMESPACE_BYTES : 0);
|
|
66
|
+
const hitsUrlCeiling = buffer.length >= ctx.maxUrlsPerFile;
|
|
67
|
+
const hitsByteCeiling = buffer.length > 0 && bufferBytes + addedBytes > ctx.maxBytesPerFile;
|
|
68
|
+
if (hitsUrlCeiling || hitsByteCeiling) await flush();
|
|
69
|
+
const addsNamespaceNow = entryHasAlternates && !bufferHasAlternates;
|
|
70
|
+
buffer.push(resolved);
|
|
71
|
+
bufferBytes += blockBytes + (addsNamespaceNow ? NAMESPACE_BYTES : 0);
|
|
72
|
+
if (addsNamespaceNow) bufferHasAlternates = true;
|
|
73
|
+
ctx.routes.record(resolved.route);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
await flush();
|
|
77
|
+
if (results.length === 0) results.push({
|
|
78
|
+
fileName: shardFileName(ctx.filePrefix, group.key, 1, ctx.gzip),
|
|
79
|
+
key: group.key,
|
|
80
|
+
urls: 0,
|
|
81
|
+
bytes: 0,
|
|
82
|
+
gzipped: false
|
|
83
|
+
});
|
|
84
|
+
return results;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
//#endregion
|
|
88
|
+
export { writeShardGroup };
|
|
89
|
+
//# sourceMappingURL=sitemap-shard-writer.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sitemap-shard-writer.mjs","names":[],"sources":["../../../../../../sitemap/src/sitemap-shard-writer.ts"],"sourcesContent":["import { writeFile } from \"node:fs/promises\";\nimport { gzipSync } from \"node:zlib\";\nimport { join } from \"node:path\";\nimport { normalizeEntry } from \"./normalize-entry\";\nimport { shardFileName } from \"./shard-name\";\nimport type { ResolvedSitemapEntry, SitemapOptions } from \"./types\";\nimport type { DuplicatePathTracker } from \"./duplicate-path-tracker\";\nimport type { RouteCounter } from \"./route-counter\";\nimport type { SitemapSourceFactory } from \"./sitemap-index-types\";\nimport { buildSitemapXml, renderUrlBlock, XHTML_NAMESPACE_ATTR } from \"./xml\";\n\n/** One `addSource` group: the unnamed group merges every unkeyed call; a keyed group holds exactly one factory. */\nexport type ShardGroup = {\n readonly key?: string;\n readonly factories: readonly SitemapSourceFactory[];\n};\n\nexport type ShardWriterContext = {\n readonly tempDir: string;\n readonly baseUrl: string;\n readonly filePrefix: string;\n readonly gzip: boolean;\n readonly maxUrlsPerFile: number;\n readonly maxBytesPerFile: number;\n readonly defaults: Pick<SitemapOptions, \"changefreq\" | \"priority\" | \"lastmod\">;\n readonly duplicates: DuplicatePathTracker;\n readonly routes: RouteCounter;\n};\n\n/** A shard actually written, or the zero-url row reported for a group that produced nothing. */\nexport type ShardFile = {\n readonly fileName: string;\n readonly key?: string;\n readonly urls: number;\n readonly bytes: number;\n readonly gzipped: boolean;\n};\n\nfunction envelopeBytes(baseUrl: string): number {\n return Buffer.byteLength(buildSitemapXml([], baseUrl), \"utf8\");\n}\n\n/**\n * The extra bytes `<urlset>` gains for `xmlns:xhtml=\"…\"` once a shard holds\n * an entry with alternates. Charged separately from `envelopeBytes()` so the\n * ceiling check can add it exactly once — the moment the first alternate\n * enters the buffer — rather than missing it entirely, which would let a\n * shard's real bytes on disk exceed the ceiling it was rolled against.\n */\nconst NAMESPACE_BYTES = Buffer.byteLength(XHTML_NAMESPACE_ATTR, \"utf8\");\n\nasync function writeShardFile(\n ctx: ShardWriterContext,\n fileName: string,\n entries: readonly ResolvedSitemapEntry[],\n): Promise<{ bytes: number }> {\n const xml = buildSitemapXml(entries, ctx.baseUrl);\n const filePath = join(ctx.tempDir, fileName);\n\n if (ctx.gzip) {\n const compressed = gzipSync(Buffer.from(xml, \"utf8\"));\n\n await writeFile(filePath, compressed);\n\n return { bytes: compressed.byteLength };\n }\n\n await writeFile(filePath, xml, \"utf8\");\n\n return { bytes: Buffer.byteLength(xml, \"utf8\") };\n}\n\n/**\n * Walks one group's factories, one at a time, rolling to a new shard on\n * whichever ceiling — URL count or serialised bytes — is hit first. Holds\n * only the current shard's buffer, never the whole group.\n */\nexport async function writeShardGroup(\n group: ShardGroup,\n ctx: ShardWriterContext,\n): Promise<ShardFile[]> {\n const results: ShardFile[] = [];\n\n let buffer: ResolvedSitemapEntry[] = [];\n let bufferBytes = envelopeBytes(ctx.baseUrl);\n // Charged once, the moment the buffer's first alternate-bearing entry is added — mirrors\n // buildSitemapXml()'s own \"at least one entry has alternates\" rule for the same shard.\n let bufferHasAlternates = false;\n let ordinal = 1;\n\n const flush = async () => {\n if (buffer.length === 0) return;\n\n const fileName = shardFileName(ctx.filePrefix, group.key, ordinal, ctx.gzip);\n const { bytes } = await writeShardFile(ctx, fileName, buffer);\n\n results.push({ fileName, key: group.key, urls: buffer.length, bytes, gzipped: ctx.gzip });\n\n ordinal += 1;\n buffer = [];\n bufferBytes = envelopeBytes(ctx.baseUrl);\n bufferHasAlternates = false;\n };\n\n for (const factory of group.factories) {\n const source = await factory();\n\n for await (const rawEntry of source) {\n const resolved = normalizeEntry(rawEntry, ctx.defaults);\n\n // Duplicate paths are SKIPPED, not overridden: the earlier one is already on disk.\n if (!ctx.duplicates.attempt(resolved.path, resolved.route)) continue;\n\n const entryHasAlternates = (resolved.alternates?.length ?? 0) > 0;\n const blockBytes = Buffer.byteLength(renderUrlBlock(resolved, ctx.baseUrl), \"utf8\") + 1;\n // What this entry would add to the CURRENT shard: its own block, plus the namespace\n // attribute if this is the shard's first alternate and the buffer doesn't carry it yet.\n const addedBytes =\n blockBytes + (entryHasAlternates && !bufferHasAlternates ? NAMESPACE_BYTES : 0);\n\n const hitsUrlCeiling = buffer.length >= ctx.maxUrlsPerFile;\n // A single entry can never be split, so the byte ceiling only rolls an already-nonempty shard.\n const hitsByteCeiling = buffer.length > 0 && bufferBytes + addedBytes > ctx.maxBytesPerFile;\n\n if (hitsUrlCeiling || hitsByteCeiling) await flush();\n\n const addsNamespaceNow = entryHasAlternates && !bufferHasAlternates;\n\n buffer.push(resolved);\n bufferBytes += blockBytes + (addsNamespaceNow ? NAMESPACE_BYTES : 0);\n if (addsNamespaceNow) bufferHasAlternates = true;\n ctx.routes.record(resolved.route);\n }\n }\n\n await flush();\n\n if (results.length === 0) {\n // Reported, never written: an empty shard in an index is a section someone lost.\n results.push({\n fileName: shardFileName(ctx.filePrefix, group.key, 1, ctx.gzip),\n key: group.key,\n urls: 0,\n bytes: 0,\n gzipped: false,\n });\n }\n\n return results;\n}\n"],"mappings":";;;;;;;;AAsCA,SAAS,cAAc,SAAyB;CAC9C,OAAO,OAAO,WAAW,gBAAgB,CAAC,GAAG,OAAO,GAAG,MAAM;AAC/D;;;;;;;;AASA,MAAM,kBAAkB,OAAO,WAAW,sBAAsB,MAAM;AAEtE,eAAe,eACb,KACA,UACA,SAC4B;CAC5B,MAAM,MAAM,gBAAgB,SAAS,IAAI,OAAO;CAChD,MAAM,WAAW,KAAK,IAAI,SAAS,QAAQ;CAE3C,IAAI,IAAI,MAAM;EACZ,MAAM,aAAa,SAAS,OAAO,KAAK,KAAK,MAAM,CAAC;EAEpD,MAAM,UAAU,UAAU,UAAU;EAEpC,OAAO,EAAE,OAAO,WAAW,WAAW;CACxC;CAEA,MAAM,UAAU,UAAU,KAAK,MAAM;CAErC,OAAO,EAAE,OAAO,OAAO,WAAW,KAAK,MAAM,EAAE;AACjD;;;;;;AAOA,eAAsB,gBACpB,OACA,KACsB;CACtB,MAAM,UAAuB,CAAC;CAE9B,IAAI,SAAiC,CAAC;CACtC,IAAI,cAAc,cAAc,IAAI,OAAO;CAG3C,IAAI,sBAAsB;CAC1B,IAAI,UAAU;CAEd,MAAM,QAAQ,YAAY;EACxB,IAAI,OAAO,WAAW,GAAG;EAEzB,MAAM,WAAW,cAAc,IAAI,YAAY,MAAM,KAAK,SAAS,IAAI,IAAI;EAC3E,MAAM,EAAE,UAAU,MAAM,eAAe,KAAK,UAAU,MAAM;EAE5D,QAAQ,KAAK;GAAE;GAAU,KAAK,MAAM;GAAK,MAAM,OAAO;GAAQ;GAAO,SAAS,IAAI;EAAK,CAAC;EAExF,WAAW;EACX,SAAS,CAAC;EACV,cAAc,cAAc,IAAI,OAAO;EACvC,sBAAsB;CACxB;CAEA,KAAK,MAAM,WAAW,MAAM,WAAW;EACrC,MAAM,SAAS,MAAM,QAAQ;EAE7B,WAAW,MAAM,YAAY,QAAQ;GACnC,MAAM,WAAW,eAAe,UAAU,IAAI,QAAQ;GAGtD,IAAI,CAAC,IAAI,WAAW,QAAQ,SAAS,MAAM,SAAS,KAAK,GAAG;GAE5D,MAAM,sBAAsB,SAAS,YAAY,UAAU,KAAK;GAChE,MAAM,aAAa,OAAO,WAAW,eAAe,UAAU,IAAI,OAAO,GAAG,MAAM,IAAI;GAGtF,MAAM,aACJ,cAAc,sBAAsB,CAAC,sBAAsB,kBAAkB;GAE/E,MAAM,iBAAiB,OAAO,UAAU,IAAI;GAE5C,MAAM,kBAAkB,OAAO,SAAS,KAAK,cAAc,aAAa,IAAI;GAE5E,IAAI,kBAAkB,iBAAiB,MAAM,MAAM;GAEnD,MAAM,mBAAmB,sBAAsB,CAAC;GAEhD,OAAO,KAAK,QAAQ;GACpB,eAAe,cAAc,mBAAmB,kBAAkB;GAClE,IAAI,kBAAkB,sBAAsB;GAC5C,IAAI,OAAO,OAAO,SAAS,KAAK;EAClC;CACF;CAEA,MAAM,MAAM;CAEZ,IAAI,QAAQ,WAAW,GAErB,QAAQ,KAAK;EACX,UAAU,cAAc,IAAI,YAAY,MAAM,KAAK,GAAG,IAAI,IAAI;EAC9D,KAAK,MAAM;EACX,MAAM;EACN,OAAO;EACP,SAAS;CACX,CAAC;CAGH,OAAO;AACT"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { DuplicateReport, ResolvedSitemapEntry, RouteSummary, SitemapEntry, SitemapOptions } from "./types.mjs";
|
|
2
|
+
|
|
3
|
+
//#region ../sitemap/src/sitemap.d.ts
|
|
4
|
+
/**
|
|
5
|
+
* A bounded sitemap builder: it RETAINS every entry, which is what makes
|
|
6
|
+
* `entries()` and a repeatable `toXML()` possible, and is exactly right up to
|
|
7
|
+
* the sitemaps.org ceiling of 50,000 URLs / 50MB.
|
|
8
|
+
*
|
|
9
|
+
* Above that ceiling this is the wrong tool — the streaming writer retains
|
|
10
|
+
* nothing and emits shards plus an index instead. The two modes are separate
|
|
11
|
+
* on purpose: a class that promised not to retain entries and still offered
|
|
12
|
+
* `entries()` would be lying about one of them.
|
|
13
|
+
*/
|
|
14
|
+
declare class Sitemap {
|
|
15
|
+
private readonly baseUrl;
|
|
16
|
+
private readonly defaults;
|
|
17
|
+
/** Keyed by path: a duplicate `<loc>` makes the document invalid, so the later add wins. */
|
|
18
|
+
private readonly entriesByPath;
|
|
19
|
+
/** Insertion order per path, so `duplicates()` can name every colliding source. */
|
|
20
|
+
private readonly routesByPath;
|
|
21
|
+
/** Declared patterns, including ones that never contributed a URL. */
|
|
22
|
+
private readonly declaredRoutes;
|
|
23
|
+
constructor(options: SitemapOptions);
|
|
24
|
+
add(entry: SitemapEntry): this;
|
|
25
|
+
addMany(entries: Iterable<SitemapEntry>): this;
|
|
26
|
+
/**
|
|
27
|
+
* Names a pattern the caller EXPECTS to contribute URLs, so that one which
|
|
28
|
+
* contributes none shows up in `routes()` as a `count: 0` row instead of as
|
|
29
|
+
* silence. A dynamic route whose supplier returned nothing is the failure
|
|
30
|
+
* this package exists to make visible.
|
|
31
|
+
*/
|
|
32
|
+
declareRoute(route: string): this;
|
|
33
|
+
get size(): number;
|
|
34
|
+
entries(): readonly ResolvedSitemapEntry[];
|
|
35
|
+
routes(): readonly RouteSummary[];
|
|
36
|
+
/**
|
|
37
|
+
* Every path added more than once. The override is silent — it is not
|
|
38
|
+
* hidden: a collision between static discovery and a dynamic supplier is a
|
|
39
|
+
* real defect, and the CALLER decides whether it fails their build. This
|
|
40
|
+
* package reports; it never prints and never throws over a duplicate.
|
|
41
|
+
*/
|
|
42
|
+
duplicates(): readonly DuplicateReport[];
|
|
43
|
+
/** Pure and repeatable: calling it twice returns the same string and mutates nothing. */
|
|
44
|
+
toXML(): string;
|
|
45
|
+
/**
|
|
46
|
+
* Writes the document, creating parent directories so a clean checkout
|
|
47
|
+
* works. The write is atomic: a failed or interrupted publish leaves
|
|
48
|
+
* whatever was already at `filePath` untouched instead of truncating it.
|
|
49
|
+
*/
|
|
50
|
+
saveTo(filePath: string): Promise<void>;
|
|
51
|
+
/**
|
|
52
|
+
* Publishes the document as the whole content of `outDir`, exactly the way
|
|
53
|
+
* `SitemapIndex.saveTo` publishes a set: swapped in atomically, and marked
|
|
54
|
+
* as owned. So a site that later outgrows one file can publish an index
|
|
55
|
+
* into the same directory, and a later single file removes stale shards.
|
|
56
|
+
* Refuses a non-empty directory this package did not write
|
|
57
|
+
* (`UnownedOutputDirectoryError`). Returns the published file's path.
|
|
58
|
+
*/
|
|
59
|
+
publishTo(outDir: string, fileName?: string): Promise<string>;
|
|
60
|
+
}
|
|
61
|
+
//#endregion
|
|
62
|
+
export { Sitemap };
|
|
63
|
+
//# sourceMappingURL=sitemap.d.mts.map
|
package/esm/sitemap.mjs
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { publishAtomically } from "./atomic-publish.mjs";
|
|
2
|
+
import { atomicWriteFile } from "./atomic-write-file.mjs";
|
|
3
|
+
import { normalizeEntry } from "./normalize-entry.mjs";
|
|
4
|
+
import { normalizeBaseUrl } from "./url.mjs";
|
|
5
|
+
import { buildSitemapXml } from "./xml.mjs";
|
|
6
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
7
|
+
import { dirname, join } from "node:path";
|
|
8
|
+
|
|
9
|
+
//#region ../sitemap/src/sitemap.ts
|
|
10
|
+
/**
|
|
11
|
+
* A bounded sitemap builder: it RETAINS every entry, which is what makes
|
|
12
|
+
* `entries()` and a repeatable `toXML()` possible, and is exactly right up to
|
|
13
|
+
* the sitemaps.org ceiling of 50,000 URLs / 50MB.
|
|
14
|
+
*
|
|
15
|
+
* Above that ceiling this is the wrong tool — the streaming writer retains
|
|
16
|
+
* nothing and emits shards plus an index instead. The two modes are separate
|
|
17
|
+
* on purpose: a class that promised not to retain entries and still offered
|
|
18
|
+
* `entries()` would be lying about one of them.
|
|
19
|
+
*/
|
|
20
|
+
var Sitemap = class {
|
|
21
|
+
baseUrl;
|
|
22
|
+
defaults;
|
|
23
|
+
/** Keyed by path: a duplicate `<loc>` makes the document invalid, so the later add wins. */
|
|
24
|
+
entriesByPath = /* @__PURE__ */ new Map();
|
|
25
|
+
/** Insertion order per path, so `duplicates()` can name every colliding source. */
|
|
26
|
+
routesByPath = /* @__PURE__ */ new Map();
|
|
27
|
+
/** Declared patterns, including ones that never contributed a URL. */
|
|
28
|
+
declaredRoutes = /* @__PURE__ */ new Set();
|
|
29
|
+
constructor(options) {
|
|
30
|
+
this.baseUrl = normalizeBaseUrl(options?.baseUrl);
|
|
31
|
+
this.defaults = {
|
|
32
|
+
changefreq: options.changefreq,
|
|
33
|
+
priority: options.priority,
|
|
34
|
+
lastmod: options.lastmod
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
add(entry) {
|
|
38
|
+
const resolved = normalizeEntry(entry, this.defaults);
|
|
39
|
+
this.entriesByPath.set(resolved.path, resolved);
|
|
40
|
+
const seen = this.routesByPath.get(resolved.path);
|
|
41
|
+
if (seen) seen.push(resolved.route);
|
|
42
|
+
else this.routesByPath.set(resolved.path, [resolved.route]);
|
|
43
|
+
if (resolved.route !== void 0) this.declaredRoutes.add(resolved.route);
|
|
44
|
+
return this;
|
|
45
|
+
}
|
|
46
|
+
addMany(entries) {
|
|
47
|
+
for (const entry of entries) this.add(entry);
|
|
48
|
+
return this;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Names a pattern the caller EXPECTS to contribute URLs, so that one which
|
|
52
|
+
* contributes none shows up in `routes()` as a `count: 0` row instead of as
|
|
53
|
+
* silence. A dynamic route whose supplier returned nothing is the failure
|
|
54
|
+
* this package exists to make visible.
|
|
55
|
+
*/
|
|
56
|
+
declareRoute(route) {
|
|
57
|
+
this.declaredRoutes.add(route);
|
|
58
|
+
return this;
|
|
59
|
+
}
|
|
60
|
+
get size() {
|
|
61
|
+
return this.entriesByPath.size;
|
|
62
|
+
}
|
|
63
|
+
entries() {
|
|
64
|
+
return [...this.entriesByPath.values()];
|
|
65
|
+
}
|
|
66
|
+
routes() {
|
|
67
|
+
const counts = /* @__PURE__ */ new Map();
|
|
68
|
+
for (const route of this.declaredRoutes) counts.set(route, 0);
|
|
69
|
+
for (const entry of this.entriesByPath.values()) {
|
|
70
|
+
if (entry.route === void 0) continue;
|
|
71
|
+
counts.set(entry.route, (counts.get(entry.route) ?? 0) + 1);
|
|
72
|
+
}
|
|
73
|
+
return [...counts].map(([route, count]) => ({
|
|
74
|
+
route,
|
|
75
|
+
count
|
|
76
|
+
}));
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Every path added more than once. The override is silent — it is not
|
|
80
|
+
* hidden: a collision between static discovery and a dynamic supplier is a
|
|
81
|
+
* real defect, and the CALLER decides whether it fails their build. This
|
|
82
|
+
* package reports; it never prints and never throws over a duplicate.
|
|
83
|
+
*/
|
|
84
|
+
duplicates() {
|
|
85
|
+
const reports = [];
|
|
86
|
+
for (const [path, routes] of this.routesByPath) {
|
|
87
|
+
if (routes.length < 2) continue;
|
|
88
|
+
reports.push({
|
|
89
|
+
path,
|
|
90
|
+
count: routes.length,
|
|
91
|
+
routes: [...routes]
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
return reports;
|
|
95
|
+
}
|
|
96
|
+
/** Pure and repeatable: calling it twice returns the same string and mutates nothing. */
|
|
97
|
+
toXML() {
|
|
98
|
+
return buildSitemapXml(this.entries(), this.baseUrl);
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Writes the document, creating parent directories so a clean checkout
|
|
102
|
+
* works. The write is atomic: a failed or interrupted publish leaves
|
|
103
|
+
* whatever was already at `filePath` untouched instead of truncating it.
|
|
104
|
+
*/
|
|
105
|
+
async saveTo(filePath) {
|
|
106
|
+
await mkdir(dirname(filePath), { recursive: true });
|
|
107
|
+
await atomicWriteFile(filePath, this.toXML());
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Publishes the document as the whole content of `outDir`, exactly the way
|
|
111
|
+
* `SitemapIndex.saveTo` publishes a set: swapped in atomically, and marked
|
|
112
|
+
* as owned. So a site that later outgrows one file can publish an index
|
|
113
|
+
* into the same directory, and a later single file removes stale shards.
|
|
114
|
+
* Refuses a non-empty directory this package did not write
|
|
115
|
+
* (`UnownedOutputDirectoryError`). Returns the published file's path.
|
|
116
|
+
*/
|
|
117
|
+
async publishTo(outDir, fileName = "sitemap.xml") {
|
|
118
|
+
const xml = this.toXML();
|
|
119
|
+
await publishAtomically(outDir, async (tempDir) => {
|
|
120
|
+
await writeFile(join(tempDir, fileName), xml, "utf8");
|
|
121
|
+
return [fileName];
|
|
122
|
+
});
|
|
123
|
+
return join(outDir, fileName);
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
//#endregion
|
|
128
|
+
export { Sitemap };
|
|
129
|
+
//# sourceMappingURL=sitemap.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sitemap.mjs","names":[],"sources":["../../../../../../sitemap/src/sitemap.ts"],"sourcesContent":["import { mkdir, writeFile } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\nimport { publishAtomically } from \"./atomic-publish\";\nimport { atomicWriteFile } from \"./atomic-write-file\";\nimport { normalizeEntry } from \"./normalize-entry\";\nimport type {\n DuplicateReport,\n ResolvedSitemapEntry,\n RouteSummary,\n SitemapEntry,\n SitemapOptions,\n} from \"./types\";\nimport { normalizeBaseUrl } from \"./url\";\nimport { buildSitemapXml } from \"./xml\";\n\n/**\n * A bounded sitemap builder: it RETAINS every entry, which is what makes\n * `entries()` and a repeatable `toXML()` possible, and is exactly right up to\n * the sitemaps.org ceiling of 50,000 URLs / 50MB.\n *\n * Above that ceiling this is the wrong tool — the streaming writer retains\n * nothing and emits shards plus an index instead. The two modes are separate\n * on purpose: a class that promised not to retain entries and still offered\n * `entries()` would be lying about one of them.\n */\nexport class Sitemap {\n private readonly baseUrl: string;\n\n private readonly defaults: Pick<SitemapOptions, \"changefreq\" | \"priority\" | \"lastmod\">;\n\n /** Keyed by path: a duplicate `<loc>` makes the document invalid, so the later add wins. */\n private readonly entriesByPath = new Map<string, ResolvedSitemapEntry>();\n\n /** Insertion order per path, so `duplicates()` can name every colliding source. */\n private readonly routesByPath = new Map<string, (string | undefined)[]>();\n\n /** Declared patterns, including ones that never contributed a URL. */\n private readonly declaredRoutes = new Set<string>();\n\n public constructor(options: SitemapOptions) {\n this.baseUrl = normalizeBaseUrl(options?.baseUrl);\n this.defaults = {\n changefreq: options.changefreq,\n priority: options.priority,\n lastmod: options.lastmod,\n };\n }\n\n public add(entry: SitemapEntry): this {\n const resolved = normalizeEntry(entry, this.defaults);\n\n this.entriesByPath.set(resolved.path, resolved);\n\n const seen = this.routesByPath.get(resolved.path);\n\n if (seen) {\n seen.push(resolved.route);\n } else {\n this.routesByPath.set(resolved.path, [resolved.route]);\n }\n\n if (resolved.route !== undefined) this.declaredRoutes.add(resolved.route);\n\n return this;\n }\n\n public addMany(entries: Iterable<SitemapEntry>): this {\n for (const entry of entries) this.add(entry);\n\n return this;\n }\n\n /**\n * Names a pattern the caller EXPECTS to contribute URLs, so that one which\n * contributes none shows up in `routes()` as a `count: 0` row instead of as\n * silence. A dynamic route whose supplier returned nothing is the failure\n * this package exists to make visible.\n */\n public declareRoute(route: string): this {\n this.declaredRoutes.add(route);\n\n return this;\n }\n\n public get size(): number {\n return this.entriesByPath.size;\n }\n\n public entries(): readonly ResolvedSitemapEntry[] {\n return [...this.entriesByPath.values()];\n }\n\n public routes(): readonly RouteSummary[] {\n const counts = new Map<string, number>();\n\n for (const route of this.declaredRoutes) counts.set(route, 0);\n\n for (const entry of this.entriesByPath.values()) {\n if (entry.route === undefined) continue;\n\n counts.set(entry.route, (counts.get(entry.route) ?? 0) + 1);\n }\n\n return [...counts].map(([route, count]) => ({ route, count }));\n }\n\n /**\n * Every path added more than once. The override is silent — it is not\n * hidden: a collision between static discovery and a dynamic supplier is a\n * real defect, and the CALLER decides whether it fails their build. This\n * package reports; it never prints and never throws over a duplicate.\n */\n public duplicates(): readonly DuplicateReport[] {\n const reports: DuplicateReport[] = [];\n\n for (const [path, routes] of this.routesByPath) {\n if (routes.length < 2) continue;\n\n reports.push({ path, count: routes.length, routes: [...routes] });\n }\n\n return reports;\n }\n\n /** Pure and repeatable: calling it twice returns the same string and mutates nothing. */\n public toXML(): string {\n return buildSitemapXml(this.entries(), this.baseUrl);\n }\n\n /**\n * Writes the document, creating parent directories so a clean checkout\n * works. The write is atomic: a failed or interrupted publish leaves\n * whatever was already at `filePath` untouched instead of truncating it.\n */\n public async saveTo(filePath: string): Promise<void> {\n await mkdir(dirname(filePath), { recursive: true });\n await atomicWriteFile(filePath, this.toXML());\n }\n\n /**\n * Publishes the document as the whole content of `outDir`, exactly the way\n * `SitemapIndex.saveTo` publishes a set: swapped in atomically, and marked\n * as owned. So a site that later outgrows one file can publish an index\n * into the same directory, and a later single file removes stale shards.\n * Refuses a non-empty directory this package did not write\n * (`UnownedOutputDirectoryError`). Returns the published file's path.\n */\n public async publishTo(outDir: string, fileName = \"sitemap.xml\"): Promise<string> {\n const xml = this.toXML();\n\n await publishAtomically(outDir, async (tempDir) => {\n await writeFile(join(tempDir, fileName), xml, \"utf8\");\n\n return [fileName];\n });\n\n return join(outDir, fileName);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAyBA,IAAa,UAAb,MAAqB;CACnB,AAAiB;CAEjB,AAAiB;;CAGjB,AAAiB,gCAAgB,IAAI,IAAkC;;CAGvE,AAAiB,+BAAe,IAAI,IAAoC;;CAGxE,AAAiB,iCAAiB,IAAI,IAAY;CAElD,AAAO,YAAY,SAAyB;EAC1C,KAAK,UAAU,iBAAiB,SAAS,OAAO;EAChD,KAAK,WAAW;GACd,YAAY,QAAQ;GACpB,UAAU,QAAQ;GAClB,SAAS,QAAQ;EACnB;CACF;CAEA,AAAO,IAAI,OAA2B;EACpC,MAAM,WAAW,eAAe,OAAO,KAAK,QAAQ;EAEpD,KAAK,cAAc,IAAI,SAAS,MAAM,QAAQ;EAE9C,MAAM,OAAO,KAAK,aAAa,IAAI,SAAS,IAAI;EAEhD,IAAI,MACF,KAAK,KAAK,SAAS,KAAK;OAExB,KAAK,aAAa,IAAI,SAAS,MAAM,CAAC,SAAS,KAAK,CAAC;EAGvD,IAAI,SAAS,UAAU,QAAW,KAAK,eAAe,IAAI,SAAS,KAAK;EAExE,OAAO;CACT;CAEA,AAAO,QAAQ,SAAuC;EACpD,KAAK,MAAM,SAAS,SAAS,KAAK,IAAI,KAAK;EAE3C,OAAO;CACT;;;;;;;CAQA,AAAO,aAAa,OAAqB;EACvC,KAAK,eAAe,IAAI,KAAK;EAE7B,OAAO;CACT;CAEA,IAAW,OAAe;EACxB,OAAO,KAAK,cAAc;CAC5B;CAEA,AAAO,UAA2C;EAChD,OAAO,CAAC,GAAG,KAAK,cAAc,OAAO,CAAC;CACxC;CAEA,AAAO,SAAkC;EACvC,MAAM,yBAAS,IAAI,IAAoB;EAEvC,KAAK,MAAM,SAAS,KAAK,gBAAgB,OAAO,IAAI,OAAO,CAAC;EAE5D,KAAK,MAAM,SAAS,KAAK,cAAc,OAAO,GAAG;GAC/C,IAAI,MAAM,UAAU,QAAW;GAE/B,OAAO,IAAI,MAAM,QAAQ,OAAO,IAAI,MAAM,KAAK,KAAK,KAAK,CAAC;EAC5D;EAEA,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY;GAAE;GAAO;EAAM,EAAE;CAC/D;;;;;;;CAQA,AAAO,aAAyC;EAC9C,MAAM,UAA6B,CAAC;EAEpC,KAAK,MAAM,CAAC,MAAM,WAAW,KAAK,cAAc;GAC9C,IAAI,OAAO,SAAS,GAAG;GAEvB,QAAQ,KAAK;IAAE;IAAM,OAAO,OAAO;IAAQ,QAAQ,CAAC,GAAG,MAAM;GAAE,CAAC;EAClE;EAEA,OAAO;CACT;;CAGA,AAAO,QAAgB;EACrB,OAAO,gBAAgB,KAAK,QAAQ,GAAG,KAAK,OAAO;CACrD;;;;;;CAOA,MAAa,OAAO,UAAiC;EACnD,MAAM,MAAM,QAAQ,QAAQ,GAAG,EAAE,WAAW,KAAK,CAAC;EAClD,MAAM,gBAAgB,UAAU,KAAK,MAAM,CAAC;CAC9C;;;;;;;;;CAUA,MAAa,UAAU,QAAgB,WAAW,eAAgC;EAChF,MAAM,MAAM,KAAK,MAAM;EAEvB,MAAM,kBAAkB,QAAQ,OAAO,YAAY;GACjD,MAAM,UAAU,KAAK,SAAS,QAAQ,GAAG,KAAK,MAAM;GAEpD,OAAO,CAAC,QAAQ;EAClB,CAAC;EAED,OAAO,KAAK,QAAQ,QAAQ;CAC9B;AACF"}
|