@thallylabs/migrate 0.2.27 → 0.2.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -0
- package/dist/index.d.ts +4 -360
- package/dist/index.js +1288 -33
- package/dist/index.js.map +1 -1
- package/dist/svg-policy.d.ts +16 -0
- package/dist/svg-policy.js +308 -0
- package/dist/svg-policy.js.map +1 -0
- package/dist/url-migration-CDjLZVaN.d.ts +512 -0
- package/dist/url-migration.d.ts +1 -0
- package/dist/url-migration.js +3872 -0
- package/dist/url-migration.js.map +1 -0
- package/package.json +13 -2
package/dist/index.js
CHANGED
|
@@ -5382,14 +5382,14 @@ function buildNavigationFromPages(pages, options = {}) {
|
|
|
5382
5382
|
const ids = pages.filter((page) => !page.locale || page.locale === "en").map((page) => page.navigationId);
|
|
5383
5383
|
const ordered = [...new Set(ids)];
|
|
5384
5384
|
if (options.topLevelTabs) {
|
|
5385
|
-
const
|
|
5385
|
+
const sourceNavigation2 = (options.topLevelNavigation ?? []).flatMap((entry) => {
|
|
5386
5386
|
const sectionPages = ordered.filter((id) => id === entry.section || id.startsWith(`${entry.section}/`));
|
|
5387
5387
|
const pageId = ordered.includes(entry.pageId) ? entry.pageId : preferredLandingPage(entry.section, sectionPages) ?? sectionPages[0];
|
|
5388
5388
|
return pageId ? [{ ...entry, pageId }] : [];
|
|
5389
5389
|
});
|
|
5390
|
-
if (
|
|
5390
|
+
if (sourceNavigation2.length > 1) {
|
|
5391
5391
|
const claimedIds = /* @__PURE__ */ new Set();
|
|
5392
|
-
const tabs =
|
|
5392
|
+
const tabs = sourceNavigation2.map((entry) => {
|
|
5393
5393
|
const pageIds = ordered.filter((id) => {
|
|
5394
5394
|
const matches = id === entry.pageId || id === entry.section || id.startsWith(`${entry.section}/`);
|
|
5395
5395
|
if (matches) claimedIds.add(id);
|
|
@@ -7431,10 +7431,48 @@ function validateIntegrations(integrations) {
|
|
|
7431
7431
|
return messages;
|
|
7432
7432
|
}
|
|
7433
7433
|
|
|
7434
|
+
// src/url-images.ts
|
|
7435
|
+
function migrationImageUrl(value, base) {
|
|
7436
|
+
const url = new URL(value, base);
|
|
7437
|
+
if (!["http:", "https:"].includes(url.protocol) || url.username || url.password) throw new Error("Responsive image URL is not a credential-free HTTP(S) URL.");
|
|
7438
|
+
return url.toString();
|
|
7439
|
+
}
|
|
7440
|
+
function migrationImageCandidates(value, base, resolve4 = true) {
|
|
7441
|
+
if (value.length > 64e3) throw new Error("Responsive image candidate list exceeds its safety bound.");
|
|
7442
|
+
const result = [];
|
|
7443
|
+
let index = 0;
|
|
7444
|
+
while (index < value.length) {
|
|
7445
|
+
while (index < value.length && /[\t\n\f\r ,]/.test(value[index])) index++;
|
|
7446
|
+
if (index === value.length) break;
|
|
7447
|
+
const start = index;
|
|
7448
|
+
while (index < value.length && !/[\t\n\f\r ]/.test(value[index])) index++;
|
|
7449
|
+
let url = value.slice(start, index);
|
|
7450
|
+
const hasTerminatingComma = url.endsWith(",");
|
|
7451
|
+
if (hasTerminatingComma) url = url.replace(/,+$/, "");
|
|
7452
|
+
const descriptorStart = index;
|
|
7453
|
+
if (!hasTerminatingComma) while (index < value.length && value[index] !== ",") index++;
|
|
7454
|
+
const descriptor = hasTerminatingComma ? "" : value.slice(descriptorStart, index).trim();
|
|
7455
|
+
if (descriptor && !/^(?:[1-9]\d*w|(?:\d+(?:\.\d+)?|\.\d+)x)$/.test(descriptor)) throw new Error("Responsive image has an unsupported descriptor.");
|
|
7456
|
+
if (descriptor.endsWith("x") && Number(descriptor.slice(0, -1)) <= 0) throw new Error("Responsive image density must be positive.");
|
|
7457
|
+
const resolved = migrationImageUrl(url, base);
|
|
7458
|
+
result.push({ url: resolve4 ? resolved : url, descriptor });
|
|
7459
|
+
if (result.length > 128) throw new Error("Responsive image has too many candidates.");
|
|
7460
|
+
if (value[index] === ",") index++;
|
|
7461
|
+
}
|
|
7462
|
+
if (!result.length) throw new Error("Responsive image has no candidates.");
|
|
7463
|
+
return result;
|
|
7464
|
+
}
|
|
7465
|
+
function renderMigrationSrcset(candidates) {
|
|
7466
|
+
return candidates.map((candidate) => `${candidate.url}${candidate.descriptor ? ` ${candidate.descriptor}` : ""}`).join(", ");
|
|
7467
|
+
}
|
|
7468
|
+
function migrationImageAttribute(value) {
|
|
7469
|
+
return value.replace(/&/g, "&").replace(/"/g, """).replace(/</g, "<").replace(/>/g, ">").replace(/\{/g, "{").replace(/\}/g, "}").replace(/\r/g, " ").replace(/\n/g, " ");
|
|
7470
|
+
}
|
|
7471
|
+
|
|
7434
7472
|
// src/url.ts
|
|
7435
7473
|
var DEFAULT_MAX_PAGES = 1e3;
|
|
7436
7474
|
var DEFAULT_MAX_TOTAL_BYTES = 1e8;
|
|
7437
|
-
var MAX_DISCOVERED_URLS =
|
|
7475
|
+
var MAX_DISCOVERED_URLS = 1e5;
|
|
7438
7476
|
var MAX_LOCAL_RESPONSE_BYTES = 2e6;
|
|
7439
7477
|
var MAX_SITEMAP_DOCUMENTS = 25;
|
|
7440
7478
|
var PORTABLE_MDX_COMPONENTS = /* @__PURE__ */ new Set([
|
|
@@ -7502,7 +7540,9 @@ var PORTABLE_MDX_COMPONENTS = /* @__PURE__ */ new Set([
|
|
|
7502
7540
|
"View",
|
|
7503
7541
|
"Visibility",
|
|
7504
7542
|
"Warning",
|
|
7505
|
-
"YouTube"
|
|
7543
|
+
"YouTube",
|
|
7544
|
+
"picture",
|
|
7545
|
+
"source"
|
|
7506
7546
|
]);
|
|
7507
7547
|
var LOCALE_CODES = /* @__PURE__ */ new Set([
|
|
7508
7548
|
"ar",
|
|
@@ -7944,6 +7984,20 @@ function sanitizeMarkdownUrls(value) {
|
|
|
7944
7984
|
hasUnsafeMdxSyntax = true;
|
|
7945
7985
|
return;
|
|
7946
7986
|
}
|
|
7987
|
+
if (node.name === "source" || node.name === "picture") {
|
|
7988
|
+
const allowed = node.name === "picture" ? [] : ["src", "srcSet", "srcset", "sizes", "media", "type", "width", "height", "id"];
|
|
7989
|
+
if (!allowed.includes(attribute.name) || typeof attribute.value !== "string") {
|
|
7990
|
+
hasUnsafeMdxSyntax = true;
|
|
7991
|
+
return;
|
|
7992
|
+
}
|
|
7993
|
+
try {
|
|
7994
|
+
if (/^srcset$/i.test(attribute.name)) migrationImageCandidates(decodeHtmlAttribute(attribute.value), new URL("https://migration.invalid"));
|
|
7995
|
+
if (attribute.name === "src") migrationImageUrl(decodeHtmlAttribute(attribute.value), new URL("https://migration.invalid"));
|
|
7996
|
+
} catch {
|
|
7997
|
+
hasUnsafeMdxSyntax = true;
|
|
7998
|
+
return;
|
|
7999
|
+
}
|
|
8000
|
+
}
|
|
7947
8001
|
}
|
|
7948
8002
|
}
|
|
7949
8003
|
for (const child of node.children ?? []) validateMdxSyntax(child);
|
|
@@ -8272,15 +8326,28 @@ function mergeEmbeddedOpenApi(fragments) {
|
|
|
8272
8326
|
if (tags.size > 0) combined.tags = [...tags.values()];
|
|
8273
8327
|
return new TextEncoder().encode(stringifyYaml2(combined));
|
|
8274
8328
|
}
|
|
8275
|
-
function htmlPage(document, id) {
|
|
8329
|
+
function htmlPage(document, id, options = {}) {
|
|
8276
8330
|
const $ = load2(document.body);
|
|
8277
8331
|
const links = $('nav a[href], main a[href], article a[href], [role="main"] a[href]').map((_index, element) => $(element).attr("href") ?? "").get().filter(Boolean);
|
|
8278
|
-
const
|
|
8332
|
+
const scopedTitle = options.omitDocumentHeading ? $('main article, article, main, [role="main"]').first().find("h1").first().text().trim() : "";
|
|
8333
|
+
const title = scopedTitle || $("main h1, article h1, h1").first().text().trim() || $('meta[property="og:title"]').attr("content")?.trim() || $("title").text().split("|")[0].trim();
|
|
8279
8334
|
const description = $('meta[name="description"]').attr("content")?.trim() || $('meta[property="og:description"]').attr("content")?.trim();
|
|
8280
8335
|
const content = $('main article, article, main, [role="main"]').first();
|
|
8281
8336
|
const root = content.length ? content : $("body");
|
|
8282
|
-
|
|
8337
|
+
let documentHeadingAnchor = "";
|
|
8338
|
+
const excludedElements = options.omitDocumentHeading ? "script,style,noscript,nav,footer,aside,form,button,svg,iframe,object,embed,link,meta" : "script,style,noscript,nav,header,footer,aside,form,button,svg,iframe,object,embed,link,meta";
|
|
8339
|
+
root.find(excludedElements).remove();
|
|
8283
8340
|
root.find(".theme-doc-version-badge,.theme-doc-breadcrumbs,.theme-doc-toc-mobile,.table-of-contents,.hash-link").remove();
|
|
8341
|
+
if (options.omitDocumentHeading) {
|
|
8342
|
+
const heading = root.find("h1").first();
|
|
8343
|
+
if (heading.length && heading.text().trim() === title) {
|
|
8344
|
+
const anchorId = heading.attr("id");
|
|
8345
|
+
if (anchorId) documentHeadingAnchor = `<a id="${migrationImageAttribute(anchorId)}" />
|
|
8346
|
+
|
|
8347
|
+
`;
|
|
8348
|
+
heading.remove();
|
|
8349
|
+
}
|
|
8350
|
+
}
|
|
8284
8351
|
root.find("a[href]").each((_index, element) => {
|
|
8285
8352
|
const href = $(element).attr("href");
|
|
8286
8353
|
if (!href) return;
|
|
@@ -8311,6 +8378,53 @@ function htmlPage(document, id) {
|
|
|
8311
8378
|
});
|
|
8312
8379
|
const turndown = new TurndownService({ codeBlockStyle: "fenced", emDelimiter: "_", headingStyle: "atx" });
|
|
8313
8380
|
turndown.remove(["script", "style", "noscript"]);
|
|
8381
|
+
const anchor = (element) => {
|
|
8382
|
+
const value = element.getAttribute("id");
|
|
8383
|
+
return options.preserveAnchors && value ? `<a id="${value.replace(/&/g, "&").replace(/"/g, """).replace(/</g, "<").replace(/\{/g, "{")}" />
|
|
8384
|
+
|
|
8385
|
+
` : "";
|
|
8386
|
+
};
|
|
8387
|
+
if (options.preserveAnchors) turndown.addRule("migration-source-anchor", {
|
|
8388
|
+
filter: (node) => node.hasAttribute("id") && node.nodeName !== "PRE",
|
|
8389
|
+
replacement: (content2, node) => {
|
|
8390
|
+
const element = node;
|
|
8391
|
+
const heading = element.nodeName.match(/^H([1-6])$/);
|
|
8392
|
+
return `
|
|
8393
|
+
|
|
8394
|
+
${anchor(element)}${heading ? `${"#".repeat(Number(heading[1]))} ` : ""}${content2}
|
|
8395
|
+
|
|
8396
|
+
`;
|
|
8397
|
+
}
|
|
8398
|
+
});
|
|
8399
|
+
if (options.preserveResponsiveImages) {
|
|
8400
|
+
const image = (element) => {
|
|
8401
|
+
const tag = element.nodeName.toLowerCase();
|
|
8402
|
+
const attributes = [];
|
|
8403
|
+
for (const name of ["src", "srcset", "sizes", "alt", "title", "media", "type", "width", "height", "id"]) {
|
|
8404
|
+
let value = element.getAttribute(name);
|
|
8405
|
+
if (value === null) continue;
|
|
8406
|
+
if (name === "src") value = migrationImageUrl(value, document.finalUrl);
|
|
8407
|
+
if (name === "srcset") value = renderMigrationSrcset(migrationImageCandidates(value, document.finalUrl));
|
|
8408
|
+
const prop = name === "srcset" ? "srcSet" : name;
|
|
8409
|
+
attributes.push(`${prop}="${migrationImageAttribute(value)}"`);
|
|
8410
|
+
}
|
|
8411
|
+
return `<${tag}${attributes.length ? ` ${attributes.join(" ")}` : ""} />`;
|
|
8412
|
+
};
|
|
8413
|
+
turndown.addRule("migration-responsive-image", {
|
|
8414
|
+
filter: (node) => ["IMG", "SOURCE"].includes(node.nodeName),
|
|
8415
|
+
replacement: (_content, node) => image(node)
|
|
8416
|
+
});
|
|
8417
|
+
turndown.addRule("migration-responsive-picture", {
|
|
8418
|
+
filter: "picture",
|
|
8419
|
+
replacement: (content2) => `
|
|
8420
|
+
|
|
8421
|
+
<picture>
|
|
8422
|
+
${content2.trim()}
|
|
8423
|
+
</picture>
|
|
8424
|
+
|
|
8425
|
+
`
|
|
8426
|
+
});
|
|
8427
|
+
}
|
|
8314
8428
|
turndown.addRule("docusaurus-admonition", {
|
|
8315
8429
|
filter: (node) => node.nodeName === "DIV" && node.classList.contains("theme-admonition"),
|
|
8316
8430
|
replacement: (_content, node) => {
|
|
@@ -8325,8 +8439,8 @@ function htmlPage(document, id) {
|
|
|
8325
8439
|
const markdown = turndown.turndown(clone.html() ?? "").trim();
|
|
8326
8440
|
return `
|
|
8327
8441
|
|
|
8328
|
-
<${tag}>
|
|
8329
|
-
${titleText ? `**${titleText}**
|
|
8442
|
+
${anchor(element)}<${tag}>
|
|
8443
|
+
${titleText ? `**${escapeMarkdownText(titleText)}**
|
|
8330
8444
|
|
|
8331
8445
|
` : ""}${markdown}
|
|
8332
8446
|
</${tag}>
|
|
@@ -8341,17 +8455,22 @@ ${titleText ? `**${titleText}**
|
|
|
8341
8455
|
const fragment = load2(element.outerHTML);
|
|
8342
8456
|
const container = fragment(".tabs-container").first();
|
|
8343
8457
|
const labels = container.children("ul.tabs").children("li").toArray().map((item) => fragment(item).text().trim() || "Tab");
|
|
8344
|
-
const panels = container.
|
|
8458
|
+
const panels = container.find('[role="tabpanel"]').toArray().filter((panel) => fragment(panel).closest(".tabs-container")[0] === container[0]);
|
|
8345
8459
|
if (panels.length === 0) return _content;
|
|
8346
8460
|
const tabs = panels.map((panel, index) => {
|
|
8347
|
-
const
|
|
8461
|
+
const panelId = fragment(panel).attr("id");
|
|
8462
|
+
const tab = panelId ? container.find('[role="tab"]').toArray().find((item) => fragment(item).attr("aria-controls") === panelId) : void 0;
|
|
8463
|
+
const title2 = migrationImageAttribute((tab ? fragment(tab).text().trim() : labels[index]) || `Tab ${index + 1}`);
|
|
8464
|
+
const panelAnchor = options.preserveAnchors && panelId ? `<a id="${migrationImageAttribute(panelId)}" />
|
|
8465
|
+
|
|
8466
|
+
` : "";
|
|
8348
8467
|
return `<Tab title="${title2}">
|
|
8349
|
-
${turndown.turndown(fragment(panel).html() ?? "").trim()}
|
|
8468
|
+
${panelAnchor}${turndown.turndown(fragment(panel).html() ?? "").trim()}
|
|
8350
8469
|
</Tab>`;
|
|
8351
8470
|
});
|
|
8352
8471
|
return `
|
|
8353
8472
|
|
|
8354
|
-
<Tabs>
|
|
8473
|
+
${anchor(element)}<Tabs>
|
|
8355
8474
|
${tabs.join("\n\n")}
|
|
8356
8475
|
</Tabs>
|
|
8357
8476
|
|
|
@@ -8390,7 +8509,7 @@ ${normalized.slice(1).map((row) => `| ${row.join(" | ")} |`).join("\n")}
|
|
|
8390
8509
|
const fence = "`".repeat(Math.max(3, longestFence + 1));
|
|
8391
8510
|
return `
|
|
8392
8511
|
|
|
8393
|
-
${fence}${language}
|
|
8512
|
+
${anchor(element)}${fence}${language}
|
|
8394
8513
|
${code}
|
|
8395
8514
|
${fence}
|
|
8396
8515
|
|
|
@@ -8398,8 +8517,8 @@ ${fence}
|
|
|
8398
8517
|
}
|
|
8399
8518
|
});
|
|
8400
8519
|
const escape = turndown.escape.bind(turndown);
|
|
8401
|
-
turndown.escape = (text2) => escape(text2).replace(/[<{]/g, (match) => `\\${match}`);
|
|
8402
|
-
const body = turndown.turndown(root.html() ?? "").trim()
|
|
8520
|
+
turndown.escape = (text2) => escape(text2).replace(/[<{}]/g, (match) => `\\${match}`);
|
|
8521
|
+
const body = `${documentHeadingAnchor}${turndown.turndown(root.html() ?? "").trim()}`;
|
|
8403
8522
|
if (!title || body.length < 40 && $("article").length === 0) return { page: null, links };
|
|
8404
8523
|
return {
|
|
8405
8524
|
page: {
|
|
@@ -8556,13 +8675,13 @@ function docusaurusNavigationFromSnapshots(snapshots, pages, scopePath) {
|
|
|
8556
8675
|
});
|
|
8557
8676
|
mergedSnapshots.sort((left, right) => pageIdsFor(right).length - pageIdsFor(left).length);
|
|
8558
8677
|
const tabs = mergedSnapshots.flatMap((snapshot, snapshotIndex) => {
|
|
8559
|
-
function
|
|
8678
|
+
function convert2(items) {
|
|
8560
8679
|
const converted2 = [];
|
|
8561
8680
|
for (const item of items) {
|
|
8562
8681
|
const routeId = item.href ? docusaurusRouteIdForUrl(new URL(item.href), scopePath) : null;
|
|
8563
8682
|
const page = routeId && imported.has(routeId) && !claimed.has(routeId) ? routeId : null;
|
|
8564
8683
|
if (page) claimed.add(page);
|
|
8565
|
-
const children = item.items ?
|
|
8684
|
+
const children = item.items ? convert2(item.items) : [];
|
|
8566
8685
|
if (children.length > 0 || item.isCategory) {
|
|
8567
8686
|
converted2.push({ group: item.label, pages: page ? [page, ...children] : children });
|
|
8568
8687
|
continue;
|
|
@@ -8571,7 +8690,7 @@ function docusaurusNavigationFromSnapshots(snapshots, pages, scopePath) {
|
|
|
8571
8690
|
}
|
|
8572
8691
|
return converted2;
|
|
8573
8692
|
}
|
|
8574
|
-
const converted =
|
|
8693
|
+
const converted = convert2(snapshot.items);
|
|
8575
8694
|
const loosePages = converted.filter((item) => typeof item === "string");
|
|
8576
8695
|
const groups = converted.filter((item) => typeof item !== "string");
|
|
8577
8696
|
if (loosePages.length > 0) groups.unshift({ group: "Overview", pages: loosePages });
|
|
@@ -8676,7 +8795,7 @@ async function safeFetch(fetcher, url, accept) {
|
|
|
8676
8795
|
async function migrateUrl(options) {
|
|
8677
8796
|
const source = validateMigrationUrl(options.sourceUrl);
|
|
8678
8797
|
const fetcher = options.fetcher ?? defaultMigrationFetcher;
|
|
8679
|
-
const maxPages = Math.max(1, Math.min(options.maxPages ?? DEFAULT_MAX_PAGES,
|
|
8798
|
+
const maxPages = Math.max(1, Math.min(options.maxPages ?? DEFAULT_MAX_PAGES, 1e4));
|
|
8680
8799
|
const concurrency = Math.max(1, Math.min(options.concurrency ?? 5, 10));
|
|
8681
8800
|
const maxTotalBytes = Math.max(1e6, Math.min(options.maxTotalBytes ?? DEFAULT_MAX_TOTAL_BYTES, 5e8));
|
|
8682
8801
|
const submittedScopePath = docsScopePath(source);
|
|
@@ -8835,12 +8954,15 @@ async function migrateUrl(options) {
|
|
|
8835
8954
|
if (!isInScope(document.finalUrl, source, scopePath)) {
|
|
8836
8955
|
return { candidate, page: null, links: [], failure: false };
|
|
8837
8956
|
}
|
|
8957
|
+
if (/\/(?:llms(?:-full)?|robots)\.txt$|\/sitemap[^/]*\.xml$/i.test(document.finalUrl.pathname) || /(?:xml|json)/i.test(document.contentType)) {
|
|
8958
|
+
return { candidate, page: null, links: markdownLinks(document.body), base: document.finalUrl, failure: false };
|
|
8959
|
+
}
|
|
8838
8960
|
const isMarkdown = /(?:markdown|text\/plain)/i.test(document.contentType) || /\.mdx?$/i.test(document.finalUrl.pathname);
|
|
8839
8961
|
if (!isMarkdown && !/\.mdx?$/i.test(candidate.pathname)) {
|
|
8840
8962
|
const markdownUrl = new URL(candidate);
|
|
8841
8963
|
markdownUrl.pathname = `${markdownUrl.pathname.replace(/\/$/, "")}.md`;
|
|
8842
8964
|
const markdownDocument = await safeFetch(fetcher, markdownUrl, "text/markdown,text/plain");
|
|
8843
|
-
if (markdownDocument && isInScope(markdownDocument.finalUrl, source, scopePath) && /(?:markdown|text\/plain)/i.test(markdownDocument.contentType)) {
|
|
8965
|
+
if (markdownDocument && isInScope(markdownDocument.finalUrl, source, scopePath) && !/\/(?:llms(?:-full)?|robots)\.txt$|\/sitemap[^/]*\.xml$/i.test(markdownDocument.finalUrl.pathname) && /(?:markdown|text\/plain)/i.test(markdownDocument.contentType)) {
|
|
8844
8966
|
document = markdownDocument;
|
|
8845
8967
|
}
|
|
8846
8968
|
}
|
|
@@ -10340,12 +10462,12 @@ function apiOperationLinkMap(sources, docsConfig) {
|
|
|
10340
10462
|
const tagSegment = mintlifyOperationSlugSegment(operation.tag ?? "default");
|
|
10341
10463
|
const mintlifyTag = mintlifyUrlSlug(operation.tag ?? "default");
|
|
10342
10464
|
const mintlifyLeaf = mintlifyUrlSlug(operation.summary?.trim() || operation.operationId?.trim() || `${operation.method} ${operation.path}`);
|
|
10343
|
-
const
|
|
10465
|
+
const canonical2 = `${source.prefix}/${mintlifyTag}/${mintlifyLeaf}`.replace(/^\/+|\/+$/g, "");
|
|
10344
10466
|
if (mintlifyTag && mintlifyLeaf) {
|
|
10345
|
-
if (!mintlifyPages.has(
|
|
10346
|
-
else if (mintlifyPages.get(
|
|
10467
|
+
if (!mintlifyPages.has(canonical2)) mintlifyPages.set(canonical2, thallyHref);
|
|
10468
|
+
else if (mintlifyPages.get(canonical2) !== thallyHref && !duplicatePages.includes(canonical2)) duplicatePages.push(canonical2);
|
|
10347
10469
|
}
|
|
10348
|
-
if (mintlifyTag && mintlifyLeaf && !map.has(
|
|
10470
|
+
if (mintlifyTag && mintlifyLeaf && !map.has(canonical2.toLowerCase())) map.set(canonical2.toLowerCase(), thallyHref);
|
|
10349
10471
|
const tagRoot = `${source.prefix}/${mintlifyTag}`.replace(/^\/+|\/+$/g, "");
|
|
10350
10472
|
if (mintlifyTag && !mintlifyPages.has(tagRoot)) mintlifyPages.set(tagRoot, thallyHref);
|
|
10351
10473
|
if (mintlifyTag && !map.has(tagRoot.toLowerCase())) map.set(tagRoot.toLowerCase(), thallyHref);
|
|
@@ -10820,16 +10942,16 @@ function preserveLinkedAnchors(pages) {
|
|
|
10820
10942
|
const requested = /* @__PURE__ */ new Map();
|
|
10821
10943
|
const crossPageRequests = /* @__PURE__ */ new Set();
|
|
10822
10944
|
const addTarget = (target, currentId) => {
|
|
10823
|
-
const
|
|
10824
|
-
if (
|
|
10945
|
+
const hash3 = target.indexOf("#");
|
|
10946
|
+
if (hash3 < 0) return;
|
|
10825
10947
|
let fragment;
|
|
10826
10948
|
try {
|
|
10827
|
-
fragment = decodeURIComponent(target.slice(
|
|
10949
|
+
fragment = decodeURIComponent(target.slice(hash3 + 1));
|
|
10828
10950
|
} catch {
|
|
10829
10951
|
return;
|
|
10830
10952
|
}
|
|
10831
10953
|
if (!/^[\p{L}\p{N}_.:$-]{1,120}$/u.test(fragment)) return;
|
|
10832
|
-
const path = target.slice(0,
|
|
10954
|
+
const path = target.slice(0, hash3).split("?")[0];
|
|
10833
10955
|
const id = target.startsWith("#") ? currentId : path.startsWith("/") && !path.startsWith("//") ? path.replace(/^\//, "").replace(/\/$/, "") || "introduction" : "";
|
|
10834
10956
|
if (!pageById.has(id)) return;
|
|
10835
10957
|
if (!requested.has(id)) requested.set(id, /* @__PURE__ */ new Set());
|
|
@@ -13442,8 +13564,1137 @@ function renderMigrationFiles(bundle, options = {}) {
|
|
|
13442
13564
|
];
|
|
13443
13565
|
}
|
|
13444
13566
|
|
|
13445
|
-
// src/
|
|
13567
|
+
// src/url-run.ts
|
|
13446
13568
|
import { createHash as createHash3 } from "crypto";
|
|
13569
|
+
import { load as load3 } from "cheerio";
|
|
13570
|
+
import remarkParse4 from "remark-parse";
|
|
13571
|
+
import remarkMdx4 from "remark-mdx";
|
|
13572
|
+
import { unified as unified4 } from "unified";
|
|
13573
|
+
|
|
13574
|
+
// src/svg-policy.ts
|
|
13575
|
+
import { Parser } from "htmlparser2";
|
|
13576
|
+
var SVG_NAMESPACE = "http://www.w3.org/2000/svg";
|
|
13577
|
+
var XLINK_NAMESPACE = "http://www.w3.org/1999/xlink";
|
|
13578
|
+
var ALLOWED_ELEMENTS = /* @__PURE__ */ new Set([
|
|
13579
|
+
"svg",
|
|
13580
|
+
"g",
|
|
13581
|
+
"defs",
|
|
13582
|
+
"symbol",
|
|
13583
|
+
"use",
|
|
13584
|
+
"title",
|
|
13585
|
+
"desc",
|
|
13586
|
+
"path",
|
|
13587
|
+
"rect",
|
|
13588
|
+
"circle",
|
|
13589
|
+
"ellipse",
|
|
13590
|
+
"line",
|
|
13591
|
+
"polyline",
|
|
13592
|
+
"polygon",
|
|
13593
|
+
"text",
|
|
13594
|
+
"tspan",
|
|
13595
|
+
"textpath",
|
|
13596
|
+
"lineargradient",
|
|
13597
|
+
"radialgradient",
|
|
13598
|
+
"stop",
|
|
13599
|
+
"clippath",
|
|
13600
|
+
"mask",
|
|
13601
|
+
"pattern",
|
|
13602
|
+
"marker",
|
|
13603
|
+
"filter",
|
|
13604
|
+
"feblend",
|
|
13605
|
+
"fecolormatrix",
|
|
13606
|
+
"fecomponenttransfer",
|
|
13607
|
+
"fecomposite",
|
|
13608
|
+
"feconvolvematrix",
|
|
13609
|
+
"fediffuselighting",
|
|
13610
|
+
"fedisplacementmap",
|
|
13611
|
+
"fedistantlight",
|
|
13612
|
+
"fedropshadow",
|
|
13613
|
+
"feflood",
|
|
13614
|
+
"fefunca",
|
|
13615
|
+
"fefuncb",
|
|
13616
|
+
"fefuncg",
|
|
13617
|
+
"fefuncr",
|
|
13618
|
+
"fegaussianblur",
|
|
13619
|
+
"femerge",
|
|
13620
|
+
"femergenode",
|
|
13621
|
+
"femorphology",
|
|
13622
|
+
"feoffset",
|
|
13623
|
+
"fepointlight",
|
|
13624
|
+
"fespecularlighting",
|
|
13625
|
+
"fespotlight",
|
|
13626
|
+
"fetile",
|
|
13627
|
+
"feturbulence"
|
|
13628
|
+
]);
|
|
13629
|
+
var ALLOWED_ATTRIBUTES = /* @__PURE__ */ new Set([
|
|
13630
|
+
"xmlns",
|
|
13631
|
+
"xmlns:xlink",
|
|
13632
|
+
"xml:space",
|
|
13633
|
+
"version",
|
|
13634
|
+
"baseprofile",
|
|
13635
|
+
"id",
|
|
13636
|
+
"class",
|
|
13637
|
+
"viewbox",
|
|
13638
|
+
"preserveaspectratio",
|
|
13639
|
+
"width",
|
|
13640
|
+
"height",
|
|
13641
|
+
"x",
|
|
13642
|
+
"y",
|
|
13643
|
+
"x1",
|
|
13644
|
+
"y1",
|
|
13645
|
+
"x2",
|
|
13646
|
+
"y2",
|
|
13647
|
+
"cx",
|
|
13648
|
+
"cy",
|
|
13649
|
+
"r",
|
|
13650
|
+
"rx",
|
|
13651
|
+
"ry",
|
|
13652
|
+
"d",
|
|
13653
|
+
"points",
|
|
13654
|
+
"pathlength",
|
|
13655
|
+
"transform",
|
|
13656
|
+
"transform-origin",
|
|
13657
|
+
"fill",
|
|
13658
|
+
"fill-opacity",
|
|
13659
|
+
"fill-rule",
|
|
13660
|
+
"stroke",
|
|
13661
|
+
"stroke-width",
|
|
13662
|
+
"stroke-opacity",
|
|
13663
|
+
"stroke-linecap",
|
|
13664
|
+
"stroke-linejoin",
|
|
13665
|
+
"stroke-miterlimit",
|
|
13666
|
+
"stroke-dasharray",
|
|
13667
|
+
"stroke-dashoffset",
|
|
13668
|
+
"opacity",
|
|
13669
|
+
"color",
|
|
13670
|
+
"display",
|
|
13671
|
+
"visibility",
|
|
13672
|
+
"overflow",
|
|
13673
|
+
"clip-path",
|
|
13674
|
+
"clip-rule",
|
|
13675
|
+
"clippathunits",
|
|
13676
|
+
"mask",
|
|
13677
|
+
"maskunits",
|
|
13678
|
+
"maskcontentunits",
|
|
13679
|
+
"filter",
|
|
13680
|
+
"filterunits",
|
|
13681
|
+
"primitiveunits",
|
|
13682
|
+
"marker-start",
|
|
13683
|
+
"marker-mid",
|
|
13684
|
+
"marker-end",
|
|
13685
|
+
"markerwidth",
|
|
13686
|
+
"markerheight",
|
|
13687
|
+
"markerunits",
|
|
13688
|
+
"refx",
|
|
13689
|
+
"refy",
|
|
13690
|
+
"orient",
|
|
13691
|
+
"patternunits",
|
|
13692
|
+
"patterncontentunits",
|
|
13693
|
+
"patterntransform",
|
|
13694
|
+
"gradientunits",
|
|
13695
|
+
"gradienttransform",
|
|
13696
|
+
"spreadmethod",
|
|
13697
|
+
"fx",
|
|
13698
|
+
"fy",
|
|
13699
|
+
"fr",
|
|
13700
|
+
"offset",
|
|
13701
|
+
"stop-color",
|
|
13702
|
+
"stop-opacity",
|
|
13703
|
+
"font-family",
|
|
13704
|
+
"font-size",
|
|
13705
|
+
"font-weight",
|
|
13706
|
+
"font-style",
|
|
13707
|
+
"font-variant",
|
|
13708
|
+
"letter-spacing",
|
|
13709
|
+
"word-spacing",
|
|
13710
|
+
"text-anchor",
|
|
13711
|
+
"dominant-baseline",
|
|
13712
|
+
"alignment-baseline",
|
|
13713
|
+
"baseline-shift",
|
|
13714
|
+
"text-decoration",
|
|
13715
|
+
"dx",
|
|
13716
|
+
"dy",
|
|
13717
|
+
"rotate",
|
|
13718
|
+
"textlength",
|
|
13719
|
+
"lengthadjust",
|
|
13720
|
+
"startoffset",
|
|
13721
|
+
"writing-mode",
|
|
13722
|
+
"direction",
|
|
13723
|
+
"unicode-bidi",
|
|
13724
|
+
"shape-rendering",
|
|
13725
|
+
"text-rendering",
|
|
13726
|
+
"image-rendering",
|
|
13727
|
+
"color-interpolation",
|
|
13728
|
+
"color-interpolation-filters",
|
|
13729
|
+
"vector-effect",
|
|
13730
|
+
"paint-order",
|
|
13731
|
+
"mix-blend-mode",
|
|
13732
|
+
"isolation",
|
|
13733
|
+
"in",
|
|
13734
|
+
"in2",
|
|
13735
|
+
"result",
|
|
13736
|
+
"stddeviation",
|
|
13737
|
+
"mode",
|
|
13738
|
+
"type",
|
|
13739
|
+
"values",
|
|
13740
|
+
"tablevalues",
|
|
13741
|
+
"slope",
|
|
13742
|
+
"intercept",
|
|
13743
|
+
"amplitude",
|
|
13744
|
+
"exponent",
|
|
13745
|
+
"operator",
|
|
13746
|
+
"k1",
|
|
13747
|
+
"k2",
|
|
13748
|
+
"k3",
|
|
13749
|
+
"k4",
|
|
13750
|
+
"radius",
|
|
13751
|
+
"scale",
|
|
13752
|
+
"xchannelselector",
|
|
13753
|
+
"ychannelselector",
|
|
13754
|
+
"basefrequency",
|
|
13755
|
+
"numoctaves",
|
|
13756
|
+
"seed",
|
|
13757
|
+
"stitchtiles",
|
|
13758
|
+
"flood-color",
|
|
13759
|
+
"flood-opacity",
|
|
13760
|
+
"lighting-color",
|
|
13761
|
+
"surfacescale",
|
|
13762
|
+
"diffuseconstant",
|
|
13763
|
+
"specularconstant",
|
|
13764
|
+
"specularexponent",
|
|
13765
|
+
"kernelmatrix",
|
|
13766
|
+
"kernelunitlength",
|
|
13767
|
+
"order",
|
|
13768
|
+
"divisor",
|
|
13769
|
+
"bias",
|
|
13770
|
+
"targetx",
|
|
13771
|
+
"targety",
|
|
13772
|
+
"edgemode",
|
|
13773
|
+
"preservealpha",
|
|
13774
|
+
"azimuth",
|
|
13775
|
+
"elevation",
|
|
13776
|
+
"z",
|
|
13777
|
+
"pointsatx",
|
|
13778
|
+
"pointsaty",
|
|
13779
|
+
"pointsatz",
|
|
13780
|
+
"limitingconeangle",
|
|
13781
|
+
"aria-hidden",
|
|
13782
|
+
"aria-label",
|
|
13783
|
+
"role",
|
|
13784
|
+
"focusable",
|
|
13785
|
+
// Validated below: only in-document fragment references.
|
|
13786
|
+
"href",
|
|
13787
|
+
"xlink:href"
|
|
13788
|
+
]);
|
|
13789
|
+
function hasExternalReference(value) {
|
|
13790
|
+
if (value.includes("\\")) return true;
|
|
13791
|
+
const compact = value.replace(/[\u0000- ]/gu, "").toLowerCase();
|
|
13792
|
+
if (/(?:java|vb)script:|data:|expression\(|@import|<|>/u.test(compact)) {
|
|
13793
|
+
return true;
|
|
13794
|
+
}
|
|
13795
|
+
for (const match of compact.matchAll(/url\(([^)]*)\)/gu)) {
|
|
13796
|
+
if (!/^["']?#[a-z0-9_.:-]+["']?$/u.test(match[1])) return true;
|
|
13797
|
+
}
|
|
13798
|
+
return compact.replace(/url\(([^)]*)\)/gu, "").includes("url(");
|
|
13799
|
+
}
|
|
13800
|
+
function utf8SvgText(content) {
|
|
13801
|
+
const hasUtf8Bom = content[0] === 239 && content[1] === 187 && content[2] === 191;
|
|
13802
|
+
const body = hasUtf8Bom ? content.subarray(3) : content;
|
|
13803
|
+
if (body[0] === 254 && body[1] === 255 || body[0] === 255 && body[1] === 254 || body[0] === 0 && body[1] === 0 || body.includes(0)) {
|
|
13804
|
+
return null;
|
|
13805
|
+
}
|
|
13806
|
+
const text2 = body.toString("utf8");
|
|
13807
|
+
if (text2.includes("\uFFFD")) return null;
|
|
13808
|
+
const declaration = /^<\?xml\b[^>]*\?>/iu.exec(text2);
|
|
13809
|
+
if (declaration) {
|
|
13810
|
+
const encoding = /\bencoding\s*=\s*["']([^"']*)["']/iu.exec(declaration[0]);
|
|
13811
|
+
if (encoding && !/^utf-?8$/iu.test(encoding[1].trim())) return null;
|
|
13812
|
+
} else if (/<\?xml\b/iu.test(text2)) {
|
|
13813
|
+
return null;
|
|
13814
|
+
}
|
|
13815
|
+
return text2;
|
|
13816
|
+
}
|
|
13817
|
+
function isPassiveSvg(content) {
|
|
13818
|
+
const text2 = utf8SvgText(Buffer.from(content.buffer, content.byteOffset, content.byteLength));
|
|
13819
|
+
if (text2 === null) return false;
|
|
13820
|
+
let isPassive = true;
|
|
13821
|
+
let depth = 0;
|
|
13822
|
+
let sawRoot = false;
|
|
13823
|
+
const openStarts = [];
|
|
13824
|
+
const fail = () => {
|
|
13825
|
+
isPassive = false;
|
|
13826
|
+
};
|
|
13827
|
+
const parser3 = new Parser(
|
|
13828
|
+
{
|
|
13829
|
+
onopentag(name, attributes) {
|
|
13830
|
+
const element = name.toLowerCase();
|
|
13831
|
+
if (depth === 0) {
|
|
13832
|
+
if (sawRoot || element !== "svg") fail();
|
|
13833
|
+
sawRoot = true;
|
|
13834
|
+
}
|
|
13835
|
+
depth += 1;
|
|
13836
|
+
openStarts.push(parser3.startIndex);
|
|
13837
|
+
if (element.includes(":") || !ALLOWED_ELEMENTS.has(element)) fail();
|
|
13838
|
+
for (const [rawName, value] of Object.entries(attributes)) {
|
|
13839
|
+
const attribute = rawName.toLowerCase();
|
|
13840
|
+
if (!ALLOWED_ATTRIBUTES.has(attribute)) {
|
|
13841
|
+
fail();
|
|
13842
|
+
continue;
|
|
13843
|
+
}
|
|
13844
|
+
if (attribute === "xmlns" && value !== SVG_NAMESPACE) fail();
|
|
13845
|
+
if (attribute === "xmlns:xlink" && value !== XLINK_NAMESPACE) fail();
|
|
13846
|
+
if ((attribute === "href" || attribute === "xlink:href") && !/^#[A-Za-z0-9_.:-]+$/u.test(value.trim())) {
|
|
13847
|
+
fail();
|
|
13848
|
+
}
|
|
13849
|
+
if (hasExternalReference(value)) fail();
|
|
13850
|
+
}
|
|
13851
|
+
},
|
|
13852
|
+
onclosetag(_name, isImplied) {
|
|
13853
|
+
depth -= 1;
|
|
13854
|
+
const start = openStarts.pop();
|
|
13855
|
+
if (isImplied && start !== parser3.startIndex) fail();
|
|
13856
|
+
},
|
|
13857
|
+
// DOCTYPE (and with it custom entities) or any other declaration other
|
|
13858
|
+
// than the XML prolog.
|
|
13859
|
+
onprocessinginstruction(name) {
|
|
13860
|
+
if (name.toLowerCase() !== "?xml") fail();
|
|
13861
|
+
},
|
|
13862
|
+
oncdatastart: fail,
|
|
13863
|
+
ontext(value) {
|
|
13864
|
+
if (depth === 0 && value.trim()) fail();
|
|
13865
|
+
},
|
|
13866
|
+
onerror: fail
|
|
13867
|
+
},
|
|
13868
|
+
{ xmlMode: true, decodeEntities: true, recognizeSelfClosing: true }
|
|
13869
|
+
);
|
|
13870
|
+
try {
|
|
13871
|
+
parser3.write(text2);
|
|
13872
|
+
parser3.end();
|
|
13873
|
+
} catch {
|
|
13874
|
+
return false;
|
|
13875
|
+
}
|
|
13876
|
+
return isPassive && sawRoot && depth === 0;
|
|
13877
|
+
}
|
|
13878
|
+
|
|
13879
|
+
// src/url-asset-cache.ts
|
|
13880
|
+
function createMigrationAssetCache() {
|
|
13881
|
+
const settled = /* @__PURE__ */ new Map();
|
|
13882
|
+
const pending = /* @__PURE__ */ new Map();
|
|
13883
|
+
const remember = (url, asset) => {
|
|
13884
|
+
settled.delete(url);
|
|
13885
|
+
settled.set(url, asset);
|
|
13886
|
+
if (settled.size > 480) settled.delete(settled.keys().next().value);
|
|
13887
|
+
return asset;
|
|
13888
|
+
};
|
|
13889
|
+
const get = (url) => {
|
|
13890
|
+
const asset = settled.get(url);
|
|
13891
|
+
if (asset) return Promise.resolve(remember(url, asset));
|
|
13892
|
+
return pending.get(url);
|
|
13893
|
+
};
|
|
13894
|
+
return {
|
|
13895
|
+
remember,
|
|
13896
|
+
get,
|
|
13897
|
+
has(url) {
|
|
13898
|
+
return settled.has(url) || pending.has(url);
|
|
13899
|
+
},
|
|
13900
|
+
get size() {
|
|
13901
|
+
return settled.size + pending.size;
|
|
13902
|
+
},
|
|
13903
|
+
load(url, acquire) {
|
|
13904
|
+
const existing = get(url);
|
|
13905
|
+
if (existing) return existing;
|
|
13906
|
+
const request2 = Promise.resolve().then(acquire).then((asset) => {
|
|
13907
|
+
pending.delete(url);
|
|
13908
|
+
return remember(url, asset);
|
|
13909
|
+
}, (error) => {
|
|
13910
|
+
pending.delete(url);
|
|
13911
|
+
throw error;
|
|
13912
|
+
});
|
|
13913
|
+
pending.set(url, request2);
|
|
13914
|
+
return request2;
|
|
13915
|
+
},
|
|
13916
|
+
delete(url) {
|
|
13917
|
+
settled.delete(url);
|
|
13918
|
+
}
|
|
13919
|
+
};
|
|
13920
|
+
}
|
|
13921
|
+
|
|
13922
|
+
// src/url-journal-clone.ts
|
|
13923
|
+
function cloneUrlMigrationCheckpoint(checkpoint) {
|
|
13924
|
+
const ancestors = /* @__PURE__ */ new Set();
|
|
13925
|
+
const clone = (value) => {
|
|
13926
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
13927
|
+
if (typeof value === "number") return Number.isFinite(value) ? value === 0 ? 0 : value : null;
|
|
13928
|
+
if (typeof value === "undefined" || typeof value === "function" || typeof value === "symbol") return void 0;
|
|
13929
|
+
if (typeof value !== "object") throw new TypeError("Checkpoint must contain JSON-safe values.");
|
|
13930
|
+
if (ancestors.has(value)) throw new TypeError("Checkpoint must not contain circular references.");
|
|
13931
|
+
if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw new TypeError("Checkpoint must contain plain JSON containers.");
|
|
13932
|
+
ancestors.add(value);
|
|
13933
|
+
try {
|
|
13934
|
+
if (Array.isArray(value)) return Array.from(value, (entry) => clone(entry) ?? null);
|
|
13935
|
+
return Object.fromEntries(Object.entries(value).flatMap(([key, entry]) => {
|
|
13936
|
+
const copied = clone(entry);
|
|
13937
|
+
return copied === void 0 ? [] : [[key, copied]];
|
|
13938
|
+
}));
|
|
13939
|
+
} finally {
|
|
13940
|
+
ancestors.delete(value);
|
|
13941
|
+
}
|
|
13942
|
+
};
|
|
13943
|
+
return clone(checkpoint);
|
|
13944
|
+
}
|
|
13945
|
+
|
|
13946
|
+
// src/url-run.ts
|
|
13947
|
+
var CONVERTER_VERSION = "url-evidence-v2";
|
|
13948
|
+
var parser2 = unified4().use(remarkParse4).use(remarkMdx4);
|
|
13949
|
+
var ACCEPT = "text/markdown,text/html,application/xhtml+xml;q=0.9";
|
|
13950
|
+
var MAX_RESPONSE_BYTES = 8e6;
|
|
13951
|
+
var MAX_ASSET_BYTES2 = 1e7;
|
|
13952
|
+
var MAX_RECORDS = 1e5;
|
|
13953
|
+
var LOCALES = /* @__PURE__ */ new Set(["ar", "cs", "da", "de", "el", "en", "es", "fi", "fr", "he", "hi", "hu", "id", "it", "ja", "ko", "ms", "nl", "no", "pl", "pt", "pt-br", "ro", "ru", "sk", "sv", "th", "tr", "uk", "vi", "zh", "zh-cn", "zh-hans", "zh-hant"]);
|
|
13954
|
+
function hash2(value) {
|
|
13955
|
+
return createHash3("sha256").update(value).digest("hex");
|
|
13956
|
+
}
|
|
13957
|
+
function pageHash(page) {
|
|
13958
|
+
const canonical2 = (value) => Array.isArray(value) ? value.map(canonical2) : value !== null && typeof value === "object" ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([key, field]) => [key, canonical2(field)])) : value;
|
|
13959
|
+
return hash2(JSON.stringify(canonical2(page)));
|
|
13960
|
+
}
|
|
13961
|
+
function inScope(url, source, scope) {
|
|
13962
|
+
return url.origin === source.origin && !url.username && !url.password && (scope === "/" || url.pathname === scope || url.pathname.startsWith(`${scope}/`));
|
|
13963
|
+
}
|
|
13964
|
+
function isDiscoveryIndex(url, source) {
|
|
13965
|
+
return url.origin === source.origin && /\/(?:llms(?:-full)?|robots)\.txt$|\/sitemap[^/]*\.xml$/i.test(url.pathname);
|
|
13966
|
+
}
|
|
13967
|
+
function canonical(value, base) {
|
|
13968
|
+
try {
|
|
13969
|
+
const url = validateMigrationUrl(new URL(value, base).toString());
|
|
13970
|
+
url.pathname = url.pathname.replace(/\/+$/, "") || "/";
|
|
13971
|
+
return url.toString();
|
|
13972
|
+
} catch {
|
|
13973
|
+
return null;
|
|
13974
|
+
}
|
|
13975
|
+
}
|
|
13976
|
+
function classifyMigrationResponse(document) {
|
|
13977
|
+
if (document.status && document.status >= 400) return "unsupported";
|
|
13978
|
+
if (/\/(?:llms(?:-full)?|robots)\.txt$|\/sitemap[^/]*\.xml$|\/search(?:\.html)?$/i.test(document.finalUrl.pathname) || /(?:xml|json)/i.test(document.contentType)) return "index";
|
|
13979
|
+
if (/^(?:image|audio|video)\//i.test(document.contentType) || /\.(?:pdf|zip|png|jpe?g|webp|gif|svg|ico|mp4|webm)$/i.test(document.finalUrl.pathname)) return "asset";
|
|
13980
|
+
return /(?:html|markdown|text\/plain)/i.test(document.contentType) || /\.mdx?$/i.test(document.finalUrl.pathname) ? "page" : "unsupported";
|
|
13981
|
+
}
|
|
13982
|
+
function markdownNodes(body) {
|
|
13983
|
+
const nodes = [];
|
|
13984
|
+
const visit = (node) => {
|
|
13985
|
+
nodes.push(node);
|
|
13986
|
+
for (const child of node.children ?? []) visit(child);
|
|
13987
|
+
};
|
|
13988
|
+
visit(parser2.parse(body));
|
|
13989
|
+
return nodes;
|
|
13990
|
+
}
|
|
13991
|
+
function markdownCode(body) {
|
|
13992
|
+
return markdownNodes(body).filter((node) => node.type === "code").map((node) => node.value ?? "");
|
|
13993
|
+
}
|
|
13994
|
+
function alternativeBlocks(body, baseline) {
|
|
13995
|
+
const blocks = (value) => (parser2.parse(value).children ?? []).map((node) => value.slice(node.position.start.offset, node.position.end.offset));
|
|
13996
|
+
const originals = new Set(blocks(baseline));
|
|
13997
|
+
return blocks(body).filter((block) => !originals.has(block)).join("\n\n");
|
|
13998
|
+
}
|
|
13999
|
+
function linksOf(document) {
|
|
14000
|
+
if (/html/i.test(document.contentType)) {
|
|
14001
|
+
const $ = load3(document.body);
|
|
14002
|
+
const root = $('main article,article,main,[role="main"]').first();
|
|
14003
|
+
root.find("nav,footer,aside").remove();
|
|
14004
|
+
return [...root.find("a[href]").toArray(), ...$('link[rel="sitemap"][href],link[rel="alternate"][href]').toArray()].map((node) => $(node).attr("href")).filter(Boolean);
|
|
14005
|
+
}
|
|
14006
|
+
if (/xml/i.test(document.contentType) || /<urlset|<sitemapindex/i.test(document.body)) {
|
|
14007
|
+
const $ = load3(document.body, { xmlMode: true });
|
|
14008
|
+
return $("loc").toArray().map((node) => $(node).text().trim());
|
|
14009
|
+
}
|
|
14010
|
+
return [
|
|
14011
|
+
...markdownNodes(document.body).flatMap((node) => node.url ? [node.url] : []),
|
|
14012
|
+
...[...document.body.matchAll(/^Sitemap:\s*(\S+)/gmi)].map((match) => match[1])
|
|
14013
|
+
];
|
|
14014
|
+
}
|
|
14015
|
+
function htmlEvidence(document) {
|
|
14016
|
+
const $ = load3(document.body);
|
|
14017
|
+
const root = $('main article,article,main,[role="main"]').first();
|
|
14018
|
+
const article = root.length ? root : $("body");
|
|
14019
|
+
article.find("nav,footer,aside,form,script,style").remove();
|
|
14020
|
+
return {
|
|
14021
|
+
anchors: article.find("[id]").toArray().map((node) => $(node).attr("id")).filter(Boolean),
|
|
14022
|
+
code: article.find("pre").toArray().map((node) => {
|
|
14023
|
+
const lines = $(node).find(".token-line");
|
|
14024
|
+
return lines.length ? lines.toArray().map((line) => $(line).text()).join("\n") : $(node).text();
|
|
14025
|
+
})
|
|
14026
|
+
};
|
|
14027
|
+
}
|
|
14028
|
+
function evidenceOf(capture) {
|
|
14029
|
+
const technical = /html/i.test(capture.contentType) ? htmlEvidence(capture) : { code: markdownCode(capture.body), anchors: markdownHeadingAnchors(capture.body) };
|
|
14030
|
+
return {
|
|
14031
|
+
sha256: hash2(capture.body),
|
|
14032
|
+
byteLength: Buffer.byteLength(capture.body),
|
|
14033
|
+
finalUrl: capture.finalUrl.toString(),
|
|
14034
|
+
contentType: capture.contentType,
|
|
14035
|
+
...capture.artifactRef ? { artifactRef: capture.artifactRef } : {},
|
|
14036
|
+
states: (capture.states ?? []).map((state) => ({ id: state.id, label: state.label, sha256: hash2(state.response.body) })),
|
|
14037
|
+
...technical,
|
|
14038
|
+
links: linksOf(capture)
|
|
14039
|
+
};
|
|
14040
|
+
}
|
|
14041
|
+
function markdownHeadingAnchors(body) {
|
|
14042
|
+
const seen = /* @__PURE__ */ new Map();
|
|
14043
|
+
return markdownNodes(body).filter((node) => node.type === "heading").map((node) => {
|
|
14044
|
+
const text2 = (value) => value.value ?? (value.children ?? []).map(text2).join("");
|
|
14045
|
+
const base = text2(node).toLowerCase().replace(/[^\p{L}\p{N}\s_-]/gu, "").replace(/\s/g, "-");
|
|
14046
|
+
const occurrence = seen.get(base) ?? 0;
|
|
14047
|
+
seen.set(base, occurrence + 1);
|
|
14048
|
+
return occurrence ? `${base}-${occurrence}` : base;
|
|
14049
|
+
});
|
|
14050
|
+
}
|
|
14051
|
+
function sourceNavigation(capture) {
|
|
14052
|
+
if (capture.navigation) {
|
|
14053
|
+
const resolve4 = (nodes, depth = 0) => {
|
|
14054
|
+
if (depth > 32 || nodes.length > 1e4) throw new Error("Source navigation exceeds its depth or width safety bound.");
|
|
14055
|
+
return nodes.map((node) => ({ label: node.label, ...node.url ? { url: new URL(node.url, capture.finalUrl).toString() } : {}, ...node.children ? { children: resolve4(node.children, depth + 1) } : {} }));
|
|
14056
|
+
};
|
|
14057
|
+
return resolve4(capture.navigation);
|
|
14058
|
+
}
|
|
14059
|
+
if (!/html/i.test(capture.contentType)) return [];
|
|
14060
|
+
const $ = load3(capture.body);
|
|
14061
|
+
const candidates = ['.theme-doc-sidebar-menu,[aria-label*="sidebar" i]', "aside", '[role="navigation"],nav'];
|
|
14062
|
+
const root = candidates.map((selector) => $(selector).filter((_index, node) => $(node).find("a[href]").length > 0 && $(node).closest("header").length === 0).first()).find((candidate) => candidate.length) ?? $("aside").first();
|
|
14063
|
+
if (!root.length) return [];
|
|
14064
|
+
const walk2 = (element, excluded) => {
|
|
14065
|
+
const result = [];
|
|
14066
|
+
$(element).children().each((_index, node) => {
|
|
14067
|
+
if (node === excluded) return;
|
|
14068
|
+
const item = $(node);
|
|
14069
|
+
if (item.is("a[href]")) {
|
|
14070
|
+
result.push({ label: item.text().replace(/\s+/g, " ").trim(), url: new URL(item.attr("href"), capture.finalUrl).toString() });
|
|
14071
|
+
} else if (item.is("li,details") || item.attr("data-migration-group") !== void 0 || item.children("h2,h3,h4,h5,h6").length > 0) {
|
|
14072
|
+
const directLink = item.children("a[href]").first();
|
|
14073
|
+
const direct = directLink.length ? directLink : item.children("div").children("a[href]").first();
|
|
14074
|
+
const label = direct.text().trim() || item.children("summary,button,span,h2,h3,h4,h5,h6").first().text().trim();
|
|
14075
|
+
const children = walk2(node, direct[0]);
|
|
14076
|
+
if (label) result.push({ label, ...direct.length ? { url: new URL(direct.attr("href"), capture.finalUrl).toString() } : {}, ...children.length ? { children } : {} });
|
|
14077
|
+
else result.push(...children);
|
|
14078
|
+
} else result.push(...walk2(node, excluded));
|
|
14079
|
+
});
|
|
14080
|
+
return result;
|
|
14081
|
+
};
|
|
14082
|
+
try {
|
|
14083
|
+
return walk2(root);
|
|
14084
|
+
} catch {
|
|
14085
|
+
return [];
|
|
14086
|
+
}
|
|
14087
|
+
}
|
|
14088
|
+
function idForUrl(url, scope) {
|
|
14089
|
+
const relative7 = scope === "/" ? url.pathname.slice(1) : url.pathname.slice(scope.length).replace(/^\//, "");
|
|
14090
|
+
const id = pageIdFromReference(relative7 || "introduction", true);
|
|
14091
|
+
return id && `${id}${url.search ? `--${hash2(url.search).slice(0, 12)}` : ""}`;
|
|
14092
|
+
}
|
|
14093
|
+
function convert(document, id) {
|
|
14094
|
+
if (/html/i.test(document.contentType)) {
|
|
14095
|
+
const technical = htmlEvidence(document);
|
|
14096
|
+
const page = htmlPage(document, id, { preserveAnchors: true, preserveResponsiveImages: true, omitDocumentHeading: true }).page;
|
|
14097
|
+
if (!page) return null;
|
|
14098
|
+
page.descriptionPlacement = "body";
|
|
14099
|
+
const nodes = markdownNodes(page.body).filter((node) => node.type === "code");
|
|
14100
|
+
if (nodes.length === technical.code.length) {
|
|
14101
|
+
for (let index = nodes.length - 1; index >= 0; index--) {
|
|
14102
|
+
const node = nodes[index];
|
|
14103
|
+
const start = node.position?.start.offset;
|
|
14104
|
+
const end = node.position?.end.offset;
|
|
14105
|
+
if (start === void 0 || end === void 0) continue;
|
|
14106
|
+
const original = page.body.slice(start, end);
|
|
14107
|
+
const info = original.split("\n", 1)[0].replace(/^`+|^~+/, "");
|
|
14108
|
+
const code = technical.code[index];
|
|
14109
|
+
const fence = "`".repeat(Math.max(3, ...[...code.matchAll(/`+/g)].map((match) => match[0].length + 1)));
|
|
14110
|
+
page.body = `${page.body.slice(0, start)}${fence}${info}
|
|
14111
|
+
${code}
|
|
14112
|
+
${fence}${page.body.slice(end)}`;
|
|
14113
|
+
}
|
|
14114
|
+
}
|
|
14115
|
+
return page;
|
|
14116
|
+
}
|
|
14117
|
+
const safe = sanitizeRemoteMarkdown(document.body);
|
|
14118
|
+
return safe === null ? null : markdownPage({ ...document, body: safe }, id, "generic").page;
|
|
14119
|
+
}
|
|
14120
|
+
function assetReferences(capture) {
|
|
14121
|
+
if (!/html/i.test(capture.contentType)) {
|
|
14122
|
+
return markdownNodes(capture.body).filter((node) => node.type === "image").flatMap((node) => node.url ? [node.url] : []);
|
|
14123
|
+
}
|
|
14124
|
+
const $ = load3(capture.body);
|
|
14125
|
+
const root = $('main article,article,main,[role="main"]').first();
|
|
14126
|
+
return [
|
|
14127
|
+
...root.find("img[src],video[poster],a[download][href]").toArray().map((node) => $(node).attr("src") ?? $(node).attr("poster") ?? $(node).attr("href")),
|
|
14128
|
+
...root.find("[srcset]").toArray().flatMap((node) => migrationImageCandidates($(node).attr("srcset") ?? "", capture.finalUrl).map((candidate) => candidate.url))
|
|
14129
|
+
].filter(Boolean);
|
|
14130
|
+
}
|
|
14131
|
+
function unresolvedStaticControls(body) {
|
|
14132
|
+
const $ = load3(body);
|
|
14133
|
+
const root = $('main article,article,main,[role="main"]').first();
|
|
14134
|
+
const tabs = root.find('[role="tab"]').toArray();
|
|
14135
|
+
for (const tab of tabs) {
|
|
14136
|
+
const controls = $(tab).attr("aria-controls");
|
|
14137
|
+
const panels = controls ? root.find("[id]").filter((_index, node) => $(node).attr("id") === controls) : root.find('[role="tabpanel"]').eq(tabs.indexOf(tab));
|
|
14138
|
+
if (panels.length !== 1 || !panels.text().trim() && !panels.find("pre,img,table,svg").length || panels.find('[aria-busy="true"],[role="progressbar"]').length) return true;
|
|
14139
|
+
}
|
|
14140
|
+
return root.find("details:not([open])").toArray().some((node) => {
|
|
14141
|
+
const clone = $(node).clone();
|
|
14142
|
+
clone.children("summary").remove();
|
|
14143
|
+
return !clone.text().trim() && !clone.find("pre,img,table,svg").length;
|
|
14144
|
+
});
|
|
14145
|
+
}
|
|
14146
|
+
async function defaultAssetFetcher(url, signal) {
|
|
14147
|
+
const response = await fetch(url, { signal: signal ?? AbortSignal.timeout(15e3) });
|
|
14148
|
+
if (!response.ok) throw new Error(`Asset returned ${response.status}.`);
|
|
14149
|
+
if (Number(response.headers.get("content-length")) > MAX_ASSET_BYTES2) throw new Error("Asset exceeds size limit.");
|
|
14150
|
+
const reader = response.body?.getReader();
|
|
14151
|
+
const chunks = [];
|
|
14152
|
+
let size = 0;
|
|
14153
|
+
if (reader) while (true) {
|
|
14154
|
+
const next = await reader.read();
|
|
14155
|
+
if (next.done) break;
|
|
14156
|
+
size += next.value.length;
|
|
14157
|
+
if (size > MAX_ASSET_BYTES2) {
|
|
14158
|
+
await reader.cancel();
|
|
14159
|
+
throw new Error("Asset exceeds size limit.");
|
|
14160
|
+
}
|
|
14161
|
+
chunks.push(next.value);
|
|
14162
|
+
}
|
|
14163
|
+
return { url: response.url || url.toString(), contentType: response.headers.get("content-type") ?? "", content: Buffer.concat(chunks) };
|
|
14164
|
+
}
|
|
14165
|
+
function acceptedAsset(asset) {
|
|
14166
|
+
if (asset.content.byteLength > MAX_ASSET_BYTES2) return null;
|
|
14167
|
+
const mime = asset.contentType.split(";", 1)[0].toLowerCase().trim();
|
|
14168
|
+
if (mime === "image/svg+xml") return isPassiveSvg(asset.content) ? "svg" : null;
|
|
14169
|
+
const signature = Buffer.from(asset.content.slice(0, 16));
|
|
14170
|
+
const extensions = { "image/png": "png", "image/jpeg": "jpg", "image/gif": "gif", "image/webp": "webp", "application/pdf": "pdf", "image/avif": "avif" };
|
|
14171
|
+
const extension = extensions[mime];
|
|
14172
|
+
if (!extension) return null;
|
|
14173
|
+
const isSignatureValid = extension === "png" ? signature.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) : extension === "jpg" ? signature[0] === 255 && signature[1] === 216 : extension === "gif" ? /^GIF8[79]a/.test(signature.toString()) : extension === "webp" ? signature.toString("ascii", 0, 4) === "RIFF" && signature.toString("ascii", 8, 12) === "WEBP" : extension === "pdf" ? signature.toString("ascii", 0, 5) === "%PDF-" : signature.toString("ascii", 4, 8) === "ftyp" && /avif|avis/.test(signature.toString("ascii", 8));
|
|
14174
|
+
return isSignatureValid ? extension : null;
|
|
14175
|
+
}
|
|
14176
|
+
function validRecord(record) {
|
|
14177
|
+
if (record.outcome !== "imported") return true;
|
|
14178
|
+
return Boolean(record.page && record.pageSha256 === pageHash(record.page) && record.evidence && pageIdFromReference(record.page.id, true) === record.page.id && pageIdFromReference(record.page.navigationId, true) === record.page.navigationId && sanitizeRemoteMarkdown(record.page.body.replace(/^<a id="[^"<]*" \/>$/gm, "")) !== null && /^[a-f0-9]{64}$/.test(record.evidence.sha256) && record.assets?.every((asset) => /^migration-assets\/[a-f0-9]{64}\.[a-z0-9]+$/.test(asset.path) && hash2(Buffer.from(asset.base64, "base64")) === asset.sha256));
|
|
14179
|
+
}
|
|
14180
|
+
function navigationConfig(navigation, records) {
|
|
14181
|
+
const project2 = (nodes) => nodes.flatMap((node) => {
|
|
14182
|
+
const record = node.url ? records.get(canonical(node.url, node.url) ?? "") : void 0;
|
|
14183
|
+
const target = record?.outcome === "alias" ? records.get(record.aliasOf ?? "") : record;
|
|
14184
|
+
const page = target?.page;
|
|
14185
|
+
const children = project2(node.children ?? []);
|
|
14186
|
+
if (children.length) {
|
|
14187
|
+
const contains = (entries) => entries.some((entry) => typeof entry === "string" ? entry === page?.navigationId : contains(entry.pages));
|
|
14188
|
+
return [{ group: node.label || "Documentation", pages: [...page && !contains(children) ? [page.navigationId] : [], ...children] }];
|
|
14189
|
+
}
|
|
14190
|
+
return page ? [page.navigationId] : [];
|
|
14191
|
+
});
|
|
14192
|
+
const pages = project2(navigation);
|
|
14193
|
+
return pages.length ? { tabs: [{ tab: "Documentation", pages }] } : null;
|
|
14194
|
+
}
|
|
14195
|
+
function observedLocale(document, id, scopePath) {
|
|
14196
|
+
const issues = [];
|
|
14197
|
+
const canonicalLanguage = (value) => {
|
|
14198
|
+
if (!/^[a-z]{2,3}(?:-[a-z0-9]{2,8}){0,4}$/i.test(value)) return void 0;
|
|
14199
|
+
try {
|
|
14200
|
+
return Intl.getCanonicalLocales(value)[0];
|
|
14201
|
+
} catch {
|
|
14202
|
+
return void 0;
|
|
14203
|
+
}
|
|
14204
|
+
};
|
|
14205
|
+
const candidates = [];
|
|
14206
|
+
if (document.locale) {
|
|
14207
|
+
const locale = canonicalLanguage(document.locale);
|
|
14208
|
+
if (!locale) throw new Error("Capture locale is not a portable language code.");
|
|
14209
|
+
candidates.push({ source: "capture", value: locale });
|
|
14210
|
+
}
|
|
14211
|
+
const urlLocale = [id.split("/", 1)[0], ...scopePath.split("/"), document.finalUrl.pathname.split("/").filter(Boolean)[0]].find((segment) => segment && LOCALES.has(segment.toLowerCase()));
|
|
14212
|
+
if (urlLocale) candidates.push({ source: "URL", value: canonicalLanguage(urlLocale) });
|
|
14213
|
+
const declared = /html/i.test(document.contentType) ? load3(document.body)("html").attr("lang")?.trim() : void 0;
|
|
14214
|
+
if (declared) {
|
|
14215
|
+
const locale = canonicalLanguage(declared);
|
|
14216
|
+
if (locale) candidates.push({ source: "html lang", value: locale });
|
|
14217
|
+
else issues.push({ code: "unsupported-component", source: document.finalUrl.href, message: "The source declares an unsupported HTML language; review its locale configuration." });
|
|
14218
|
+
}
|
|
14219
|
+
if (new Set(candidates.map((candidate) => candidate.value)).size > 1) {
|
|
14220
|
+
issues.push({ code: "unsupported-component", source: document.finalUrl.href, message: `Observed locale metadata disagrees (${candidates.map((candidate) => `${candidate.source}: ${candidate.value}`).join(", ")}); review the source language.` });
|
|
14221
|
+
}
|
|
14222
|
+
return { locale: candidates[0]?.value, issues };
|
|
14223
|
+
}
|
|
14224
|
+
function localeProjection(checkpoint) {
|
|
14225
|
+
const pages = checkpoint.records.flatMap((record) => record.page ? [record.page] : []);
|
|
14226
|
+
const canonicalLocale = (page) => {
|
|
14227
|
+
try {
|
|
14228
|
+
return page.locale ? Intl.getCanonicalLocales(page.locale)[0] : void 0;
|
|
14229
|
+
} catch {
|
|
14230
|
+
return void 0;
|
|
14231
|
+
}
|
|
14232
|
+
};
|
|
14233
|
+
const locales = [...new Set(pages.map(canonicalLocale).filter((locale) => Boolean(locale)))];
|
|
14234
|
+
const defaultLocale = pages.some((page) => !page.locale) || locales.includes("en") ? "en" : locales[0] ?? "en";
|
|
14235
|
+
const pageId = (page) => {
|
|
14236
|
+
const locale = canonicalLocale(page);
|
|
14237
|
+
return locale ? `${locale === defaultLocale ? "" : `${locale}/`}${page.navigationId}` : page.id;
|
|
14238
|
+
};
|
|
14239
|
+
const route = (page) => {
|
|
14240
|
+
const locale = canonicalLocale(page);
|
|
14241
|
+
const prefix = locale && locale !== defaultLocale ? `/${locale}` : "";
|
|
14242
|
+
return `${prefix}${page.navigationId === "introduction" ? "/" : `/${page.navigationId}`}`;
|
|
14243
|
+
};
|
|
14244
|
+
return { canonicalLocale, locales, defaultLocale, pageId, route };
|
|
14245
|
+
}
|
|
14246
|
+
function validateUrlMigration(checkpoint, cancelled = false) {
|
|
14247
|
+
const issues = [];
|
|
14248
|
+
const counts = { pending: 0, imported: 0, excluded: 0, failed: 0, alias: 0 };
|
|
14249
|
+
const records = new Map(checkpoint.records.map((record) => [record.url, record]));
|
|
14250
|
+
const source = new URL(checkpoint.sourceUrl);
|
|
14251
|
+
const projection = localeProjection(checkpoint);
|
|
14252
|
+
const primaryIds = new Set(checkpoint.records.flatMap((record) => record.page && (!record.page.locale || projection.canonicalLocale(record.page) === projection.defaultLocale) ? [record.page.navigationId] : []));
|
|
14253
|
+
const outputIds = /* @__PURE__ */ new Set();
|
|
14254
|
+
const resolve4 = (url) => {
|
|
14255
|
+
const record = records.get(canonical(url, checkpoint.sourceUrl) ?? "");
|
|
14256
|
+
return record?.outcome === "alias" ? records.get(record.aliasOf ?? "") : record;
|
|
14257
|
+
};
|
|
14258
|
+
for (const record of checkpoint.records) {
|
|
14259
|
+
counts[record.outcome]++;
|
|
14260
|
+
issues.push(...record.issues ?? []);
|
|
14261
|
+
if (record.outcome === "pending" || record.outcome === "failed") issues.push({ code: "missing-page", source: record.url, message: record.reason ?? `Inventory entry is ${record.outcome}.` });
|
|
14262
|
+
if (!record.page || !record.evidence) continue;
|
|
14263
|
+
const outputId = projection.pageId(record.page);
|
|
14264
|
+
if (outputIds.has(outputId)) issues.push({ code: "collision", source: record.url, message: `Multiple source pages map to runtime path ${outputId}.` });
|
|
14265
|
+
outputIds.add(outputId);
|
|
14266
|
+
if (record.page.locale && projection.canonicalLocale(record.page) !== projection.defaultLocale && !primaryIds.has(record.page.navigationId)) {
|
|
14267
|
+
issues.push({ code: "missing-page", source: record.url, message: `Secondary locale ${record.page.locale} requires a primary ${projection.defaultLocale} page at ${record.page.navigationId}; the runtime cannot serve an orphan translation.` });
|
|
14268
|
+
}
|
|
14269
|
+
const actual = markdownCode(record.page.body);
|
|
14270
|
+
if (JSON.stringify(actual) !== JSON.stringify(record.evidence.code)) issues.push({ code: "code-mismatch", source: record.url, message: "Source code payloads differ from converted code payloads." });
|
|
14271
|
+
for (const value of record.evidence.links) {
|
|
14272
|
+
let link;
|
|
14273
|
+
try {
|
|
14274
|
+
link = new URL(value, record.evidence.finalUrl);
|
|
14275
|
+
} catch {
|
|
14276
|
+
continue;
|
|
14277
|
+
}
|
|
14278
|
+
if (!inScope(link, source, checkpoint.scopePath)) continue;
|
|
14279
|
+
if (/\.(?:png|jpe?g|gif|svg|webp|avif|pdf|zip|mp4|webm|css|js|json|ya?ml)$/i.test(link.pathname)) continue;
|
|
14280
|
+
const destination = resolve4(link.toString());
|
|
14281
|
+
if (!destination || !["imported", "excluded"].includes(destination.outcome)) {
|
|
14282
|
+
issues.push({ code: "missing-page", source: record.url, message: `Unresolved internal destination: ${link}` });
|
|
14283
|
+
} else if (link.hash && destination.outcome === "imported") {
|
|
14284
|
+
let anchor = link.hash.slice(1);
|
|
14285
|
+
try {
|
|
14286
|
+
anchor = decodeURIComponent(anchor);
|
|
14287
|
+
} catch {
|
|
14288
|
+
}
|
|
14289
|
+
if (!destination.evidence?.anchors.includes(anchor)) issues.push({ code: "missing-anchor", source: record.url, message: `Missing source anchor: ${link}` });
|
|
14290
|
+
}
|
|
14291
|
+
}
|
|
14292
|
+
}
|
|
14293
|
+
const visit = (nodes) => {
|
|
14294
|
+
for (const node of nodes) {
|
|
14295
|
+
if (node.url) {
|
|
14296
|
+
const url = new URL(node.url, source);
|
|
14297
|
+
if (inScope(url, source, checkpoint.scopePath) && !resolve4(url.toString())?.page) issues.push({ code: "navigation-missing", source: node.url, message: `Source navigation target ${node.label} was not imported.` });
|
|
14298
|
+
}
|
|
14299
|
+
visit(node.children ?? []);
|
|
14300
|
+
}
|
|
14301
|
+
};
|
|
14302
|
+
visit(checkpoint.navigation);
|
|
14303
|
+
if (!counts.imported) issues.push({ code: "missing-page", source: checkpoint.sourceUrl, message: "No readable article pages were imported." });
|
|
14304
|
+
return { status: cancelled ? "cancelled" : issues.length ? "partial" : "passed", issues, counts };
|
|
14305
|
+
}
|
|
14306
|
+
async function runUrlMigration(options) {
|
|
14307
|
+
const source = validateMigrationUrl(options.sourceUrl);
|
|
14308
|
+
const sourceUrl = canonical(source.toString(), source.toString());
|
|
14309
|
+
const requestedScope = options.scopePath ?? (source.pathname.endsWith("/") ? source.pathname : source.pathname.slice(0, source.pathname.lastIndexOf("/")) || "/");
|
|
14310
|
+
let scopeEnd = requestedScope.length;
|
|
14311
|
+
while (scopeEnd > 0 && requestedScope.charCodeAt(scopeEnd - 1) === 47) scopeEnd--;
|
|
14312
|
+
const scopePath = requestedScope.slice(0, scopeEnd) || "/";
|
|
14313
|
+
if (!scopePath.startsWith("/") || scopePath.includes("?") || scopePath.includes("#") || !inScope(source, source, scopePath)) throw new Error("The selected documentation scope must contain the source URL.");
|
|
14314
|
+
const maxPages = options.maxPages ?? 1e3;
|
|
14315
|
+
if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > 1e4) throw new Error("maxPages must be between 1 and 10000.");
|
|
14316
|
+
const concurrency = Math.max(1, Math.min(32, Math.floor(options.concurrency ?? 5)));
|
|
14317
|
+
const batchSize = Math.max(1, Math.min(500, Math.floor(options.batchSize ?? 50)));
|
|
14318
|
+
if (!Number.isFinite(concurrency) || !Number.isFinite(batchSize) || options.maxCaptures !== void 0 && (!Number.isInteger(options.maxCaptures) || options.maxCaptures < 1) || options.maxTotalBytes !== void 0 && (!Number.isFinite(options.maxTotalBytes) || options.maxTotalBytes < 1)) throw new Error("Migration batch, concurrency and byte limits must be positive finite numbers.");
|
|
14319
|
+
const captureVersion = options.captureVersion ?? "fetch-v1";
|
|
14320
|
+
if (options.checkpoint && options.checkpoint.records.length > MAX_RECORDS) throw new Error("Checkpoint inventory exceeds its bound.");
|
|
14321
|
+
const checkpoint = options.checkpoint ? cloneUrlMigrationCheckpoint(options.checkpoint) : { schemaVersion: 1, converterVersion: CONVERTER_VERSION, sourceUrl, scopePath, captureVersion, records: [], navigation: [] };
|
|
14322
|
+
if (checkpoint.schemaVersion !== 1 || checkpoint.converterVersion !== CONVERTER_VERSION || checkpoint.sourceUrl !== sourceUrl || checkpoint.scopePath !== scopePath || checkpoint.captureVersion !== captureVersion) throw new Error("Checkpoint does not match this source, scope or capture/converter version.");
|
|
14323
|
+
if (checkpoint.records.length > MAX_RECORDS) throw new Error("Checkpoint inventory exceeds its bound.");
|
|
14324
|
+
const records = /* @__PURE__ */ new Map();
|
|
14325
|
+
for (const record of checkpoint.records) {
|
|
14326
|
+
if (canonical(record.url, sourceUrl) !== record.url || !inScope(new URL(record.url), source, scopePath) && !isDiscoveryIndex(new URL(record.url), source)) throw new Error("Checkpoint contains an unsafe inventory URL.");
|
|
14327
|
+
if (records.has(record.url)) throw new Error("Checkpoint contains a duplicate inventory URL.");
|
|
14328
|
+
if (!validRecord(record) || record.outcome === "failed" && options.retryFailed !== false || record.outcome !== "pending" && options.reconcile && !await options.reconcile(record)) {
|
|
14329
|
+
record.outcome = "pending";
|
|
14330
|
+
delete record.page;
|
|
14331
|
+
delete record.pageSha256;
|
|
14332
|
+
delete record.evidence;
|
|
14333
|
+
delete record.assets;
|
|
14334
|
+
}
|
|
14335
|
+
if (record.page && pageIdFromReference(record.page.id, true) !== record.page.id) throw new Error("Checkpoint contains an unsafe page path.");
|
|
14336
|
+
records.set(record.url, record);
|
|
14337
|
+
}
|
|
14338
|
+
const pending = [...records.values()].filter((record) => record.outcome === "pending");
|
|
14339
|
+
let changed = [];
|
|
14340
|
+
let navigationChanged = false;
|
|
14341
|
+
const enqueue = (value, base, discoveredBy) => {
|
|
14342
|
+
const url = canonical(value, base);
|
|
14343
|
+
if (!url || !inScope(new URL(url), source, scopePath) && !isDiscoveryIndex(new URL(url), source)) return;
|
|
14344
|
+
const existing = records.get(url);
|
|
14345
|
+
if (existing) {
|
|
14346
|
+
if (!existing.discoveredBy.includes(discoveredBy)) {
|
|
14347
|
+
existing.discoveredBy.push(discoveredBy);
|
|
14348
|
+
changed.push(existing);
|
|
14349
|
+
}
|
|
14350
|
+
return;
|
|
14351
|
+
}
|
|
14352
|
+
if (records.size >= MAX_RECORDS) throw new Error("Discovered URL inventory exceeded the 100000 URL safety bound.");
|
|
14353
|
+
const record = { url, discoveredBy: [discoveredBy], outcome: "pending" };
|
|
14354
|
+
records.set(url, record);
|
|
14355
|
+
checkpoint.records.push(record);
|
|
14356
|
+
pending.push(record);
|
|
14357
|
+
changed.push(record);
|
|
14358
|
+
};
|
|
14359
|
+
enqueue(sourceUrl, sourceUrl, "submitted");
|
|
14360
|
+
for (const path of ["llms.txt", "sitemap.xml", "robots.txt"]) enqueue(`${scopePath === "/" ? "" : scopePath}/${path}`, sourceUrl, "discovery-index");
|
|
14361
|
+
if (scopePath !== "/") for (const path of ["llms.txt", "sitemap.xml", "robots.txt"]) enqueue(`/${path}`, sourceUrl, "discovery-index");
|
|
14362
|
+
const fetcher = options.fetcher ?? defaultMigrationFetcher;
|
|
14363
|
+
const capture = options.captureProvider?.capture.bind(options.captureProvider) ?? (async ({ url }) => fetcher(url, { accept: ACCEPT }));
|
|
14364
|
+
const assetFetcher = options.assetFetcher ?? (!options.fetcher && !options.captureProvider ? defaultAssetFetcher : void 0);
|
|
14365
|
+
const usedIds = new Map([...records.values()].filter((record) => record.page).map((record) => [record.page.id, record.url]));
|
|
14366
|
+
const finalUrls = new Map([...records.values()].filter((record) => record.evidence && record.page).map((record) => [canonical(record.evidence.finalUrl, sourceUrl), record.url]));
|
|
14367
|
+
let imported = usedIds.size;
|
|
14368
|
+
const assetPayloads = new Map(checkpoint.records.flatMap((record) => (record.assets ?? []).map((asset) => [asset.sha256, asset])));
|
|
14369
|
+
const assetBytes = /* @__PURE__ */ new Map();
|
|
14370
|
+
const retainAsset = (asset) => {
|
|
14371
|
+
if (!acceptedAsset(asset)) throw new Error("Asset MIME/signature is unsupported or unsafe to serve.");
|
|
14372
|
+
const sha256 = hash2(asset.content);
|
|
14373
|
+
const content = assetBytes.get(sha256) ?? Buffer.from(asset.content);
|
|
14374
|
+
assetBytes.set(sha256, content);
|
|
14375
|
+
return { ...asset, content };
|
|
14376
|
+
};
|
|
14377
|
+
const assetRequests = createMigrationAssetCache();
|
|
14378
|
+
for (const record of checkpoint.records) for (const asset of record.assets ?? []) {
|
|
14379
|
+
if (!assetRequests.has(asset.url)) {
|
|
14380
|
+
const content = assetBytes.get(asset.sha256) ?? Buffer.from(asset.base64, "base64");
|
|
14381
|
+
assetBytes.set(asset.sha256, content);
|
|
14382
|
+
assetRequests.remember(asset.url, { url: asset.url, contentType: asset.contentType, content });
|
|
14383
|
+
}
|
|
14384
|
+
}
|
|
14385
|
+
let totalBytes = checkpoint.records.reduce((sum, record) => sum + (record.evidence?.byteLength ?? 0), 0) + [...assetPayloads.values()].reduce((sum, asset) => sum + Buffer.byteLength(asset.base64, "base64"), 0);
|
|
14386
|
+
let budgetExhausted = false;
|
|
14387
|
+
let cursor = 0;
|
|
14388
|
+
const flush = async () => {
|
|
14389
|
+
const distinct = [...new Map(changed.map((record) => [record.url, record])).values()];
|
|
14390
|
+
if (options.onCheckpoint) for (let index = 0; index < distinct.length || index === 0 && navigationChanged; index += batchSize) {
|
|
14391
|
+
await options.onCheckpoint({ schemaVersion: 1, converterVersion: CONVERTER_VERSION, scopePath, captureVersion, sourceUrl, records: distinct.slice(index, index + batchSize), ...navigationChanged && index === 0 ? { navigation: checkpoint.navigation } : {} });
|
|
14392
|
+
}
|
|
14393
|
+
changed = [];
|
|
14394
|
+
navigationChanged = false;
|
|
14395
|
+
};
|
|
14396
|
+
const process2 = async (record) => {
|
|
14397
|
+
if (options.signal?.aborted) return;
|
|
14398
|
+
try {
|
|
14399
|
+
delete record.reason;
|
|
14400
|
+
const document = await capture({ url: new URL(record.url), signal: options.signal });
|
|
14401
|
+
if (options.signal?.aborted) return;
|
|
14402
|
+
if (Buffer.byteLength(document.body) > MAX_RESPONSE_BYTES) throw new Error("Captured response exceeds the 8 MB safety bound.");
|
|
14403
|
+
const final = canonical(document.finalUrl.toString(), sourceUrl);
|
|
14404
|
+
const role = classifyMigrationResponse(document);
|
|
14405
|
+
if (!inScope(document.finalUrl, source, scopePath) && !(role === "index" && isDiscoveryIndex(document.finalUrl, source))) {
|
|
14406
|
+
record.outcome = "excluded";
|
|
14407
|
+
record.reason = "Redirected outside the selected origin/path.";
|
|
14408
|
+
return;
|
|
14409
|
+
}
|
|
14410
|
+
if (document.status && document.status >= 400) throw new Error(`Source page returned HTTP ${document.status}.`);
|
|
14411
|
+
if (role !== "page") {
|
|
14412
|
+
totalBytes += Buffer.byteLength(document.body);
|
|
14413
|
+
if (totalBytes > (options.maxTotalBytes ?? 5e8)) {
|
|
14414
|
+
budgetExhausted = true;
|
|
14415
|
+
throw new Error("Migration exceeded its content byte budget.");
|
|
14416
|
+
}
|
|
14417
|
+
record.outcome = "excluded";
|
|
14418
|
+
record.reason = `Final response role: ${role}.`;
|
|
14419
|
+
const links = role === "index" ? linksOf(document) : [];
|
|
14420
|
+
record.evidence = { sha256: hash2(document.body), byteLength: Buffer.byteLength(document.body), finalUrl: final, contentType: document.contentType, ...document.artifactRef ? { artifactRef: document.artifactRef } : {}, states: [], code: [], anchors: [], links };
|
|
14421
|
+
for (const link of links) enqueue(link, final, record.url);
|
|
14422
|
+
return;
|
|
14423
|
+
}
|
|
14424
|
+
if (imported >= maxPages) {
|
|
14425
|
+
record.outcome = "excluded";
|
|
14426
|
+
record.reason = "A discovery endpoint resolved to an article after the selected page limit.";
|
|
14427
|
+
return;
|
|
14428
|
+
}
|
|
14429
|
+
const already = finalUrls.get(final);
|
|
14430
|
+
if (already && already !== record.url) {
|
|
14431
|
+
record.outcome = "alias";
|
|
14432
|
+
record.aliasOf = already;
|
|
14433
|
+
return;
|
|
14434
|
+
}
|
|
14435
|
+
finalUrls.set(final, record.url);
|
|
14436
|
+
const id = idForUrl(document.finalUrl, scopePath);
|
|
14437
|
+
if (!id) throw new Error("Source URL has no safe destination path.");
|
|
14438
|
+
if (usedIds.has(id) && usedIds.get(id) !== record.url) {
|
|
14439
|
+
record.outcome = "failed";
|
|
14440
|
+
record.reason = `Destination collision for ${id}.`;
|
|
14441
|
+
return;
|
|
14442
|
+
}
|
|
14443
|
+
usedIds.set(id, record.url);
|
|
14444
|
+
const evidence = evidenceOf(document);
|
|
14445
|
+
const page = convert(document, id);
|
|
14446
|
+
if (!page) {
|
|
14447
|
+
record.outcome = "failed";
|
|
14448
|
+
record.reason = "Content could not be represented as safe inert MDX.";
|
|
14449
|
+
return;
|
|
14450
|
+
}
|
|
14451
|
+
const states = document.states ?? [];
|
|
14452
|
+
if (states.length > 64) throw new Error("Capture contains too many alternative states.");
|
|
14453
|
+
for (const state of states) {
|
|
14454
|
+
if (Buffer.byteLength(state.response.body) > MAX_RESPONSE_BYTES) throw new Error("Alternative state exceeds the 8 MB safety bound.");
|
|
14455
|
+
if (!inScope(state.response.finalUrl, source, scopePath) || classifyMigrationResponse(state.response) !== "page") throw new Error("Capture state escaped its selected scope or article role.");
|
|
14456
|
+
const alternative = convert(state.response, id);
|
|
14457
|
+
if (!alternative) throw new Error(`Alternative state ${state.id} is not inert content.`);
|
|
14458
|
+
const stateCode = /html/i.test(state.response.contentType) ? htmlEvidence(state.response).code : markdownCode(state.response.body);
|
|
14459
|
+
evidence.states.find((entry) => entry.id === state.id).code = stateCode;
|
|
14460
|
+
const difference = alternativeBlocks(alternative.body, page.body);
|
|
14461
|
+
evidence.code.push(...markdownCode(difference));
|
|
14462
|
+
evidence.links.push(...linksOf(state.response));
|
|
14463
|
+
evidence.anchors.push(.../html/i.test(state.response.contentType) ? htmlEvidence(state.response).anchors : markdownHeadingAnchors(state.response.body));
|
|
14464
|
+
if (difference) page.body += `
|
|
14465
|
+
|
|
14466
|
+
### ${state.label.replace(/[<>{}#\n\r]/g, "")}
|
|
14467
|
+
|
|
14468
|
+
${difference}`;
|
|
14469
|
+
}
|
|
14470
|
+
totalBytes += Buffer.byteLength(document.body) + states.reduce((sum, state) => sum + Buffer.byteLength(state.response.body), 0);
|
|
14471
|
+
evidence.byteLength = Buffer.byteLength(document.body) + states.reduce((sum, state) => sum + Buffer.byteLength(state.response.body), 0);
|
|
14472
|
+
if (totalBytes > (options.maxTotalBytes ?? 5e8)) {
|
|
14473
|
+
budgetExhausted = true;
|
|
14474
|
+
throw new Error("Migration exceeded its content byte budget.");
|
|
14475
|
+
}
|
|
14476
|
+
record.assets = [];
|
|
14477
|
+
record.issues = [];
|
|
14478
|
+
if (/html/i.test(document.contentType)) {
|
|
14479
|
+
const $ = load3(document.body);
|
|
14480
|
+
const root = $('main article,article,main,[role="main"]').first();
|
|
14481
|
+
root.find("nav,footer,aside,.theme-doc-breadcrumbs,.theme-doc-toc-mobile,.table-of-contents,.hash-link").remove();
|
|
14482
|
+
root.find('button svg,a.hash-link svg,[aria-hidden="true"] svg,svg[aria-hidden="true"],svg[data-lucide],svg[data-icon]').remove();
|
|
14483
|
+
if (root.find("h1").length > 1) record.issues.push({ code: "unsupported-component", source: record.url, message: "The article contains multiple level-one headings; review their hierarchy." });
|
|
14484
|
+
const unsupported = [...new Set(root.find("iframe,object,embed,canvas,svg,form,input,textarea,select,math,video,audio").toArray().map((node) => node.tagName))];
|
|
14485
|
+
if (unsupported.length) record.issues.push({ code: "unsupported-component", source: record.url, message: `Rendered content requires explicit review for: ${unsupported.join(", ")}.` });
|
|
14486
|
+
if (!states.length && unresolvedStaticControls(document.body)) record.issues.push({ code: "unsupported-component", source: record.url, message: "An interactive alternative has no complete captured panel or disclosure content." });
|
|
14487
|
+
}
|
|
14488
|
+
const suppliedAssets = new Map((document.assets ?? []).map((asset) => [canonical(asset.url, final), asset]));
|
|
14489
|
+
const assetUrls = [...new Set([...assetReferences(document), ...states.flatMap((state) => assetReferences(state.response))].map((url) => canonical(url, final)).filter((url) => Boolean(url)))];
|
|
14490
|
+
if (assetUrls.length > 256 || (document.assets?.length ?? 0) > 256) throw new Error("Capture exceeds the 256 assets per page safety bound.");
|
|
14491
|
+
for (const url of assetUrls) {
|
|
14492
|
+
try {
|
|
14493
|
+
let assetRequest = assetRequests.get(url);
|
|
14494
|
+
if (!suppliedAssets.has(url) && !assetRequest && assetFetcher) {
|
|
14495
|
+
assetRequest = assetRequests.load(url, () => assetFetcher(new URL(url), options.signal).then(retainAsset));
|
|
14496
|
+
}
|
|
14497
|
+
const supplied = suppliedAssets.get(url);
|
|
14498
|
+
const asset = supplied ? retainAsset(supplied) : assetRequest ? await assetRequest : null;
|
|
14499
|
+
if (!asset) throw new Error("Capture host did not supply asset bytes or a binary fetch boundary.");
|
|
14500
|
+
const extension = acceptedAsset(asset);
|
|
14501
|
+
if (!extension) throw new Error("Asset MIME/signature is unsupported or unsafe to serve.");
|
|
14502
|
+
const sha256 = hash2(asset.content);
|
|
14503
|
+
const existing = assetPayloads.get(sha256);
|
|
14504
|
+
if (!existing) {
|
|
14505
|
+
totalBytes += asset.content.byteLength;
|
|
14506
|
+
if (totalBytes > (options.maxTotalBytes ?? 5e8)) {
|
|
14507
|
+
budgetExhausted = true;
|
|
14508
|
+
throw new Error("Migration exceeded its content and asset byte budget.");
|
|
14509
|
+
}
|
|
14510
|
+
}
|
|
14511
|
+
const path = `migration-assets/${sha256}.${extension}`;
|
|
14512
|
+
const receipt = { url, path, sha256, contentType: asset.contentType, base64: existing?.base64 ?? Buffer.from(asset.content).toString("base64") };
|
|
14513
|
+
record.assets.push(receipt);
|
|
14514
|
+
if (!existing) assetPayloads.set(sha256, receipt);
|
|
14515
|
+
if (!assetRequests.has(url)) assetRequests.remember(url, asset);
|
|
14516
|
+
page.body = rewriteMarkdownDestinations(page.body, (destination) => canonical(destination, final) === url ? `/${path}${new URL(destination, final).hash}` : destination);
|
|
14517
|
+
} catch (error) {
|
|
14518
|
+
assetRequests.delete(url);
|
|
14519
|
+
record.issues.push({ code: "asset-failed", source: record.url, message: `${url}: ${error instanceof Error ? error.message : String(error)}` });
|
|
14520
|
+
}
|
|
14521
|
+
}
|
|
14522
|
+
if (options.signal?.aborted) return;
|
|
14523
|
+
if (sanitizeRemoteMarkdown(page.body.replace(/^<a id="[^"<]*" \/>$/gm, "")) === null) throw new Error("Converted content did not pass inert MDX validation.");
|
|
14524
|
+
record.evidence = evidence;
|
|
14525
|
+
record.page = page;
|
|
14526
|
+
const language = observedLocale(document, page.id, scopePath);
|
|
14527
|
+
record.issues.push(...language.issues);
|
|
14528
|
+
const canonicalLocale = language.locale;
|
|
14529
|
+
if (canonicalLocale) {
|
|
14530
|
+
page.locale = canonicalLocale;
|
|
14531
|
+
const segments = page.id.split("/");
|
|
14532
|
+
const localeIndex = segments.findIndex((segment) => segment.toLowerCase() === canonicalLocale.toLowerCase());
|
|
14533
|
+
page.navigationId = segments.filter((_segment, index) => index !== localeIndex).join("/") || "introduction";
|
|
14534
|
+
page.id = `${canonicalLocale}/${page.navigationId}`;
|
|
14535
|
+
if (usedIds.has(page.id) && usedIds.get(page.id) !== record.url) throw new Error(`Destination collision for locale page ${page.id}.`);
|
|
14536
|
+
usedIds.delete(id);
|
|
14537
|
+
usedIds.set(page.id, record.url);
|
|
14538
|
+
}
|
|
14539
|
+
record.version = document.version ?? [...page.id.split("/"), ...document.finalUrl.pathname.split("/")].find((segment) => /^v\d+(?:\.\d+)*$/i.test(segment));
|
|
14540
|
+
record.pageSha256 = pageHash(page);
|
|
14541
|
+
record.outcome = "imported";
|
|
14542
|
+
imported++;
|
|
14543
|
+
const navigation = checkpoint.navigation.length ? [] : sourceNavigation(document);
|
|
14544
|
+
if (!checkpoint.navigation.length && navigation.length) {
|
|
14545
|
+
checkpoint.navigation = navigation;
|
|
14546
|
+
navigationChanged = true;
|
|
14547
|
+
}
|
|
14548
|
+
for (const link of evidence.links) {
|
|
14549
|
+
try {
|
|
14550
|
+
const url = new URL(link, final);
|
|
14551
|
+
if (!/\.(?:png|jpe?g|gif|svg|webp|avif|pdf|zip|mp4|webm|css|js)$/i.test(url.pathname)) enqueue(link, final, record.url);
|
|
14552
|
+
} catch {
|
|
14553
|
+
}
|
|
14554
|
+
}
|
|
14555
|
+
const visit = (nodes) => {
|
|
14556
|
+
for (const node of nodes) {
|
|
14557
|
+
if (node.url) enqueue(node.url, final, "source-navigation");
|
|
14558
|
+
visit(node.children ?? []);
|
|
14559
|
+
}
|
|
14560
|
+
};
|
|
14561
|
+
visit(navigation);
|
|
14562
|
+
} catch (error) {
|
|
14563
|
+
if (options.signal?.aborted) return;
|
|
14564
|
+
const wasImported = record.outcome === "imported";
|
|
14565
|
+
record.outcome = record.discoveredBy.every((origin) => origin === "discovery-index") ? "excluded" : "failed";
|
|
14566
|
+
record.reason = error instanceof Error ? error.message : String(error);
|
|
14567
|
+
if (record.page) {
|
|
14568
|
+
if (wasImported) imported--;
|
|
14569
|
+
usedIds.delete(record.page.id);
|
|
14570
|
+
}
|
|
14571
|
+
delete record.page;
|
|
14572
|
+
delete record.pageSha256;
|
|
14573
|
+
delete record.assets;
|
|
14574
|
+
} finally {
|
|
14575
|
+
if (record.outcome !== "pending") {
|
|
14576
|
+
if (options.onRecord) await options.onRecord(record);
|
|
14577
|
+
changed.push(record);
|
|
14578
|
+
}
|
|
14579
|
+
}
|
|
14580
|
+
};
|
|
14581
|
+
while (cursor < pending.length && !options.signal?.aborted && !budgetExhausted && cursor < (options.maxCaptures ?? Infinity)) {
|
|
14582
|
+
if (imported >= maxPages) {
|
|
14583
|
+
const index = pending.findIndex((record, position) => position >= cursor && isDiscoveryIndex(new URL(record.url), source));
|
|
14584
|
+
if (index < 0) break;
|
|
14585
|
+
pending.splice(cursor, 0, pending.splice(index, 1)[0]);
|
|
14586
|
+
}
|
|
14587
|
+
const batch = pending.slice(cursor, cursor + Math.min(concurrency, imported >= maxPages ? 1 : maxPages - imported, (options.maxCaptures ?? Infinity) - cursor));
|
|
14588
|
+
cursor += batch.length;
|
|
14589
|
+
await Promise.all(batch.map(process2));
|
|
14590
|
+
if (changed.length >= batchSize) await flush();
|
|
14591
|
+
}
|
|
14592
|
+
await flush();
|
|
14593
|
+
const projection = localeProjection(checkpoint);
|
|
14594
|
+
const routeMap = /* @__PURE__ */ new Map();
|
|
14595
|
+
for (const record of checkpoint.records) {
|
|
14596
|
+
const target = record.outcome === "alias" ? records.get(record.aliasOf ?? "") : record;
|
|
14597
|
+
if (target?.page) {
|
|
14598
|
+
const route = projection.route(target.page);
|
|
14599
|
+
routeMap.set(record.url, route);
|
|
14600
|
+
if (target.evidence) routeMap.set(canonical(target.evidence.finalUrl, sourceUrl), route);
|
|
14601
|
+
}
|
|
14602
|
+
}
|
|
14603
|
+
const navigationLabels = /* @__PURE__ */ new Map();
|
|
14604
|
+
const navigationIssues = [];
|
|
14605
|
+
const labelPages = (nodes) => {
|
|
14606
|
+
for (const node of nodes) {
|
|
14607
|
+
const entry = node.url ? records.get(canonical(node.url, sourceUrl) ?? "") : void 0;
|
|
14608
|
+
const target = entry?.outcome === "alias" ? records.get(entry.aliasOf ?? "") : entry;
|
|
14609
|
+
if (target?.page && node.label && !node.children?.length) {
|
|
14610
|
+
const previous = navigationLabels.get(target.url);
|
|
14611
|
+
if (previous && previous !== node.label) navigationIssues.push({ code: "unsupported-component", source: target.url, message: `This page has multiple source navigation labels (${previous}, ${node.label}); review the shared sidebar title.` });
|
|
14612
|
+
else navigationLabels.set(target.url, node.label);
|
|
14613
|
+
}
|
|
14614
|
+
labelPages(node.children ?? []);
|
|
14615
|
+
}
|
|
14616
|
+
};
|
|
14617
|
+
labelPages(checkpoint.navigation);
|
|
14618
|
+
const pages = checkpoint.records.flatMap((record) => record.page ? [{ ...record.page, id: projection.pageId(record.page), ...record.page.locale ? { locale: projection.canonicalLocale(record.page) } : {}, ...navigationLabels.has(record.url) ? { navTitle: navigationLabels.get(record.url) } : {}, body: rewriteMarkdownDestinations(record.page.body, (value) => {
|
|
14619
|
+
let url;
|
|
14620
|
+
try {
|
|
14621
|
+
url = new URL(value, record.evidence.finalUrl);
|
|
14622
|
+
} catch {
|
|
14623
|
+
return value;
|
|
14624
|
+
}
|
|
14625
|
+
const route = routeMap.get(canonical(url.toString(), sourceUrl));
|
|
14626
|
+
return route ? `${route}${url.hash}` : value;
|
|
14627
|
+
}) }] : []);
|
|
14628
|
+
const validation = validateUrlMigration(checkpoint, Boolean(options.signal?.aborted));
|
|
14629
|
+
validation.issues.push(...navigationIssues);
|
|
14630
|
+
if (navigationIssues.length && validation.status === "passed") validation.status = "partial";
|
|
14631
|
+
const assets = /* @__PURE__ */ new Map();
|
|
14632
|
+
for (const record of checkpoint.records) for (const asset of record.assets ?? []) {
|
|
14633
|
+
if (!assets.has(asset.path)) {
|
|
14634
|
+
const content = assetBytes.get(asset.sha256) ?? Buffer.from(asset.base64, "base64");
|
|
14635
|
+
assets.set(asset.path, { path: asset.path, content: Buffer.from(content.buffer, content.byteOffset, content.byteLength) });
|
|
14636
|
+
}
|
|
14637
|
+
}
|
|
14638
|
+
const bundle = {
|
|
14639
|
+
sourceUrl,
|
|
14640
|
+
sourceKind: "url",
|
|
14641
|
+
platform: options.platform ?? "generic",
|
|
14642
|
+
pages,
|
|
14643
|
+
assets: [...assets.values()],
|
|
14644
|
+
docsConfig: navigationConfig(checkpoint.navigation, records) ?? buildNavigationFromPages(pages),
|
|
14645
|
+
warnings: validation.issues.map((issue) => ({ code: issue.code === "missing-page" ? "missing-page" : "unsupported-config", message: issue.message, source: issue.source })),
|
|
14646
|
+
urlMigrationValidation: validation,
|
|
14647
|
+
stats: { discovered: records.size, imported: pages.length, skipped: records.size - pages.length }
|
|
14648
|
+
};
|
|
14649
|
+
if (projection.locales.length) {
|
|
14650
|
+
const locales = [.../* @__PURE__ */ new Set([projection.defaultLocale, ...projection.locales])];
|
|
14651
|
+
bundle.docsConfig.i18n = { defaultLocale: projection.defaultLocale, locales: locales.map((code) => ({ code, label: code === "en" ? "English" : code })) };
|
|
14652
|
+
}
|
|
14653
|
+
const stopReason = options.signal?.aborted ? "cancelled" : budgetExhausted ? "byte-limit" : cursor < pending.length && cursor >= (options.maxCaptures ?? Infinity) ? "capture-limit" : imported >= maxPages && cursor < pending.length ? "page-limit" : cursor < pending.length ? "capture-limit" : "complete";
|
|
14654
|
+
return { bundle, checkpoint, validation, stopReason };
|
|
14655
|
+
}
|
|
14656
|
+
function rewriteMarkdownDestinations(body, rewrite) {
|
|
14657
|
+
const edits = markdownNodes(body).filter((node) => node.url && node.position).map((node) => {
|
|
14658
|
+
const start = node.position.start.offset;
|
|
14659
|
+
const end = node.position.end.offset;
|
|
14660
|
+
const original = body.slice(start, end);
|
|
14661
|
+
const replacement = rewrite(node.url);
|
|
14662
|
+
if (replacement === node.url) return null;
|
|
14663
|
+
const index = original.lastIndexOf(node.url);
|
|
14664
|
+
if (index < 0) return null;
|
|
14665
|
+
return { start: start + index, end: start + index + node.url.length, replacement };
|
|
14666
|
+
}).filter((edit) => edit !== null).sort((a, b) => b.start - a.start);
|
|
14667
|
+
for (const node of markdownNodes(body)) for (const attribute of node.attributes ?? []) {
|
|
14668
|
+
if (!attribute.name || !["src", "href", "srcSet", "srcset", "img", "primaryHref", "secondaryHref"].includes(attribute.name) || !attribute.position) continue;
|
|
14669
|
+
let value = attribute.value;
|
|
14670
|
+
if (typeof value !== "string") {
|
|
14671
|
+
try {
|
|
14672
|
+
value = JSON.parse(value?.value ?? "");
|
|
14673
|
+
} catch {
|
|
14674
|
+
continue;
|
|
14675
|
+
}
|
|
14676
|
+
}
|
|
14677
|
+
if (typeof value !== "string") continue;
|
|
14678
|
+
let replacement;
|
|
14679
|
+
if (/^srcset$/i.test(attribute.name)) {
|
|
14680
|
+
const candidates = migrationImageCandidates(value, new URL("https://migration.invalid"), false);
|
|
14681
|
+
replacement = renderMigrationSrcset(candidates.map((candidate) => ({ ...candidate, url: rewrite(candidate.url) })));
|
|
14682
|
+
if (replacement === renderMigrationSrcset(candidates)) continue;
|
|
14683
|
+
} else {
|
|
14684
|
+
replacement = rewrite(value);
|
|
14685
|
+
if (replacement === value) continue;
|
|
14686
|
+
}
|
|
14687
|
+
const start = attribute.position.start.offset, end = attribute.position.end.offset;
|
|
14688
|
+
if (start === void 0 || end === void 0) continue;
|
|
14689
|
+
edits.push({ start, end, replacement: `${attribute.name}="${migrationImageAttribute(replacement)}"` });
|
|
14690
|
+
}
|
|
14691
|
+
edits.sort((a, b) => b.start - a.start);
|
|
14692
|
+
for (const edit of edits) body = body.slice(0, edit.start) + edit.replacement + body.slice(edit.end);
|
|
14693
|
+
return body;
|
|
14694
|
+
}
|
|
14695
|
+
|
|
14696
|
+
// src/remote-api.ts
|
|
14697
|
+
import { createHash as createHash4 } from "crypto";
|
|
13447
14698
|
import { lookup } from "dns/promises";
|
|
13448
14699
|
import { request } from "https";
|
|
13449
14700
|
import { isIP } from "net";
|
|
@@ -13606,7 +14857,7 @@ async function hydrateRemoteApiSpecs(bundle, fetcher) {
|
|
|
13606
14857
|
const body = pruned.content;
|
|
13607
14858
|
if (reference.withheldOperations?.length) warnings.push({ code: "gated-page", message: sharedSpecMessage(reference.url, pruned.excluded), source: reference.url });
|
|
13608
14859
|
const operations = parseOpenApi(body, extension);
|
|
13609
|
-
const filename = `openapi-${
|
|
14860
|
+
const filename = `openapi-${createHash4("sha256").update(reference.url).digest("hex").slice(0, 12)}.${extension}`;
|
|
13610
14861
|
assets.push({ path: `openapi/${filename}`, content: body, projectRelative: true });
|
|
13611
14862
|
for (const page of pages) {
|
|
13612
14863
|
const ref = page.openapi ? splitOpenApiRef(page.openapi) : null;
|
|
@@ -13652,6 +14903,7 @@ export {
|
|
|
13652
14903
|
SOURCE_REF_MAX_REPOSITORY_KB,
|
|
13653
14904
|
addMintlifyDirectoryRedirects,
|
|
13654
14905
|
buildNavigationFromPages,
|
|
14906
|
+
classifyMigrationResponse,
|
|
13655
14907
|
cloneGitHubRepository,
|
|
13656
14908
|
defaultMigrationFetcher,
|
|
13657
14909
|
detectRepositoryDocsDir,
|
|
@@ -13659,6 +14911,7 @@ export {
|
|
|
13659
14911
|
gitmodulePaths,
|
|
13660
14912
|
hydrateRemoteApiSpecs,
|
|
13661
14913
|
importSourceRef,
|
|
14914
|
+
isPassiveSvg,
|
|
13662
14915
|
mergeMigrationConfig,
|
|
13663
14916
|
migrateRepository,
|
|
13664
14917
|
migrateUrl,
|
|
@@ -13673,8 +14926,10 @@ export {
|
|
|
13673
14926
|
readFernConfig,
|
|
13674
14927
|
readMintlifyConfig,
|
|
13675
14928
|
renderMigrationFiles,
|
|
14929
|
+
runUrlMigration,
|
|
13676
14930
|
sourceRefOversizeReason,
|
|
13677
14931
|
validateIntegrations,
|
|
13678
|
-
validateMigrationUrl
|
|
14932
|
+
validateMigrationUrl,
|
|
14933
|
+
validateUrlMigration
|
|
13679
14934
|
};
|
|
13680
14935
|
//# sourceMappingURL=index.js.map
|