@thallylabs/migrate 0.2.30 → 0.2.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -8
- package/dist/index.js +1107 -506
- package/dist/index.js.map +1 -1
- package/dist/{url-migration-CDjLZVaN.d.ts → url-migration-Coy62q1L.d.ts} +15 -1
- package/dist/url-migration.d.ts +1 -1
- package/dist/url-migration.js +801 -200
- package/dist/url-migration.js.map +1 -1
- package/package.json +1 -1
package/dist/url-migration.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// src/url-run.ts
|
|
2
2
|
import { createHash } from "crypto";
|
|
3
|
-
import { load as
|
|
3
|
+
import { load as load3 } from "cheerio";
|
|
4
4
|
import remarkParse3 from "remark-parse";
|
|
5
5
|
import remarkMdx3 from "remark-mdx";
|
|
6
6
|
import { unified as unified3 } from "unified";
|
|
@@ -390,20 +390,20 @@ function utf8SvgText(content) {
|
|
|
390
390
|
if (body[0] === 254 && body[1] === 255 || body[0] === 255 && body[1] === 254 || body[0] === 0 && body[1] === 0 || body.includes(0)) {
|
|
391
391
|
return null;
|
|
392
392
|
}
|
|
393
|
-
const
|
|
394
|
-
if (
|
|
395
|
-
const declaration = /^<\?xml\b[^>]*\?>/iu.exec(
|
|
393
|
+
const text2 = body.toString("utf8");
|
|
394
|
+
if (text2.includes("\uFFFD")) return null;
|
|
395
|
+
const declaration = /^<\?xml\b[^>]*\?>/iu.exec(text2);
|
|
396
396
|
if (declaration) {
|
|
397
397
|
const encoding = /\bencoding\s*=\s*["']([^"']*)["']/iu.exec(declaration[0]);
|
|
398
398
|
if (encoding && !/^utf-?8$/iu.test(encoding[1].trim())) return null;
|
|
399
|
-
} else if (/<\?xml\b/iu.test(
|
|
399
|
+
} else if (/<\?xml\b/iu.test(text2)) {
|
|
400
400
|
return null;
|
|
401
401
|
}
|
|
402
|
-
return
|
|
402
|
+
return text2;
|
|
403
403
|
}
|
|
404
404
|
function isPassiveSvg(content) {
|
|
405
|
-
const
|
|
406
|
-
if (
|
|
405
|
+
const text2 = utf8SvgText(Buffer.from(content.buffer, content.byteOffset, content.byteLength));
|
|
406
|
+
if (text2 === null) return false;
|
|
407
407
|
let isPassive = true;
|
|
408
408
|
let depth = 0;
|
|
409
409
|
let sawRoot = false;
|
|
@@ -455,7 +455,7 @@ function isPassiveSvg(content) {
|
|
|
455
455
|
{ xmlMode: true, decodeEntities: true, recognizeSelfClosing: true }
|
|
456
456
|
);
|
|
457
457
|
try {
|
|
458
|
-
parser3.write(
|
|
458
|
+
parser3.write(text2);
|
|
459
459
|
parser3.end();
|
|
460
460
|
} catch {
|
|
461
461
|
return false;
|
|
@@ -530,6 +530,274 @@ function cloneUrlMigrationCheckpoint(checkpoint) {
|
|
|
530
530
|
return clone(checkpoint);
|
|
531
531
|
}
|
|
532
532
|
|
|
533
|
+
// src/url-navigation.ts
|
|
534
|
+
import { load } from "cheerio";
|
|
535
|
+
var MAX_CAPTURE_BYTES = 8e6;
|
|
536
|
+
var MAX_JSON_BYTES = 2e6;
|
|
537
|
+
var MAX_NODES = 1e4;
|
|
538
|
+
var MAX_DEPTH = 32;
|
|
539
|
+
var MAX_STRING_LENGTH = 8192;
|
|
540
|
+
function object(value) {
|
|
541
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
542
|
+
}
|
|
543
|
+
function text(value) {
|
|
544
|
+
return typeof value === "string" && value.length <= MAX_STRING_LENGTH && value.trim() ? value.trim() : void 0;
|
|
545
|
+
}
|
|
546
|
+
function target(value, base) {
|
|
547
|
+
const reference = text(value);
|
|
548
|
+
if (!reference) return void 0;
|
|
549
|
+
try {
|
|
550
|
+
const url = new URL(reference, base);
|
|
551
|
+
return ["https:", "http:"].includes(url.protocol) && !url.username && !url.password ? url.toString() : void 0;
|
|
552
|
+
} catch {
|
|
553
|
+
return void 0;
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
function readJson(value) {
|
|
557
|
+
if (Buffer.byteLength(value) > MAX_JSON_BYTES) return void 0;
|
|
558
|
+
let depth = 0;
|
|
559
|
+
let quoted = false;
|
|
560
|
+
let escaped = false;
|
|
561
|
+
for (const char of value) {
|
|
562
|
+
if (quoted) {
|
|
563
|
+
if (escaped) escaped = false;
|
|
564
|
+
else if (char === "\\") escaped = true;
|
|
565
|
+
else if (char === '"') quoted = false;
|
|
566
|
+
} else if (char === '"') quoted = true;
|
|
567
|
+
else if (char === "[" || char === "{") {
|
|
568
|
+
if (++depth > 96) return void 0;
|
|
569
|
+
} else if (char === "]" || char === "}") depth--;
|
|
570
|
+
}
|
|
571
|
+
try {
|
|
572
|
+
return JSON.parse(value);
|
|
573
|
+
} catch {
|
|
574
|
+
return void 0;
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
function flightRecords(stream) {
|
|
578
|
+
const bytes = Buffer.from(stream);
|
|
579
|
+
const records = [];
|
|
580
|
+
let cursor = 0;
|
|
581
|
+
while (cursor < bytes.length) {
|
|
582
|
+
const colon = bytes.indexOf(58, cursor);
|
|
583
|
+
if (colon < 0 || colon - cursor > 16 || !/^[a-f\d]*$/i.test(bytes.toString("utf8", cursor, colon))) break;
|
|
584
|
+
cursor = colon + 1;
|
|
585
|
+
if (bytes[cursor] === 84) {
|
|
586
|
+
const comma = bytes.indexOf(44, cursor + 1);
|
|
587
|
+
const length = comma < 0 ? "" : bytes.toString("utf8", cursor + 1, comma);
|
|
588
|
+
if (!/^[a-f\d]{1,8}$/i.test(length)) break;
|
|
589
|
+
const end2 = comma + 1 + Number.parseInt(length, 16);
|
|
590
|
+
if (end2 > bytes.length) break;
|
|
591
|
+
cursor = end2;
|
|
592
|
+
continue;
|
|
593
|
+
}
|
|
594
|
+
const newline = bytes.indexOf(10, cursor);
|
|
595
|
+
const end = newline < 0 ? bytes.length : newline;
|
|
596
|
+
if (bytes[cursor] === 91 || bytes[cursor] === 123) records.push(readJson(bytes.toString("utf8", cursor, end)));
|
|
597
|
+
cursor = end + 1;
|
|
598
|
+
}
|
|
599
|
+
return records;
|
|
600
|
+
}
|
|
601
|
+
function mintlifyNavigation(body, base, locale) {
|
|
602
|
+
const $ = load(body);
|
|
603
|
+
let stream = "";
|
|
604
|
+
const records = [];
|
|
605
|
+
$("script").each((_index, script) => {
|
|
606
|
+
const source = $(script).text();
|
|
607
|
+
const call = /^\s*self\.__next_f\.push\(\s*(\[[\s\S]*\])\s*\)\s*;?\s*$/.exec(source);
|
|
608
|
+
if (call) {
|
|
609
|
+
const value = readJson(call[1]);
|
|
610
|
+
if (Array.isArray(value) && value[0] === 1 && typeof value[1] === "string") stream += value[1];
|
|
611
|
+
} else if ($(script).attr("type") === "application/json") {
|
|
612
|
+
records.push(readJson(source));
|
|
613
|
+
}
|
|
614
|
+
});
|
|
615
|
+
records.push(...flightRecords(stream));
|
|
616
|
+
const candidates = [];
|
|
617
|
+
let visited = 0;
|
|
618
|
+
const visit = (value, depth = 0) => {
|
|
619
|
+
if (++visited > MAX_NODES * 5 || depth > 96) return;
|
|
620
|
+
if (Array.isArray(value)) {
|
|
621
|
+
for (const child of value) visit(child, depth + 1);
|
|
622
|
+
} else if (object(value)) {
|
|
623
|
+
for (const key of ["scopedNav", "docsNavWithMetadata", "anonymizedNav"]) {
|
|
624
|
+
if (object(value[key])) candidates.push({ value: value[key], rank: key === "scopedNav" ? 3 : 2 });
|
|
625
|
+
}
|
|
626
|
+
if (object(value.docsConfig) && object(value.docsConfig.navigation)) {
|
|
627
|
+
candidates.push({ value: value.docsConfig.navigation, rank: 1 });
|
|
628
|
+
}
|
|
629
|
+
for (const child of Object.values(value)) visit(child, depth + 1);
|
|
630
|
+
}
|
|
631
|
+
};
|
|
632
|
+
records.forEach((record) => visit(record));
|
|
633
|
+
const language = locale ?? text($("html").attr("lang"));
|
|
634
|
+
for (const candidate of candidates.sort((a, b) => b.rank - a.rank)) {
|
|
635
|
+
let count = 0;
|
|
636
|
+
const walk = (value, depth = 0, activeLocale) => {
|
|
637
|
+
if (depth > MAX_DEPTH || ++count > MAX_NODES) throw new Error("Source navigation exceeds its safety bound.");
|
|
638
|
+
if (typeof value === "string") {
|
|
639
|
+
const url2 = target(value, new URL("/", base));
|
|
640
|
+
return url2 ? [{ kind: "page", label: value, url: url2 }] : [];
|
|
641
|
+
}
|
|
642
|
+
if (Array.isArray(value)) return value.flatMap((child) => walk(child, depth + 1, activeLocale));
|
|
643
|
+
if (!object(value)) return [];
|
|
644
|
+
if (Array.isArray(value.languages)) {
|
|
645
|
+
const languages = value.languages.filter(object);
|
|
646
|
+
const current = new URL(base);
|
|
647
|
+
current.hash = "";
|
|
648
|
+
const routeScore = (item) => {
|
|
649
|
+
let visited2 = 0;
|
|
650
|
+
let matches = false;
|
|
651
|
+
let populated = 0;
|
|
652
|
+
const scan = (node, depth2 = 0) => {
|
|
653
|
+
if (++visited2 > MAX_NODES || depth2 > MAX_DEPTH) return;
|
|
654
|
+
if (Array.isArray(node)) {
|
|
655
|
+
for (const child of node) scan(child, depth2 + 1);
|
|
656
|
+
return;
|
|
657
|
+
}
|
|
658
|
+
if (!object(node)) return;
|
|
659
|
+
const href = target(node.href, base);
|
|
660
|
+
if (href) {
|
|
661
|
+
populated++;
|
|
662
|
+
const url2 = new URL(href);
|
|
663
|
+
url2.hash = "";
|
|
664
|
+
if (url2.toString() === current.toString()) matches = true;
|
|
665
|
+
}
|
|
666
|
+
for (const key of ["tabs", "groups", "pages", "dropdowns", "anchors", "versions"]) {
|
|
667
|
+
if (Array.isArray(node[key])) scan(node[key], depth2 + 1);
|
|
668
|
+
}
|
|
669
|
+
};
|
|
670
|
+
scan(item);
|
|
671
|
+
return matches ? populated : 0;
|
|
672
|
+
};
|
|
673
|
+
const matching = languages.map((item) => ({ item, score: routeScore(item) })).filter((candidate2) => candidate2.score > 0).sort((a, b) => b.score - a.score);
|
|
674
|
+
const selected = matching[0]?.item ?? languages.find((item) => text(item.language)?.toLowerCase() === language?.toLowerCase()) ?? languages.find((item) => text(item.language)?.toLowerCase() === "en") ?? languages[0];
|
|
675
|
+
return selected ? walk({ ...selected, languages: void 0 }, depth + 1, text(selected.language)) : [];
|
|
676
|
+
}
|
|
677
|
+
const label = text(value.tab) ?? text(value.group) ?? text(value.title) ?? text(value.label);
|
|
678
|
+
const kind = text(value.tab) ? "tab" : text(value.group) ? "group" : "page";
|
|
679
|
+
const children = ["tabs", "groups", "pages", "dropdowns", "anchors", "versions"].flatMap((key) => Array.isArray(value[key]) ? walk(value[key], depth + 1, activeLocale) : []);
|
|
680
|
+
const url = target(value.href, base);
|
|
681
|
+
if (!label) return children;
|
|
682
|
+
if (!url && !children.length) return [];
|
|
683
|
+
return [{
|
|
684
|
+
kind,
|
|
685
|
+
label,
|
|
686
|
+
...url ? { url } : {},
|
|
687
|
+
...children.length ? { children } : {},
|
|
688
|
+
...kind === "tab" && activeLocale ? { locale: activeLocale } : {}
|
|
689
|
+
}];
|
|
690
|
+
};
|
|
691
|
+
try {
|
|
692
|
+
const nodes = walk(candidate.value);
|
|
693
|
+
if (nodes.length) return nodes;
|
|
694
|
+
} catch {
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
return [];
|
|
698
|
+
}
|
|
699
|
+
function resolveNavigation(nodes, base) {
|
|
700
|
+
let count = 0;
|
|
701
|
+
const walk = (items, depth = 0) => {
|
|
702
|
+
if (depth > MAX_DEPTH || !Array.isArray(items)) throw new Error("Invalid source navigation.");
|
|
703
|
+
return items.flatMap((node) => {
|
|
704
|
+
if (++count > MAX_NODES || !object(node)) throw new Error("Source navigation exceeds its safety bound.");
|
|
705
|
+
const label = text(node.label);
|
|
706
|
+
if (!label) return [];
|
|
707
|
+
const url = target(node.url, base);
|
|
708
|
+
const children = node.children ? walk(node.children, depth + 1) : [];
|
|
709
|
+
return [{
|
|
710
|
+
label,
|
|
711
|
+
...url ? { url } : {},
|
|
712
|
+
...children.length ? { children } : {},
|
|
713
|
+
...["tab", "group", "page"].includes(node.kind ?? "") ? { kind: node.kind } : {},
|
|
714
|
+
...text(node.locale) ? { locale: text(node.locale) } : {}
|
|
715
|
+
}];
|
|
716
|
+
});
|
|
717
|
+
};
|
|
718
|
+
return walk(nodes);
|
|
719
|
+
}
|
|
720
|
+
function extractSourceNavigation(capture) {
|
|
721
|
+
if (capture.navigation) return resolveNavigation(capture.navigation, capture.finalUrl);
|
|
722
|
+
if (!/html/i.test(capture.contentType) || Buffer.byteLength(capture.body) > MAX_CAPTURE_BYTES) return [];
|
|
723
|
+
const mintlify = mintlifyNavigation(capture.body, capture.finalUrl, capture.locale);
|
|
724
|
+
if (mintlify.length) return mintlify;
|
|
725
|
+
const $ = load(capture.body);
|
|
726
|
+
const candidates = ['.theme-doc-sidebar-menu,[aria-label*="sidebar" i]', "aside", '[role="navigation"],nav'];
|
|
727
|
+
const root = candidates.map((selector) => $(selector).filter((_index, node) => $(node).find("a[href]").length > 0 && $(node).closest("header").length === 0).first()).find((candidate) => candidate.length);
|
|
728
|
+
if (!root) return [];
|
|
729
|
+
let count = 0;
|
|
730
|
+
const walk = (element, excluded, depth = 0) => {
|
|
731
|
+
if (depth > MAX_DEPTH) throw new Error("Source navigation exceeds its safety bound.");
|
|
732
|
+
const result = [];
|
|
733
|
+
$(element).children().each((_index, node) => {
|
|
734
|
+
if (++count > MAX_NODES) throw new Error("Source navigation exceeds its safety bound.");
|
|
735
|
+
if (node === excluded) return;
|
|
736
|
+
const item = $(node);
|
|
737
|
+
if (item.is("a[href]")) {
|
|
738
|
+
const label = item.text().replace(/\s+/g, " ").trim();
|
|
739
|
+
const url = target(item.attr("href"), capture.finalUrl);
|
|
740
|
+
if (label && url) result.push({ label, url });
|
|
741
|
+
} else if (item.is("li,details") || item.attr("data-migration-group") !== void 0 || item.children("h2,h3,h4,h5,h6").length > 0) {
|
|
742
|
+
const directLink = item.children("a[href]").first();
|
|
743
|
+
const direct = directLink.length ? directLink : item.children("div").children("a[href]").first();
|
|
744
|
+
const label = direct.text().trim() || item.children("summary,button,span,h2,h3,h4,h5,h6").first().text().trim();
|
|
745
|
+
const children = walk(node, direct[0], depth + 1);
|
|
746
|
+
const url = target(direct.attr("href"), capture.finalUrl);
|
|
747
|
+
if (label && (url || children.length)) result.push({ label, ...url ? { url } : {}, ...children.length ? { children } : {} });
|
|
748
|
+
else result.push(...children);
|
|
749
|
+
} else result.push(...walk(node, excluded, depth + 1));
|
|
750
|
+
});
|
|
751
|
+
return result;
|
|
752
|
+
};
|
|
753
|
+
try {
|
|
754
|
+
return walk(root);
|
|
755
|
+
} catch {
|
|
756
|
+
return [];
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
function placeholder(node) {
|
|
760
|
+
const children = node.children ?? [];
|
|
761
|
+
return node.kind === "tab" && children.length === 1 && children[0].kind === "page" && children[0].label === node.label && !children[0].children?.length;
|
|
762
|
+
}
|
|
763
|
+
function mergeSourceNavigation(existing, incoming) {
|
|
764
|
+
let count = 0;
|
|
765
|
+
const merge = (left, right, depth = 0) => {
|
|
766
|
+
if (depth > MAX_DEPTH || (count += left.length + right.length) > MAX_NODES * 4) throw new Error("Source navigation exceeds its safety bound.");
|
|
767
|
+
const result = left.map((node) => ({ ...node }));
|
|
768
|
+
const identity = (node) => JSON.stringify([
|
|
769
|
+
node.kind,
|
|
770
|
+
node.label,
|
|
771
|
+
node.locale,
|
|
772
|
+
node.kind === "tab" || node.kind === "group" ? void 0 : node.url
|
|
773
|
+
]);
|
|
774
|
+
const positions = /* @__PURE__ */ new Map();
|
|
775
|
+
result.forEach((node, index) => {
|
|
776
|
+
const key = identity(node);
|
|
777
|
+
const siblings = positions.get(key) ?? [];
|
|
778
|
+
siblings.push(index);
|
|
779
|
+
positions.set(key, siblings);
|
|
780
|
+
});
|
|
781
|
+
const offsets = /* @__PURE__ */ new Map();
|
|
782
|
+
for (const next of right) {
|
|
783
|
+
const key = identity(next);
|
|
784
|
+
const offset = offsets.get(key) ?? 0;
|
|
785
|
+
const index = positions.get(key)?.[offset];
|
|
786
|
+
offsets.set(key, offset + 1);
|
|
787
|
+
if (index === void 0) {
|
|
788
|
+
result.push({ ...next });
|
|
789
|
+
continue;
|
|
790
|
+
}
|
|
791
|
+
const previous = result[index];
|
|
792
|
+
if (placeholder(previous) && !placeholder(next)) result[index] = { ...next };
|
|
793
|
+
else if (!placeholder(previous) && placeholder(next)) continue;
|
|
794
|
+
else result[index] = { ...previous, ...next, ...previous.children || next.children ? { children: merge(previous.children ?? [], next.children ?? [], depth + 1) } : {} };
|
|
795
|
+
}
|
|
796
|
+
return result;
|
|
797
|
+
};
|
|
798
|
+
return merge(existing, incoming);
|
|
799
|
+
}
|
|
800
|
+
|
|
533
801
|
// src/url-images.ts
|
|
534
802
|
function migrationImageUrl(value, base) {
|
|
535
803
|
const url = new URL(value, base);
|
|
@@ -569,7 +837,7 @@ function migrationImageAttribute(value) {
|
|
|
569
837
|
}
|
|
570
838
|
|
|
571
839
|
// src/url.ts
|
|
572
|
-
import { load } from "cheerio";
|
|
840
|
+
import { load as load2 } from "cheerio";
|
|
573
841
|
import remarkMdx2 from "remark-mdx";
|
|
574
842
|
import remarkParse2 from "remark-parse";
|
|
575
843
|
import TurndownService from "turndown";
|
|
@@ -845,9 +1113,9 @@ function collectRefs(root) {
|
|
|
845
1113
|
return;
|
|
846
1114
|
}
|
|
847
1115
|
if (ts.isAwaitExpression(node) && !inAsync) awaitOutsideAsync = true;
|
|
848
|
-
const
|
|
849
|
-
if (
|
|
850
|
-
let base =
|
|
1116
|
+
const target2 = ts.isBinaryExpression(node) && node.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && node.operatorToken.kind <= ts.SyntaxKind.LastAssignment ? node.left : (ts.isPrefixUnaryExpression(node) || ts.isPostfixUnaryExpression(node)) && (node.operator === ts.SyntaxKind.PlusPlusToken || node.operator === ts.SyntaxKind.MinusMinusToken) ? node.operand : void 0;
|
|
1117
|
+
if (target2) {
|
|
1118
|
+
let base = target2;
|
|
851
1119
|
while (ts.isPropertyAccessExpression(base) || ts.isElementAccessExpression(base)) base = base.expression;
|
|
852
1120
|
if (ts.isIdentifier(base) && !bound(base.text, scopes)) assigned.push(base.text);
|
|
853
1121
|
}
|
|
@@ -968,8 +1236,8 @@ function pageScopeNames(esmSources) {
|
|
|
968
1236
|
for (const statement of parse(source).statements) {
|
|
969
1237
|
for (const local of importLocals(statement.getText())) names.add(local);
|
|
970
1238
|
if (ts.isImportDeclaration(statement)) continue;
|
|
971
|
-
const
|
|
972
|
-
if (
|
|
1239
|
+
const target2 = ts.isExportDeclaration(statement) || ts.isExportAssignment(statement) ? void 0 : statement;
|
|
1240
|
+
if (target2) for (const name of declaredNames(target2).names) names.add(name);
|
|
973
1241
|
}
|
|
974
1242
|
}
|
|
975
1243
|
return names;
|
|
@@ -1265,14 +1533,14 @@ ${trimmed.slice(0, marker).trimEnd()}`;
|
|
|
1265
1533
|
function headingIdMarker(id) {
|
|
1266
1534
|
return `{/* #${id} */}`;
|
|
1267
1535
|
}
|
|
1268
|
-
function mintlifyHeadingSlug(
|
|
1269
|
-
return
|
|
1536
|
+
function mintlifyHeadingSlug(text2) {
|
|
1537
|
+
return text2.replace(/\u200b/g, "").normalize("NFC").trim().toLowerCase().replace(/[.\s]+/g, "-").replace(/-{2,}/g, "-").replace(/^-|-$/g, "").replace(/[^\p{L}\p{M}\p{N}_\-+&/\u2019\u201c\u201d\u2014\u2013\u2192]/gu, "");
|
|
1270
1538
|
}
|
|
1271
|
-
function thallyHeadingSlug(
|
|
1272
|
-
return
|
|
1539
|
+
function thallyHeadingSlug(text2) {
|
|
1540
|
+
return text2.normalize("NFC").toLowerCase().replace(/[^\p{L}\p{M}\p{N}]+/gu, "-").replace(/(^-|-$)/g, "");
|
|
1273
1541
|
}
|
|
1274
|
-
function typesetQuotes(
|
|
1275
|
-
return
|
|
1542
|
+
function typesetQuotes(text2) {
|
|
1543
|
+
return text2.split(/(`[^`]*`|<[^>]*>|\]\([^)]*\))/).map((part, index) => {
|
|
1276
1544
|
if (index % 2) return part;
|
|
1277
1545
|
return part.replace(/(^|[\s([{\u2014\u2013])"/g, "$1\u201C").replace(/"/g, "\u201D").replace(/(^|[\s([{\u2014\u2013\u201c])'(?=\S)/g, "$1\u2018").replace(/'/g, "\u2019");
|
|
1278
1546
|
}).join("");
|
|
@@ -1295,9 +1563,9 @@ function markMintlifyHeadings(body) {
|
|
|
1295
1563
|
const explicit = heading[2].match(/^(.*?)([ \t]*\{\/\*[ \t]*#\S+[ \t]*\*\/\})$/);
|
|
1296
1564
|
const source = typesetQuotes(explicit ? explicit[1] : heading[2]);
|
|
1297
1565
|
if (explicit) return `${heading[1]}${source}${explicit[2]}`;
|
|
1298
|
-
const
|
|
1299
|
-
const id = mintlifyHeadingSlug(
|
|
1300
|
-
return `${heading[1]}${source}${id && id !== thallyHeadingSlug(
|
|
1566
|
+
const text2 = plainHeadingText(source);
|
|
1567
|
+
const id = mintlifyHeadingSlug(text2);
|
|
1568
|
+
return `${heading[1]}${source}${id && id !== thallyHeadingSlug(text2) ? ` ${headingIdMarker(id)}` : ""}`;
|
|
1301
1569
|
}).join("\n");
|
|
1302
1570
|
}
|
|
1303
1571
|
var GLOBAL_DOCUSARUS_COMPONENTS = /* @__PURE__ */ new Set([
|
|
@@ -1319,8 +1587,8 @@ function leadingPageHeading(body) {
|
|
|
1319
1587
|
const root = descriptionParser.parse(body);
|
|
1320
1588
|
const first = root.children?.find((node) => node.type !== "mdxjsEsm" && node.type !== "html" && !(node.type === "mdxFlowExpression" && node.data?.estree?.body.length === 0));
|
|
1321
1589
|
if (first?.type !== "heading" || first.depth !== 1) return void 0;
|
|
1322
|
-
const
|
|
1323
|
-
const title =
|
|
1590
|
+
const text2 = (node) => node.type === "mdxTextExpression" || node.type === "mdxFlowExpression" ? "" : node.value ?? (node.children ?? []).map(text2).join("");
|
|
1591
|
+
const title = text2(first).replace(/\s+/g, " ").trim();
|
|
1324
1592
|
const start = first.position?.start.offset;
|
|
1325
1593
|
const end = first.position?.end.offset;
|
|
1326
1594
|
return title && start !== void 0 && end !== void 0 ? { title, start, end } : void 0;
|
|
@@ -1349,8 +1617,8 @@ function splitFrontmatterBlock(raw) {
|
|
|
1349
1617
|
function maskCode(body) {
|
|
1350
1618
|
const source = body.replace(/\u0000/g, "");
|
|
1351
1619
|
const blocks = [];
|
|
1352
|
-
const stash = (
|
|
1353
|
-
blocks.push(
|
|
1620
|
+
const stash = (text2) => {
|
|
1621
|
+
blocks.push(text2);
|
|
1354
1622
|
return `\0${blocks.length - 1}\0`;
|
|
1355
1623
|
};
|
|
1356
1624
|
const lines = [];
|
|
@@ -1377,7 +1645,7 @@ function maskCode(body) {
|
|
|
1377
1645
|
if (fenceBuffer.length) lines.push(stash(fenceBuffer.join("\n")));
|
|
1378
1646
|
return {
|
|
1379
1647
|
masked: lines.join("\n"),
|
|
1380
|
-
unmask: (
|
|
1648
|
+
unmask: (text2) => text2.replace(/\u0000(\d+)\u0000/g, (_match, index) => blocks[Number(index)]),
|
|
1381
1649
|
openFence: codeFence
|
|
1382
1650
|
};
|
|
1383
1651
|
}
|
|
@@ -1385,8 +1653,8 @@ function replaceOutsideCode(body, transform) {
|
|
|
1385
1653
|
const { masked, unmask } = maskCode(body);
|
|
1386
1654
|
return unmask(transform(masked));
|
|
1387
1655
|
}
|
|
1388
|
-
function mdxComment(
|
|
1389
|
-
return `{/*${
|
|
1656
|
+
function mdxComment(text2) {
|
|
1657
|
+
return `{/*${text2.replace(/\*\//g, "* /")}*/}`;
|
|
1390
1658
|
}
|
|
1391
1659
|
function normalizeFernFileTrees(body) {
|
|
1392
1660
|
return replaceOutsideCode(body, (whole) => whole.replace(
|
|
@@ -1793,12 +2061,12 @@ function maskCodeRegions(body) {
|
|
|
1793
2061
|
}).join("\n");
|
|
1794
2062
|
}
|
|
1795
2063
|
var UNTERMINATED = "unterminated";
|
|
1796
|
-
function readTagEnd(
|
|
1797
|
-
if (from <
|
|
2064
|
+
function readTagEnd(text2, from) {
|
|
2065
|
+
if (from < text2.length && !/[\s/>]/.test(text2[from])) return null;
|
|
1798
2066
|
let depth = 0;
|
|
1799
2067
|
let quote = "";
|
|
1800
|
-
for (let i = from; i <
|
|
1801
|
-
const char =
|
|
2068
|
+
for (let i = from; i < text2.length; i++) {
|
|
2069
|
+
const char = text2[i];
|
|
1802
2070
|
if (quote) {
|
|
1803
2071
|
if (char === "\\" && depth > 0) i++;
|
|
1804
2072
|
else if (char === quote) quote = "";
|
|
@@ -1811,8 +2079,8 @@ function readTagEnd(text, from) {
|
|
|
1811
2079
|
} else if (depth === 0) {
|
|
1812
2080
|
if (char === "<") return null;
|
|
1813
2081
|
if (char === ">") {
|
|
1814
|
-
const selfClosing =
|
|
1815
|
-
return { end: i + 1, selfClosing, attributes:
|
|
2082
|
+
const selfClosing = text2[i - 1] === "/" && i > from;
|
|
2083
|
+
return { end: i + 1, selfClosing, attributes: text2.slice(from, selfClosing ? i - 1 : i) };
|
|
1816
2084
|
}
|
|
1817
2085
|
}
|
|
1818
2086
|
}
|
|
@@ -1925,7 +2193,7 @@ function removeUndefinedExpressions(body, warn) {
|
|
|
1925
2193
|
visit(tree);
|
|
1926
2194
|
if (edits.length === 0) return body;
|
|
1927
2195
|
warn?.(`Expressions that use ${[...removed].map((name) => `"${name}"`).join(", ")} were removed so the page builds: ${removed.size === 1 ? "the name is" : "the names are"} not defined on this page. Define ${removed.size === 1 ? "the name" : "the names"} or escape the braces as \\{ \\} to show the text.`);
|
|
1928
|
-
return edits.sort((a, b) => b.start - a.start).reduce((
|
|
2196
|
+
return edits.sort((a, b) => b.start - a.start).reduce((text2, edit) => text2.slice(0, edit.start) + text2.slice(edit.end), body);
|
|
1929
2197
|
}
|
|
1930
2198
|
function normalizeFlowParagraphContainers(body) {
|
|
1931
2199
|
if (!body.includes("<p")) return body;
|
|
@@ -1951,7 +2219,7 @@ function normalizeFlowParagraphContainers(body) {
|
|
|
1951
2219
|
for (const child of node.children ?? []) visit(child);
|
|
1952
2220
|
}
|
|
1953
2221
|
visit(tree);
|
|
1954
|
-
return edits.sort((a, b) => b.offset - a.offset).reduce((
|
|
2222
|
+
return edits.sort((a, b) => b.offset - a.offset).reduce((text2, edit) => text2.slice(0, edit.offset) + edit.value + text2.slice(edit.offset + 1), body);
|
|
1955
2223
|
}
|
|
1956
2224
|
function normalizeMdx(body, platform, unwrapMdxCodeBlocks = true) {
|
|
1957
2225
|
const runFern = platform === "fern";
|
|
@@ -2168,7 +2436,12 @@ var PORTABLE_MDX_COMPONENTS = /* @__PURE__ */ new Set([
|
|
|
2168
2436
|
]);
|
|
2169
2437
|
var PORTABLE_URL_PROPS = "href|src|img|primaryHref|secondaryHref";
|
|
2170
2438
|
var PORTABLE_URL_PROP_NAMES = new Set(PORTABLE_URL_PROPS.split("|"));
|
|
2439
|
+
var MINTLIFY_ARTICLE_LABELS = [
|
|
2440
|
+
{ baseUrl: "Select base URL", permalink: "Navigate to header" },
|
|
2441
|
+
{ baseUrl: "\u9009\u62E9\u57FA\u7840URL", permalink: "\u5BFC\u822A\u5230\u6807\u9898" }
|
|
2442
|
+
];
|
|
2171
2443
|
var remoteMarkdownParser = unified2().use(remarkParse2).use(remarkMdx2);
|
|
2444
|
+
var embeddedOpenApiParser = unified2().use(remarkParse2);
|
|
2172
2445
|
function validateMigrationUrl(value) {
|
|
2173
2446
|
let url;
|
|
2174
2447
|
try {
|
|
@@ -2301,7 +2574,7 @@ function isSafePortableUrl(value) {
|
|
|
2301
2574
|
}
|
|
2302
2575
|
}
|
|
2303
2576
|
function decodeHtmlAttribute(value) {
|
|
2304
|
-
const document =
|
|
2577
|
+
const document = load2(`<i data-thally-value="${value.replace(/"/g, """)}"></i>`);
|
|
2305
2578
|
return document("i").attr("data-thally-value") ?? "";
|
|
2306
2579
|
}
|
|
2307
2580
|
function escapeMarkdownText(value, escapeTableSeparator = false) {
|
|
@@ -2318,6 +2591,43 @@ function escapeMarkdownText(value, escapeTableSeparator = false) {
|
|
|
2318
2591
|
}
|
|
2319
2592
|
return escaped;
|
|
2320
2593
|
}
|
|
2594
|
+
function portableMarkdownTableCell(value) {
|
|
2595
|
+
let output = "";
|
|
2596
|
+
let hasOddBackslashes = false;
|
|
2597
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
2598
|
+
const character = value[index];
|
|
2599
|
+
if (character === "\\") {
|
|
2600
|
+
output += character;
|
|
2601
|
+
hasOddBackslashes = !hasOddBackslashes;
|
|
2602
|
+
continue;
|
|
2603
|
+
}
|
|
2604
|
+
if (character === "|" && !hasOddBackslashes) output += "\\";
|
|
2605
|
+
if (character === "\r" || character === "\n") {
|
|
2606
|
+
output += " ";
|
|
2607
|
+
if (character === "\r" && value[index + 1] === "\n") index += 1;
|
|
2608
|
+
} else output += character;
|
|
2609
|
+
hasOddBackslashes = false;
|
|
2610
|
+
}
|
|
2611
|
+
return output.trim();
|
|
2612
|
+
}
|
|
2613
|
+
function needsLiteralTableCode(value) {
|
|
2614
|
+
let hasOddBackslashes = false;
|
|
2615
|
+
for (const character of value) {
|
|
2616
|
+
if (character === "\\") hasOddBackslashes = !hasOddBackslashes;
|
|
2617
|
+
else {
|
|
2618
|
+
if (character === "|" && hasOddBackslashes) return true;
|
|
2619
|
+
hasOddBackslashes = false;
|
|
2620
|
+
}
|
|
2621
|
+
}
|
|
2622
|
+
return false;
|
|
2623
|
+
}
|
|
2624
|
+
function literalMdxCodeText(value) {
|
|
2625
|
+
let output = "";
|
|
2626
|
+
for (const character of value) {
|
|
2627
|
+
output += /[A-Za-z0-9 ]/.test(character) ? character : `&#${character.codePointAt(0)};`;
|
|
2628
|
+
}
|
|
2629
|
+
return output;
|
|
2630
|
+
}
|
|
2321
2631
|
function markdownLinkDestination(value) {
|
|
2322
2632
|
try {
|
|
2323
2633
|
return encodeURI(value).replace(/\(/g, "%28").replace(/\)/g, "%29").replace(/&/g, "&");
|
|
@@ -2350,7 +2660,7 @@ function replacePortableHtmlImages(value) {
|
|
|
2350
2660
|
if (start < cursor) continue;
|
|
2351
2661
|
const end = htmlOpeningTagEnd(value, start + match[0].length);
|
|
2352
2662
|
if (end === null) continue;
|
|
2353
|
-
const fragment =
|
|
2663
|
+
const fragment = load2(value.slice(start, end));
|
|
2354
2664
|
const image = fragment("img").first();
|
|
2355
2665
|
const source = image.attr("src");
|
|
2356
2666
|
const destination = source && isSafePortableUrl(source) ? markdownLinkDestination(source) : null;
|
|
@@ -2375,7 +2685,7 @@ function replacePortableHtmlAnchors(value) {
|
|
|
2375
2685
|
const closing = closingMatch.exec(value);
|
|
2376
2686
|
if (!closing) continue;
|
|
2377
2687
|
const end = closing.index + closing[0].length;
|
|
2378
|
-
const fragment =
|
|
2688
|
+
const fragment = load2(value.slice(start, end));
|
|
2379
2689
|
const anchor = fragment("a").first();
|
|
2380
2690
|
const href = anchor.attr("href");
|
|
2381
2691
|
const label = escapeMarkdownText(anchor.text().replace(/\s+/g, " ").trim());
|
|
@@ -2405,6 +2715,11 @@ function sanitizeMarkdownUrls(value) {
|
|
|
2405
2715
|
return;
|
|
2406
2716
|
}
|
|
2407
2717
|
if (node.type === "mdxJsxTextElement" || node.type === "mdxJsxFlowElement") {
|
|
2718
|
+
const isLiteralCode = node.type === "mdxJsxTextElement" && node.name === "code" && !node.attributes?.length && (node.children ?? []).every((child) => child.type === "text");
|
|
2719
|
+
if (!isLiteralCode && (!node.name || !PORTABLE_MDX_COMPONENTS.has(node.name))) {
|
|
2720
|
+
hasUnsafeMdxSyntax = true;
|
|
2721
|
+
return;
|
|
2722
|
+
}
|
|
2408
2723
|
for (const attribute of node.attributes ?? []) {
|
|
2409
2724
|
if (attribute.type !== "mdxJsxAttribute" || !attribute.name || /^on[A-Z]/.test(attribute.name)) {
|
|
2410
2725
|
hasUnsafeMdxSyntax = true;
|
|
@@ -2565,10 +2880,6 @@ function sanitizeRemoteMarkdown(body) {
|
|
|
2565
2880
|
}
|
|
2566
2881
|
}).replace(/\\[{}]/g, "");
|
|
2567
2882
|
if (hasUnsafeExpression || /[{}]/.test(withoutStaticProps) || /\bon[A-Z][A-Za-z]*\s*=/g.test(withoutStaticProps) || hasUnsafePortableUrlProp(executable)) return null;
|
|
2568
|
-
for (const match of withoutStaticProps.matchAll(/<\/?([A-Za-z][A-Za-z0-9.]*)(?:\s|>|\/)/g)) {
|
|
2569
|
-
const component = match[1].split(".", 1)[0];
|
|
2570
|
-
if (!PORTABLE_MDX_COMPONENTS.has(component)) return null;
|
|
2571
|
-
}
|
|
2572
2883
|
return urlSafeMarkdown;
|
|
2573
2884
|
}
|
|
2574
2885
|
function remoteMarkdownMetadata(body) {
|
|
@@ -2610,22 +2921,45 @@ function stripMintlifyDocumentationIndex(body) {
|
|
|
2610
2921
|
return lines.slice(index).join("\n");
|
|
2611
2922
|
}
|
|
2612
2923
|
function extractEmbeddedOpenApi(content) {
|
|
2613
|
-
|
|
2614
|
-
const match = fencePattern.exec(content);
|
|
2615
|
-
if (!match) return { content };
|
|
2924
|
+
if (Buffer.byteLength(content, "utf8") > MAX_LOCAL_RESPONSE_BYTES) return { content };
|
|
2616
2925
|
try {
|
|
2617
|
-
const
|
|
2926
|
+
const root = embeddedOpenApiParser.parse(content);
|
|
2927
|
+
const candidates = (root.children ?? []).flatMap((node2) => {
|
|
2928
|
+
if (node2.type !== "code" || !/^(?:yaml|yml|json)$/i.test(node2.lang ?? "")) return [];
|
|
2929
|
+
const metadata = node2.meta?.match(/^(?:(\S+\.(?:yaml|yml|json))\s+)?(delete|get|head|options|patch|post|put|trace)\s+(\/\S+)\s*$/i);
|
|
2930
|
+
return metadata ? [{ node: node2, method: metadata[2].toUpperCase(), path: metadata[3] }] : [];
|
|
2931
|
+
});
|
|
2932
|
+
if (candidates.length !== 1) return { content };
|
|
2933
|
+
const { node, method, path } = candidates[0];
|
|
2934
|
+
const start = node.position?.start.offset;
|
|
2935
|
+
const end = node.position?.end.offset;
|
|
2936
|
+
if (start === void 0 || end === void 0) return { content };
|
|
2937
|
+
const lines = content.slice(start, end).split(/\r?\n/);
|
|
2938
|
+
const opening = lines[0].match(/^ {0,3}(`{3,}|~{3,})/);
|
|
2939
|
+
const closing = lines.at(-1)?.match(/^ {0,3}(`{3,}|~{3,})[\t ]*$/);
|
|
2940
|
+
if (!opening || !closing || opening[1][0] !== closing[1][0] || closing[1].length < opening[1].length) return { content };
|
|
2941
|
+
const parsed = parseYaml2(node.value ?? "", { maxAliasCount: 50 });
|
|
2618
2942
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return { content };
|
|
2619
2943
|
const document = parsed;
|
|
2620
2944
|
if (typeof document.openapi !== "string" && typeof document.swagger !== "string") return { content };
|
|
2621
|
-
const method = match[3].toUpperCase();
|
|
2622
|
-
const path = match[4];
|
|
2623
2945
|
const paths = document.paths;
|
|
2624
2946
|
if (!paths || typeof paths !== "object" || Array.isArray(paths)) return { content };
|
|
2947
|
+
if (!Object.prototype.hasOwnProperty.call(paths, path)) return { content };
|
|
2625
2948
|
const pathItem = paths[path];
|
|
2626
|
-
if (!pathItem || typeof pathItem !== "object" || Array.isArray(pathItem) || !(
|
|
2627
|
-
const
|
|
2628
|
-
|
|
2949
|
+
if (!pathItem || typeof pathItem !== "object" || Array.isArray(pathItem) || !Object.prototype.hasOwnProperty.call(pathItem, method.toLowerCase())) return { content };
|
|
2950
|
+
const operation = pathItem[method.toLowerCase()];
|
|
2951
|
+
if (!operation || typeof operation !== "object" || Array.isArray(operation)) return { content };
|
|
2952
|
+
let values = 0;
|
|
2953
|
+
let textLength = 0;
|
|
2954
|
+
const serialized = JSON.stringify(document, (_key, value) => {
|
|
2955
|
+
values += 1;
|
|
2956
|
+
if (typeof value === "string") textLength += value.length;
|
|
2957
|
+
if (values > 1e5 || textLength > MAX_LOCAL_RESPONSE_BYTES || typeof value === "number" && !Number.isFinite(value)) throw new Error("OpenAPI document exceeds portable data limits.");
|
|
2958
|
+
return value;
|
|
2959
|
+
});
|
|
2960
|
+
if (Buffer.byteLength(serialized, "utf8") > MAX_LOCAL_RESPONSE_BYTES) return { content };
|
|
2961
|
+
const before = content.slice(0, start).replace(/(?:^|\n)##\s+OpenAPI\s*\n\s*$/i, "\n");
|
|
2962
|
+
const after = content.slice(end);
|
|
2629
2963
|
return {
|
|
2630
2964
|
content: `${before}${after}`.trim(),
|
|
2631
2965
|
openapi: `${method} ${path}`,
|
|
@@ -2676,14 +3010,74 @@ function markdownPage(document, id, platform, warn) {
|
|
|
2676
3010
|
openApiFragment: embeddedOpenApi.fragment
|
|
2677
3011
|
};
|
|
2678
3012
|
}
|
|
3013
|
+
function migrationHtmlArticle(body) {
|
|
3014
|
+
const $ = load2(body);
|
|
3015
|
+
const root = ["main article", "article", "main", '[role="main"]', "body"].map((selector) => $(selector).first()).find((candidate) => candidate.length);
|
|
3016
|
+
root.find([
|
|
3017
|
+
"script",
|
|
3018
|
+
"style",
|
|
3019
|
+
"noscript",
|
|
3020
|
+
"nav",
|
|
3021
|
+
"footer",
|
|
3022
|
+
"aside",
|
|
3023
|
+
".theme-doc-version-badge",
|
|
3024
|
+
".theme-doc-breadcrumbs",
|
|
3025
|
+
".theme-doc-toc-mobile",
|
|
3026
|
+
".table-of-contents",
|
|
3027
|
+
".chat-assistant-floating-input",
|
|
3028
|
+
".chat-assistant-sheet",
|
|
3029
|
+
"#chat-assistant-sheet",
|
|
3030
|
+
"#chat-assistant-textarea"
|
|
3031
|
+
].join(",")).remove();
|
|
3032
|
+
const permalinkSelectors = MINTLIFY_ARTICLE_LABELS.map((labels) => `a[aria-label="${labels.permalink}"]`);
|
|
3033
|
+
root.find([...permalinkSelectors, "h1 .hash-link", "h2 .hash-link", "h3 .hash-link", "h4 .hash-link", "h5 .hash-link", "h6 .hash-link"].join(",")).each((_index, link) => {
|
|
3034
|
+
const parent = $(link).parent();
|
|
3035
|
+
$(link).remove();
|
|
3036
|
+
if (parent.is("div") && !parent.text().trim() && parent.children().length === 0) parent.remove();
|
|
3037
|
+
});
|
|
3038
|
+
root.find('svg.lucide-chevrons-up-down,summary[data-component-part="expandable-button"] svg[data-component-part="icon-svg"]').remove();
|
|
3039
|
+
root.find('li.task-list-item > input[type="checkbox"][disabled]').each((_index, element) => {
|
|
3040
|
+
const isChecked = $(element).attr("checked") !== void 0;
|
|
3041
|
+
$(element).replaceWith($("<span></span>").attr("data-thally-static-task-checkbox", isChecked ? "checked" : "unchecked").text(isChecked ? "[x] " : "[ ] "));
|
|
3042
|
+
});
|
|
3043
|
+
root.find(MINTLIFY_ARTICLE_LABELS.map((labels) => `select[aria-label="${labels.baseUrl}"]`).join(",")).each((_index, element) => {
|
|
3044
|
+
const select = $(element);
|
|
3045
|
+
const options = select.children("option").toArray();
|
|
3046
|
+
if (!options.length || options.length !== select.children().length) return;
|
|
3047
|
+
const urls = [];
|
|
3048
|
+
for (const option of options) {
|
|
3049
|
+
if ($(option).children().length) return;
|
|
3050
|
+
const label = $(option).text().trim();
|
|
3051
|
+
if (!/^https:\/\//i.test(label) || /[\s\u0000-\u001f\u007f]/.test(label)) return;
|
|
3052
|
+
try {
|
|
3053
|
+
const url = new URL(label);
|
|
3054
|
+
if (url.protocol !== "https:" || url.username || url.password) return;
|
|
3055
|
+
urls.push({ label, href: url.href, isSelected: $(option).attr("selected") !== void 0 });
|
|
3056
|
+
} catch {
|
|
3057
|
+
return;
|
|
3058
|
+
}
|
|
3059
|
+
}
|
|
3060
|
+
const projection = $("<div data-thally-static-base-urls></div>");
|
|
3061
|
+
const sourceId = select.attr("id");
|
|
3062
|
+
if (sourceId) projection.attr("id", sourceId);
|
|
3063
|
+
projection.append($("<p></p>").text(select.attr("aria-label")));
|
|
3064
|
+
const list = $("<ul></ul>");
|
|
3065
|
+
for (const url of urls) {
|
|
3066
|
+
const item = $("<li></li>").append($("<a></a>").attr("href", url.href).text(url.label));
|
|
3067
|
+
if (url.isSelected) item.append(" (selected)");
|
|
3068
|
+
list.append(item);
|
|
3069
|
+
}
|
|
3070
|
+
projection.append(list);
|
|
3071
|
+
select.replaceWith(projection);
|
|
3072
|
+
});
|
|
3073
|
+
return { $, root };
|
|
3074
|
+
}
|
|
2679
3075
|
function htmlPage(document, id, options = {}) {
|
|
2680
|
-
const
|
|
2681
|
-
const links =
|
|
2682
|
-
const scopedTitle = options.omitDocumentHeading ?
|
|
3076
|
+
const { $, root } = migrationHtmlArticle(document.body);
|
|
3077
|
+
const links = root.find("a[href]").map((_index, element) => $(element).attr("href") ?? "").get().filter(Boolean);
|
|
3078
|
+
const scopedTitle = options.omitDocumentHeading ? root.find("h1").first().text().trim() : "";
|
|
2683
3079
|
const title = scopedTitle || $("main h1, article h1, h1").first().text().trim() || $('meta[property="og:title"]').attr("content")?.trim() || $("title").text().split("|")[0].trim();
|
|
2684
3080
|
const description = $('meta[name="description"]').attr("content")?.trim() || $('meta[property="og:description"]').attr("content")?.trim();
|
|
2685
|
-
const content = $('main article, article, main, [role="main"]').first();
|
|
2686
|
-
const root = content.length ? content : $("body");
|
|
2687
3081
|
let documentHeadingAnchor = "";
|
|
2688
3082
|
const excludedElements = options.omitDocumentHeading ? "script,style,noscript,nav,footer,aside,form,button,svg,iframe,object,embed,link,meta" : "script,style,noscript,nav,header,footer,aside,form,button,svg,iframe,object,embed,link,meta";
|
|
2689
3083
|
root.find(excludedElements).remove();
|
|
@@ -2697,6 +3091,9 @@ function htmlPage(document, id, options = {}) {
|
|
|
2697
3091
|
`;
|
|
2698
3092
|
heading.remove();
|
|
2699
3093
|
}
|
|
3094
|
+
root.find("h1").each((_index, element) => {
|
|
3095
|
+
element.name = "h2";
|
|
3096
|
+
});
|
|
2700
3097
|
}
|
|
2701
3098
|
root.find("a[href]").each((_index, element) => {
|
|
2702
3099
|
const href = $(element).attr("href");
|
|
@@ -2726,8 +3123,20 @@ function htmlPage(document, id, options = {}) {
|
|
|
2726
3123
|
$(element).removeAttr("src");
|
|
2727
3124
|
}
|
|
2728
3125
|
});
|
|
2729
|
-
const turndown = new TurndownService({ codeBlockStyle: "fenced", emDelimiter: "_", headingStyle: "atx" });
|
|
3126
|
+
const turndown = new TurndownService({ codeBlockStyle: "fenced", emDelimiter: "_", headingStyle: "atx", preformattedCode: true });
|
|
3127
|
+
let isConvertingTableCell = false;
|
|
2730
3128
|
turndown.remove(["script", "style", "noscript"]);
|
|
3129
|
+
turndown.addRule("migration-literal-table-code", {
|
|
3130
|
+
filter: (node) => isConvertingTableCell && node.nodeName === "CODE" && node.parentNode?.nodeName !== "PRE" && needsLiteralTableCode(node.textContent ?? ""),
|
|
3131
|
+
// A strict, attribute-free text leaf preserves the payload that GFM's
|
|
3132
|
+
// escaped-pipe code-span grammar cannot represent without changing it.
|
|
3133
|
+
replacement: (_content, node) => `<code>${literalMdxCodeText(node.textContent ?? "")}</code>`
|
|
3134
|
+
});
|
|
3135
|
+
turndown.addRule("migration-static-task-checkbox", {
|
|
3136
|
+
filter: (node) => node.nodeName === "SPAN" && ["checked", "unchecked"].includes(node.getAttribute("data-thally-static-task-checkbox") ?? ""),
|
|
3137
|
+
// Emit literal task-list syntax rather than escaping its brackets as copy.
|
|
3138
|
+
replacement: (_content, node) => node.getAttribute("data-thally-static-task-checkbox") === "checked" ? "[x] " : "[ ] "
|
|
3139
|
+
});
|
|
2731
3140
|
const anchor = (element) => {
|
|
2732
3141
|
const value = element.getAttribute("id");
|
|
2733
3142
|
return options.preserveAnchors && value ? `<a id="${value.replace(/&/g, "&").replace(/"/g, """).replace(/</g, "<").replace(/\{/g, "{")}" />
|
|
@@ -2736,12 +3145,12 @@ function htmlPage(document, id, options = {}) {
|
|
|
2736
3145
|
};
|
|
2737
3146
|
if (options.preserveAnchors) turndown.addRule("migration-source-anchor", {
|
|
2738
3147
|
filter: (node) => node.hasAttribute("id") && node.nodeName !== "PRE",
|
|
2739
|
-
replacement: (
|
|
3148
|
+
replacement: (content, node) => {
|
|
2740
3149
|
const element = node;
|
|
2741
3150
|
const heading = element.nodeName.match(/^H([1-6])$/);
|
|
2742
3151
|
return `
|
|
2743
3152
|
|
|
2744
|
-
${anchor(element)}${heading ? `${"#".repeat(Number(heading[1]))} ` :
|
|
3153
|
+
${anchor(element)}${heading ? `${"#".repeat(Number(heading[1]))} ${content.trim()}` : content}
|
|
2745
3154
|
|
|
2746
3155
|
`;
|
|
2747
3156
|
}
|
|
@@ -2766,10 +3175,10 @@ ${anchor(element)}${heading ? `${"#".repeat(Number(heading[1]))} ` : ""}${conten
|
|
|
2766
3175
|
});
|
|
2767
3176
|
turndown.addRule("migration-responsive-picture", {
|
|
2768
3177
|
filter: "picture",
|
|
2769
|
-
replacement: (
|
|
3178
|
+
replacement: (content) => `
|
|
2770
3179
|
|
|
2771
3180
|
<picture>
|
|
2772
|
-
${
|
|
3181
|
+
${content.trim()}
|
|
2773
3182
|
</picture>
|
|
2774
3183
|
|
|
2775
3184
|
`
|
|
@@ -2781,7 +3190,7 @@ ${content2.trim()}
|
|
|
2781
3190
|
const element = node;
|
|
2782
3191
|
const className = element.className;
|
|
2783
3192
|
const tag = /admonition-(?:danger)/.test(className) ? "Error" : /admonition-(?:caution|warning)/.test(className) ? "Warning" : /admonition-info/.test(className) ? "Info" : "Note";
|
|
2784
|
-
const fragment =
|
|
3193
|
+
const fragment = load2(element.outerHTML);
|
|
2785
3194
|
const clone = fragment(".theme-admonition").first();
|
|
2786
3195
|
const title2 = clone.find('[class*="admonitionHeading"],.admonition-heading').first();
|
|
2787
3196
|
const titleText = title2.text().trim();
|
|
@@ -2802,7 +3211,7 @@ ${titleText ? `**${escapeMarkdownText(titleText)}**
|
|
|
2802
3211
|
filter: (node) => node.nodeName === "DIV" && node.classList.contains("tabs-container"),
|
|
2803
3212
|
replacement: (_content, node) => {
|
|
2804
3213
|
const element = node;
|
|
2805
|
-
const fragment =
|
|
3214
|
+
const fragment = load2(element.outerHTML);
|
|
2806
3215
|
const container = fragment(".tabs-container").first();
|
|
2807
3216
|
const labels = container.children("ul.tabs").children("li").toArray().map((item) => fragment(item).text().trim() || "Tab");
|
|
2808
3217
|
const panels = container.find('[role="tabpanel"]').toArray().filter((panel) => fragment(panel).closest(".tabs-container")[0] === container[0]);
|
|
@@ -2830,11 +3239,16 @@ ${tabs.join("\n\n")}
|
|
|
2830
3239
|
turndown.addRule("gfm-table", {
|
|
2831
3240
|
filter: "table",
|
|
2832
3241
|
replacement: (_content, node) => {
|
|
2833
|
-
const fragment =
|
|
2834
|
-
const rows = fragment("tr").toArray().map((row) => fragment(row).children("th,td").toArray().map((cell) =>
|
|
2835
|
-
|
|
2836
|
-
true
|
|
2837
|
-
|
|
3242
|
+
const fragment = load2(node.outerHTML);
|
|
3243
|
+
const rows = fragment("tr").toArray().map((row) => fragment(row).children("th,td").toArray().map((cell) => {
|
|
3244
|
+
const wasConvertingTableCell = isConvertingTableCell;
|
|
3245
|
+
isConvertingTableCell = true;
|
|
3246
|
+
try {
|
|
3247
|
+
return portableMarkdownTableCell(turndown.turndown(fragment(cell).html() ?? ""));
|
|
3248
|
+
} finally {
|
|
3249
|
+
isConvertingTableCell = wasConvertingTableCell;
|
|
3250
|
+
}
|
|
3251
|
+
})).filter((row) => row.length > 0);
|
|
2838
3252
|
if (rows.length === 0) return "";
|
|
2839
3253
|
const width = Math.max(...rows.map((row) => row.length));
|
|
2840
3254
|
const normalized = rows.map((row) => [...row, ...Array(width - row.length).fill("")]);
|
|
@@ -2851,7 +3265,7 @@ ${normalized.slice(1).map((row) => `| ${row.join(" | ")} |`).join("\n")}
|
|
|
2851
3265
|
filter: (node) => node.nodeName === "PRE",
|
|
2852
3266
|
replacement: (_content, node) => {
|
|
2853
3267
|
const element = node;
|
|
2854
|
-
const fragment =
|
|
3268
|
+
const fragment = load2(element.outerHTML);
|
|
2855
3269
|
const tokenLines = fragment(".token-line").toArray();
|
|
2856
3270
|
const code = (tokenLines.length > 0 ? tokenLines.map((line) => fragment(line).text()).join("\n") : fragment("pre").text()).replace(/\n$/, "");
|
|
2857
3271
|
const language = fragment("code").attr("class")?.match(/(?:language-|lang-)([A-Za-z0-9_+-]+)/)?.[1] ?? "";
|
|
@@ -2867,7 +3281,7 @@ ${fence}
|
|
|
2867
3281
|
}
|
|
2868
3282
|
});
|
|
2869
3283
|
const escape = turndown.escape.bind(turndown);
|
|
2870
|
-
turndown.escape = (
|
|
3284
|
+
turndown.escape = (text2) => isConvertingTableCell ? escape(text2.replace(/[<>{}]/g, (character) => ({ "<": "<", ">": ">", "{": "{", "}": "}" })[character])) : escape(text2).replace(/[<{}]/g, (match) => `\\${match}`);
|
|
2871
3285
|
const body = `${documentHeadingAnchor}${turndown.turndown(root.html() ?? "").trim()}`;
|
|
2872
3286
|
if (!title || body.length < 40 && $("article").length === 0) return { page: null, links };
|
|
2873
3287
|
return {
|
|
@@ -2885,7 +3299,7 @@ ${fence}
|
|
|
2885
3299
|
}
|
|
2886
3300
|
|
|
2887
3301
|
// src/url-run.ts
|
|
2888
|
-
var CONVERTER_VERSION = "url-evidence-
|
|
3302
|
+
var CONVERTER_VERSION = "url-evidence-v4";
|
|
2889
3303
|
var parser = unified3().use(remarkParse3).use(remarkMdx3);
|
|
2890
3304
|
var ACCEPT = "text/markdown,text/html,application/xhtml+xml;q=0.9";
|
|
2891
3305
|
var MAX_RESPONSE_BYTES = 8e6;
|
|
@@ -2895,9 +3309,12 @@ var LOCALES = /* @__PURE__ */ new Set(["ar", "cs", "da", "de", "el", "en", "es",
|
|
|
2895
3309
|
function hash(value) {
|
|
2896
3310
|
return createHash("sha256").update(value).digest("hex");
|
|
2897
3311
|
}
|
|
3312
|
+
function canonicalJson(value) {
|
|
3313
|
+
const canonical2 = (value2) => Array.isArray(value2) ? value2.map(canonical2) : value2 !== null && typeof value2 === "object" ? Object.fromEntries(Object.entries(value2).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([key, field]) => [key, canonical2(field)])) : value2;
|
|
3314
|
+
return JSON.stringify(canonical2(value));
|
|
3315
|
+
}
|
|
2898
3316
|
function pageHash(page) {
|
|
2899
|
-
|
|
2900
|
-
return hash(JSON.stringify(canonical2(page)));
|
|
3317
|
+
return hash(canonicalJson(page));
|
|
2901
3318
|
}
|
|
2902
3319
|
function inScope(url, source, scope) {
|
|
2903
3320
|
return url.origin === source.origin && !url.username && !url.password && (scope === "/" || url.pathname === scope || url.pathname.startsWith(`${scope}/`));
|
|
@@ -2916,7 +3333,7 @@ function canonical(value, base) {
|
|
|
2916
3333
|
}
|
|
2917
3334
|
function classifyMigrationResponse(document) {
|
|
2918
3335
|
if (document.status && document.status >= 400) return "unsupported";
|
|
2919
|
-
if (/\/(?:llms(?:-full)?|robots)\.txt$|\/sitemap[^/]*\.xml$|\/search(?:\.html)?$/i.test(document.finalUrl.pathname) || /(?:xml|json)/i.test(document.contentType)) return "index";
|
|
3336
|
+
if (/\/(?:llms(?:-full)?|robots)\.txt$|\/sitemap[^/]*\.xml$|\/search(?:\.html)?$/i.test(document.finalUrl.pathname) || /\.(?:json|ya?ml)$/i.test(document.finalUrl.pathname) || /(?:xml|json)/i.test(document.contentType)) return "index";
|
|
2920
3337
|
if (/^(?:image|audio|video)\//i.test(document.contentType) || /\.(?:pdf|zip|png|jpe?g|webp|gif|svg|ico|mp4|webm)$/i.test(document.finalUrl.pathname)) return "asset";
|
|
2921
3338
|
return /(?:html|markdown|text\/plain)/i.test(document.contentType) || /\.mdx?$/i.test(document.finalUrl.pathname) ? "page" : "unsupported";
|
|
2922
3339
|
}
|
|
@@ -2938,14 +3355,14 @@ function alternativeBlocks(body, baseline) {
|
|
|
2938
3355
|
return blocks(body).filter((block) => !originals.has(block)).join("\n\n");
|
|
2939
3356
|
}
|
|
2940
3357
|
function linksOf(document) {
|
|
3358
|
+
if (/json|ya?ml/i.test(document.contentType) || /\.(?:json|ya?ml)$/i.test(document.finalUrl.pathname)) return [];
|
|
2941
3359
|
if (/html/i.test(document.contentType)) {
|
|
2942
|
-
const
|
|
2943
|
-
const root = $('main article,article,main,[role="main"]').first();
|
|
3360
|
+
const { $, root } = migrationHtmlArticle(document.body);
|
|
2944
3361
|
root.find("nav,footer,aside").remove();
|
|
2945
3362
|
return [...root.find("a[href]").toArray(), ...$('link[rel="sitemap"][href],link[rel="alternate"][href]').toArray()].map((node) => $(node).attr("href")).filter(Boolean);
|
|
2946
3363
|
}
|
|
2947
3364
|
if (/xml/i.test(document.contentType) || /<urlset|<sitemapindex/i.test(document.body)) {
|
|
2948
|
-
const $ =
|
|
3365
|
+
const $ = load3(document.body, { xmlMode: true });
|
|
2949
3366
|
return $("loc").toArray().map((node) => $(node).text().trim());
|
|
2950
3367
|
}
|
|
2951
3368
|
return [
|
|
@@ -2954,8 +3371,7 @@ function linksOf(document) {
|
|
|
2954
3371
|
];
|
|
2955
3372
|
}
|
|
2956
3373
|
function htmlEvidence(document) {
|
|
2957
|
-
const
|
|
2958
|
-
const root = $('main article,article,main,[role="main"]').first();
|
|
3374
|
+
const { $, root } = migrationHtmlArticle(document.body);
|
|
2959
3375
|
const article = root.length ? root : $("body");
|
|
2960
3376
|
article.find("nav,footer,aside,form,script,style").remove();
|
|
2961
3377
|
return {
|
|
@@ -2982,55 +3398,35 @@ function evidenceOf(capture) {
|
|
|
2982
3398
|
function markdownHeadingAnchors(body) {
|
|
2983
3399
|
const seen = /* @__PURE__ */ new Map();
|
|
2984
3400
|
return markdownNodes(body).filter((node) => node.type === "heading").map((node) => {
|
|
2985
|
-
const
|
|
2986
|
-
const base =
|
|
3401
|
+
const text2 = (value) => value.value ?? (value.children ?? []).map(text2).join("");
|
|
3402
|
+
const base = text2(node).toLowerCase().replace(/[^\p{L}\p{N}\s_-]/gu, "").replace(/\s/g, "-");
|
|
2987
3403
|
const occurrence = seen.get(base) ?? 0;
|
|
2988
3404
|
seen.set(base, occurrence + 1);
|
|
2989
3405
|
return occurrence ? `${base}-${occurrence}` : base;
|
|
2990
3406
|
});
|
|
2991
3407
|
}
|
|
2992
3408
|
function sourceNavigation(capture) {
|
|
2993
|
-
|
|
2994
|
-
const resolve3 = (nodes, depth = 0) => {
|
|
2995
|
-
if (depth > 32 || nodes.length > 1e4) throw new Error("Source navigation exceeds its depth or width safety bound.");
|
|
2996
|
-
return nodes.map((node) => ({ label: node.label, ...node.url ? { url: new URL(node.url, capture.finalUrl).toString() } : {}, ...node.children ? { children: resolve3(node.children, depth + 1) } : {} }));
|
|
2997
|
-
};
|
|
2998
|
-
return resolve3(capture.navigation);
|
|
2999
|
-
}
|
|
3000
|
-
if (!/html/i.test(capture.contentType)) return [];
|
|
3001
|
-
const $ = load2(capture.body);
|
|
3002
|
-
const candidates = ['.theme-doc-sidebar-menu,[aria-label*="sidebar" i]', "aside", '[role="navigation"],nav'];
|
|
3003
|
-
const root = candidates.map((selector) => $(selector).filter((_index, node) => $(node).find("a[href]").length > 0 && $(node).closest("header").length === 0).first()).find((candidate) => candidate.length) ?? $("aside").first();
|
|
3004
|
-
if (!root.length) return [];
|
|
3005
|
-
const walk = (element, excluded) => {
|
|
3006
|
-
const result = [];
|
|
3007
|
-
$(element).children().each((_index, node) => {
|
|
3008
|
-
if (node === excluded) return;
|
|
3009
|
-
const item = $(node);
|
|
3010
|
-
if (item.is("a[href]")) {
|
|
3011
|
-
result.push({ label: item.text().replace(/\s+/g, " ").trim(), url: new URL(item.attr("href"), capture.finalUrl).toString() });
|
|
3012
|
-
} else if (item.is("li,details") || item.attr("data-migration-group") !== void 0 || item.children("h2,h3,h4,h5,h6").length > 0) {
|
|
3013
|
-
const directLink = item.children("a[href]").first();
|
|
3014
|
-
const direct = directLink.length ? directLink : item.children("div").children("a[href]").first();
|
|
3015
|
-
const label = direct.text().trim() || item.children("summary,button,span,h2,h3,h4,h5,h6").first().text().trim();
|
|
3016
|
-
const children = walk(node, direct[0]);
|
|
3017
|
-
if (label) result.push({ label, ...direct.length ? { url: new URL(direct.attr("href"), capture.finalUrl).toString() } : {}, ...children.length ? { children } : {} });
|
|
3018
|
-
else result.push(...children);
|
|
3019
|
-
} else result.push(...walk(node, excluded));
|
|
3020
|
-
});
|
|
3021
|
-
return result;
|
|
3022
|
-
};
|
|
3023
|
-
try {
|
|
3024
|
-
return walk(root);
|
|
3025
|
-
} catch {
|
|
3026
|
-
return [];
|
|
3027
|
-
}
|
|
3409
|
+
return extractSourceNavigation(capture);
|
|
3028
3410
|
}
|
|
3029
3411
|
function idForUrl(url, scope) {
|
|
3030
3412
|
const relative4 = scope === "/" ? url.pathname.slice(1) : url.pathname.slice(scope.length).replace(/^\//, "");
|
|
3031
3413
|
const id = pageIdFromReference(relative4 || "introduction", true);
|
|
3032
3414
|
return id && `${id}${url.search ? `--${hash(url.search).slice(0, 12)}` : ""}`;
|
|
3033
3415
|
}
|
|
3416
|
+
function articleIdentity(document) {
|
|
3417
|
+
if (!/html/i.test(document.contentType)) return document.finalUrl;
|
|
3418
|
+
const $ = load3(document.body);
|
|
3419
|
+
const reference = $('link[rel="canonical"][href]').first().attr("href");
|
|
3420
|
+
if (!reference) return document.finalUrl;
|
|
3421
|
+
try {
|
|
3422
|
+
const target2 = new URL(reference, document.finalUrl);
|
|
3423
|
+
const source = document.finalUrl;
|
|
3424
|
+
const directory = (path) => path.replace(/\/index\/?$/i, "").replace(/\/$/, "");
|
|
3425
|
+
return target2.origin === source.origin && !target2.username && !target2.password && target2.search === source.search && directory(target2.pathname) === directory(source.pathname) ? target2 : source;
|
|
3426
|
+
} catch {
|
|
3427
|
+
return document.finalUrl;
|
|
3428
|
+
}
|
|
3429
|
+
}
|
|
3034
3430
|
function convert(document, id) {
|
|
3035
3431
|
if (/html/i.test(document.contentType)) {
|
|
3036
3432
|
const technical = htmlEvidence(document);
|
|
@@ -3048,9 +3444,20 @@ function convert(document, id) {
|
|
|
3048
3444
|
const info = original.split("\n", 1)[0].replace(/^`+|^~+/, "");
|
|
3049
3445
|
const code = technical.code[index];
|
|
3050
3446
|
const fence = "`".repeat(Math.max(3, ...[...code.matchAll(/`+/g)].map((match) => match[0].length + 1)));
|
|
3447
|
+
const lineStart = page.body.lastIndexOf("\n", start - 1) + 1;
|
|
3448
|
+
const prefix = page.body.slice(lineStart, start);
|
|
3449
|
+
let hasContainerPrefix = true;
|
|
3450
|
+
for (const character of prefix) {
|
|
3451
|
+
if (character !== " " && character !== " " && character !== ">") {
|
|
3452
|
+
hasContainerPrefix = false;
|
|
3453
|
+
break;
|
|
3454
|
+
}
|
|
3455
|
+
}
|
|
3456
|
+
const indent = hasContainerPrefix ? prefix : "";
|
|
3457
|
+
const payload = code.split("\n").map((line) => `${indent}${line}`).join("\n");
|
|
3051
3458
|
page.body = `${page.body.slice(0, start)}${fence}${info}
|
|
3052
|
-
${
|
|
3053
|
-
${fence}${page.body.slice(end)}`;
|
|
3459
|
+
${payload}
|
|
3460
|
+
${indent}${fence}${page.body.slice(end)}`;
|
|
3054
3461
|
}
|
|
3055
3462
|
}
|
|
3056
3463
|
return page;
|
|
@@ -3062,16 +3469,14 @@ function assetReferences(capture) {
|
|
|
3062
3469
|
if (!/html/i.test(capture.contentType)) {
|
|
3063
3470
|
return markdownNodes(capture.body).filter((node) => node.type === "image").flatMap((node) => node.url ? [node.url] : []);
|
|
3064
3471
|
}
|
|
3065
|
-
const
|
|
3066
|
-
const root = $('main article,article,main,[role="main"]').first();
|
|
3472
|
+
const { $, root } = migrationHtmlArticle(capture.body);
|
|
3067
3473
|
return [
|
|
3068
3474
|
...root.find("img[src],video[poster],a[download][href]").toArray().map((node) => $(node).attr("src") ?? $(node).attr("poster") ?? $(node).attr("href")),
|
|
3069
3475
|
...root.find("[srcset]").toArray().flatMap((node) => migrationImageCandidates($(node).attr("srcset") ?? "", capture.finalUrl).map((candidate) => candidate.url))
|
|
3070
3476
|
].filter(Boolean);
|
|
3071
3477
|
}
|
|
3072
|
-
function unresolvedStaticControls(body) {
|
|
3073
|
-
const
|
|
3074
|
-
const root = $('main article,article,main,[role="main"]').first();
|
|
3478
|
+
function unresolvedStaticControls(body, hasApiSchema = false) {
|
|
3479
|
+
const { $, root } = migrationHtmlArticle(body);
|
|
3075
3480
|
const tabs = root.find('[role="tab"]').toArray();
|
|
3076
3481
|
for (const tab of tabs) {
|
|
3077
3482
|
const controls = $(tab).attr("aria-controls");
|
|
@@ -3079,6 +3484,7 @@ function unresolvedStaticControls(body) {
|
|
|
3079
3484
|
if (panels.length !== 1 || !panels.text().trim() && !panels.find("pre,img,table,svg").length || panels.find('[aria-busy="true"],[role="progressbar"]').length) return true;
|
|
3080
3485
|
}
|
|
3081
3486
|
return root.find("details:not([open])").toArray().some((node) => {
|
|
3487
|
+
if (hasApiSchema && $(node).is(".expandable") && $(node).children('summary[data-component-part="expandable-button"]').length) return false;
|
|
3082
3488
|
const clone = $(node).clone();
|
|
3083
3489
|
clone.children("summary").remove();
|
|
3084
3490
|
return !clone.text().trim() && !clone.find("pre,img,table,svg").length;
|
|
@@ -3116,33 +3522,85 @@ function acceptedAsset(asset) {
|
|
|
3116
3522
|
}
|
|
3117
3523
|
function validRecord(record) {
|
|
3118
3524
|
if (record.outcome !== "imported") return true;
|
|
3119
|
-
return Boolean(record.page && record.pageSha256 === pageHash(record.page) && record.evidence && pageIdFromReference(record.page.id, true) === record.page.id && pageIdFromReference(record.page.navigationId, true) === record.page.navigationId && sanitizeRemoteMarkdown(record.page.body.replace(/^<a id="[^"<]*" \/>$/gm, "")) !== null && /^[a-f0-9]{64}$/.test(record.evidence.sha256) && record.assets?.every((asset) => /^migration-assets\/[a-f0-9]{64}\.[a-z0-9]+$/.test(asset.path) && hash(Buffer.from(asset.base64, "base64")) === asset.sha256));
|
|
3525
|
+
return Boolean(record.page && record.pageSha256 === pageHash(record.page) && record.evidence && pageIdFromReference(record.page.id, true) === record.page.id && pageIdFromReference(record.page.navigationId, true) === record.page.navigationId && sanitizeRemoteMarkdown(record.page.body.replace(/^<a id="[^"<]*" \/>$/gm, "")) !== null && /^[a-f0-9]{64}$/.test(record.evidence.sha256) && (!record.apiSpecification || hasCompleteLocalApiSchema(record.apiSpecification.document) && record.apiSpecification.path === `openapi/migration-${record.apiSpecification.sha256}.json` && hash(canonicalJson(record.apiSpecification.document)) === record.apiSpecification.sha256) && record.assets?.every((asset) => /^migration-assets\/[a-f0-9]{64}\.[a-z0-9]+$/.test(asset.path) && hash(Buffer.from(asset.base64, "base64")) === asset.sha256));
|
|
3526
|
+
}
|
|
3527
|
+
function hasCompleteLocalApiSchema(document) {
|
|
3528
|
+
let visited = 0;
|
|
3529
|
+
const visit = (value, depth = 0) => {
|
|
3530
|
+
if (++visited > 1e5 || depth > 96) return false;
|
|
3531
|
+
if (typeof value === "number" && !Number.isFinite(value)) return false;
|
|
3532
|
+
if (!value || typeof value !== "object") return true;
|
|
3533
|
+
if (Array.isArray(value)) return value.every((child) => visit(child, depth + 1));
|
|
3534
|
+
const object2 = value;
|
|
3535
|
+
if (Object.prototype.hasOwnProperty.call(object2, "$ref")) {
|
|
3536
|
+
if (typeof object2.$ref !== "string" || !object2.$ref.startsWith("#/")) return false;
|
|
3537
|
+
let target2 = document;
|
|
3538
|
+
try {
|
|
3539
|
+
for (const encoded of object2.$ref.slice(2).split("/")) {
|
|
3540
|
+
const key = decodeURIComponent(encoded).replace(/~1/g, "/").replace(/~0/g, "~");
|
|
3541
|
+
if (!target2 || typeof target2 !== "object" || !Object.prototype.hasOwnProperty.call(target2, key)) return false;
|
|
3542
|
+
target2 = target2[key];
|
|
3543
|
+
}
|
|
3544
|
+
} catch {
|
|
3545
|
+
return false;
|
|
3546
|
+
}
|
|
3547
|
+
}
|
|
3548
|
+
return Object.values(object2).every((child) => visit(child, depth + 1));
|
|
3549
|
+
};
|
|
3550
|
+
return visit(document);
|
|
3551
|
+
}
|
|
3552
|
+
function resolveMigrationRecord(records, record) {
|
|
3553
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3554
|
+
while (record?.outcome === "alias") {
|
|
3555
|
+
if (seen.has(record.url) || seen.size >= 32) return void 0;
|
|
3556
|
+
seen.add(record.url);
|
|
3557
|
+
record = records.get(record.aliasOf ?? "");
|
|
3558
|
+
}
|
|
3559
|
+
return record;
|
|
3120
3560
|
}
|
|
3121
|
-
function navigationConfig(
|
|
3561
|
+
function navigationConfig(checkpoint, records) {
|
|
3562
|
+
const navigation = checkpoint.navigation;
|
|
3563
|
+
const projection = localeProjection(checkpoint);
|
|
3122
3564
|
const project = (nodes) => nodes.flatMap((node) => {
|
|
3123
3565
|
const record = node.url ? records.get(canonical(node.url, node.url) ?? "") : void 0;
|
|
3124
|
-
const
|
|
3125
|
-
const page =
|
|
3566
|
+
const target2 = resolveMigrationRecord(records, record);
|
|
3567
|
+
const page = target2?.page;
|
|
3126
3568
|
const children = project(node.children ?? []);
|
|
3127
3569
|
if (children.length) {
|
|
3128
|
-
const contains = (entries) => entries.some((entry) => typeof entry === "string" ? entry === page
|
|
3129
|
-
return [{ group: node.label || "Documentation", pages: [...page && !contains(children) ? [
|
|
3570
|
+
const contains = (entries) => entries.some((entry) => typeof entry === "string" ? entry === (page && projection.navigationId(page)) : contains(entry.pages));
|
|
3571
|
+
return [{ group: node.label || "Documentation", pages: [...page && !contains(children) ? [projection.navigationId(page)] : [], ...children] }];
|
|
3130
3572
|
}
|
|
3131
|
-
return page ? [
|
|
3573
|
+
return page ? [projection.navigationId(page)] : [];
|
|
3132
3574
|
});
|
|
3575
|
+
const tabs = navigation.filter((node) => node.kind === "tab");
|
|
3576
|
+
if (tabs.length) {
|
|
3577
|
+
const locales = [...new Set(tabs.map((tab) => tab.locale ?? projection.defaultLocale))];
|
|
3578
|
+
const localizedTabs = (locale) => tabs.filter((tab) => (tab.locale ?? projection.defaultLocale) === locale).flatMap((tab) => {
|
|
3579
|
+
const pages2 = project(tab.children ?? []);
|
|
3580
|
+
return pages2.length || tab.url ? [{ tab: tab.label, ...pages2.length ? { pages: pages2 } : {}, ...tab.url ? { href: tab.url } : {} }] : [];
|
|
3581
|
+
});
|
|
3582
|
+
const primary = localizedTabs(projection.defaultLocale);
|
|
3583
|
+
return { tabs: primary.length ? primary : localizedTabs(locales[0]), ...locales.length > 1 ? {
|
|
3584
|
+
i18n: {
|
|
3585
|
+
defaultLocale: projection.defaultLocale,
|
|
3586
|
+
locales: locales.map((code) => ({ code, label: code === "en" ? "English" : code })),
|
|
3587
|
+
navigation: Object.fromEntries(locales.map((locale) => [locale, localizedTabs(locale)]))
|
|
3588
|
+
}
|
|
3589
|
+
} : {} };
|
|
3590
|
+
}
|
|
3133
3591
|
const pages = project(navigation);
|
|
3134
3592
|
return pages.length ? { tabs: [{ tab: "Documentation", pages }] } : null;
|
|
3135
3593
|
}
|
|
3594
|
+
function canonicalLanguage(value) {
|
|
3595
|
+
if (!/^[a-z]{2,3}(?:-[a-z0-9]{2,8}){0,4}$/i.test(value)) return void 0;
|
|
3596
|
+
try {
|
|
3597
|
+
return Intl.getCanonicalLocales(value)[0];
|
|
3598
|
+
} catch {
|
|
3599
|
+
return void 0;
|
|
3600
|
+
}
|
|
3601
|
+
}
|
|
3136
3602
|
function observedLocale(document, id, scopePath) {
|
|
3137
3603
|
const issues = [];
|
|
3138
|
-
const canonicalLanguage = (value) => {
|
|
3139
|
-
if (!/^[a-z]{2,3}(?:-[a-z0-9]{2,8}){0,4}$/i.test(value)) return void 0;
|
|
3140
|
-
try {
|
|
3141
|
-
return Intl.getCanonicalLocales(value)[0];
|
|
3142
|
-
} catch {
|
|
3143
|
-
return void 0;
|
|
3144
|
-
}
|
|
3145
|
-
};
|
|
3146
3604
|
const candidates = [];
|
|
3147
3605
|
if (document.locale) {
|
|
3148
3606
|
const locale = canonicalLanguage(document.locale);
|
|
@@ -3151,7 +3609,7 @@ function observedLocale(document, id, scopePath) {
|
|
|
3151
3609
|
}
|
|
3152
3610
|
const urlLocale = [id.split("/", 1)[0], ...scopePath.split("/"), document.finalUrl.pathname.split("/").filter(Boolean)[0]].find((segment) => segment && LOCALES.has(segment.toLowerCase()));
|
|
3153
3611
|
if (urlLocale) candidates.push({ source: "URL", value: canonicalLanguage(urlLocale) });
|
|
3154
|
-
const declared = /html/i.test(document.contentType) ?
|
|
3612
|
+
const declared = /html/i.test(document.contentType) ? load3(document.body)("html").attr("lang")?.trim() : void 0;
|
|
3155
3613
|
if (declared) {
|
|
3156
3614
|
const locale = canonicalLanguage(declared);
|
|
3157
3615
|
if (locale) candidates.push({ source: "html lang", value: locale });
|
|
@@ -3164,37 +3622,73 @@ function observedLocale(document, id, scopePath) {
|
|
|
3164
3622
|
}
|
|
3165
3623
|
function localeProjection(checkpoint) {
|
|
3166
3624
|
const pages = checkpoint.records.flatMap((record) => record.page ? [record.page] : []);
|
|
3167
|
-
const
|
|
3625
|
+
const declaredLocale = (page) => {
|
|
3168
3626
|
try {
|
|
3169
3627
|
return page.locale ? Intl.getCanonicalLocales(page.locale)[0] : void 0;
|
|
3170
3628
|
} catch {
|
|
3171
3629
|
return void 0;
|
|
3172
3630
|
}
|
|
3173
3631
|
};
|
|
3632
|
+
const declaredLocales = [...new Set(pages.map(declaredLocale).filter((locale) => Boolean(locale)))];
|
|
3633
|
+
const defaultLocale = pages.some((page) => !page.locale) || declaredLocales.includes("en") ? "en" : declaredLocales[0] ?? "en";
|
|
3634
|
+
const records = new Map(checkpoint.records.map((record) => [record.url, record]));
|
|
3635
|
+
const memberships = /* @__PURE__ */ new Map();
|
|
3636
|
+
const observe = (nodes, parentLocale) => {
|
|
3637
|
+
for (const node of nodes) {
|
|
3638
|
+
const locale = node.locale ? canonicalLanguage(node.locale) : parentLocale;
|
|
3639
|
+
const entry = node.url ? records.get(canonical(node.url, checkpoint.sourceUrl) ?? "") : void 0;
|
|
3640
|
+
const page = resolveMigrationRecord(records, entry)?.page;
|
|
3641
|
+
if (page && locale) {
|
|
3642
|
+
const languages = memberships.get(page) ?? /* @__PURE__ */ new Set();
|
|
3643
|
+
languages.add(locale);
|
|
3644
|
+
memberships.set(page, languages);
|
|
3645
|
+
}
|
|
3646
|
+
observe(node.children ?? [], locale);
|
|
3647
|
+
}
|
|
3648
|
+
};
|
|
3649
|
+
observe(checkpoint.navigation);
|
|
3650
|
+
const primaryIds = new Set(pages.filter((page) => !declaredLocale(page) || declaredLocale(page) === defaultLocale).filter((page) => memberships.get(page)?.has(defaultLocale)).map((page) => page.navigationId));
|
|
3651
|
+
const remapped = /* @__PURE__ */ new Map();
|
|
3652
|
+
for (const page of pages) {
|
|
3653
|
+
const languages = memberships.get(page);
|
|
3654
|
+
if (languages?.size !== 1) continue;
|
|
3655
|
+
const locale = [...languages][0];
|
|
3656
|
+
const prefix = `${locale.toLowerCase()}-`;
|
|
3657
|
+
if (locale === defaultLocale || !page.navigationId.toLowerCase().startsWith(prefix)) continue;
|
|
3658
|
+
const navigationId2 = page.navigationId.slice(prefix.length);
|
|
3659
|
+
if (primaryIds.has(navigationId2)) remapped.set(page, { locale, navigationId: navigationId2 });
|
|
3660
|
+
}
|
|
3661
|
+
const canonicalLocale = (page) => remapped.get(page)?.locale ?? declaredLocale(page);
|
|
3662
|
+
const navigationId = (page) => remapped.get(page)?.navigationId ?? page.navigationId;
|
|
3174
3663
|
const locales = [...new Set(pages.map(canonicalLocale).filter((locale) => Boolean(locale)))];
|
|
3175
|
-
const defaultLocale = pages.some((page) => !page.locale) || locales.includes("en") ? "en" : locales[0] ?? "en";
|
|
3176
3664
|
const pageId = (page) => {
|
|
3177
3665
|
const locale = canonicalLocale(page);
|
|
3178
|
-
return locale ? `${locale === defaultLocale ? "" : `${locale}/`}${page
|
|
3666
|
+
return locale ? `${locale === defaultLocale ? "" : `${locale}/`}${navigationId(page)}` : page.id;
|
|
3179
3667
|
};
|
|
3180
3668
|
const route = (page) => {
|
|
3181
3669
|
const locale = canonicalLocale(page);
|
|
3182
3670
|
const prefix = locale && locale !== defaultLocale ? `/${locale}` : "";
|
|
3183
|
-
return `${prefix}${page
|
|
3671
|
+
return `${prefix}${navigationId(page) === "introduction" ? "/" : `/${navigationId(page)}`}`;
|
|
3184
3672
|
};
|
|
3185
|
-
|
|
3673
|
+
const redirects = checkpoint.records.flatMap((record) => {
|
|
3674
|
+
if (!record.page || !remapped.has(record.page)) return [];
|
|
3675
|
+
const source = new URL(record.url);
|
|
3676
|
+
return source.search || !/^\/[A-Za-z0-9/_~.%+-]*$/.test(source.pathname) ? [] : [{ source: source.pathname, destination: route(record.page), permanent: true }];
|
|
3677
|
+
});
|
|
3678
|
+
return { canonicalLocale, navigationId, locales, defaultLocale, pageId, route, redirects };
|
|
3186
3679
|
}
|
|
3187
3680
|
function validateUrlMigration(checkpoint, cancelled = false) {
|
|
3188
3681
|
const issues = [];
|
|
3682
|
+
const warnings = [];
|
|
3189
3683
|
const counts = { pending: 0, imported: 0, excluded: 0, failed: 0, alias: 0 };
|
|
3190
3684
|
const records = new Map(checkpoint.records.map((record) => [record.url, record]));
|
|
3191
3685
|
const source = new URL(checkpoint.sourceUrl);
|
|
3192
3686
|
const projection = localeProjection(checkpoint);
|
|
3193
|
-
const primaryIds = new Set(checkpoint.records.flatMap((record) => record.page && (!record.page
|
|
3687
|
+
const primaryIds = new Set(checkpoint.records.flatMap((record) => record.page && (!projection.canonicalLocale(record.page) || projection.canonicalLocale(record.page) === projection.defaultLocale) ? [projection.navigationId(record.page)] : []));
|
|
3194
3688
|
const outputIds = /* @__PURE__ */ new Set();
|
|
3195
3689
|
const resolve3 = (url) => {
|
|
3196
3690
|
const record = records.get(canonical(url, checkpoint.sourceUrl) ?? "");
|
|
3197
|
-
return
|
|
3691
|
+
return resolveMigrationRecord(records, record);
|
|
3198
3692
|
};
|
|
3199
3693
|
for (const record of checkpoint.records) {
|
|
3200
3694
|
counts[record.outcome]++;
|
|
@@ -3204,8 +3698,8 @@ function validateUrlMigration(checkpoint, cancelled = false) {
|
|
|
3204
3698
|
const outputId = projection.pageId(record.page);
|
|
3205
3699
|
if (outputIds.has(outputId)) issues.push({ code: "collision", source: record.url, message: `Multiple source pages map to runtime path ${outputId}.` });
|
|
3206
3700
|
outputIds.add(outputId);
|
|
3207
|
-
if (record.page
|
|
3208
|
-
issues.push({ code: "missing-page", source: record.url, message: `Secondary locale ${record.page
|
|
3701
|
+
if (projection.canonicalLocale(record.page) && projection.canonicalLocale(record.page) !== projection.defaultLocale && !primaryIds.has(projection.navigationId(record.page))) {
|
|
3702
|
+
issues.push({ code: "missing-page", source: record.url, message: `Secondary locale ${projection.canonicalLocale(record.page)} requires a primary ${projection.defaultLocale} page at ${projection.navigationId(record.page)}; the runtime cannot serve an orphan translation.` });
|
|
3209
3703
|
}
|
|
3210
3704
|
const actual = markdownCode(record.page.body);
|
|
3211
3705
|
if (JSON.stringify(actual) !== JSON.stringify(record.evidence.code)) issues.push({ code: "code-mismatch", source: record.url, message: "Source code payloads differ from converted code payloads." });
|
|
@@ -3227,22 +3721,37 @@ function validateUrlMigration(checkpoint, cancelled = false) {
|
|
|
3227
3721
|
anchor = decodeURIComponent(anchor);
|
|
3228
3722
|
} catch {
|
|
3229
3723
|
}
|
|
3230
|
-
if (!destination.evidence?.anchors.includes(anchor))
|
|
3724
|
+
if (!destination.evidence?.anchors.includes(anchor)) {
|
|
3725
|
+
warnings.push({ code: "source-link-warning", source: record.url, message: `The original source link refers to an absent source anchor: ${link}` });
|
|
3726
|
+
} else if (destination.page) {
|
|
3727
|
+
const nodes = markdownNodes(destination.page.body);
|
|
3728
|
+
const explicit = nodes.flatMap((node) => node.name === "a" ? (node.attributes ?? []).flatMap((attribute) => attribute.name === "id" && typeof attribute.value === "string" ? [attribute.value] : []) : []);
|
|
3729
|
+
if (!explicit.includes(anchor) && !markdownHeadingAnchors(destination.page.body).includes(anchor)) {
|
|
3730
|
+
issues.push({ code: "missing-anchor", source: record.url, message: `Conversion lost a source anchor: ${link}` });
|
|
3731
|
+
}
|
|
3732
|
+
}
|
|
3231
3733
|
}
|
|
3232
3734
|
}
|
|
3233
3735
|
}
|
|
3234
|
-
const visit = (nodes) => {
|
|
3736
|
+
const visit = (nodes, parentLocale) => {
|
|
3235
3737
|
for (const node of nodes) {
|
|
3738
|
+
const locale = node.locale ? canonicalLanguage(node.locale) : parentLocale;
|
|
3236
3739
|
if (node.url) {
|
|
3237
3740
|
const url = new URL(node.url, source);
|
|
3238
|
-
if (inScope(url, source, checkpoint.scopePath)
|
|
3741
|
+
if (inScope(url, source, checkpoint.scopePath)) {
|
|
3742
|
+
const page = resolve3(url.toString())?.page;
|
|
3743
|
+
if (!page) issues.push({ code: "navigation-missing", source: node.url, message: `Source navigation target ${node.label} was not imported.` });
|
|
3744
|
+
else if (locale && locale !== projection.defaultLocale && !outputIds.has(`${locale}/${projection.navigationId(page)}`)) {
|
|
3745
|
+
issues.push({ code: "navigation-missing", source: node.url, message: `The ${locale} sidebar target ${node.label} has no authored localized runtime page.` });
|
|
3746
|
+
}
|
|
3747
|
+
}
|
|
3239
3748
|
}
|
|
3240
|
-
visit(node.children ?? []);
|
|
3749
|
+
visit(node.children ?? [], locale);
|
|
3241
3750
|
}
|
|
3242
3751
|
};
|
|
3243
3752
|
visit(checkpoint.navigation);
|
|
3244
3753
|
if (!counts.imported) issues.push({ code: "missing-page", source: checkpoint.sourceUrl, message: "No readable article pages were imported." });
|
|
3245
|
-
return { status: cancelled ? "cancelled" : issues.length ? "partial" : "passed", issues, counts };
|
|
3754
|
+
return { status: cancelled ? "cancelled" : issues.length ? "partial" : "passed", issues, counts, ...warnings.length ? { warnings } : {} };
|
|
3246
3755
|
}
|
|
3247
3756
|
async function runUrlMigration(options) {
|
|
3248
3757
|
const source = validateMigrationUrl(options.sourceUrl);
|
|
@@ -3272,6 +3781,11 @@ async function runUrlMigration(options) {
|
|
|
3272
3781
|
delete record.pageSha256;
|
|
3273
3782
|
delete record.evidence;
|
|
3274
3783
|
delete record.assets;
|
|
3784
|
+
delete record.apiSpecification;
|
|
3785
|
+
delete record.version;
|
|
3786
|
+
delete record.issues;
|
|
3787
|
+
delete record.reason;
|
|
3788
|
+
delete record.aliasOf;
|
|
3275
3789
|
}
|
|
3276
3790
|
if (record.page && pageIdFromReference(record.page.id, true) !== record.page.id) throw new Error("Checkpoint contains an unsafe page path.");
|
|
3277
3791
|
records.set(record.url, record);
|
|
@@ -3362,13 +3876,24 @@ async function runUrlMigration(options) {
|
|
|
3362
3876
|
navigationChanged = false;
|
|
3363
3877
|
};
|
|
3364
3878
|
const process = async (record) => {
|
|
3365
|
-
if (options.signal?.aborted) return;
|
|
3879
|
+
if (options.signal?.aborted || record.outcome !== "pending") return;
|
|
3880
|
+
let claimedId;
|
|
3881
|
+
let claimedFinal;
|
|
3882
|
+
let markdownRepresentation;
|
|
3366
3883
|
try {
|
|
3367
3884
|
delete record.reason;
|
|
3885
|
+
delete record.apiSpecification;
|
|
3886
|
+
const represented = resolveMigrationRecord(records, record.representationOf ? records.get(record.representationOf) : void 0);
|
|
3887
|
+
if (represented?.outcome === "imported") {
|
|
3888
|
+
record.outcome = "alias";
|
|
3889
|
+
record.aliasOf = represented.url;
|
|
3890
|
+
return;
|
|
3891
|
+
}
|
|
3368
3892
|
const document = await capture({ url: new URL(record.url), signal: options.signal });
|
|
3369
3893
|
if (options.signal?.aborted) return;
|
|
3370
3894
|
if (Buffer.byteLength(document.body) > MAX_RESPONSE_BYTES) throw new Error("Captured response exceeds the 8 MB safety bound.");
|
|
3371
|
-
const
|
|
3895
|
+
const identity = articleIdentity(document);
|
|
3896
|
+
const final = canonical(identity.toString(), sourceUrl);
|
|
3372
3897
|
const role = classifyMigrationResponse(document);
|
|
3373
3898
|
if (!inScope(document.finalUrl, source, scopePath) && !(role === "index" && isDiscoveryIndex(document.finalUrl, source))) {
|
|
3374
3899
|
record.outcome = "excluded";
|
|
@@ -3389,6 +3914,30 @@ async function runUrlMigration(options) {
|
|
|
3389
3914
|
for (const link of links) enqueue(link, final, record.url);
|
|
3390
3915
|
return;
|
|
3391
3916
|
}
|
|
3917
|
+
if (/html/i.test(document.contentType)) {
|
|
3918
|
+
const $ = load3(document.body);
|
|
3919
|
+
$('link[rel="alternate"][type="text/markdown"][href]').each((_index, node) => {
|
|
3920
|
+
const alternate = canonical($(node).attr("href"), final);
|
|
3921
|
+
if (!alternate) return;
|
|
3922
|
+
const target2 = new URL(alternate), primary = new URL(final);
|
|
3923
|
+
const directory = (path) => path.replace(/\.mdx?$/i, "").replace(/\/index\/?$/i, "").replace(/\/$/, "");
|
|
3924
|
+
if (target2.origin !== primary.origin || target2.search !== primary.search || directory(target2.pathname) !== directory(primary.pathname) || !/\.mdx?$/i.test(target2.pathname)) return;
|
|
3925
|
+
enqueue(alternate, final, "declared-markdown-representation");
|
|
3926
|
+
const sibling = records.get(alternate);
|
|
3927
|
+
sibling.representationOf = record.url;
|
|
3928
|
+
markdownRepresentation ??= alternate;
|
|
3929
|
+
changed.push(sibling);
|
|
3930
|
+
});
|
|
3931
|
+
}
|
|
3932
|
+
const observedNavigation = sourceNavigation(document);
|
|
3933
|
+
if (observedNavigation.length && !(checkpoint.navigation.some((node) => node.kind === "tab") && !observedNavigation.some((node) => node.kind === "tab"))) {
|
|
3934
|
+
const merged = mergeSourceNavigation(checkpoint.navigation, observedNavigation);
|
|
3935
|
+
if (JSON.stringify(merged) !== JSON.stringify(checkpoint.navigation)) {
|
|
3936
|
+
checkpoint.navigation = merged;
|
|
3937
|
+
navigationChanged = true;
|
|
3938
|
+
}
|
|
3939
|
+
enqueueNavigation(observedNavigation, final);
|
|
3940
|
+
}
|
|
3392
3941
|
if (imported >= maxPages) {
|
|
3393
3942
|
record.outcome = "excluded";
|
|
3394
3943
|
record.reason = "A discovery endpoint resolved to an article after the selected page limit.";
|
|
@@ -3401,14 +3950,20 @@ async function runUrlMigration(options) {
|
|
|
3401
3950
|
return;
|
|
3402
3951
|
}
|
|
3403
3952
|
finalUrls.set(final, record.url);
|
|
3404
|
-
|
|
3953
|
+
claimedFinal = final;
|
|
3954
|
+
const id = idForUrl(identity, scopePath);
|
|
3405
3955
|
if (!id) throw new Error("Source URL has no safe destination path.");
|
|
3406
|
-
|
|
3956
|
+
const language = observedLocale(document, id, scopePath);
|
|
3957
|
+
const locale = language.locale;
|
|
3958
|
+
const navigationId = locale ? id.split("/").filter((segment) => segment.toLowerCase() !== locale.toLowerCase()).join("/") || "introduction" : id;
|
|
3959
|
+
const identityId = locale ? `${locale}/${navigationId}` : id;
|
|
3960
|
+
if (usedIds.has(identityId) && usedIds.get(identityId) !== record.url) {
|
|
3407
3961
|
record.outcome = "failed";
|
|
3408
|
-
record.reason = `Destination collision for ${
|
|
3962
|
+
record.reason = `Destination collision for ${identityId}.`;
|
|
3409
3963
|
return;
|
|
3410
3964
|
}
|
|
3411
|
-
usedIds.set(
|
|
3965
|
+
usedIds.set(identityId, record.url);
|
|
3966
|
+
claimedId = identityId;
|
|
3412
3967
|
const evidence = evidenceOf(document);
|
|
3413
3968
|
const page = convert(document, id);
|
|
3414
3969
|
if (!page) {
|
|
@@ -3416,6 +3971,23 @@ async function runUrlMigration(options) {
|
|
|
3416
3971
|
record.reason = "Content could not be represented as safe inert MDX.";
|
|
3417
3972
|
return;
|
|
3418
3973
|
}
|
|
3974
|
+
const hasSchemaChoice = /html/i.test(document.contentType) && migrationHtmlArticle(document.body).root.find('select[aria-label="Select schema type"],select[aria-label="\u9009\u62E9\u67B6\u6784\u7C7B\u578B"]').length > 0;
|
|
3975
|
+
if (/html/i.test(document.contentType) && markdownRepresentation && (unresolvedStaticControls(document.body) || hasSchemaChoice)) {
|
|
3976
|
+
const schemaCapture = await capture({ url: new URL(markdownRepresentation), signal: options.signal });
|
|
3977
|
+
if (options.signal?.aborted) return;
|
|
3978
|
+
if (schemaCapture.status && schemaCapture.status >= 400 || !/text\/(?:markdown|plain)/i.test(schemaCapture.contentType)) throw new Error("API schema representation is not a successful Markdown response.");
|
|
3979
|
+
totalBytes += Buffer.byteLength(schemaCapture.body);
|
|
3980
|
+
if (Buffer.byteLength(schemaCapture.body) > MAX_RESPONSE_BYTES || totalBytes > (options.maxTotalBytes ?? 5e8)) throw new Error("API schema capture exceeds its content byte budget.");
|
|
3981
|
+
const expected = new URL(markdownRepresentation);
|
|
3982
|
+
if (schemaCapture.finalUrl.origin !== expected.origin || schemaCapture.finalUrl.pathname !== expected.pathname || schemaCapture.finalUrl.search !== expected.search) throw new Error("API schema representation escaped its declared identity.");
|
|
3983
|
+
const fragment = extractEmbeddedOpenApi(schemaCapture.body).fragment;
|
|
3984
|
+
if (fragment && hasCompleteLocalApiSchema(fragment.document)) {
|
|
3985
|
+
const sha256 = hash(canonicalJson(fragment.document));
|
|
3986
|
+
const path = `openapi/migration-${sha256}.json`;
|
|
3987
|
+
record.apiSpecification = { path, sha256, document: fragment.document };
|
|
3988
|
+
page.openapi = `${path} ${fragment.method} ${fragment.path}`;
|
|
3989
|
+
}
|
|
3990
|
+
}
|
|
3419
3991
|
const states = document.states ?? [];
|
|
3420
3992
|
if (states.length > 64) throw new Error("Capture contains too many alternative states.");
|
|
3421
3993
|
for (const state of states) {
|
|
@@ -3444,14 +4016,16 @@ ${difference}`;
|
|
|
3444
4016
|
record.assets = [];
|
|
3445
4017
|
record.issues = [];
|
|
3446
4018
|
if (/html/i.test(document.contentType)) {
|
|
3447
|
-
const
|
|
3448
|
-
const root = $('main article,article,main,[role="main"]').first();
|
|
4019
|
+
const { $, root } = migrationHtmlArticle(document.body);
|
|
3449
4020
|
root.find("nav,footer,aside,.theme-doc-breadcrumbs,.theme-doc-toc-mobile,.table-of-contents,.hash-link").remove();
|
|
3450
4021
|
root.find('button svg,a.hash-link svg,[aria-hidden="true"] svg,svg[aria-hidden="true"],svg[data-lucide],svg[data-icon]').remove();
|
|
3451
|
-
if (
|
|
4022
|
+
if (record.apiSpecification) {
|
|
4023
|
+
root.find('select[aria-label="Select schema type"],select[aria-label="\u9009\u62E9\u67B6\u6784\u7C7B\u578B"]').remove();
|
|
4024
|
+
root.find('.option-dropdown svg[data-component-part="icon-svg"]').remove();
|
|
4025
|
+
}
|
|
3452
4026
|
const unsupported = [...new Set(root.find("iframe,object,embed,canvas,svg,form,input,textarea,select,math,video,audio").toArray().map((node) => node.tagName))];
|
|
3453
4027
|
if (unsupported.length) record.issues.push({ code: "unsupported-component", source: record.url, message: `Rendered content requires explicit review for: ${unsupported.join(", ")}.` });
|
|
3454
|
-
if (!states.length && unresolvedStaticControls(document.body)) record.issues.push({ code: "unsupported-component", source: record.url, message: "An interactive alternative has no complete captured panel or disclosure content." });
|
|
4028
|
+
if (!states.length && unresolvedStaticControls(document.body, Boolean(record.apiSpecification))) record.issues.push({ code: "unsupported-component", source: record.url, message: "An interactive alternative has no complete captured panel or disclosure content." });
|
|
3455
4029
|
}
|
|
3456
4030
|
const suppliedAssets = new Map((document.assets ?? []).map((asset) => [canonical(asset.url, final), asset]));
|
|
3457
4031
|
const assetUrls = [...new Set([...assetReferences(document), ...states.flatMap((state) => assetReferences(state.response))].map((url) => canonical(url, final)).filter((url) => Boolean(url)))];
|
|
@@ -3491,30 +4065,18 @@ ${difference}`;
|
|
|
3491
4065
|
if (sanitizeRemoteMarkdown(page.body.replace(/^<a id="[^"<]*" \/>$/gm, "")) === null) throw new Error("Converted content did not pass inert MDX validation.");
|
|
3492
4066
|
record.evidence = evidence;
|
|
3493
4067
|
record.page = page;
|
|
3494
|
-
const language = observedLocale(document, page.id, scopePath);
|
|
3495
4068
|
record.issues.push(...language.issues);
|
|
3496
4069
|
const canonicalLocale = language.locale;
|
|
3497
4070
|
if (canonicalLocale) {
|
|
3498
4071
|
page.locale = canonicalLocale;
|
|
3499
|
-
|
|
3500
|
-
const localeIndex = segments.findIndex((segment) => segment.toLowerCase() === canonicalLocale.toLowerCase());
|
|
3501
|
-
page.navigationId = segments.filter((_segment, index) => index !== localeIndex).join("/") || "introduction";
|
|
4072
|
+
page.navigationId = navigationId;
|
|
3502
4073
|
page.id = `${canonicalLocale}/${page.navigationId}`;
|
|
3503
|
-
if (usedIds.has(page.id) && usedIds.get(page.id) !== record.url) throw new Error(`Destination collision for locale page ${page.id}.`);
|
|
3504
|
-
usedIds.delete(id);
|
|
3505
|
-
usedIds.set(page.id, record.url);
|
|
3506
4074
|
}
|
|
3507
4075
|
record.version = document.version ?? [...page.id.split("/"), ...document.finalUrl.pathname.split("/")].find((segment) => /^v\d+(?:\.\d+)*$/i.test(segment));
|
|
3508
4076
|
record.pageSha256 = pageHash(page);
|
|
3509
4077
|
record.outcome = "imported";
|
|
3510
4078
|
imported++;
|
|
3511
|
-
const navigation = checkpoint.navigation.length ? [] : sourceNavigation(document);
|
|
3512
|
-
if (!checkpoint.navigation.length && navigation.length) {
|
|
3513
|
-
checkpoint.navigation = navigation;
|
|
3514
|
-
navigationChanged = true;
|
|
3515
|
-
}
|
|
3516
4079
|
enqueueArticleLinks(evidence.links, final, record.url);
|
|
3517
|
-
enqueueNavigation(navigation, final);
|
|
3518
4080
|
} catch (error) {
|
|
3519
4081
|
if (options.signal?.aborted) return;
|
|
3520
4082
|
const wasImported = record.outcome === "imported";
|
|
@@ -3527,13 +4089,43 @@ ${difference}`;
|
|
|
3527
4089
|
delete record.page;
|
|
3528
4090
|
delete record.pageSha256;
|
|
3529
4091
|
delete record.assets;
|
|
4092
|
+
delete record.apiSpecification;
|
|
3530
4093
|
} finally {
|
|
4094
|
+
if (record.outcome !== "imported") {
|
|
4095
|
+
if (claimedId && usedIds.get(claimedId) === record.url) usedIds.delete(claimedId);
|
|
4096
|
+
if (claimedFinal && finalUrls.get(claimedFinal) === record.url) finalUrls.delete(claimedFinal);
|
|
4097
|
+
}
|
|
3531
4098
|
if (record.outcome !== "pending") {
|
|
3532
4099
|
if (options.onRecord) await options.onRecord(record);
|
|
3533
4100
|
changed.push(record);
|
|
3534
4101
|
}
|
|
3535
4102
|
}
|
|
3536
4103
|
};
|
|
4104
|
+
const settleRepresentations = () => {
|
|
4105
|
+
for (const record of records.values()) {
|
|
4106
|
+
if (!record.representationOf) continue;
|
|
4107
|
+
const primary = resolveMigrationRecord(records, records.get(record.representationOf));
|
|
4108
|
+
if (!primary) continue;
|
|
4109
|
+
const winner = primary.outcome === "imported" ? primary : record.outcome === "imported" && primary.outcome === "failed" ? record : void 0;
|
|
4110
|
+
if (!winner) continue;
|
|
4111
|
+
const alternate = winner === primary ? record : primary;
|
|
4112
|
+
if (alternate.outcome === "alias" && alternate.aliasOf === winner.url) continue;
|
|
4113
|
+
if (alternate.outcome === "imported") {
|
|
4114
|
+
imported--;
|
|
4115
|
+
if (alternate.page && usedIds.get(alternate.page.id) === alternate.url) usedIds.delete(alternate.page.id);
|
|
4116
|
+
}
|
|
4117
|
+
alternate.outcome = "alias";
|
|
4118
|
+
alternate.aliasOf = winner.url;
|
|
4119
|
+
delete alternate.reason;
|
|
4120
|
+
delete alternate.page;
|
|
4121
|
+
delete alternate.pageSha256;
|
|
4122
|
+
delete alternate.assets;
|
|
4123
|
+
delete alternate.issues;
|
|
4124
|
+
delete alternate.apiSpecification;
|
|
4125
|
+
changed.push(alternate);
|
|
4126
|
+
}
|
|
4127
|
+
};
|
|
4128
|
+
settleRepresentations();
|
|
3537
4129
|
while (cursor < pending.length && !options.signal?.aborted && !budgetExhausted && cursor < (options.maxCaptures ?? Infinity)) {
|
|
3538
4130
|
if (imported >= maxPages) {
|
|
3539
4131
|
const index = pending.findIndex((record, position) => position >= cursor && isDiscoveryIndex(new URL(record.url), source));
|
|
@@ -3543,17 +4135,18 @@ ${difference}`;
|
|
|
3543
4135
|
const batch = pending.slice(cursor, cursor + Math.min(concurrency, imported >= maxPages ? 1 : maxPages - imported, (options.maxCaptures ?? Infinity) - cursor));
|
|
3544
4136
|
cursor += batch.length;
|
|
3545
4137
|
await Promise.all(batch.map(process));
|
|
4138
|
+
settleRepresentations();
|
|
3546
4139
|
if (changed.length >= batchSize) await flush();
|
|
3547
4140
|
}
|
|
3548
4141
|
await flush();
|
|
3549
4142
|
const projection = localeProjection(checkpoint);
|
|
3550
4143
|
const routeMap = /* @__PURE__ */ new Map();
|
|
3551
4144
|
for (const record of checkpoint.records) {
|
|
3552
|
-
const
|
|
3553
|
-
if (
|
|
3554
|
-
const route = projection.route(
|
|
4145
|
+
const target2 = resolveMigrationRecord(records, record);
|
|
4146
|
+
if (target2?.page) {
|
|
4147
|
+
const route = projection.route(target2.page);
|
|
3555
4148
|
routeMap.set(record.url, route);
|
|
3556
|
-
if (
|
|
4149
|
+
if (target2.evidence) routeMap.set(canonical(target2.evidence.finalUrl, sourceUrl), route);
|
|
3557
4150
|
}
|
|
3558
4151
|
}
|
|
3559
4152
|
const navigationLabels = /* @__PURE__ */ new Map();
|
|
@@ -3561,17 +4154,17 @@ ${difference}`;
|
|
|
3561
4154
|
const labelPages = (nodes) => {
|
|
3562
4155
|
for (const node of nodes) {
|
|
3563
4156
|
const entry = node.url ? records.get(canonical(node.url, sourceUrl) ?? "") : void 0;
|
|
3564
|
-
const
|
|
3565
|
-
if (
|
|
3566
|
-
const previous = navigationLabels.get(
|
|
3567
|
-
if (previous && previous !== node.label) navigationIssues.push({ code: "unsupported-component", source:
|
|
3568
|
-
else navigationLabels.set(
|
|
4157
|
+
const target2 = resolveMigrationRecord(records, entry);
|
|
4158
|
+
if (target2?.page && node.label && !node.children?.length) {
|
|
4159
|
+
const previous = navigationLabels.get(target2.url);
|
|
4160
|
+
if (previous && previous !== node.label) navigationIssues.push({ code: "unsupported-component", source: target2.url, message: `This page has multiple source navigation labels (${previous}, ${node.label}); review the shared sidebar title.` });
|
|
4161
|
+
else navigationLabels.set(target2.url, node.label);
|
|
3569
4162
|
}
|
|
3570
4163
|
labelPages(node.children ?? []);
|
|
3571
4164
|
}
|
|
3572
4165
|
};
|
|
3573
4166
|
labelPages(checkpoint.navigation);
|
|
3574
|
-
const pages = checkpoint.records.flatMap((record) => record.page ? [{ ...record.page, id: projection.pageId(record.page), ...record.page
|
|
4167
|
+
const pages = checkpoint.records.flatMap((record) => record.page ? [{ ...record.page, id: projection.pageId(record.page), navigationId: projection.navigationId(record.page), ...projection.canonicalLocale(record.page) ? { locale: projection.canonicalLocale(record.page) } : {}, ...navigationLabels.has(record.url) ? { navTitle: navigationLabels.get(record.url) } : {}, body: rewriteMarkdownDestinations(record.page.body, (value) => {
|
|
3575
4168
|
let url;
|
|
3576
4169
|
try {
|
|
3577
4170
|
url = new URL(value, record.evidence.finalUrl);
|
|
@@ -3591,20 +4184,28 @@ ${difference}`;
|
|
|
3591
4184
|
assets.set(asset.path, { path: asset.path, content: Buffer.from(content.buffer, content.byteOffset, content.byteLength) });
|
|
3592
4185
|
}
|
|
3593
4186
|
}
|
|
4187
|
+
for (const record of checkpoint.records) if (record.page && record.apiSpecification) {
|
|
4188
|
+
const spec = record.apiSpecification;
|
|
4189
|
+
assets.set(spec.path, { path: spec.path, projectRelative: true, content: Buffer.from(canonicalJson(spec.document)) });
|
|
4190
|
+
}
|
|
3594
4191
|
const bundle = {
|
|
3595
4192
|
sourceUrl,
|
|
3596
4193
|
sourceKind: "url",
|
|
3597
4194
|
platform: options.platform ?? "generic",
|
|
3598
4195
|
pages,
|
|
3599
4196
|
assets: [...assets.values()],
|
|
3600
|
-
docsConfig: navigationConfig(checkpoint
|
|
3601
|
-
warnings: validation.issues.map((issue) => ({ code: issue.code === "missing-page" ? "missing-page" : "unsupported-config", message: issue.message, source: issue.source })),
|
|
4197
|
+
docsConfig: navigationConfig(checkpoint, records) ?? buildNavigationFromPages(pages),
|
|
4198
|
+
warnings: [...validation.issues, ...validation.warnings ?? []].map((issue) => ({ code: issue.code === "missing-page" ? "missing-page" : "unsupported-config", message: issue.message, source: issue.source })),
|
|
3602
4199
|
urlMigrationValidation: validation,
|
|
3603
4200
|
stats: { discovered: records.size, imported: pages.length, skipped: records.size - pages.length }
|
|
3604
4201
|
};
|
|
4202
|
+
if (projection.redirects.length) bundle.docsConfig.redirects = projection.redirects;
|
|
3605
4203
|
if (projection.locales.length) {
|
|
3606
4204
|
const locales = [.../* @__PURE__ */ new Set([projection.defaultLocale, ...projection.locales])];
|
|
3607
|
-
bundle.docsConfig.i18n = { defaultLocale: projection.defaultLocale, locales: locales.map((code) => ({ code, label: code === "en" ? "English" : code })) };
|
|
4205
|
+
bundle.docsConfig.i18n = { ...bundle.docsConfig.i18n, defaultLocale: projection.defaultLocale, locales: locales.map((code) => ({ code, label: code === "en" ? "English" : code })) };
|
|
4206
|
+
}
|
|
4207
|
+
for (const spec of assets.values()) if (spec.projectRelative && spec.path.startsWith("openapi/")) {
|
|
4208
|
+
bundle.docsConfig.tabs.push({ tab: `Migration API ${spec.path.slice(-69, -5)}`, hidden: true, api: { source: spec.path, navigation: false } });
|
|
3608
4209
|
}
|
|
3609
4210
|
const stopReason = options.signal?.aborted ? "cancelled" : budgetExhausted ? "byte-limit" : cursor < pending.length && cursor >= (options.maxCaptures ?? Infinity) ? "capture-limit" : imported >= maxPages && cursor < pending.length ? "page-limit" : cursor < pending.length ? "capture-limit" : "complete";
|
|
3610
4211
|
return { bundle, checkpoint, validation, stopReason };
|
|
@@ -3694,7 +4295,7 @@ var DOCUSAURUS_THEME_SHIMS = {
|
|
|
3694
4295
|
};
|
|
3695
4296
|
function applyReplacements(source, replacements) {
|
|
3696
4297
|
return replacements.sort((a, b) => b.start - a.start).reduce(
|
|
3697
|
-
(
|
|
4298
|
+
(text2, edit) => text2.slice(0, edit.start) + edit.value + text2.slice(edit.end),
|
|
3698
4299
|
source
|
|
3699
4300
|
);
|
|
3700
4301
|
}
|
|
@@ -3823,18 +4424,18 @@ function mergeMigrationConfig(existing, incoming) {
|
|
|
3823
4424
|
names.add(tab.tab.toLowerCase());
|
|
3824
4425
|
continue;
|
|
3825
4426
|
}
|
|
3826
|
-
const
|
|
3827
|
-
if (!
|
|
4427
|
+
const target2 = tabs.find((item) => item.tab.toLowerCase() === tab.tab.toLowerCase());
|
|
4428
|
+
if (!target2) continue;
|
|
3828
4429
|
const incomingRootNodes = [
|
|
3829
4430
|
...uniqueTab.pages ?? [],
|
|
3830
4431
|
...uniqueTab.groups ?? []
|
|
3831
4432
|
];
|
|
3832
4433
|
if (uniqueTab.pages?.length) {
|
|
3833
|
-
|
|
3834
|
-
delete
|
|
4434
|
+
target2.pages = [...target2.pages ?? target2.groups ?? [], ...incomingRootNodes];
|
|
4435
|
+
delete target2.groups;
|
|
3835
4436
|
} else if (uniqueTab.groups?.length) {
|
|
3836
|
-
if (
|
|
3837
|
-
else
|
|
4437
|
+
if (target2.pages) target2.pages = [...target2.pages, ...uniqueTab.groups];
|
|
4438
|
+
else target2.groups = [...target2.groups ?? [], ...uniqueTab.groups];
|
|
3838
4439
|
}
|
|
3839
4440
|
}
|
|
3840
4441
|
const changelog = existing.tabs.find((tab) => tab.tab.toLowerCase() === "changelog") ?? incoming.tabs.find((tab) => tab.tab.toLowerCase() === "changelog");
|