@thallylabs/migrate 0.2.41 → 0.2.42

package/README.md

@@ -10,6 +10,12 @@ ordered navigation and binary assets), or uses ordinary local fetching. Hosted
 callers must inject their own network-safe text/capture and binary boundaries.
 No browser, model, credentials, sandbox or publishing provider is bundled here.
 
+Default page requests advertise HTML to retain rendered content and declared
+alternates. An explicit same-origin, same-route Markdown alternate shares the
+article's destination; the rendered article takes precedence even when the
+Markdown URL was discovered first or restored from an interrupted checkpoint.
+Independent pages still report destination collisions.
+
 Server applications should import `@thallylabs/migrate/svg-policy` for the passive
 SVG validator, or `@thallylabs/migrate/url-migration` for the URL runner, renderer
 and their types. These independent entries avoid tracing repository cloning and
@@ -99,8 +105,13 @@ Missing primary counterparts and projected path collisions require review; the
 engine does not invent translations. Fresh HTML captures also observe `html lang`; conflicting or unsupported
 language evidence requires review. Capture adapters can supply `locale` when
 no language is declared in HTML or identified by the scoped URL path.
-The `url-evidence-v6` converter rejects earlier journals: replay retained raw
+The `url-evidence-v7` converter rejects earlier journals: replay retained raw
 captures into a new journal rather than relabelling previously hashed pages.
+Distinct observed redirect routes can retain their own sidebar titles through
+noindex page metadata and portable internal redirects to the captured article.
+Conflicting labels on the same route still require review. Markdown image and
+link destinations are localized by source spans, including escaped query strings,
+without changing matching labels, titles, or fenced examples.
 
 `stopReason` distinguishes a resumable `capture-limit` from hard `page-limit`
 or `byte-limit` outcomes. Hosts must not automatically retry a hard limit as an

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { M as MigrationPage, a as MigrationAsset, b as MigrationNavigationGroup, R as RenderedMigrationFile, c as MigrationWarning, d as MigrationFetcher, e as MigrationPlatform, f as MigrationBundle, g as MigrationDocsConfig, E as EndpointTabSlot } from './url-run-mSKFqHx_.js';
-export { h as MigrationCapture, i as MigrationCaptureProvider, j as MigrationCaptureState, k as MigrationCapturedAsset, l as MigrationEvidence, m as MigrationFetchRequest, n as MigrationFetchResponse, o as MigrationJournalAsset, p as MigrationNavigationOccurrence, q as MigrationNavigationTab, U as UrlMigrationCheckpoint, r as UrlMigrationCheckpointDelta, s as UrlMigrationIssue, t as UrlMigrationRecord, u as UrlMigrationRun, v as UrlMigrationRunOptions, w as UrlMigrationValidation, x as classifyMigrationResponse, y as mergeMigrationConfig, z as renderMigrationFiles, A as runUrlMigration, B as validateUrlMigration } from './url-run-mSKFqHx_.js';
+import { M as MigrationPage, a as MigrationAsset, b as MigrationNavigationGroup, R as RenderedMigrationFile, c as MigrationWarning, d as MigrationFetcher, e as MigrationPlatform, f as MigrationBundle, g as MigrationDocsConfig, E as EndpointTabSlot } from './url-run-CPzQfjRW.js';
+export { h as MigrationCapture, i as MigrationCaptureProvider, j as MigrationCaptureState, k as MigrationCapturedAsset, l as MigrationEvidence, m as MigrationFetchRequest, n as MigrationFetchResponse, o as MigrationJournalAsset, p as MigrationNavigationOccurrence, q as MigrationNavigationTab, U as UrlMigrationCheckpoint, r as UrlMigrationCheckpointDelta, s as UrlMigrationIssue, t as UrlMigrationRecord, u as UrlMigrationRun, v as UrlMigrationRunOptions, w as UrlMigrationValidation, x as classifyMigrationResponse, y as mergeMigrationConfig, z as renderMigrationFiles, A as runUrlMigration, B as validateUrlMigration } from './url-run-CPzQfjRW.js';
 export { isPassiveSvg } from './svg-policy.js';
 
 /**

package/dist/index.js

@@ -7860,7 +7860,10 @@ var VIDEO_ATTRIBUTES = /* @__PURE__ */ new Set([
   "width",
   "height",
   "preload",
-  "aria-label"
+  "aria-label",
+  // Publisher asset provenance does not change playback and is never emitted.
+  // Keep this explicit: arbitrary data attributes may drive custom controls.
+  "data-path"
 ]);
 var YOUTUBE_ATTRIBUTES = /* @__PURE__ */ new Set([
   "src",
@@ -16119,6 +16122,7 @@ function renderMigrationFiles(bundle, options = {}) {
 // src/url-run.ts
 import { createHash as createHash6 } from "crypto";
 import { load as load8 } from "cheerio";
+import { parse as parse2, postprocess, preprocess } from "micromark";
 import remarkParse5 from "remark-parse";
 import remarkMdx5 from "remark-mdx";
 import { unified as unified5 } from "unified";
@@ -17197,9 +17201,9 @@ function apiOperationCodeControls(body, specification, operation) {
 }
 
 // src/url-run.ts
-var CONVERTER_VERSION = "url-evidence-v6";
+var CONVERTER_VERSION = "url-evidence-v7";
 var parser3 = unified5().use(remarkParse5).use(remarkMdx5);
-var ACCEPT = "text/markdown,text/html,application/xhtml+xml;q=0.9";
+var ACCEPT = "text/html,application/xhtml+xml";
 var MAX_RESPONSE_BYTES = 8e6;
 var MAX_ASSET_BYTES2 = 1e7;
 var MAX_RECORDS = 1e5;
@@ -17229,6 +17233,11 @@ function canonical(value, base) {
     return null;
   }
 }
+function sameRouteMarkdownRepresentation(alternate, primary) {
+  const target2 = new URL(alternate), source = new URL(primary);
+  const directory2 = (path) => path.replace(/\.mdx?$/i, "").replace(/\/index\/?$/i, "").replace(/\/$/, "");
+  return target2.origin === source.origin && target2.search === source.search && directory2(target2.pathname) === directory2(source.pathname) && /\.mdx?$/i.test(target2.pathname);
+}
 function classifyMigrationResponse(document) {
   if (document.status && document.status >= 400) return "unsupported";
   if (/\/(?:llms(?:-full)?|robots)\.txt$|\/sitemap[^/]*\.xml$|\/search(?:\.html)?$/i.test(document.finalUrl.pathname) || /\.(?:json|ya?ml)$/i.test(document.finalUrl.pathname) || /(?:xml|json)/i.test(document.contentType)) return "index";
@@ -17479,7 +17488,7 @@ function resolveMigrationRecord(records, record) {
   }
   return record;
 }
-function navigationConfig(checkpoint, records) {
+function navigationConfig(checkpoint, records, occurrencePages = /* @__PURE__ */ new Map()) {
   const navigation = checkpoint.navigation;
   const shortcuts = [];
   const collectShortcuts = (nodes) => {
@@ -17504,7 +17513,7 @@ function navigationConfig(checkpoint, records) {
   const project2 = (nodes) => nodes.flatMap((node) => {
     const record = node.url ? records.get(canonical(node.url, node.url) ?? "") : void 0;
     const target2 = resolveMigrationRecord(records, record);
-    const page = target2?.page;
+    const page = occurrencePages.get(node) ?? target2?.page;
     const children = project2(node.children ?? []);
     if (children.length) {
       const contains = (entries) => entries.some((entry) => typeof entry === "string" ? entry === (page && projection.navigationId(page)) : contains(entry.pages));
@@ -17804,8 +17813,19 @@ async function runUrlMigration(options) {
     if (!record.evidence) continue;
     const final = canonical(record.evidence.finalUrl, sourceUrl);
     if (!final) continue;
-    if (record.outcome === "imported") enqueueArticleLinks(record.evidence.links, final, record.url);
-    else if (record.outcome === "excluded") {
+    if (record.outcome === "imported") {
+      enqueueArticleLinks(record.evidence.links, final, record.url);
+      for (const reference of record.evidence.markdownRepresentations ?? []) {
+        const alternate = canonical(reference, final);
+        if (!alternate || !sameRouteMarkdownRepresentation(alternate, final)) continue;
+        enqueue(alternate, final, "declared-markdown-representation");
+        const sibling = records.get(alternate);
+        if (sibling && sibling.representationOf !== record.url) {
+          sibling.representationOf = record.url;
+          changed.push(sibling);
+        }
+      }
+    } else if (record.outcome === "excluded") {
       if (classifyMigrationResponse({ finalUrl: new URL(final), contentType: record.evidence.contentType, body: "" }) === "index") {
         for (const link of record.evidence.links) enqueue(link, final, record.url);
       }
@@ -17888,6 +17908,7 @@ async function runUrlMigration(options) {
     let claimedId;
     let claimedFinal;
     let markdownRepresentation;
+    const markdownRepresentations = [];
     let exactKey;
     try {
       delete record.reason;
@@ -17980,13 +18001,13 @@ async function runUrlMigration(options) {
         $('link[rel="alternate"][type="text/markdown"][href]').each((_index, node) => {
           const alternate = canonical($(node).attr("href"), final);
           if (!alternate) return;
-          const target2 = new URL(alternate), primary = new URL(final);
-          const directory2 = (path) => path.replace(/\.mdx?$/i, "").replace(/\/index\/?$/i, "").replace(/\/$/, "");
-          if (target2.origin !== primary.origin || target2.search !== primary.search || directory2(target2.pathname) !== directory2(primary.pathname) || !/\.mdx?$/i.test(target2.pathname)) return;
+          if (!sameRouteMarkdownRepresentation(alternate, final)) return;
           enqueue(alternate, final, "declared-markdown-representation");
           const sibling = records.get(alternate);
+          if (!sibling) return;
           sibling.representationOf = record.url;
           markdownRepresentation ??= alternate;
+          markdownRepresentations.push(alternate);
           changed.push(sibling);
         });
       }
@@ -17999,7 +18020,8 @@ async function runUrlMigration(options) {
         }
         enqueueNavigation(observedNavigation, final);
       }
-      if (imported >= maxPages) {
+      const replacesRepresentation = imported >= maxPages && markdownRepresentations.some((url) => records.get(url)?.page);
+      if (imported >= maxPages && !replacesRepresentation) {
         record.outcome = "excluded";
         record.reason = "A discovery endpoint resolved to an article after the selected page limit.";
         return;
@@ -18018,14 +18040,17 @@ async function runUrlMigration(options) {
       const locale = language.locale;
       const navigationId = locale ? id.split("/").filter((segment) => segment.toLowerCase() !== locale.toLowerCase()).join("/") || "introduction" : id;
       const identityId = locale ? `${locale}/${navigationId}` : id;
-      if (usedIds.has(identityId) && usedIds.get(identityId) !== record.url) {
+      const owner = records.get(usedIds.get(identityId) ?? "");
+      const sharesDeclaredRepresentation = owner && (owner.representationOf === record.url || record.representationOf === owner.url);
+      if (owner && owner !== record && !sharesDeclaredRepresentation) {
         record.outcome = "failed";
         record.reason = `Destination collision for ${identityId}.`;
         return;
       }
-      usedIds.set(identityId, record.url);
+      if (!owner || owner === record || owner.representationOf === record.url) usedIds.set(identityId, record.url);
       claimedId = identityId;
       const evidence = evidenceOf(document);
+      if (markdownRepresentations.length) evidence.markdownRepresentations = markdownRepresentations;
       const page = convert(document, id);
       if (!page) {
         record.outcome = "failed";
@@ -18236,6 +18261,7 @@ ${difference}`;
       if (!primary || primary === record) continue;
       const winner = primary.outcome === "imported" ? primary : record.outcome === "imported" && primary.outcome === "failed" ? record : void 0;
       if (!winner) continue;
+      if (winner.page) usedIds.set(winner.page.id, winner.url);
       const alternate = winner === primary ? record : primary;
       if (alternate.outcome === "alias" && alternate.aliasOf === winner.url) continue;
       if (alternate.outcome === "imported") {
@@ -18257,7 +18283,17 @@ ${difference}`;
   settleArticleIdentities();
   while (cursor < pending.length && !options.signal?.aborted && !budgetExhausted && cursor < (options.maxCaptures ?? Infinity)) {
     if (imported >= maxPages) {
-      const index = pending.findIndex((record, position) => position >= cursor && isDiscoveryIndex(new URL(record.url), source));
+      const index = pending.findIndex((record, position) => {
+        if (position < cursor || record.outcome !== "pending") return false;
+        const url = new URL(record.url);
+        if (isDiscoveryIndex(url, source)) return true;
+        if (/\.mdx?$/i.test(url.pathname)) return false;
+        return [".md", ".mdx", "/index.md", "/index.mdx"].some((extension) => {
+          const alternate = new URL(url);
+          alternate.pathname = alternate.pathname.replace(/\/index$/i, "") + extension;
+          return records.get(alternate.href)?.outcome === "imported";
+        });
+      });
       if (index < 0) break;
       pending.splice(cursor, 0, pending.splice(index, 1)[0]);
     }
@@ -18281,19 +18317,60 @@ ${difference}`;
   }
   const navigationLabels = /* @__PURE__ */ new Map();
   const navigationIssues = [];
+  const navigationOccurrences = /* @__PURE__ */ new Map();
   const labelPages = (nodes) => {
     for (const node of nodes) {
       const entry = node.url ? records.get(canonical(node.url, sourceUrl) ?? "") : void 0;
       const target2 = resolveMigrationRecord(records, entry);
-      if (target2?.page && node.label && !node.children?.length && !entry?.aliasAnchor) {
-        const previous = navigationLabels.get(target2.url);
-        if (previous && previous !== node.label) navigationIssues.push({ code: "unsupported-component", source: target2.url, message: `This page has multiple source navigation labels (${previous}, ${node.label}); review the shared sidebar title.` });
-        else navigationLabels.set(target2.url, node.label);
+      if (entry && target2?.page && node.label && !node.children?.length && !entry.aliasAnchor) {
+        const occurrences = navigationOccurrences.get(target2) ?? [];
+        occurrences.push({ node, entry });
+        navigationOccurrences.set(target2, occurrences);
       }
       labelPages(node.children ?? []);
     }
   };
   labelPages(checkpoint.navigation);
+  const occurrencePages = /* @__PURE__ */ new Map();
+  const navigationAliases = /* @__PURE__ */ new Map();
+  const occupiedPageIds = new Set(checkpoint.records.flatMap((record) => record.page ? [projection.pageId(record.page)] : []));
+  const navigationRedirects = [];
+  for (const [target2, occurrences] of navigationOccurrences) {
+    const final = target2.evidence && canonical(target2.evidence.finalUrl, sourceUrl);
+    const primary = occurrences.find(({ entry }) => entry.url === final) ?? occurrences.find(({ entry }) => entry === target2) ?? occurrences[0];
+    navigationLabels.set(target2.url, primary.node.label);
+    for (const { node, entry } of occurrences) {
+      if (node.label === primary.node.label) continue;
+      const existing = navigationAliases.get(entry.url);
+      if (existing?.page.navTitle === node.label && existing.target === target2) {
+        occurrencePages.set(node, existing.page);
+        continue;
+      }
+      const sourceRoute = new URL(entry.url);
+      const id = inScope(sourceRoute, source, scopePath) ? idForUrl(sourceRoute, scopePath) : null;
+      const locale = projection.canonicalLocale(target2.page);
+      let navigationId = id && (locale ? id.split("/").filter((segment) => segment.toLowerCase() !== locale.toLowerCase()).join("/") || "introduction" : id);
+      const legacyPrefix = locale && `${locale.toLowerCase()}-`;
+      if (navigationId && legacyPrefix && projection.navigationId(target2.page) !== target2.page.navigationId && navigationId.toLowerCase().startsWith(legacyPrefix)) navigationId = navigationId.slice(legacyPrefix.length);
+      const page = navigationId ? {
+        ...target2.page,
+        id: locale ? `${locale}/${navigationId}` : navigationId,
+        navigationId,
+        ...locale ? { locale } : {},
+        navTitle: node.label,
+        source: entry.url,
+        noindex: true
+      } : void 0;
+      if (!page || existing || occupiedPageIds.has(projection.pageId(page))) {
+        navigationIssues.push({ code: "unsupported-component", source: target2.url, message: `This page has multiple source navigation labels (${primary.node.label}, ${node.label}); their source routes cannot retain distinct sidebar titles safely.` });
+        continue;
+      }
+      occupiedPageIds.add(projection.pageId(page));
+      navigationAliases.set(entry.url, { page, target: target2 });
+      occurrencePages.set(node, page);
+      navigationRedirects.push({ source: projection.route(page), destination: projection.route(target2.page), permanent: true });
+    }
+  }
   const pages = checkpoint.records.flatMap((record) => record.page ? [{ ...record.page, id: projection.pageId(record.page), navigationId: projection.navigationId(record.page), ...projection.canonicalLocale(record.page) ? { locale: projection.canonicalLocale(record.page) } : {}, ...navigationLabels.has(record.url) ? { navTitle: navigationLabels.get(record.url) } : {}, body: rewriteMarkdownDestinations(record.page.body, (value) => {
     let url;
     try {
@@ -18314,6 +18391,10 @@ ${difference}`;
     const hasBrokenSourceAnchor = Boolean(url.hash && destination?.outcome === "imported" && destination.evidence && !destination.evidence.anchors.includes(anchor));
     return `${route}${hasBrokenSourceAnchor ? "" : effectiveHash}`;
   }) }] : []);
+  const projectedPages = new Map(pages.map((page) => [page.id, page]));
+  for (const { page, target: target2 } of navigationAliases.values()) {
+    pages.push({ ...page, id: projection.pageId(page), body: projectedPages.get(projection.pageId(target2.page)).body });
+  }
   const validation = validateUrlMigration(checkpoint, Boolean(options.signal?.aborted));
   validation.issues.push(...navigationIssues);
   if (navigationIssues.length && validation.status === "passed") validation.status = "partial";
@@ -18334,12 +18415,12 @@ ${difference}`;
     platform: options.platform ?? "generic",
     pages,
     assets: [...assets.values()],
-    docsConfig: navigationConfig(checkpoint, records) ?? buildNavigationFromPages(pages),
+    docsConfig: navigationConfig(checkpoint, records, occurrencePages) ?? buildNavigationFromPages(pages),
     warnings: [...validation.issues, ...validation.warnings ?? []].map((issue) => ({ code: issue.code === "missing-page" ? "missing-page" : "unsupported-config", message: issue.message, source: issue.source })),
     urlMigrationValidation: validation,
     stats: { discovered: records.size, imported: pages.length, skipped: records.size - pages.length }
   };
-  if (projection.redirects.length) bundle.docsConfig.redirects = projection.redirects;
+  if (projection.redirects.length || navigationRedirects.length) bundle.docsConfig.redirects = [...projection.redirects, ...navigationRedirects];
   if (projection.locales.length) {
     const locales = [.../* @__PURE__ */ new Set([projection.defaultLocale, ...projection.locales])];
     bundle.docsConfig.i18n = { ...bundle.docsConfig.i18n, defaultLocale: projection.defaultLocale, locales: locales.map((code) => ({ code, label: code === "en" ? "English" : code })) };
@@ -18357,9 +18438,11 @@ function rewriteMarkdownDestinations(body, rewrite) {
     const original = body.slice(start, end);
     const replacement = rewrite(node.url);
     if (replacement === node.url) return null;
-    const index = original.lastIndexOf(node.url);
-    if (index < 0) return null;
-    return { start: start + index, end: start + index + node.url.length, replacement };
+    const destinations = postprocess(parse2().document().write(preprocess()(original, void 0, true))).filter(([event, token]) => event === "enter" && ["resourceDestinationString", "definitionDestinationString", "autolinkProtocol", "autolinkEmail"].includes(token.type));
+    const destination = destinations.at(-1)?.[1];
+    if (!destination) return null;
+    const escaped = replacement.replace(/[\\&()<>]/g, "\\$&").replace(/[\t\n\r ]/g, (character) => encodeURIComponent(character));
+    return { start: start + destination.start.offset, end: start + destination.end.offset, replacement: escaped };
   }).filter((edit) => edit !== null).sort((a, b) => b.start - a.start);
   for (const node of markdownNodes(body)) for (const attribute of node.attributes ?? []) {
     if (!attribute.name || !["src", "poster", "href", "srcSet", "srcset", "img", "primaryHref", "secondaryHref"].includes(attribute.name) || !attribute.position) continue;