@thallylabs/migrate 0.1.0 → 0.1.2

package/dist/index.js

@@ -29,7 +29,6 @@ function normalizeMdx(body) {
 function parseMarkdownPage(input) {
   const parsed = matter(input.raw);
   const body = normalizeMdx(parsed.content).trim();
-  if (parsed.data.openapi && !body) return null;
   const keywords = Array.isArray(parsed.data.keywords) ? parsed.data.keywords.filter((value) => typeof value === "string") : [];
   const title = typeof parsed.data.title === "string" && parsed.data.title.trim() ? parsed.data.title.trim() : titleFromId(input.navigationId ?? input.id);
   const description = typeof parsed.data.description === "string" && parsed.data.description.trim() ? parsed.data.description.trim() : firstParagraph(body);
@@ -40,6 +39,7 @@ function parseMarkdownPage(input) {
     title,
     description,
     keywords,
+    openapi: typeof parsed.data.openapi === "string" ? parsed.data.openapi.trim() : void 0,
     body,
     source: input.source
   };
@@ -308,9 +308,91 @@ function projectMintlifyNavigation(config) {
 function titleCase(value) {
   return value.split(/[-_]/).map((word) => ["api", "cli", "sdk", "ui"].includes(word.toLowerCase()) ? word.toUpperCase() : word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
 }
-function buildNavigationFromPages(pages) {
+var TOP_LEVEL_LABELS = {
+  "api-reference": "API Reference",
+  cli: "CLI",
+  faqs: "FAQs",
+  introduction: "Overview",
+  "mcp-server": "MCP Server",
+  sdk: "SDK"
+};
+function topLevelLabel(segment) {
+  return TOP_LEVEL_LABELS[segment] ?? titleCase(segment);
+}
+function groupsWithinSection(section, pageIds, sectionLabel = topLevelLabel(section)) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const id of pageIds) {
+    const relative4 = id === "introduction" ? "" : id.startsWith(`${section}/`) ? id.slice(section.length + 1) : id;
+    const nestedSegment = relative4.includes("/") ? relative4.split("/", 1)[0] : "overview";
+    const group = groups.get(nestedSegment) ?? [];
+    group.push(id);
+    groups.set(nestedSegment, group);
+  }
+  return [...groups].map(([segment, pages]) => ({
+    group: segment === "overview" ? sectionLabel : titleCase(segment),
+    pages
+  }));
+}
+function preferredLandingPage(section, pageIds) {
+  const candidates = [
+    section === "introduction" ? "introduction" : void 0,
+    `${section}/overview`,
+    `${section}/introduction`,
+    section
+  ];
+  return candidates.find((candidate) => Boolean(candidate && pageIds.includes(candidate)));
+}
+function buildNavigationFromPages(pages, options = {}) {
   const ids = pages.filter((page) => !page.locale || page.locale === "en").map((page) => page.navigationId);
   const ordered = [...new Set(ids)];
+  if (options.topLevelTabs) {
+    const sourceNavigation = (options.topLevelNavigation ?? []).flatMap((entry) => {
+      const sectionPages = ordered.filter((id) => id === entry.section || id.startsWith(`${entry.section}/`));
+      const pageId = ordered.includes(entry.pageId) ? entry.pageId : preferredLandingPage(entry.section, sectionPages) ?? sectionPages[0];
+      return pageId ? [{ ...entry, pageId }] : [];
+    });
+    if (sourceNavigation.length > 1) {
+      const claimedIds = /* @__PURE__ */ new Set();
+      const tabs = sourceNavigation.map((entry) => {
+        const pageIds = ordered.filter((id) => {
+          const matches = id === entry.pageId || id === entry.section || id.startsWith(`${entry.section}/`);
+          if (matches) claimedIds.add(id);
+          return matches;
+        });
+        return { entry, pageIds };
+      });
+      tabs[0].pageIds.push(...ordered.filter((id) => !claimedIds.has(id)));
+      return {
+        tabs: tabs.map(({ entry, pageIds }) => ({
+          tab: entry.label,
+          href: entry.pageId === "introduction" ? "/" : `/${entry.pageId}`,
+          groups: groupsWithinSection(entry.section, [...new Set(pageIds)], entry.label)
+        }))
+      };
+    }
+    const sectionNames = [...new Set(ordered.filter((id) => id.includes("/")).map((id) => id.split("/", 1)[0]))];
+    if (sectionNames.length > 1) {
+      const defaultSection = sectionNames.includes("introduction") ? "introduction" : sectionNames[0];
+      const sections = /* @__PURE__ */ new Map();
+      for (const id of ordered) {
+        const section = id.includes("/") ? id.split("/", 1)[0] : defaultSection;
+        const bucket = sections.get(section) ?? [];
+        bucket.push(id);
+        sections.set(section, bucket);
+      }
+      return {
+        tabs: [...sections].map(([section, pageIds]) => {
+          const label = topLevelLabel(section);
+          const landingPage = preferredLandingPage(section, pageIds);
+          return {
+            tab: label,
+            ...landingPage ? { href: landingPage === "introduction" ? "/" : `/${landingPage}` } : {},
+            groups: groupsWithinSection(section, pageIds)
+          };
+        })
+      };
+    }
+  }
   const buckets = /* @__PURE__ */ new Map();
   for (const id of ordered) {
     const segment = id.includes("/") ? id.split("/", 1)[0] : "overview";
@@ -700,6 +782,7 @@ function renderPage(bundle, page) {
     `title: ${yamlString(page.title)}`,
     `description: ${yamlString(page.description)}`,
     page.keywords.length > 0 ? `keywords: [${page.keywords.map(yamlString).join(", ")}]` : null,
+    page.openapi ? `openapi: ${yamlString(page.openapi)}` : null,
     bundle.sourceKind === "url" ? `source: ${yamlString(page.source)}` : null,
     "---",
     "",
@@ -778,6 +861,7 @@ function renderMigrationFiles(bundle, options = {}) {
 // src/url.ts
 import { load } from "cheerio";
 import TurndownService from "turndown";
+import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
 var DEFAULT_MAX_PAGES = 1e3;
 var DEFAULT_MAX_TOTAL_BYTES = 1e8;
 var MAX_DISCOVERED_URLS = 5e3;
@@ -857,8 +941,12 @@ var LOCALE_CODES = /* @__PURE__ */ new Set([
   "tr",
   "uk",
   "vi",
-  "zh"
+  "zh",
+  "pt-br",
+  "zh-hans",
+  "zh-hant"
 ]);
+var PORTABLE_URL_PROPS = "href|src|img|primaryHref|secondaryHref";
 function validateMigrationUrl(value) {
   let url;
   try {
@@ -921,6 +1009,27 @@ function docsScopePath(source) {
   const segments = path.split("/").filter(Boolean);
   return `/${segments.length > 1 ? segments[0] : segments.join("/")}`;
 }
+function machineIndexScopePath(source, documents) {
+  const candidates = /* @__PURE__ */ new Set();
+  for (const document of documents) {
+    if (!document) continue;
+    const references = [...headerLinks(document)];
+    if (/html/i.test(document.contentType)) {
+      const $ = load(document.body);
+      references.push(...$("link[href], a[href]").map((_index, element) => $(element).attr("href") ?? "").get().filter((href) => /\/(?:llms(?:-full)?\.txt|sitemap\.xml)(?:[?#]|$)/i.test(href)));
+    }
+    for (const reference of references) {
+      try {
+        const indexUrl = new URL(reference, document.finalUrl);
+        if (indexUrl.origin !== source.origin) continue;
+        const scope = indexUrl.pathname.replace(/\/(?:llms(?:-full)?\.txt|sitemap\.xml)$/i, "") || "/";
+        if (isInScope(source, source, scope)) candidates.add(scope);
+      } catch {
+      }
+    }
+  }
+  return [...candidates].sort((left, right) => right.length - left.length)[0] ?? null;
+}
 function isInScope(url, source, scopePath) {
   if (url.origin !== source.origin || !["http:", "https:"].includes(url.protocol)) return false;
   const path = url.pathname.replace(/\/+$/, "") || "/";
@@ -932,7 +1041,9 @@ function normalizeCandidate(value, base, source, scopePath) {
     const url = new URL(value, base);
     url.hash = "";
     if (!isInScope(url, source, scopePath)) return null;
-    if (/\.(?:avif|bmp|gif|ico|jpe?g|mp[34]|pdf|png|svg|webm|webp|zip)$/i.test(url.pathname)) return null;
+    if (url.pathname.startsWith("/cdn-cgi/")) return null;
+    if (/\/(?:llms(?:-full)?|robots)\.txt$/i.test(url.pathname) || /\/sitemap\.xml$/i.test(url.pathname)) return null;
+    if (/\.(?:avif|bmp|gif|ico|jpe?g|mp[34]|pdf|png|svg|ya?ml|webm|webp|zip)$/i.test(url.pathname) || /\/(?:openapi|swagger|asyncapi)[^/]*\.json$/i.test(url.pathname)) return null;
     return url;
   } catch {
     return null;
@@ -951,9 +1062,19 @@ function pageIdForUrl(url, scopePath) {
   }
   return pageIdFromReference(path.replace(/^\/+/, "") || "introduction");
 }
+function canonicalLocalizedPageId(id) {
+  const [segment, ...rest] = id.split("/");
+  if (LOCALE_CODES.has(segment)) return rest.length > 0 ? id : `${segment}/introduction`;
+  for (const locale of LOCALE_CODES) {
+    if (segment !== `${locale}-api-reference`) continue;
+    return [locale, "api-reference", ...rest].join("/");
+  }
+  return id;
+}
 function detectUrlPlatform(document) {
-  const value = `${document.body.slice(0, 2e5)} ${document.headers?.["x-powered-by"] ?? ""}`.toLowerCase();
-  if (value.includes("mintlify") || value.includes("__mintlify")) return "mintlify";
+  const headers = Object.entries(document.headers ?? {}).map(([key, value2]) => `${key}:${value2 ?? ""}`).join(" ");
+  const value = `${document.body.slice(0, 2e5)} ${headers}`.toLowerCase();
+  if (value.includes("__mintlify") || value.includes("/mintlify-assets/") || value.includes("x-mintlify-") || value.includes("/_mintlify/")) return "mintlify";
   if (value.includes("docusaurus") || value.includes("__docusaurus")) return "docusaurus";
   if (value.includes("gitbook") || value.includes("gitbook.io")) return "gitbook";
   if (value.includes("nextra")) return "nextra";
@@ -982,10 +1103,122 @@ function withoutFencedCode(body) {
     return fenceCharacter ? "" : line;
   }).join("\n");
 }
+function eventHandlerPropEnd(value, start) {
+  const opening = value[start];
+  if (opening === '"' || opening === "'") {
+    for (let index = start + 1; index < value.length; index += 1) {
+      if (value[index] === opening && value[index - 1] !== "\\") return index + 1;
+    }
+    return null;
+  }
+  if (opening !== "{") {
+    const match = value.slice(start).match(/^[^\s>]+/);
+    return match ? start + match[0].length : null;
+  }
+  let depth = 0;
+  let quote = "";
+  for (let index = start; index < value.length; index += 1) {
+    const character = value[index];
+    if (quote) {
+      if (character === quote && value[index - 1] !== "\\") quote = "";
+      continue;
+    }
+    if (character === '"' || character === "'" || character === "`") {
+      quote = character;
+      continue;
+    }
+    if (character === "{") depth += 1;
+    if (character === "}") {
+      depth -= 1;
+      if (depth === 0) return index + 1;
+    }
+  }
+  return null;
+}
+function stripEventHandlerProps(line) {
+  const matcher = /\s+on[A-Z][A-Za-z0-9]*\s*=\s*/g;
+  let cursor = 0;
+  let result = "";
+  for (const match of line.matchAll(matcher)) {
+    const start = match.index;
+    const valueStart = start + match[0].length;
+    const end = eventHandlerPropEnd(line, valueStart);
+    if (end === null) continue;
+    result += line.slice(cursor, start);
+    cursor = end;
+  }
+  return cursor === 0 ? line : `${result}${line.slice(cursor)}`;
+}
+function hasUnsafeUrlScheme(value) {
+  const normalized = value.trim().replace(/[\u0000-\u0020]/g, "").replace(/&(?:#x0*3a|#0*58|colon);/gi, ":");
+  return /^(?:javascript|data|vbscript):/i.test(normalized);
+}
+function hasUnsafePortableUrlProp(value) {
+  const quotedProps = new RegExp(`\\b(?:${PORTABLE_URL_PROPS})\\s*=\\s*(['"])([^'"\\r\\n]*)\\1`, "gi");
+  for (const match of value.matchAll(quotedProps)) {
+    if (hasUnsafeUrlScheme(match[2])) return true;
+  }
+  const expressionProps = new RegExp(`\\b(?:${PORTABLE_URL_PROPS})\\s*=\\s*\\{\\s*("(?:\\\\.|[^"\\\\])*")\\s*\\}`, "gi");
+  for (const match of value.matchAll(expressionProps)) {
+    try {
+      if (hasUnsafeUrlScheme(JSON.parse(match[1]))) return true;
+    } catch {
+      return true;
+    }
+  }
+  return false;
+}
+function portableHtmlToMarkdown(value) {
+  return value.replace(/<span\b[^>]*>|<\/span>/gi, "").replace(/<a\b([^>]*)>([\s\S]*?)<\/a>/gi, (_original, attributes, label) => {
+    const href = attributes.match(/\bhref\s*=\s*(['"])(.*?)\1/i)?.[2];
+    const plainLabel = label.replace(/<[^>]+>/g, "").trim();
+    if (!href || hasUnsafeUrlScheme(href)) return plainLabel;
+    return `[${plainLabel.replace(/]/g, "\\]")}](${href})`;
+  }).replace(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi, (_original, level, content) => {
+    return `
+${"#".repeat(Number(level))} ${content.trim()}
+`;
+  }).replace(/<p\b[^>]*>([\s\S]*?)<\/p>/gi, (_original, content) => `
+${content.trim()}
+`).replace(/<br\s*\/?\s*>/gi, "\n").replace(/<\/?(?:article|div|main|section)\b[^>]*>/gi, "");
+}
+function transformMarkdownProse(body, transform) {
+  const output = [];
+  let prose = [];
+  let fenceCharacter = "";
+  let fenceLength = 0;
+  const flushProse = () => {
+    if (prose.length === 0) return;
+    output.push(transform(prose.join("\n")));
+    prose = [];
+  };
+  for (const line of body.split("\n")) {
+    const fence = line.match(/^\s{0,3}(`{3,}|~{3,})(.*)$/);
+    if (fence && !fenceCharacter) {
+      flushProse();
+      fenceCharacter = fence[1][0];
+      fenceLength = fence[1].length;
+      output.push(line);
+      continue;
+    }
+    if (fenceCharacter) {
+      output.push(line);
+      if (fence && fence[1][0] === fenceCharacter && fence[1].length >= fenceLength && fence[2].trim() === "") {
+        fenceCharacter = "";
+        fenceLength = 0;
+      }
+      continue;
+    }
+    prose.push(line);
+  }
+  flushProse();
+  return output.join("\n");
+}
 function sanitizeRemoteMarkdown(body) {
   let fenceCharacter = "";
   let fenceLength = 0;
-  const sanitized = body.split("\n").map((line) => {
+  const portableBody = transformMarkdownProse(body, portableHtmlToMarkdown);
+  const sanitized = portableBody.split("\n").map((line) => {
     const fence = line.match(/^\s{0,3}(`{3,}|~{3,})(.*)$/);
     if (fence && !fenceCharacter) {
       fenceCharacter = fence[1][0];
@@ -998,12 +1231,14 @@ function sanitizeRemoteMarkdown(body) {
       return line;
     }
     if (fenceCharacter) return line;
-    return line.replace(/<\/?div\b[^>]*>/gi, "").replace(/<img\s+([^>]*?)\/?\s*>/gi, (original, attributes) => {
+    return portableHtmlToMarkdown(stripEventHandlerProps(line.replace(/!\[([^\]]*)\]\(\s*([^)]+)\)/g, (original, alternative, destination) => {
+      return hasUnsafeUrlScheme(destination.replace(/^<|>$/g, "")) ? alternative : original;
+    }).replace(/<img\s+([^>]*?)\/?\s*>/gi, (original, attributes) => {
       const source = attributes.match(/\bsrc\s*=\s*(['"])(.*?)\1/i)?.[2];
       const alternative = attributes.match(/\balt\s*=\s*(['"])(.*?)\1/i)?.[2] ?? "";
-      if (!source || /^(?:javascript|data):/i.test(source)) return "";
+      if (!source || hasUnsafeUrlScheme(source)) return "";
       return `![${alternative.replace(/]/g, "\\]")}](${source})`;
-    });
+    })));
   }).join("\n");
   const executable = withoutFencedCode(sanitized).replace(/\{\/\*[\s\S]*?\*\/\}/g, "").replace(/<(?:https?:\/\/|mailto:)[^>]+>/gi, "").replace(/(`+)[\s\S]*?\1/g, "");
   if (/^\s*(?:import|export)\s/m.test(executable)) return null;
@@ -1017,27 +1252,35 @@ function sanitizeRemoteMarkdown(body) {
       return original;
     }
   }).replace(/\\[{}]/g, "");
-  if (hasUnsafeExpression || /[{}]/.test(withoutStaticProps) || /\bon[A-Z][A-Za-z]*\s*=/g.test(withoutStaticProps) || /\b(?:href|src)\s*=\s*(['"])\s*(?:javascript|data):/gi.test(withoutStaticProps)) return null;
+  if (hasUnsafeExpression || /[{}]/.test(withoutStaticProps) || /\bon[A-Z][A-Za-z]*\s*=/g.test(withoutStaticProps) || hasUnsafePortableUrlProp(executable)) return null;
   for (const match of withoutStaticProps.matchAll(/<\/?([A-Za-z][A-Za-z0-9.]*)(?:\s|>|\/)/g)) {
     const component = match[1].split(".", 1)[0];
     if (!PORTABLE_MDX_COMPONENTS.has(component)) return null;
   }
   return sanitized;
 }
-function migratedHref(href, currentUrl, source, scopePath, importedIds) {
+function migratedHref(href, currentUrl, source, scopePath, importedIds, sourceHomePageId) {
   if (/^(?:javascript|data|vbscript):/i.test(href)) return "#";
   if (href.startsWith("#") || /^(?:mailto|tel):/i.test(href)) return href;
   try {
     const target = new URL(href, currentUrl);
     if (target.origin !== source.origin) return href;
+    if (scopePath === "/" && candidateIdentity(target) === candidateIdentity(source)) {
+      return `/${target.search}${target.hash}`;
+    }
     let path = target.pathname;
     if (scopePath !== "/" && (path === scopePath || path.startsWith(`${scopePath}/`))) {
       path = path.slice(scopePath.length);
     }
     const id = pageIdFromReference(path.replace(/^\/+/, "") || "introduction");
-    const localizedId = id && LOCALE_CODES.has(id) ? `${id}/introduction` : id;
-    if (localizedId && importedIds.has(localizedId)) {
-      const destination = localizedId === "introduction" ? "/" : `/${localizedId.replace(/\/introduction$/, "")}`;
+    const localizedId = id ? canonicalLocalizedPageId(id) : id;
+    const destinationId = localizedId === sourceHomePageId ? "introduction" : localizedId;
+    let importedDestinationId = destinationId && importedIds.has(destinationId) ? destinationId : void 0;
+    if (destinationId && !importedDestinationId) {
+      importedDestinationId = [`${destinationId}/overview`, `${destinationId}/introduction`].find((candidate) => importedIds.has(candidate)) ?? [...importedIds].find((candidate) => candidate.startsWith(`${destinationId}/`));
+    }
+    if (importedDestinationId) {
+      const destination = importedDestinationId === "introduction" ? "/" : `/${importedDestinationId}`;
       return `${destination}${target.search}${target.hash}`;
     }
     return target.toString();
@@ -1045,13 +1288,13 @@ function migratedHref(href, currentUrl, source, scopePath, importedIds) {
     return href;
   }
 }
-function rewriteInternalLinks(body, currentUrl, source, scopePath, importedIds) {
+function rewriteInternalLinks(body, currentUrl, source, scopePath, importedIds, sourceHomePageId) {
   const markdown = body.replace(/(?<!!)\[([^\]]+)\]\(([^)\s]+)(\s+['"][^'"]*['"])?\)/g, (original, label, href, title = "") => {
-    const destination = migratedHref(href, currentUrl, source, scopePath, importedIds);
+    const destination = migratedHref(href, currentUrl, source, scopePath, importedIds, sourceHomePageId);
     return destination === href ? original : `[${label}](${destination}${title})`;
   });
   return markdown.replace(/\bhref=(['"])([^'"]+)\1/g, (_original, quote, href) => {
-    const destination = migratedHref(href, currentUrl, source, scopePath, importedIds);
+    const destination = migratedHref(href, currentUrl, source, scopePath, importedIds, sourceHomePageId);
     return `href=${quote}${destination}${quote}`;
   });
 }
@@ -1080,19 +1323,126 @@ function remoteMarkdownMetadata(body) {
     description: scalar("description")
   };
 }
+function stripMintlifyDocumentationIndex(body) {
+  const lines = body.split(/\r?\n/);
+  if (!/^>\s*##\s+Documentation Index\s*$/i.test(lines[0] ?? "") || !/^>.*\/llms(?:-full)?\.txt/i.test(lines[1] ?? "") || !/^>.*discover all available pages/i.test(lines[2] ?? "")) return body;
+  let index = 3;
+  while (index < lines.length && /^>/.test(lines[index])) index += 1;
+  while (index < lines.length && lines[index].trim() === "") index += 1;
+  return lines.slice(index).join("\n");
+}
+function extractEmbeddedOpenApi(content) {
+  const fencePattern = /^\s{0,3}(`{3,}|~{3,})(?:yaml|yml|json)\s+(\S+\.(?:yaml|yml|json))\s+(delete|get|head|options|patch|post|put|trace)\s+(\/\S+)\s*\r?\n([\s\S]*?)^\s{0,3}\1\s*$/im;
+  const match = fencePattern.exec(content);
+  if (!match) return { content };
+  try {
+    const parsed = parseYaml(match[5], { maxAliasCount: 50 });
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return { content };
+    const document = parsed;
+    if (typeof document.openapi !== "string" && typeof document.swagger !== "string") return { content };
+    const method = match[3].toUpperCase();
+    const path = match[4];
+    const paths = document.paths;
+    if (!paths || typeof paths !== "object" || Array.isArray(paths)) return { content };
+    const pathItem = paths[path];
+    if (!pathItem || typeof pathItem !== "object" || Array.isArray(pathItem) || !(match[3].toLowerCase() in pathItem)) return { content };
+    const before = content.slice(0, match.index).replace(/(?:^|\n)##\s+OpenAPI\s*\n\s*$/i, "\n");
+    const after = content.slice(match.index + match[0].length);
+    return {
+      content: `${before}${after}`.trim(),
+      openapi: `${method} ${path}`,
+      fragment: {
+        method,
+        path,
+        document
+      }
+    };
+  } catch {
+    return { content };
+  }
+}
 function markdownPage(document, id) {
   const parsed = remoteMarkdownMetadata(document.body);
-  const titleMatch = parsed.content.match(/^#\s+(.+)$/m);
+  let content = stripMintlifyDocumentationIndex(parsed.content).trimStart();
+  const titleMatch = content.match(/^#\s+(.+)\r?\n/);
   const title = parsed.title ?? titleMatch?.[1] ?? id.split("/").at(-1) ?? "Introduction";
+  if (titleMatch) content = content.slice(titleMatch[0].length).trimStart();
+  let description = parsed.description;
+  if (!description) {
+    const quoteMatch = content.match(/^(>[^\r\n]*(?:\r?\n>[^\r\n]*)*)\r?\n/);
+    if (quoteMatch) {
+      description = plainDescription(quoteMatch[1].replace(/^>\s?/gm, ""));
+      content = content.slice(quoteMatch[0].length).trimStart();
+    }
+  }
+  const embeddedOpenApi = extractEmbeddedOpenApi(content);
   const raw = [
     "---",
     `title: ${JSON.stringify(title)}`,
-    ...parsed.description ? [`description: ${JSON.stringify(parsed.description)}`] : [],
+    ...description ? [`description: ${JSON.stringify(description)}`] : [],
+    ...embeddedOpenApi.openapi ? [`openapi: ${JSON.stringify(embeddedOpenApi.openapi)}`] : [],
     "---",
     "",
-    parsed.content
+    embeddedOpenApi.content
   ].join("\n");
-  return parseMarkdownPage({ id, raw, source: document.finalUrl.toString() });
+  return {
+    page: parseMarkdownPage({ id, raw, source: document.finalUrl.toString() }),
+    openApiFragment: embeddedOpenApi.fragment
+  };
+}
+function mergeRecord(target, source) {
+  if (!source || typeof source !== "object" || Array.isArray(source)) return;
+  for (const [key, value] of Object.entries(source)) {
+    if (["__proto__", "constructor", "prototype"].includes(key)) continue;
+    if (!Object.prototype.hasOwnProperty.call(target, key)) {
+      target[key] = value;
+      continue;
+    }
+    const current = target[key];
+    if (current && value && typeof current === "object" && typeof value === "object" && !Array.isArray(current) && !Array.isArray(value)) {
+      mergeRecord(current, value);
+    }
+  }
+}
+function mergeEmbeddedOpenApi(fragments) {
+  const first = fragments[0];
+  if (!first) return null;
+  const combined = {
+    ...first.document,
+    paths: {},
+    components: {}
+  };
+  const combinedPaths = combined.paths;
+  const combinedComponents = combined.components;
+  const tags = /* @__PURE__ */ new Map();
+  for (const fragment of fragments) {
+    const sourcePaths = fragment.document.paths;
+    const sourcePathItem = sourcePaths?.[fragment.path];
+    if (!sourcePathItem || typeof sourcePathItem !== "object" || Array.isArray(sourcePathItem)) continue;
+    const sourceOperation = sourcePathItem[fragment.method.toLowerCase()];
+    if (!sourceOperation || typeof sourceOperation !== "object" || Array.isArray(sourceOperation)) continue;
+    const operation = { ...sourceOperation };
+    if (!operation.servers && Array.isArray(fragment.document.servers)) operation.servers = fragment.document.servers;
+    if (!operation.security && Array.isArray(fragment.document.security)) operation.security = fragment.document.security;
+    const existingPath = combinedPaths[fragment.path];
+    const pathItem = existingPath && typeof existingPath === "object" && !Array.isArray(existingPath) ? existingPath : {};
+    for (const sharedKey of ["parameters", "servers", "summary", "description"]) {
+      const sharedValue = sourcePathItem[sharedKey];
+      if (sharedValue !== void 0 && pathItem[sharedKey] === void 0) pathItem[sharedKey] = sharedValue;
+    }
+    pathItem[fragment.method.toLowerCase()] = operation;
+    combinedPaths[fragment.path] = pathItem;
+    mergeRecord(combinedComponents, fragment.document.components);
+    for (const tag of Array.isArray(fragment.document.tags) ? fragment.document.tags : []) {
+      if (!tag || typeof tag !== "object" || Array.isArray(tag)) continue;
+      const name = tag.name;
+      if (typeof name === "string" && !tags.has(name)) tags.set(name, tag);
+    }
+  }
+  if (Object.keys(combinedPaths).length === 0) return null;
+  if (Object.keys(combinedComponents).length === 0) delete combined.components;
+  if (tags.size > 0) combined.tags = [...tags.values()];
+  return new TextEncoder().encode(stringifyYaml(combined));
 }
 function htmlPage(document, id) {
   const $ = load(document.body);
@@ -1166,6 +1516,48 @@ ${fence}
     links
   };
 }
+function htmlNavigationLinks(document) {
+  const $ = load(document.body);
+  return $('.nav-tabs a[href], nav a[href], aside a[href], [role="navigation"] a[href]').map((_index, element) => $(element).attr("href") ?? "").get().filter(Boolean);
+}
+function htmlTopLevelNavigation(document, source, scopePath) {
+  const $ = load(document.body);
+  const collectCandidates = (selector) => {
+    const candidates2 = [];
+    $(selector).each((_containerIndex, container) => {
+      const entries = [];
+      const seenSections = /* @__PURE__ */ new Set();
+      let anchorCount = 0;
+      $(container).find("a[href]").each((_index, element) => {
+        const label = $(element).text().replace(/\s+/g, " ").trim();
+        const href = $(element).attr("href");
+        if (!label || !href) return;
+        try {
+          const target = new URL(href, document.finalUrl);
+          if (!isInScope(target, source, scopePath)) return;
+          anchorCount += 1;
+          const targetId = canonicalLocalizedPageId(pageIdForUrl(target, scopePath) ?? "");
+          const pageId = targetId;
+          const section = targetId.split("/", 1)[0] || "introduction";
+          if (!pageId || seenSections.has(section)) return;
+          seenSections.add(section);
+          entries.push({ section, label, pageId });
+        } catch {
+        }
+      });
+      if (entries.length > 1) {
+        candidates2.push({ entries, anchorCount });
+      }
+    });
+    return candidates2;
+  };
+  const tabCandidates = collectCandidates(".nav-tabs");
+  const candidates = tabCandidates.length > 0 ? tabCandidates : collectCandidates('nav, [role="navigation"]');
+  return candidates.sort((left, right) => {
+    const densityDifference = right.entries.length / right.anchorCount - left.entries.length / left.anchorCount;
+    return densityDifference || right.entries.length - left.entries.length;
+  })[0]?.entries ?? [];
+}
 function markdownLinks(body) {
   return [...body.matchAll(/\[[^\]]+\]\(([^)\s]+)(?:\s+['"][^'"]*['"])?\)/g)].map((match) => match[1]);
 }
@@ -1199,12 +1591,17 @@ async function migrateUrl(options) {
   const maxPages = Math.max(1, Math.min(options.maxPages ?? DEFAULT_MAX_PAGES, 1e3));
   const concurrency = Math.max(1, Math.min(options.concurrency ?? 5, 10));
   const maxTotalBytes = Math.max(1e6, Math.min(options.maxTotalBytes ?? DEFAULT_MAX_TOTAL_BYTES, 5e8));
-  const scopePath = docsScopePath(source);
+  const submittedScopePath = docsScopePath(source);
   const initial = await fetcher(source, { accept: "text/markdown,text/html,application/xhtml+xml;q=0.9" });
-  if (!isInScope(initial.finalUrl, source, scopePath)) {
+  if (!isInScope(initial.finalUrl, source, submittedScopePath)) {
     throw new Error("The documentation URL redirected outside the submitted docs origin or path.");
   }
-  const platform = detectUrlPlatform(initial);
+  const htmlProbe = /html/i.test(initial.contentType) ? initial : await safeFetch(fetcher, source, "text/html,application/xhtml+xml");
+  const scopedHtmlProbe = htmlProbe && isInScope(htmlProbe.finalUrl, source, submittedScopePath) ? htmlProbe : null;
+  const platform = detectUrlPlatform(scopedHtmlProbe ?? initial);
+  const scopePath = platform === "mintlify" ? machineIndexScopePath(source, [initial, scopedHtmlProbe]) ?? submittedScopePath : submittedScopePath;
+  const sourceTopLevelNavigation = platform === "mintlify" && scopedHtmlProbe && /html/i.test(scopedHtmlProbe.contentType) ? htmlTopLevelNavigation(scopedHtmlProbe, source, scopePath) : void 0;
+  const sourceHomePageId = sourceTopLevelNavigation?.[0]?.pageId ?? (platform === "mintlify" && scopePath === "/" ? canonicalLocalizedPageId(pageIdForUrl(source, scopePath) ?? "") || void 0 : void 0);
   const cache = /* @__PURE__ */ new Map([[source.toString(), initial]]);
   const queue = [source];
   const queued = /* @__PURE__ */ new Set([candidateIdentity(source)]);
@@ -1217,6 +1614,9 @@ async function migrateUrl(options) {
     queued.add(candidateIdentity(url));
     queue.push(url);
   }
+  if (scopedHtmlProbe && /html/i.test(scopedHtmlProbe.contentType)) {
+    for (const link of htmlNavigationLinks(scopedHtmlProbe)) enqueue(link, scopedHtmlProbe.finalUrl);
+  }
   async function discoverDocumentLinks(document, depth = 0) {
     const links = sitemapLinks(document.body);
     for (const link of links) {
@@ -1282,7 +1682,9 @@ async function migrateUrl(options) {
     }
   }
   const pages = [];
+  const openApiFragments = [];
   const seenIds = /* @__PURE__ */ new Set();
+  const seenSources = /* @__PURE__ */ new Set();
   const visited = /* @__PURE__ */ new Set();
   let importedBytes = 0;
   let isByteBudgetExhausted = false;
@@ -1320,6 +1722,8 @@ async function migrateUrl(options) {
         const htmlDocument = await safeFetch(fetcher, htmlUrl, "text/html,application/xhtml+xml");
         if (htmlDocument && isInScope(htmlDocument.finalUrl, source, scopePath) && /html/i.test(htmlDocument.contentType)) {
           document = htmlDocument;
+        } else {
+          return { candidate, page: null, links: [], failure: false };
         }
       } else if (sanitizedMarkdown !== null) {
         document = { ...document, body: sanitizedMarkdown };
@@ -1330,12 +1734,15 @@ async function migrateUrl(options) {
         return { candidate, page: null, links: [], failure: false, budgetExceeded: true };
       }
       importedBytes += responseBytes;
-      const id = pageIdForUrl(document.finalUrl, scopePath) ?? pageIdForUrl(candidate, scopePath);
+      const discoveredId = pageIdForUrl(document.finalUrl, scopePath) ?? pageIdForUrl(candidate, scopePath);
+      const id = discoveredId && canonicalLocalizedPageId(discoveredId) === sourceHomePageId ? "introduction" : discoveredId;
       if (!id) return { candidate, page: null, links: [], failure: false };
       if (/(?:markdown|text\/plain)/i.test(document.contentType) || /\.mdx?$/i.test(document.finalUrl.pathname)) {
+        const markdown = markdownPage(document, id);
         return {
           candidate,
-          page: markdownPage(document, id),
+          page: markdown.page,
+          openApiFragment: markdown.openApiFragment,
           links: markdownLinks(document.body),
           base: document.finalUrl,
           failure: false
@@ -1350,9 +1757,15 @@ async function migrateUrl(options) {
         continue;
       }
       if (result.budgetExceeded) continue;
-      if (result.page && !seenIds.has(result.page.id) && pages.length < maxPages) {
+      if (result.page) result.page.id = canonicalLocalizedPageId(result.page.id);
+      const sourceIdentity = result.page ? candidateIdentity(new URL(result.page.source)) : null;
+      if (result.page && sourceIdentity && !seenSources.has(sourceIdentity) && !seenIds.has(result.page.id) && pages.length < maxPages) {
         pages.push(result.page);
+        if ("openApiFragment" in result && result.openApiFragment) {
+          openApiFragments.push(result.openApiFragment);
+        }
         seenIds.add(result.page.id);
+        seenSources.add(sourceIdentity);
       }
       for (const link of result.links) enqueue(link, result.base ?? result.candidate);
     }
@@ -1368,14 +1781,41 @@ async function migrateUrl(options) {
   }
   const importedIds = new Set(pages.map((page) => page.id));
   for (const page of pages) {
-    page.body = rewriteInternalLinks(page.body, new URL(page.source), source, scopePath, importedIds);
+    page.body = rewriteInternalLinks(
+      page.body,
+      new URL(page.source),
+      source,
+      scopePath,
+      importedIds,
+      sourceHomePageId
+    );
   }
   if (isByteBudgetExhausted) {
     warnings.push({ code: "limit-reached", message: `Import stopped after reaching the ${Math.round(maxTotalBytes / 1e6)} MB content budget.` });
   } else if (queue.length > 0 || queued.size >= MAX_DISCOVERED_URLS) {
     warnings.push({ code: "limit-reached", message: `Import stopped after ${pages.length} pages; narrow the URL or raise the caller limit to import more.` });
   }
-  const docsConfig = buildNavigationFromPages(pages);
+  const topLevelNavigation = sourceTopLevelNavigation?.map((entry) => ({
+    ...entry,
+    pageId: entry.pageId === sourceHomePageId ? "introduction" : entry.pageId
+  }));
+  const docsConfig = buildNavigationFromPages(pages, {
+    topLevelTabs: platform === "mintlify",
+    topLevelNavigation
+  });
+  const openApiAsset = mergeEmbeddedOpenApi(openApiFragments);
+  if (openApiAsset) {
+    const operationPageIds = new Set(pages.filter((page) => page.openapi).map((page) => page.navigationId));
+    const apiTab = docsConfig.tabs.find((tab) => tab.groups?.some((group) => {
+      const containsOperation = (pages2) => pages2.some((page) => {
+        return typeof page === "string" ? operationPageIds.has(page) : containsOperation(page.pages);
+      });
+      return containsOperation(group.pages);
+    })) ?? docsConfig.tabs[0];
+    if (apiTab) {
+      apiTab.api = { source: "/openapi.yaml", navigation: false };
+    }
+  }
   const locales = [...new Set(pages.map((page) => page.locale).filter((value) => Boolean(value)))];
   if (locales.length > 0) {
     docsConfig.i18n = {
@@ -1391,7 +1831,7 @@ async function migrateUrl(options) {
     sourceKind: "url",
     platform,
     pages,
-    assets: [],
+    assets: openApiAsset ? [{ path: "openapi.yaml", content: openApiAsset }] : [],
     docsConfig,
     warnings,
     stats: {