@thallylabs/migrate 0.2.19 → 0.2.21

package/dist/index.js

@@ -263,7 +263,8 @@ function unwrapDocusaurusMdxCodeBlocks(body) {
   }
   return isMdxCodeBlock ? body : output.join("\n");
 }
-function normalizeExplicitHeadingIds(raw) {
+var HTML_ID = /^[^\s"'<>`{}&]+$/u;
+function normalizeExplicitHeadingIds(raw, warn) {
   const { front, body } = splitFrontmatterBlock(raw);
   return front + replaceOutsideCode(body, (segment) => segment.split("\n").map((line) => {
     if (!/^[ \t]{0,3}#{1,6}[ \t]+/.test(line)) return line;
@@ -272,10 +273,12 @@ function normalizeExplicitHeadingIds(raw) {
       const marker = trimmed.lastIndexOf(" {#");
       if (marker >= 0) {
         const id = trimmed.slice(marker + 3, -1);
-        if (/^[A-Za-z][A-Za-z0-9_-]*$/.test(id)) {
-          return `<a id="${id}"></a>
-${trimmed.slice(0, marker).trimEnd()}`;
-        }
+        const heading = trimmed.slice(0, marker).trimEnd();
+        if (/[{}]/.test(id)) return line;
+        if (HTML_ID.test(id)) return `<a id="${id}"></a>
+${heading}`;
+        warn?.(`Heading anchor {#${id}} is not a valid HTML id and was removed from "${heading.replace(/^\s*#+\s*/, "")}".`);
+        return heading;
       }
     }
     if (trimmed.endsWith("*/}")) {
@@ -283,7 +286,7 @@ ${trimmed.slice(0, marker).trimEnd()}`;
       if (marker >= 0) {
         const comment = trimmed.slice(marker + 4, -3).trim();
         const id = comment.startsWith("#") ? comment.slice(1) : "";
-        if (/^[A-Za-z][A-Za-z0-9_.:-]*$/.test(id)) {
+        if (HTML_ID.test(id)) {
           return `<a id="${id}"></a>
 ${trimmed.slice(0, marker).trimEnd()}`;
         }
@@ -671,13 +674,25 @@ function maskCode(body) {
   if (fenceBuffer.length) lines.push(stash(fenceBuffer.join("\n")));
   return {
     masked: lines.join("\n"),
-    unmask: (text2) => text2.replace(/\u0000(\d+)\u0000/g, (_match, index) => blocks[Number(index)])
+    unmask: (text2) => text2.replace(/\u0000(\d+)\u0000/g, (_match, index) => blocks[Number(index)]),
+    openFence: codeFence
   };
 }
+function closeOpenFence(body) {
+  const { openFence } = maskCode(body);
+  return openFence ? `${body}${body.endsWith("\n") ? "" : "\n"}${openFence}
+` : body;
+}
 function replaceOutsideCode(body, transform) {
   const { masked, unmask } = maskCode(body);
   return unmask(transform(masked));
 }
+function mdxComment(text2) {
+  return `{/*${text2.replace(/\*\//g, "* /")}*/}`;
+}
+function replaceOutsideCodeAndComments(body, transform) {
+  return replaceOutsideCode(body, (masked) => masked.split(/(\{\s*\/\*[\s\S]*?\*\/\s*\})/).map((part, index) => index % 2 ? part : transform(part)).join(""));
+}
 function rewriteFernRelativePageLinks(body, currentSourcePath, routesBySourcePath) {
   const rewrite = (target) => {
     if (!target || target.startsWith("/") || target.startsWith("#") || target.startsWith("?") || target.includes("\\") || /[\u0000-\u001f\u007f]/.test(target) || /^[A-Za-z][A-Za-z\d+.-]*:/.test(target)) return target;
@@ -739,7 +754,7 @@ function normalizeFernFileTrees(body) {
   ));
 }
 function normalizeHtmlComments(body) {
-  return replaceOutsideCode(body, (whole) => whole.replace(/<!--([\s\S]*?)-->/g, (_match, comment) => `{/*${comment}*/}`));
+  return replaceOutsideCode(body, (whole) => whole.replace(/<!--([\s\S]*?)-->/g, (_match, comment) => mdxComment(comment)));
 }
 function normalizeFernCallouts(body) {
   return replaceOutsideCode(body, (whole) => {
@@ -1230,7 +1245,7 @@ function normalizeMdx(body, platform, unwrapMdxCodeBlocks = true) {
   }
   if (runMintlify) rewritten = normalizeMintlifyParameterAnchors(rewritten);
   rewritten = replaceOutsideCode(rewritten, (segment) => {
-    let result = convertHtmlStyleAttributes(segment).replace(/<!--([\s\S]*?)-->/g, (_match, content) => `{/*${content}*/}`).replace(/<Danger(\s[^>]*)?>/g, "<Error$1>").replace(/<\/Danger>/g, "</Error>").replace(/<Warn(\s[^>]*)?>/g, "<Warning$1>").replace(/<\/Warn>/g, "</Warning>").replace(/<Check(\s[^>]*)?>/g, "<Note$1>").replace(/<\/Check>/g, "</Note>").replace(/<Tree\.Folder/g, "<Folder").replace(/<\/Tree\.Folder>/g, "</Folder>").replace(/<Tree\.File/g, "<File").replace(/<\/Tree\.File>/g, "</File>");
+    let result = convertHtmlStyleAttributes(segment).replace(/<!--([\s\S]*?)-->/g, (_match, content) => mdxComment(content)).replace(/<Danger(\s[^>]*)?>/g, "<Error$1>").replace(/<\/Danger>/g, "</Error>").replace(/<Warn(\s[^>]*)?>/g, "<Warning$1>").replace(/<\/Warn>/g, "</Warning>").replace(/<Check(\s[^>]*)?>/g, "<Note$1>").replace(/<\/Check>/g, "</Note>").replace(/<Tree\.Folder/g, "<Folder").replace(/<\/Tree\.Folder>/g, "</Folder>").replace(/<Tree\.File/g, "<File").replace(/<\/Tree\.File>/g, "</File>");
     if (runDocusaurus) {
       result = result.replace(/<TabItem\b([^>]*)>/g, (_match, attributes) => {
         const title = attributes.match(/\blabel=(?:"([^"]*)"|'([^']*)')/)?.slice(1).find(Boolean) ?? attributes.match(/\bvalue=(?:"([^"]*)"|'([^']*)')/)?.slice(1).find(Boolean) ?? "Tab";
@@ -2380,7 +2395,8 @@ function neutralizeUnresolvableImportsWithoutAst(content, currentFile, warn) {
   });
   for (const { name, specifier } of dropped) {
     warn(`Component import ${JSON.stringify(specifier)} could not be resolved because this page has invalid MDX (see the previous warning), so the import and its usage were removed to keep the site building.`, currentFile);
-    result = result.replace(new RegExp(`<${name}(?:\\s[^>]*)?/>`, "g"), `{/* Removed <${name}>: unresolved during fallback migration */}`).replace(new RegExp(`<${name}(?:\\s[^>]*)?>[\\s\\S]*?<\\/${name}>`, "g"), `{/* Removed <${name}>: unresolved during fallback migration */}`);
+    const removed = mdxComment(` Removed <${name}>: unresolved during fallback migration `);
+    result = replaceOutsideCodeAndComments(result, (segment) => segment.replace(new RegExp(`<${name}(?:\\s[^>]*)?/>`, "g"), removed).replace(new RegExp(`<${name}(?:\\s[^>]*)?>[\\s\\S]*?<\\/${name}>`, "g"), removed));
   }
   return result;
 }
@@ -3043,7 +3059,7 @@ ${inlineSource}` });
       const start = node.position.start.offset;
       const end = node.position.end.offset;
       const videoId = youtubeEmbedVideoId(node, specifier);
-      const value = videoId ? `<iframe width="560" height="315" src="https://www.youtube.com/embed/${videoId}" title="Embedded video" allowFullScreen />` : `{/* Removed <${node.name}>: unsupported import '${specifier}' */}`;
+      const value = videoId ? `<iframe width="560" height="315" src="https://www.youtube.com/embed/${videoId}" title="Embedded video" allowFullScreen />` : mdxComment(` Removed <${node.name}>: unsupported import '${specifier}' `);
       edits.push({ start, end, value });
     });
     const rendered = applyReplacements(content, edits);
@@ -6641,7 +6657,7 @@ function selectGroupWithinBudget(scanned, rank, warnings, allVersionPrefixes, la
   const dropped = ranked.slice(budget).map(({ file }) => file);
   if (warnings) {
     const droppedPages = label === "file" ? dropped.filter((file) => isDocumentationExtension(file.relativePath)) : dropped;
-    const noun = label === "file" ? "page" : "image or media file";
+    const noun = label === "file" ? "page" : label === "snippet" ? "snippet file" : "image or media file";
     const droppedVersions = /* @__PURE__ */ new Set();
     for (const file of dropped) {
       const firstSegment = file.relativePath.split("/", 1)[0];
@@ -6651,16 +6667,19 @@ function selectGroupWithinBudget(scanned, rank, warnings, allVersionPrefixes, la
     const rest = droppedPages.length - examples.length;
     warnings.push({
       code: "limit-reached",
-      message: `This repository has more than ${budget} ${label === "file" ? "files" : "assets"}, so only the first ${budget} (in navigation order, default version first) were migrated. ${droppedPages.length} ${noun}(s) were left out` + (examples.length > 0 ? `: ${examples.join(", ")}${rest > 0 ? `, and ${rest} more` : ""}` : "") + (droppedVersions.size > 0 ? ` (versions: ${[...droppedVersions].slice(0, 5).join(", ")}${droppedVersions.size > 5 ? `, and ${droppedVersions.size - 5} more` : ""})` : "") + (label === "file" ? ". To include them, run the migration on a smaller part of the repository with --docs-dir." : ". Copy them into public/ manually if your pages use them.")
+      message: `This repository has more than ${budget} ${label === "file" ? "files" : label === "snippet" ? "snippets" : "assets"}, so only the first ${budget} (in navigation order, default version first) were migrated. ${droppedPages.length} ${noun}(s) were left out` + (examples.length > 0 ? `: ${examples.join(", ")}${rest > 0 ? `, and ${rest} more` : ""}` : "") + (droppedVersions.size > 0 ? ` (versions: ${[...droppedVersions].slice(0, 5).join(", ")}${droppedVersions.size > 5 ? `, and ${droppedVersions.size - 5} more` : ""})` : "") + (label === "asset" ? ". Copy them into public/ manually if your pages use them." : ". To include them, run the migration on a smaller part of the repository with --docs-dir.")
     });
   }
   return ranked.slice(0, budget).map(({ file }) => file);
 }
 function selectFilesWithinBudget(scanned, rank, warnings, allVersionPrefixes, budget = MAX_SOURCE_FILES) {
   const isAsset = (file) => ASSET_EXTENSIONS.has(extname6(file.relativePath).toLowerCase());
+  const isSnippet = (file) => file.relativePath.split("/").some((segment) => SNIPPET_DIRECTORIES.has(segment.toLowerCase()));
+  const snippetFiles = scanned.filter((file) => !isAsset(file) && isSnippet(file));
   const assetFiles = scanned.filter(isAsset);
-  const otherFiles = scanned.filter((file) => !isAsset(file));
+  const otherFiles = scanned.filter((file) => !isAsset(file) && !isSnippet(file));
   return [
+    ...selectGroupWithinBudget(snippetFiles, rank, warnings, allVersionPrefixes, "snippet", budget),
     ...selectGroupWithinBudget(otherFiles, rank, warnings, allVersionPrefixes, "file", budget),
     ...selectGroupWithinBudget(assetFiles, rank, warnings, allVersionPrefixes, "asset", budget)
   ];
@@ -7159,6 +7178,13 @@ function interpolateSnippet(snippet, attributes, children) {
     });
   }).join("\n");
 }
+function isolateFences(snippet, source, start, end) {
+  const fence = /^\s*(?:`{3,}|~{3,})/;
+  const lines = snippet.split("\n");
+  const before = source.slice(source.lastIndexOf("\n", start - 1) + 1, start);
+  const after = source.slice(end, source.indexOf("\n", end) < 0 ? void 0 : source.indexOf("\n", end));
+  return `${fence.test(lines[0]) && /\S/.test(before) ? "\n" : ""}${snippet}${fence.test(lines[lines.length - 1]) && /\S/.test(after) ? "\n" : ""}`;
+}
 function snippetComponentBody(source, componentName) {
   const parsed = ts3.createSourceFile("snippet.tsx", source, ts3.ScriptTarget.Latest, true, ts3.ScriptKind.TSX);
   for (const statement of parsed.statements) {
@@ -7607,7 +7633,7 @@ function inlineMdxSnippets(raw, currentFile, repositoryRoot, warnings, depth = 0
           message: `Imported snippet ${sourcePath} could not be resolved and was left as a comment.`,
           source: relative5(repositoryRoot, currentFile).replace(/\\/g, "/")
         });
-        snippets.set(componentName, `{/* Missing imported snippet: ${sourcePath} */}`);
+        snippets.set(componentName, mdxComment(` Missing imported snippet: ${sourcePath} `));
         return "";
       }
     }
@@ -7676,48 +7702,49 @@ function inlineMdxSnippets(raw, currentFile, repositoryRoot, warnings, depth = 0
     if (declaration) preservedDeclarations.set(componentName, declaration);
     else snippets.set(componentName, snippetComponentBody(nested, componentName));
   }
-  let result = withoutImports;
-  for (const [componentName, snippet] of snippets) {
-    result = result.replace(new RegExp(`<${componentName}((?:\\s[^>]*)?)\\s*/>`, "g"), (_tag, attributes) => {
-      return interpolateSnippet(snippet, attributes);
-    }).replace(new RegExp(`<${componentName}((?:\\s[^>]*)?)>([\\s\\S]*?)<\\/${componentName}>`, "g"), (_tag, attributes, children) => {
-      return interpolateSnippet(snippet, attributes, children);
-    });
-  }
-  result = result.replace(SNIPPET_TAG_PATTERN, (_tag, doubleQuoted, singleQuoted) => {
-    const filePath = doubleQuoted ?? singleQuoted;
-    try {
-      const candidate = [
-        () => resolveWithin(siteRoot, `snippets/${filePath}`),
-        () => resolveSnippetPath(filePath, currentFile, repositoryRoot, siteRoot)
-      ].map((resolveCandidate) => {
-        try {
-          return resolveCandidate();
-        } catch {
-          return void 0;
-        }
-      }).find((path) => path !== void 0 && existsSync5(path) && lstatSync5(path).isFile());
-      if (!candidate) throw new Error("file not found");
-      const blocked = blockInline(candidate, filePath, `Snippet file="${filePath}"`);
-      if (blocked) return blocked;
-      return inlineMdxSnippets(
-        withoutFrontmatter(readFileSync5(candidate, "utf8")),
-        candidate,
-        repositoryRoot,
-        warnings,
-        depth + 1,
-        siteRoot,
-        globalAliases,
-        gate
-      );
-    } catch {
-      warnings.push({
-        code: "missing-page",
-        message: `Snippet file="${filePath}" could not be resolved and was left as a comment.`,
-        source: relative5(repositoryRoot, currentFile).replace(/\\/g, "/")
+  let result = replaceOutsideCodeAndComments(withoutImports, (segment) => {
+    for (const [componentName, snippet] of snippets) {
+      segment = segment.replace(new RegExp(`<${componentName}((?:\\s[^>]*)?)\\s*/>`, "g"), (tag, attributes, offset, source2) => {
+        return isolateFences(interpolateSnippet(snippet, attributes), source2, offset, offset + tag.length);
+      }).replace(new RegExp(`<${componentName}((?:\\s[^>]*)?)>([\\s\\S]*?)<\\/${componentName}>`, "g"), (tag, attributes, children, offset, source2) => {
+        return isolateFences(interpolateSnippet(snippet, attributes, children), source2, offset, offset + tag.length);
       });
-      return `{/* Missing snippet: ${filePath} */}`;
     }
+    return segment.replace(SNIPPET_TAG_PATTERN, (tag, doubleQuoted, singleQuoted, offset, source2) => {
+      const filePath = doubleQuoted ?? singleQuoted;
+      try {
+        const candidate = [
+          () => resolveWithin(siteRoot, `snippets/${filePath}`),
+          () => resolveSnippetPath(filePath, currentFile, repositoryRoot, siteRoot)
+        ].map((resolveCandidate) => {
+          try {
+            return resolveCandidate();
+          } catch {
+            return void 0;
+          }
+        }).find((path) => path !== void 0 && existsSync5(path) && lstatSync5(path).isFile());
+        if (!candidate) throw new Error("file not found");
+        const blocked = blockInline(candidate, filePath, `Snippet file="${filePath}"`);
+        if (blocked) return blocked;
+        return isolateFences(inlineMdxSnippets(
+          withoutFrontmatter(readFileSync5(candidate, "utf8")),
+          candidate,
+          repositoryRoot,
+          warnings,
+          depth + 1,
+          siteRoot,
+          globalAliases,
+          gate
+        ), source2, offset, offset + tag.length);
+      } catch {
+        warnings.push({
+          code: "missing-page",
+          message: `Snippet file="${filePath}" could not be resolved and was left as a comment.`,
+          source: relative5(repositoryRoot, currentFile).replace(/\\/g, "/")
+        });
+        return mdxComment(` Missing snippet: ${filePath} `);
+      }
+    });
   });
   if (preservedDeclarations.size > 0) {
     const frontmatter = result.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/);
@@ -7726,7 +7753,7 @@ function inlineMdxSnippets(raw, currentFile, repositoryRoot, warnings, depth = 0
 
 ${result.slice(prefix.length)}`;
   }
-  return depth === 0 ? hoistMdxImports(result) : result;
+  return depth === 0 ? hoistMdxImports(result) : closeOpenFence(result);
 }
 function injectOpenApiSpecs(config, specs, warnings) {
   const tabs = config.tabs.map((tab) => ({ ...tab }));
@@ -8214,7 +8241,7 @@ function migrateRepository(options) {
     const publishedText = trackPublishedRefs ? raw : void 0;
     const pageAssetReferences = [];
     if (platform === "fern" || platform === "mintlify" || platform === "docusaurus") {
-      raw = normalizeExplicitHeadingIds(raw);
+      raw = normalizeExplicitHeadingIds(raw, (message) => warnings.push({ code: "invalid-page", message, source: file.relativePath }));
     }
     if (componentMigrator) {
       const warningsBeforeTransform = warnings.length;