@mintlify/scraping 4.0.1161 → 4.0.1163

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.1161",
+  "version": "4.0.1163",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -43,7 +43,7 @@
     "format:check": "oxfmt --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.1288",
+    "@mintlify/common": "1.0.1290",
     "fast-xml-parser": "5.7.3",
     "fs-extra": "11.1.1",
     "graphql": "16.11.0",
@@ -62,9 +62,9 @@
     "zod": "3.24.0"
   },
   "devDependencies": {
-    "@mintlify/models": "0.0.376",
+    "@mintlify/models": "0.0.377",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.937",
+    "@mintlify/validation": "0.1.939",
     "@tsconfig/recommended": "1.0.2",
     "@types/hast": "3.0.4",
     "@types/mdast": "4.0.4",
@@ -87,5 +87,5 @@
       "optional": true
     }
   },
-  "gitHead": "06a4673787feb66d1471add9eca97ffec1fda176"
+  "gitHead": "0873aa5354dd9084254ffa50e997d84966aa1d65"
 }

package/src/sdk/convert.ts

@@ -1,10 +1,23 @@
 import type { Root as HastRoot, Element } from 'hast';
-import type { Root as MdastRoot } from 'mdast';
+import type {
+  Definition,
+  Heading,
+  Html,
+  Image,
+  ImageReference,
+  Link,
+  LinkReference,
+  Paragraph,
+  PhrasingContent,
+  Root as MdastRoot,
+  Text,
+} from 'mdast';
 import remarkGfm from 'remark-gfm';
 import remarkMdx from 'remark-mdx';
 import remarkParse from 'remark-parse';
 import remarkStringify from 'remark-stringify';
 import { unified, type Plugin } from 'unified';
+import type { Node, Parent } from 'unist';
 import { visit } from 'unist-util-visit';
 
 import { convertHeaderLinksToText } from '../components/link.js';
@@ -80,17 +93,191 @@ export function htmlToMdx(html: string, rewriteLink?: RewriteLink): string {
   return hastToMdx(hast, rewriteLink);
 }
 
-export function markdownToMdx(markdown: string): string {
-  const mdastTree = unified().use(remarkParse).use(remarkGfm).parse(markdown) as MdastRoot;
-  visit(mdastTree, 'html', (node: { type: string; value: string }) => {
-    node.type = 'text';
-    node.value = node.value.replace(/<[^>]*>/g, '');
+// Raw HTML is dropped and its text kept. An HTML block becomes ordinary
+// paragraphs (one per blank-line-separated run, since <pre> and comments may
+// span blank lines), so its lines are subject to the same ESM escaping as any
+// paragraph and keep their block boundaries; inline HTML becomes text within
+// its paragraph, merged with its neighbours so a paragraph that opens with a
+// tag still starts with its first word.
+export function parseMarkdown(markdown: string, minHeadingDepth?: Heading['depth']): MdastRoot {
+  // CommonMark treats \r\n and a bare \r as line endings; normalising them up
+  // front means every later split and check only has to know about \n.
+  const normalized = markdown.replace(/\r\n?/g, '\n');
+  const mdastTree = unified().use(remarkParse).use(remarkGfm).parse(normalized) as MdastRoot;
+  visit(mdastTree, 'html', (node: Html, index, parent) => {
+    const value = stripTags(node.value);
+    if (
+      !parent ||
+      index === undefined ||
+      parent.type === 'paragraph' ||
+      parent.type === 'heading'
+    ) {
+      Object.assign(node, { type: 'text', value });
+      return;
+    }
+    const paragraphs: Paragraph[] = value
+      .split(/\n[ \t]*\n/)
+      .map((chunk) => chunk.trim())
+      .filter(Boolean)
+      .map(
+        (chunk): Paragraph => ({
+          type: 'paragraph',
+          children: [{ type: 'text', value: chunk }],
+          data: { htmlBlock: true } as Paragraph['data'],
+        })
+      );
+    parent.children.splice(index, 1, ...paragraphs);
+    return index + paragraphs.length;
   });
-  return stringifyMdast(mdastTree);
+  visit(mdastTree, 'paragraph', (node: Paragraph) => {
+    node.children = mergeText(node.children);
+  });
+  if (minHeadingDepth) {
+    visit(mdastTree, 'heading', (node: Heading) => {
+      node.depth = node.depth < minHeadingDepth ? minHeadingDepth : node.depth;
+    });
+  }
+  return mdastTree;
+}
+
+// Removes every `<...>` span in one pass: from the first unmatched `<` to the
+// next `>`, the same spans as /<[^>]*>/g, without rescanning to the end of the
+// text for each `<` that never closes.
+function stripTags(value: string): string {
+  let out = '';
+  let last = 0;
+  let open = -1;
+  for (let i = 0; i < value.length; i += 1) {
+    if (value[i] === '<' && open === -1) open = i;
+    else if (value[i] === '>' && open !== -1) {
+      out += value.slice(last, open);
+      last = i + 1;
+      open = -1;
+    }
+  }
+  return out + value.slice(last);
+}
+
+// Paragraphs made from an HTML block, such as a centred <h1> title or a badge
+// row, which a README description should skip.
+export function isHtmlParagraph(node: Paragraph): boolean {
+  return (node.data as { htmlBlock?: boolean } | undefined)?.htmlBlock === true;
+}
+
+// Joins adjacent text nodes and drops empty ones; the whitespace a removed tag
+// leaves at either end of the paragraph is not content either.
+function mergeText(children: PhrasingContent[]): PhrasingContent[] {
+  const merged: PhrasingContent[] = [];
+  for (const child of children) {
+    const previous = merged[merged.length - 1];
+    if (child.type === 'text' && previous?.type === 'text') {
+      previous.value += child.value;
+    } else if (child.type !== 'text' || child.value !== '') {
+      merged.push(child);
+    }
+  }
+  const first = merged[0];
+  if (first?.type === 'text') first.value = first.value.trimStart();
+  const last = merged[merged.length - 1];
+  if (last?.type === 'text') last.value = last.value.trimEnd();
+  return merged.filter((child) => child.type !== 'text' || child.value !== '');
+}
+
+// The text a reader sees for a node: text and code verbatim, an image's alt,
+// a space for a hard line break, nothing for raw HTML.
+export function plainText(node: Node): string {
+  if (node.type === 'text' || node.type === 'inlineCode') return (node as Text).value;
+  if (node.type === 'break') return ' ';
+  if (node.type === 'image' || node.type === 'imageReference') {
+    return (node as Image).alt ?? '';
+  }
+  if ('children' in node) {
+    return (node as Parent).children.map(plainText).join('');
+  }
+  return '';
+}
+
+// A paragraph with words of its own, as opposed to one made only of badges,
+// images, or links, which is how most READMEs open.
+export function isProseParagraph(node: Paragraph): boolean {
+  return node.children.some(
+    (child) =>
+      (child.type === 'text' && child.value.trim() !== '') ||
+      child.type === 'inlineCode' ||
+      ((child.type === 'emphasis' || child.type === 'strong' || child.type === 'delete') &&
+        plainText(child).trim() !== '')
+  );
+}
+
+// Rewrites every link, image, and definition URL, or drops the node when the
+// callback returns undefined: a link becomes its text, an image its alt, and a
+// definition disappears along with the references that pointed at it.
+export function rewriteMdastLinks(
+  tree: MdastRoot,
+  rewriteLink: (node: Link | Image | Definition) => string | undefined
+): void {
+  const dropped = new Set<string>();
+  visit(tree, ['link', 'image', 'definition'], (node, index, parent) => {
+    const target = node as Link | Image | Definition;
+    const rewritten = rewriteLink(target);
+    if (rewritten !== undefined) {
+      target.url = rewritten;
+      return;
+    }
+    if (target.type === 'definition') dropped.add(target.identifier);
+    if (parent && index !== undefined) {
+      const replacement =
+        target.type === 'link'
+          ? target.children
+          : target.type === 'image'
+            ? [{ type: 'text' as const, value: target.alt ?? '' }]
+            : [];
+      parent.children.splice(index, 1, ...replacement);
+      return index + replacement.length;
+    }
+  });
+  if (dropped.size === 0) return;
+  visit(tree, ['linkReference', 'imageReference'], (node, index, parent) => {
+    const reference = node as LinkReference | ImageReference;
+    if (!dropped.has(reference.identifier) || !parent || index === undefined) return;
+    const replacement: PhrasingContent[] =
+      reference.type === 'linkReference'
+        ? reference.children
+        : [{ type: 'text', value: reference.alt ?? '' }];
+    (parent as Parent).children.splice(index, 1, ...replacement);
+    return index + replacement.length;
+  });
+}
+
+export function markdownToMdx(markdown: string, minHeadingDepth?: Heading['depth']): string {
+  return stringifyMdast(parseMarkdown(markdown, minHeadingDepth));
 }
 
-function stringifyMdast(tree: MdastRoot): string {
-  const result = unified().use(remarkMdx).use(remarkGfm).use(remarkStringify).stringify(tree);
+// MDX reads a paragraph that starts with `import` or `export` as ESM. A
+// character reference for the first letter keeps the text while making it
+// impossible for Markdown text to become JavaScript.
+function escapeEsmParagraphs(tree: MdastRoot): void {
+  visit(tree, 'paragraph', (node: Paragraph) => {
+    const first = node.children[0];
+    if (first?.type !== 'text') return;
+    const match = /^(import|export)(?=\s|$)/.exec(first.value);
+    if (!match) return;
+    first.value = first.value.slice(1);
+    node.children.unshift({ type: 'html', value: `&#${match[0].charCodeAt(0)};` });
+  });
+}
+
+// Serialises a copy, so a cached tree can be stringified more than once and
+// read for its plain text afterwards without carrying the ESM escapes. Text is
+// merged again first: a dropped link or image leaves its text beside the next
+// node, and `[imp](javascript:x)ort` must be seen as one word, `import`.
+export function stringifyMdast(tree: MdastRoot): string {
+  const copy = structuredClone(tree);
+  visit(copy, 'paragraph', (node: Paragraph) => {
+    node.children = mergeText(node.children);
+  });
+  escapeEsmParagraphs(copy);
+  const result = unified().use(remarkMdx).use(remarkGfm).use(remarkStringify).stringify(copy);
   return String(result)
     .replace(/\n{3,}/g, '\n\n')
     .trim();