@mintlify/scraping 4.0.1190 → 4.0.1191

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.1190",
+  "version": "4.0.1191",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -87,5 +87,5 @@
       "optional": true
     }
   },
-  "gitHead": "6bc228d07c5f946e1ede787856918963e868b3b8"
+  "gitHead": "78b82add935dc4b66178be74d41b67a59712367c"
 }

package/src/sdk/converters/go.ts

@@ -1,5 +1,5 @@
 import fse from 'fs-extra';
-import type { Heading, ImageReference, LinkReference, Paragraph, Root as MdastRoot } from 'mdast';
+import type { ImageReference, LinkReference, Paragraph, Root as MdastRoot } from 'mdast';
 import path from 'node:path';
 import { visit } from 'unist-util-visit';
 import { z } from 'zod';
@@ -22,6 +22,7 @@ import {
   truncate,
   type LinkedToken,
 } from '../mdx.js';
+import { parseSdkReadme, unparseableMarkdown } from '../readme.js';
 import type { SdkNavGroup, SdkPage, SdkPageTarget, SdkReference } from '../types.js';
 
 const SUPPORTED_SCHEMA_VERSION = 1;
@@ -29,7 +30,6 @@ const PACKAGE_PAGE = 'package-overview';
 const MEMBER_HEADING_DEPTH = 3;
 const MIN_COMMENT_HEADING_DEPTH = 4;
 const MAX_TYPE_ATTR = 80;
-const MIN_README_HEADING_DEPTH = 2;
 const GITHUB_MODULE = /^github\.com\/([^/]+)\/([^/]+)((?:\/[^/]+)*?)(?:\/v(?:[2-9]|[1-9]\d+))?$/;
 
 const MAX_TEXT = 1 << 20;
@@ -450,13 +450,8 @@ class GoRenderer {
   // Renders a README the way pkg.go.dev shows it, minus the sentence the page
   // shows as its description (see takeLeadingSentence).
   private readme(markdown: string, synopsis?: string): { intro: string; description?: string } {
-    if (unparseable(markdown)) return { intro: codeBlock(markdown, 'text') };
-    const tree = parseMarkdown(markdown);
-    const first = tree.children[0];
-    if (first?.type === 'heading' && first.depth === 1) tree.children.shift();
-    visit(tree, 'heading', (node: Heading) => {
-      node.depth = Math.max(node.depth, MIN_README_HEADING_DEPTH) as Heading['depth'];
-    });
+    if (unparseableMarkdown(markdown)) return { intro: codeBlock(markdown, 'text') };
+    const tree = parseSdkReadme(markdown);
     const imageDefinitions = new Set<string>();
     const linkDefinitions = new Set<string>();
     visit(tree, 'imageReference', (node: ImageReference) => {
@@ -777,7 +772,7 @@ class GoRenderer {
   }
 
   private parseDoc(markdown: string): GoDoc {
-    if (unparseable(markdown)) {
+    if (unparseableMarkdown(markdown)) {
       return {
         tree: { type: 'root', children: [{ type: 'code', lang: 'text', value: markdown }] },
       };
@@ -906,40 +901,6 @@ function dropLeadingSentence(paragraph: Paragraph): void {
   paragraph.children = [];
 }
 
-// Text past either limit is rendered as a text block instead of Markdown.
-// micromark is quadratic in unclosed brackets (80k `[a](` takes about 45 s)
-// and in balanced link and emphasis markup (40k nested brackets, `[]` pairs,
-// `[a]` references, or `*a` runs take 5 to 36 s), and remark-stringify is
-// quadratic in the characters it escapes (40k `<` or `{` take 0.7 s, minutes
-// at the 1 MiB cap). At 10k either costs well under 2 s. The largest real doc
-// seen (openai-go's README, 60 KB) has about 1,500 such characters and no more
-// than 3 unclosed brackets.
-const MAX_UNCLOSED = 1000;
-const MAX_MARKUP = 10000;
-const MARKUP = new Set(['[', ']', '*', '_', '<', '>', '{', '}', '`', '&']);
-
-function unparseable(markdown: string): boolean {
-  let square = 0;
-  let round = 0;
-  let unclosed = 0;
-  let markup = 0;
-  for (let i = 0; i < markdown.length; i += 1) {
-    const char = markdown[i];
-    if (char !== undefined && MARKUP.has(char)) markup += 1;
-    if (char === '\\') i += 1;
-    else if (char === '[') square += 1;
-    else if (char === '(') round += 1;
-    else if (char === ']') {
-      if (square > 0) square -= 1;
-      else unclosed += 1;
-    } else if (char === ')') {
-      if (round > 0) round -= 1;
-      else unclosed += 1;
-    }
-  }
-  return square + round + unclosed > MAX_UNCLOSED || markup > MAX_MARKUP;
-}
-
 const MAX_DESCRIPTION = 160;
 
 // The text up to and including the first full stop that ends a sentence, or

package/src/sdk/converters/typedoc.ts

@@ -2,8 +2,9 @@ import type { SdkSymbolKind } from '@mintlify/models';
 import fse from 'fs-extra';
 import path from 'node:path';
 
-import { escapeMdxText, markdownToMdx } from '../convert.js';
-import { inlineCode, linkedCode, responseField, truncate } from '../mdx.js';
+import { escapeMdxText, markdownToMdx, rewriteMdastLinks, stringifyMdast } from '../convert.js';
+import { inlineCode, linkedCode, responseField, safeHref, truncate } from '../mdx.js';
+import { parseSdkReadme } from '../readme.js';
 import type { SdkNavGroup, SdkPage, SdkPageTarget, SdkReference } from '../types.js';
 
 const KIND = {
@@ -60,6 +61,7 @@ type TypedocNode = {
   name: string;
   kind: number;
   comment?: Comment;
+  readme?: CommentPart[];
   children?: TypedocNode[];
   groups?: TypedocGroup[];
   categories?: TypedocCategory[];
@@ -197,7 +199,48 @@ export async function convertTypedoc(sourcePath: string): Promise<SdkReference>
     groups.push(...moduleGroups.map((group) => ({ ...group, group: `${prefix}${group.group}` })));
   }
 
-  return { pages, groups };
+  const overviewPages: SdkPage[] = [];
+  const overviewGroups: SdkNavGroup[] = [];
+  const reservedSlugs = new Set(pages.map((page) => page.slug.toLowerCase()));
+  const containers = [project, ...modules.filter((module) => module !== project)];
+  const overviews: { node: TypedocNode; slug: string }[] = [];
+  for (const container of containers) {
+    if (!container.readme?.some((part) => part.text.trim())) continue;
+    const isProject = container === project;
+    const prefix = isProject ? '' : slugify(container.name).replace(/^\.+$/, 'item');
+    const base = path.posix.join(prefix, 'package-overview');
+    let slug = base;
+    let suffix = 2;
+    while (reservedSlugs.has(slug.toLowerCase())) {
+      slug = `${base}-${suffix}`;
+      suffix += 1;
+    }
+    reservedSlugs.add(slug.toLowerCase());
+    slugById.set(container.id, slug);
+    overviews.push({ node: container, slug });
+  }
+  for (const { node: container, slug } of overviews) {
+    const isProject = container === project;
+    const declarationSlugs = new Set(
+      collectDeclarations(container).map(({ node }) => slugById.get(node.id))
+    );
+    const indexPages = pages.filter(
+      (page) => !page.target?.parent && (isProject || declarationSlugs.has(page.slug))
+    );
+    const overview = renderer.renderOverview(
+      container,
+      slug,
+      indexPages,
+      isProject ? undefined : project.name
+    );
+    overviewPages.push(overview);
+    overviewGroups.push({
+      group: isProject ? 'Overview' : `${container.name} Overview`,
+      pages: [slug],
+    });
+  }
+
+  return { pages: [...overviewPages, ...pages], groups: [...overviewGroups, ...groups] };
 }
 
 function hasCustomLayout(container: TypedocNode): boolean {
@@ -296,6 +339,31 @@ function slugify(name: string): string {
 class TypedocRenderer {
   constructor(private slugById: Map<number, string>) {}
 
+  renderOverview(node: TypedocNode, slug: string, pages: SdkPage[], namespace?: string): SdkPage {
+    const tree = parseSdkReadme(this.partsMarkdown(node.readme ?? []));
+    rewriteMdastLinks(tree, (link) => safeHref(link.url));
+    const lines: string[] = [];
+    for (const section of SECTIONS) {
+      const entries = pages.filter((page) => page.target?.kind === section.symbolKind);
+      if (entries.length === 0) continue;
+      lines.push(
+        `## ${section.group}`,
+        entries.map((page) => `- [${inlineCode(page.title)}](/${page.slug})`).join('\n')
+      );
+    }
+    return {
+      slug,
+      title: node.name,
+      intro: stringifyMdast(tree),
+      content: lines.join('\n\n'),
+      target: {
+        kind: 'package',
+        name: node.name,
+        ...(namespace ? { namespace } : {}),
+      },
+    };
+  }
+
   renderPage(
     node: TypedocNode,
     section: { dir: string; tag: string; symbolKind: SdkSymbolKind },
@@ -570,7 +638,11 @@ class TypedocRenderer {
   }
 
   private parts(parts: CommentPart[]): string {
-    const markdown = parts
+    return markdownToMdx(this.partsMarkdown(parts));
+  }
+
+  private partsMarkdown(parts: CommentPart[]): string {
+    return parts
       .map((part) => {
         if (part.kind === 'inline-tag') {
           if (typeof part.target === 'number' && this.slugById.has(part.target)) {
@@ -582,7 +654,6 @@ class TypedocRenderer {
         return part.text;
       })
       .join('');
-    return markdownToMdx(markdown);
   }
 
   private firstSentence(comment?: Comment): string | undefined {

package/src/sdk/readme.ts

@@ -0,0 +1,51 @@
+import type { Root as MdastRoot } from 'mdast';
+import { visit } from 'unist-util-visit';
+
+import { parseMarkdown } from './convert.js';
+
+export function parseSdkReadme(markdown: string): MdastRoot {
+  if (unparseableMarkdown(markdown)) {
+    return { type: 'root', children: [{ type: 'code', lang: 'text', value: markdown }] };
+  }
+  const tree = parseMarkdown(markdown);
+  const first = tree.children[0];
+  if (first?.type === 'heading' && first.depth === 1) tree.children.shift();
+  visit(tree, 'heading', (node) => {
+    if (node.depth < 2) node.depth = 2;
+  });
+  return tree;
+}
+
+// Text past either limit is rendered as a text block instead of Markdown.
+// micromark is quadratic in unclosed brackets (80k `[a](` takes about 45 s)
+// and in balanced link and emphasis markup (40k nested brackets, `[]` pairs,
+// `[a]` references, or `*a` runs take 5 to 36 s), and remark-stringify is
+// quadratic in the characters it escapes (40k `<` or `{` take 0.7 s, minutes
+// at the 1 MiB cap). At 10k either costs well under 2 s. The largest real doc
+// seen (openai-go's README, 60 KB) has about 1,500 such characters and no more
+// than 3 unclosed brackets.
+const MAX_UNCLOSED = 1000;
+const MAX_MARKUP = 10000;
+const MARKUP = new Set(['[', ']', '*', '_', '<', '>', '{', '}', '`', '&']);
+
+export function unparseableMarkdown(markdown: string): boolean {
+  let square = 0;
+  let round = 0;
+  let unclosed = 0;
+  let markup = 0;
+  for (let i = 0; i < markdown.length; i += 1) {
+    const char = markdown[i];
+    if (char !== undefined && MARKUP.has(char)) markup += 1;
+    if (char === '\\') i += 1;
+    else if (char === '[') square += 1;
+    else if (char === '(') round += 1;
+    else if (char === ']') {
+      if (square > 0) square -= 1;
+      else unclosed += 1;
+    } else if (char === ')') {
+      if (round > 0) round -= 1;
+      else unclosed += 1;
+    }
+  }
+  return square + round + unclosed > MAX_UNCLOSED || markup > MAX_MARKUP;
+}