@mintlify/scraping 4.0.1230 → 4.0.1232

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.1230",
+  "version": "4.0.1232",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -87,5 +87,5 @@
       "optional": true
     }
   },
-  "gitHead": "e929a72c0d7dc12e955ca24554b1a3d6e6171bfa"
+  "gitHead": "bd9ae94bb1b82d591ce385a24bba824f937bea7a"
 }

package/src/sdk/converters/docfx.ts

@@ -3,7 +3,10 @@ import yaml from 'js-yaml';
 import path from 'node:path';
 
 import { htmlToMdx } from '../convert.js';
-import type { SdkNavGroup, SdkPage, SdkReference } from '../types.js';
+import { identifierTokens, linkedTypeField } from '../linked-tokens.js';
+import { inlineCode, linkedCode, responseField, truncate, type LinkedToken } from '../mdx.js';
+import { nestNavGroups, type PathNavGroup } from '../navigation.js';
+import type { SdkPage, SdkReference } from '../types.js';
 
 type DocfxParameter = { id?: string; type?: string; description?: string };
 type DocfxSyntax = {
@@ -82,7 +85,7 @@ export async function convertDocfx(sourcePath: string): Promise<SdkReference> {
   ];
 
   const pages: SdkPage[] = [];
-  const groups: SdkNavGroup[] = [];
+  const groups: PathNavGroup[] = [];
   for (const namespace of namespaceOrder) {
     const ordered = orderByToc(
       typesByNamespace.get(namespace) ?? [],
@@ -91,12 +94,12 @@ export async function convertDocfx(sourcePath: string): Promise<SdkReference> {
     const namespacePages = ordered.map((type) => renderer.renderPage(type));
     pages.push(...namespacePages);
     groups.push({
-      group: namespace || 'Reference',
-      pages: namespacePages.map((page) => page.slug),
+      path: namespace ? namespace.split('.') : [],
+      group: { group: namespace || 'Reference', pages: namespacePages.map((page) => page.slug) },
     });
   }
 
-  return { pages, groups };
+  return { pages, groups: nestNavGroups(groups) };
 }
 
 async function loadManagedReferenceFiles(sourcePath: string): Promise<DocfxFile[]> {
@@ -270,7 +273,7 @@ class DocfxRenderer {
     for (const member of this.members(item).filter((child) => child.type === 'Field')) {
       const value = member.syntax?.content?.split('=')[1]?.trim() ?? '';
       lines.push(
-        this.responseField(member.name ?? member.uid, value, false, this.prose(member.summary))
+        responseField(member.name ?? member.uid, value, false, this.prose(member.summary))
       );
     }
     return lines;
@@ -292,16 +295,16 @@ class DocfxRenderer {
     if (syntax.typeParameters?.length) {
       lines.push(`${heading} Type Parameters`);
       for (const param of syntax.typeParameters) {
-        lines.push(this.responseField(param.id ?? '', '', false, this.prose(param.description)));
+        lines.push(responseField(param.id ?? '', '', false, this.prose(param.description)));
       }
     }
     if (syntax.parameters?.length) {
       lines.push(`${heading} Parameters`);
       for (const param of syntax.parameters) {
         lines.push(
-          this.responseField(
+          linkedTypeField(
             param.id ?? '',
-            this.typeName(param.type),
+            this.typeTokens(param.type),
             isOptionalParameter(param.id, syntax.content),
             this.prose(param.description)
           )
@@ -312,7 +315,10 @@ class DocfxRenderer {
     if (returnType && returnType.toLowerCase() !== 'void') {
       lines.push(`${heading} Returns`);
       const description = this.prose(syntax.return?.description);
-      lines.push(`\`${escapeInlineCode(returnType)}\`${description ? `\n\n${description}` : ''}`);
+      const linked =
+        linkedCode(this.typeTokens(syntax.return?.type)) ??
+        inlineCode(returnType.replace(/`/g, ''));
+      lines.push(`${linked}${description ? `\n\n${description}` : ''}`);
     }
     return lines;
   }
@@ -325,24 +331,14 @@ class DocfxRenderer {
     ]
       .filter(Boolean)
       .join('\n\n');
-    return this.responseField(
+    return linkedTypeField(
       member.name ?? member.uid,
-      this.typeName(member.syntax?.return?.type),
+      this.typeTokens(member.syntax?.return?.type),
       isNullable(member.syntax?.content),
       body
     );
   }
 
-  private responseField(name: string, type: string, optional: boolean, body?: string): string {
-    const typeAttr = type ? ` type=${this.jsxString(truncate(type, 80))}` : '';
-    const requiredAttr = optional ? '' : ' required';
-    return `<ResponseField name=${this.jsxString(name)}${typeAttr}${requiredAttr}>\n${indent(body || '')}\n</ResponseField>`;
-  }
-
-  private jsxString(value: string): string {
-    return `{${JSON.stringify(value)}}`;
-  }
-
   private codeBlock(code: string, lang = 'csharp'): string {
     return `\`\`\`${lang}\n${code}\n\`\`\``;
   }
@@ -413,6 +409,25 @@ class DocfxRenderer {
     return uid ? this.shortName(uid) : '';
   }
 
+  private typeTokens(uid?: string): LinkedToken[] {
+    if (!uid) return [];
+    const source = uid
+      .replace(/(^|[^\w.])`+\d+/g, '$1')
+      .replace(/\{/g, '<')
+      .replace(/\}/g, '>');
+    const tokens = identifierTokens(
+      source,
+      /[A-Za-z_]\w*(?:`+\d+)?(?:\.[A-Za-z_]\w*(?:`+\d+)?)*/g,
+      (name, index) => {
+        const arity = genericArity(source, index + name.length);
+        const slug = this.slugByUid.get(arity ? `${name}\`${arity}` : name);
+        const text = name.replace(/`+\d+/g, '').split('.').at(-1) ?? name;
+        return { text, href: slug ? `/${slug}` : undefined };
+      }
+    );
+    return tokens.some((token) => token.href) ? tokens : [{ text: this.typeName(uid) }];
+  }
+
   private firstSentence(summary?: string): string | undefined {
     if (!summary) return undefined;
     const text = decodeEntities(summary.replace(/<[^>]*>/g, ' '))
@@ -424,6 +439,19 @@ class DocfxRenderer {
   }
 }
 
+function genericArity(text: string, start: number): number {
+  if (text[start] !== '<') return 0;
+  let depth = 0;
+  let arity = 1;
+  for (let index = start + 1; index < text.length; index += 1) {
+    const character = text[index];
+    if (character === '<') depth += 1;
+    else if (character === '>' && depth-- === 0) return arity;
+    else if (character === ',' && depth === 0) arity += 1;
+  }
+  return arity;
+}
+
 function byName(left: DocfxItem, right: DocfxItem): number {
   return (left.name ?? '').localeCompare(right.name ?? '');
 }
@@ -474,18 +502,3 @@ function isOptionalParameter(id?: string, content?: string): boolean {
 function escapeRegExp(text: string): string {
   return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
 }
-
-function indent(text: string): string {
-  return text
-    .split('\n')
-    .map((line) => (line ? `  ${line}` : line))
-    .join('\n');
-}
-
-function truncate(text: string, max: number): string {
-  return text.length > max ? `${text.slice(0, max - 1)}…` : text;
-}
-
-function escapeInlineCode(text: string): string {
-  return text.replace(/`/g, '');
-}

package/src/sdk/converters/go.ts

@@ -22,6 +22,7 @@ import {
   truncate,
   type LinkedToken,
 } from '../mdx.js';
+import { nestNavGroups, type PathNavGroup } from '../navigation.js';
 import { parseSdkReadme, unparseableMarkdown } from '../readme.js';
 import type { SdkNavGroup, SdkPage, SdkPageTarget, SdkReference } from '../types.js';
 
@@ -194,14 +195,18 @@ export async function convertGo(sourcePath: string): Promise<SdkReference> {
   const renderers = artifact.packages.map((pkg) => new GoRenderer(index, pkg));
 
   const pages: SdkPage[] = [];
-  const groups: SdkNavGroup[] = [];
-  for (const renderer of renderers) {
+  const groups: PathNavGroup[] = [];
+  for (const [position, renderer] of renderers.entries()) {
     const packagePages = renderer.packagePages();
     pages.push(...packagePages, ...renderer.memberPages());
-    groups.push({ group: renderer.group, pages: packageEntries(packagePages) });
+    const dir = artifact.packages[position]?.dir ?? '.';
+    groups.push({
+      path: dir === '.' ? [] : dir.split('/'),
+      group: { group: renderer.group, pages: packageEntries(packagePages) },
+    });
   }
 
-  return { pages, groups };
+  return { pages, groups: nestNavGroups(groups) };
 }
 
 const KIND_GROUPS = [

package/src/sdk/converters/javadoc.ts

@@ -14,6 +14,7 @@ import {
   trimTokens,
 } from '../linked-tokens.js';
 import { linkedCode, type LinkedToken } from '../mdx.js';
+import { nestNavGroups } from '../navigation.js';
 import type { SdkNavGroup, SdkPage, SdkReference } from '../types.js';
 
 type JavadocKind = 'interface' | 'class' | 'enum' | 'record' | 'exception' | 'error' | 'annotation';
@@ -63,9 +64,14 @@ export async function convertJavadoc(sourcePath: string): Promise<SdkReference>
     byPackage.set(type.pkg, siblings);
   }
 
-  const groups: SdkNavGroup[] = [...byPackage.entries()]
-    .sort(([left], [right]) => left.localeCompare(right))
-    .map(([pkg, members]) => ({ group: pkg, pages: packageEntries(members) }));
+  const groups = nestNavGroups(
+    [...byPackage.entries()]
+      .sort(([left], [right]) => left.localeCompare(right))
+      .map(([pkg, members]) => ({
+        path: pkg.split('.'),
+        group: { group: pkg, pages: packageEntries(members) },
+      }))
+  );
 
   return { pages, groups };
 }

package/src/sdk/converters/phpdoc.ts

@@ -3,7 +3,10 @@ import fse from 'fs-extra';
 import path from 'node:path';
 
 import { markdownToMdx } from '../convert.js';
-import type { SdkNavGroup, SdkPage, SdkReference } from '../types.js';
+import { identifierTokens, linkedTypeField } from '../linked-tokens.js';
+import { inlineCode, linkedCode, truncate, type LinkedToken } from '../mdx.js';
+import { nestNavGroups, type PathNavGroup } from '../navigation.js';
+import type { SdkPage, SdkReference } from '../types.js';
 
 type DocTag = {
   '@_name'?: string;
@@ -118,17 +121,20 @@ export async function convertPhpdoc(sourcePath: string): Promise<SdkReference> {
   });
 
   const pages: SdkPage[] = [];
-  const groups: SdkNavGroup[] = [];
+  const groups: PathNavGroup[] = [];
   for (const groupName of groupNames) {
     const members = (byGroup.get(groupName) ?? []).sort((left, right) =>
       left.name.localeCompare(right.name)
     );
     const groupPages = members.map((entry) => renderer.renderPage(entry));
     pages.push(...groupPages);
-    groups.push({ group: groupName, pages: groupPages.map((page) => page.slug) });
+    groups.push({
+      path: members[0]?.namespace ?? [],
+      group: { group: groupName, pages: groupPages.map((page) => page.slug) },
+    });
   }
 
-  return { pages, groups };
+  return { pages, groups: nestNavGroups(groups) };
 }
 
 function collectEntries(files: PhpFile[]): Entry[] {
@@ -229,7 +235,7 @@ class PhpdocRenderer {
         ]
           .filter(Boolean)
           .join('\n\n');
-        lines.push(this.responseField(constant.name ?? '', type, false, body));
+        lines.push(linkedTypeField(constant.name ?? '', this.typeTokens(type), false, body));
       }
     }
 
@@ -254,7 +260,9 @@ class PhpdocRenderer {
         ]
           .filter(Boolean)
           .join('\n\n');
-        lines.push(this.responseField(property.name ?? '', type, isNullable(type), body));
+        lines.push(
+          linkedTypeField(property.name ?? '', this.typeTokens(type), isNullable(type), body)
+        );
       }
     }
 
@@ -305,7 +313,7 @@ class PhpdocRenderer {
         const body = paramTag?.['@_description']
           ? markdownToMdx(this.resolveInline(paramTag['@_description']))
           : '';
-        lines.push(this.responseField(arg.name ?? '', type, optional, body));
+        lines.push(linkedTypeField(arg.name ?? '', this.typeTokens(type), optional, body));
       }
     }
 
@@ -315,7 +323,8 @@ class PhpdocRenderer {
       const description = returnTag['@_description']
         ? `\n\n${markdownToMdx(this.resolveInline(returnTag['@_description']))}`
         : '';
-      lines.push(`\`${escapeInlineCode(returnTag['@_type'])}\`${description}`);
+      const returns = this.typeTokens(returnTag['@_type']);
+      lines.push(`${linkedCode(returns) ?? inlineCode(returnTag['@_type'])}${description}`);
     }
 
     const throwsTags = tagsOf(method.docblock, 'throws');
@@ -326,7 +335,7 @@ class PhpdocRenderer {
           throwsTags
             .map((tag) => {
               const description = tag['@_description'] ? ` ${tag['@_description']}` : '';
-              return `- \`${escapeInlineCode(tag['@_type'] ?? 'Exception')}\`${description}`;
+              return `- ${this.typeLink(tag['@_type'] ?? 'Exception')}${description}`;
             })
             .join('\n')
         )
@@ -415,14 +424,23 @@ class PhpdocRenderer {
     return truncate(sentence, 160);
   }
 
-  private responseField(name: string, type: string, optional: boolean, body?: string): string {
-    const typeAttr = type ? ` type=${this.jsxString(truncate(type, 80))}` : '';
-    const requiredAttr = optional ? '' : ' required';
-    return `<ResponseField name=${this.jsxString(name)}${typeAttr}${requiredAttr}>\n${indent(body || '')}\n</ResponseField>`;
+  private typeTokens(type: string): LinkedToken[] {
+    return identifierTokens(type, /\\?[A-Za-z_]\w*(?:\\[A-Za-z_]\w*)*/g, (name) => {
+      const slug = this.resolve(name);
+      return { text: name, href: slug ? `/${slug}` : undefined };
+    });
   }
 
-  private jsxString(value: string): string {
-    return `{${JSON.stringify(value)}}`;
+  private typeLink(type: string): string {
+    const spans: LinkedToken[] = [];
+    for (const token of this.typeTokens(type)) {
+      const last = spans.at(-1);
+      if (last && !last.href && !token.href) last.text += token.text;
+      else spans.push({ ...token });
+    }
+    return spans
+      .map(({ text, href }) => (href ? `[${inlineCode(text)}](${href})` : inlineCode(text)))
+      .join('');
   }
 
   private codeBlock(code: string): string {
@@ -430,17 +448,6 @@ class PhpdocRenderer {
   }
 }
 
-function indent(text: string): string {
-  return text
-    .split('\n')
-    .map((line) => (line ? `  ${line}` : line))
-    .join('\n');
-}
-
-function truncate(text: string, max: number): string {
-  return text.length > max ? `${text.slice(0, max - 1)}…` : text;
-}
-
 function escapeInlineCode(text: string): string {
   return text.replace(/`/g, '');
 }

package/src/sdk/converters/sphinx.ts

@@ -36,10 +36,13 @@ export async function convertSphinx(sourcePath: string): Promise<SdkReference> {
   const docs = splitSphinxModules(await loadDocs(sourcePath));
   const slugs = new Set(docs.map((doc) => doc.slug));
   const anchors = collectAnchors(docs);
-  return {
-    pages: docs.map((doc) => renderPage(doc, slugs, anchors)),
-    groups: buildSphinxNavigation(docs),
-  };
+  const pages = docs.map((doc) => renderPage(doc, slugs, anchors));
+  const [first, ...rest] = buildSphinxNavigation(docs);
+  const [overview, ...others] = first?.pages ?? [];
+  if (first?.group === 'Getting Started' && typeof overview === 'string' && !others.length) {
+    return { pages, groups: rest, overview };
+  }
+  return { pages, groups: first ? [first, ...rest] : [] };
 }
 
 type SphinxAnchors = Map<string, Map<string, string>>;

package/src/sdk/converters/typedoc.ts

@@ -4,6 +4,7 @@ import path from 'node:path';
 
 import { escapeMdxText, markdownToMdx, rewriteMdastLinks, stringifyMdast } from '../convert.js';
 import { inlineCode, linkedCode, responseField, safeHref, truncate } from '../mdx.js';
+import { nestNavGroups, type PathNavGroup } from '../navigation.js';
 import { parseSdkReadme } from '../readme.js';
 import type { SdkNavGroup, SdkPage, SdkPageTarget, SdkReference } from '../types.js';
 
@@ -166,9 +167,9 @@ export async function convertTypedoc(sourcePath: string): Promise<SdkReference>
   const renderer = new TypedocRenderer(slugById);
   const pages: SdkPage[] = [];
   const groups: SdkNavGroup[] = [];
+  const moduleGroupById = new Map<number, PathNavGroup>();
 
   for (const module of modules) {
-    const prefix = multiModule ? `${module.name} ` : '';
     const declarations = collectDeclarations(module);
     const kindGroups: SdkNavGroup[] = [];
     for (const section of SECTIONS) {
@@ -196,11 +197,17 @@ export async function convertTypedoc(sourcePath: string): Promise<SdkReference>
     }
     const customGroups = hasCustomLayout(module) ? layoutGroups(module, slugById) : [];
     const moduleGroups = customGroups.length ? customGroups : kindGroups;
-    groups.push(...moduleGroups.map((group) => ({ ...group, group: `${prefix}${group.group}` })));
+    if (multiModule) {
+      moduleGroupById.set(module.id, {
+        path: module.name.split('/'),
+        group: { group: module.name, pages: moduleGroups },
+      });
+    } else groups.push(...moduleGroups);
   }
 
   const overviewPages: SdkPage[] = [];
   const overviewGroups: SdkNavGroup[] = [];
+  let projectOverview: string | undefined;
   const reservedSlugs = new Set(pages.map((page) => page.slug.toLowerCase()));
   const containers = [project, ...modules.filter((module) => module !== project)];
   const overviews: { node: TypedocNode; slug: string }[] = [];
@@ -234,13 +241,22 @@ export async function convertTypedoc(sourcePath: string): Promise<SdkReference>
       isProject ? undefined : project.name
     );
     overviewPages.push(overview);
-    overviewGroups.push({
-      group: isProject ? 'Overview' : `${container.name} Overview`,
-      pages: [slug],
-    });
+    const moduleGroup = moduleGroupById.get(container.id)?.group;
+    if (isProject) projectOverview = slug;
+    else if (moduleGroup) moduleGroup.pages.unshift(slug);
+    else if (!projectOverview) projectOverview = slug;
+    else overviewGroups.push({ group: `${container.name} Overview`, pages: [slug] });
   }
 
-  return { pages: [...overviewPages, ...pages], groups: [...overviewGroups, ...groups] };
+  return {
+    pages: [...overviewPages, ...pages],
+    groups: [
+      ...overviewGroups,
+      ...groups,
+      ...nestNavGroups([...moduleGroupById.values()].filter(({ group }) => group.pages.length)),
+    ],
+    ...(projectOverview ? { overview: projectOverview } : {}),
+  };
 }
 
 function hasCustomLayout(container: TypedocNode): boolean {

package/src/sdk/linked-tokens.ts

@@ -78,14 +78,34 @@ export function topLevelDelimiter(
   return -1;
 }
 
-export function linkedTypeField(name: string, tokens: LinkedToken[], optional: boolean): string {
+export function linkedTypeField(
+  name: string,
+  tokens: LinkedToken[],
+  optional: boolean,
+  description?: string
+): string {
   const type = tokens.map((token) => token.text).join('');
   const typeHref = tokens.length === 1 ? tokens[0]?.href : undefined;
-  let body: string | undefined;
+  let linked: string | undefined;
   if (!typeHref || type.length > 80) {
-    const linked = linkedCode(tokens);
-    if (linked) body = `**Type:** ${linked}`;
-    else if (type.length > 80) body = `**Type:** ${inlineCode(type)}`;
+    linked = linkedCode(tokens) ?? (type.length > 80 ? inlineCode(type) : undefined);
   }
+  const body = [linked && `**Type:** ${linked}`, description].filter(Boolean).join('\n\n');
   return responseField(name, type, optional, body, typeHref);
 }
+
+export function identifierTokens(
+  text: string,
+  identifier: RegExp,
+  link: (name: string, index: number) => LinkedToken
+): LinkedToken[] {
+  const tokens: LinkedToken[] = [];
+  let cursor = 0;
+  for (const match of text.matchAll(identifier)) {
+    if (match.index > cursor) tokens.push({ text: text.slice(cursor, match.index) });
+    tokens.push(link(match[0], match.index));
+    cursor = match.index + match[0].length;
+  }
+  if (cursor < text.length) tokens.push({ text: text.slice(cursor) });
+  return tokens;
+}

package/src/sdk/navigation.ts

@@ -0,0 +1,22 @@
+import type { SdkNavGroup } from './types.js';
+
+export type PathNavGroup = { path: string[]; group: SdkNavGroup };
+
+export function nestNavGroups(entries: PathNavGroup[]): SdkNavGroup[] {
+  const byPath = new Map<string, SdkNavGroup>();
+  const nested = entries.map(({ path, group }) => {
+    const copy = { ...group, pages: [...group.pages] };
+    if (path.length) byPath.set(path.join('\u0000'), copy);
+    return { path, group: copy };
+  });
+  const roots: SdkNavGroup[] = [];
+  for (const { path, group } of nested) {
+    let parent: SdkNavGroup | undefined;
+    for (let length = path.length - 1; length > 0 && !parent; length -= 1) {
+      parent = byPath.get(path.slice(0, length).join('\u0000'));
+    }
+    if (parent) parent.pages.push(group);
+    else roots.push(group);
+  }
+  return roots;
+}

package/src/sdk/types.ts

@@ -29,6 +29,7 @@ export type SdkNavGroup = {
 export type SdkReference = {
   pages: SdkPage[];
   groups: SdkNavGroup[];
+  overview?: string;
 };
 
 export type SdkConverter = (sourcePath: string) => Promise<SdkReference>;