@mintlify/scraping 4.0.1110 → 4.0.1111

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.1110",
+  "version": "4.0.1111",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -87,5 +87,5 @@
       "optional": true
     }
   },
-  "gitHead": "f9869196da1eaa018c94b2e8d0c8c2cd6e082274"
+  "gitHead": "acb0c81cdceb361f88c6b238623fbf13a7457edb"
 }

package/src/sdk/converters/typedoc.ts

@@ -2,7 +2,7 @@ import type { SdkSymbolKind } from '@mintlify/models';
 import fse from 'fs-extra';
 import path from 'node:path';
 
-import { markdownToMdx } from '../convert.js';
+import { escapeMdxText, markdownToMdx } from '../convert.js';
 import type { SdkNavGroup, SdkPage, SdkPageTarget, SdkReference } from '../types.js';
 
 const KIND = {
@@ -52,12 +52,16 @@ type TypedocType = {
   head?: string;
   tail?: [TypedocType, string][];
 };
+type TypedocCategory = { title: string; children?: number[] };
+type TypedocGroup = TypedocCategory & { categories?: TypedocCategory[] };
 type TypedocNode = {
   id: number;
   name: string;
   kind: number;
   comment?: Comment;
   children?: TypedocNode[];
+  groups?: TypedocGroup[];
+  categories?: TypedocCategory[];
   signatures?: TypedocNode[];
   parameters?: TypedocNode[];
   type?: TypedocType;
@@ -70,6 +74,11 @@ type TypedocNode = {
   inheritedFrom?: { name: string };
 };
 type TypeToken = { text: string; href?: string };
+type MemberSection = {
+  title: string;
+  members: TypedocNode[];
+  categories?: { title: string; members: TypedocNode[] }[];
+};
 
 const SECTIONS: {
   kind: number;
@@ -104,6 +113,32 @@ const SECTIONS: {
   },
 ];
 
+const MAX_PLACEMENTS_PER_PAGE = 8;
+const MAX_DECLARATION_NAME_LENGTH = 512;
+
+const MEMBER_KINDS = new Set<number>([KIND.constructor, KIND.property, KIND.accessor, KIND.method]);
+
+const DEFAULT_GROUP_TITLE_BY_KIND = new Map<number, string>([
+  ...SECTIONS.map((section) => [section.kind, section.group] as const),
+  [KIND.namespace, 'Namespaces'],
+  [KIND.constructor, 'Constructors'],
+  [KIND.property, 'Properties'],
+  [KIND.accessor, 'Accessors'],
+  [KIND.method, 'Methods'],
+]);
+
+function hasCustomGroups(container: TypedocNode): boolean {
+  const kindById = new Map((container.children ?? []).map((child) => [child.id, child.kind]));
+  return (container.groups ?? []).some(
+    (group) =>
+      group.categories?.length ||
+      (group.children ?? []).some((id) => {
+        const defaultTitle = DEFAULT_GROUP_TITLE_BY_KIND.get(kindById.get(id) ?? -1);
+        return defaultTitle !== undefined && defaultTitle !== group.title;
+      })
+  );
+}
+
 export async function convertTypedoc(sourcePath: string): Promise<SdkReference> {
   const project = (await fse.readJson(sourcePath)) as TypedocNode;
   const modules =
@@ -132,6 +167,7 @@ export async function convertTypedoc(sourcePath: string): Promise<SdkReference>
   for (const module of modules) {
     const prefix = multiModule ? `${module.name} ` : '';
     const declarations = collectDeclarations(module);
+    const kindGroups: SdkNavGroup[] = [];
     for (const section of SECTIONS) {
       const entries = declarations
         .filter(({ node }) => node.kind === section.kind && !node.flags?.isPrivate)
@@ -153,13 +189,76 @@ export async function convertTypedoc(sourcePath: string): Promise<SdkReference>
           );
         pages.push(...renderer.renderMemberPages(entry.node, parentSlug, qualifiedName(entry)));
       }
-      groups.push({ group: `${prefix}${section.group}`, pages: sectionPages.map((p) => p.slug) });
+      kindGroups.push({ group: section.group, pages: sectionPages.map((p) => p.slug) });
     }
+    const customGroups = hasCustomLayout(module) ? layoutGroups(module, slugById) : [];
+    const moduleGroups = customGroups.length ? customGroups : kindGroups;
+    groups.push(...moduleGroups.map((group) => ({ ...group, group: `${prefix}${group.group}` })));
   }
 
   return { pages, groups };
 }
 
+function hasCustomLayout(container: TypedocNode): boolean {
+  if (container.categories?.length) return true;
+  if (hasCustomGroups(container)) return true;
+  return (container.children ?? []).some(
+    (child) => child.kind === KIND.namespace && hasCustomLayout(child)
+  );
+}
+
+function layoutGroups(container: TypedocNode, slugById: Map<number, string>): SdkNavGroup[] {
+  return layoutEntries(container, slugById, new Map()).filter(
+    (entry): entry is SdkNavGroup => typeof entry !== 'string'
+  );
+}
+
+function layoutEntries(
+  container: TypedocNode,
+  slugById: Map<number, string>,
+  placements: Map<number, number>
+): (string | SdkNavGroup)[] {
+  const childById = new Map((container.children ?? []).map((child) => [child.id, child]));
+  const entries = (ids: number[] = []): (string | SdkNavGroup)[] =>
+    [...new Set(ids)].flatMap((id): (string | SdkNavGroup)[] => {
+      const child = childById.get(id);
+      if (!child) return [];
+      const count = placements.get(id) ?? 0;
+      if (child.kind === KIND.namespace) {
+        if (count > 0) return [];
+        placements.set(id, 1);
+        return group(child.name, layoutEntries(child, slugById, placements));
+      }
+      const slug = slugById.get(id);
+      if (!slug) return [];
+      if (count >= MAX_PLACEMENTS_PER_PAGE) {
+        throw new Error(
+          `TypeDoc declaration "${truncate(child.name, 64)}" is listed in more than ${MAX_PLACEMENTS_PER_PAGE} navigation groups`
+        );
+      }
+      placements.set(id, count + 1);
+      return [slug];
+    });
+  const group = (title: unknown, pages: (string | SdkNavGroup)[]): SdkNavGroup[] => {
+    const label = singleLine(title);
+    return label && pages.length ? [{ group: label, pages }] : [];
+  };
+
+  if (container.categories?.length) {
+    return container.categories.flatMap((category) =>
+      group(category.title, entries(category.children))
+    );
+  }
+  return (container.groups ?? []).flatMap((entry) =>
+    group(
+      entry.title,
+      entry.categories?.length
+        ? entry.categories.flatMap((category) => group(category.title, entries(category.children)))
+        : entries(entry.children)
+    )
+  );
+}
+
 type Declaration = { node: TypedocNode; namespacePath: string[] };
 
 function qualifiedName({ node, namespacePath }: Declaration): string {
@@ -178,7 +277,18 @@ function collectDeclarations(container: TypedocNode, namespacePath: string[] = [
   return declarations;
 }
 
+function singleLine(value: unknown): string {
+  return String(value ?? '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
 function slugify(name: string): string {
+  if (name.length > MAX_DECLARATION_NAME_LENGTH) {
+    throw new Error(
+      `TypeDoc declaration name "${truncate(name, 64)}" exceeds ${MAX_DECLARATION_NAME_LENGTH} characters`
+    );
+  }
   return name.replace(/[^a-zA-Z0-9-_.]+/g, '-').replace(/^-+|-+$/g, '') || 'item';
 }
 
@@ -284,6 +394,22 @@ class TypedocRenderer {
     }
 
     const members = (node.children ?? []).filter((child) => !child.flags?.isPrivate);
+    const layout = this.memberLayout(node, members);
+    if (layout) {
+      for (const section of layout) {
+        lines.push(`## ${headingText(section.title)}`);
+        if (section.categories) {
+          for (const category of section.categories) {
+            lines.push(`### ${headingText(category.title)}`);
+            lines.push(...this.renderMembers(category.members, 4));
+          }
+        } else {
+          lines.push(...this.renderMembers(section.members, 3));
+        }
+      }
+      return lines;
+    }
+
     const constructors = members.filter((child) => child.kind === KIND.constructor);
     const properties = members
       .filter((child) => child.kind === KIND.property || child.kind === KIND.accessor)
@@ -294,36 +420,71 @@ class TypedocRenderer {
 
     for (const ctor of constructors) {
       lines.push('## Constructor');
-      lines.push(...this.renderSignatures(ctor, 3));
+      lines.push(...this.renderMembers([ctor], 3));
     }
-
     if (properties.length) {
       lines.push('## Properties');
-      for (const property of properties) {
-        const type = property.kind === KIND.accessor ? property.getSignature?.type : property.type;
-        const optional = property.flags?.isOptional === true;
-        lines.push(
-          this.typeField(
-            property.name,
-            type,
-            optional,
-            this.comment(property.comment ?? property.getSignature?.comment)
-          )
-        );
-      }
+      lines.push(...this.renderMembers(properties, 3));
     }
-
     if (methods.length) {
       lines.push('## Methods');
-      for (const method of methods) {
-        lines.push(`### ${method.name}()`);
-        lines.push(...this.renderSignatures(method, 4));
-      }
+      lines.push(...this.renderMembers(methods, 3));
     }
 
     return lines;
   }
 
+  private memberLayout(node: TypedocNode, members: TypedocNode[]): MemberSection[] | undefined {
+    const memberById = new Map(members.map((member) => [member.id, member]));
+    const placed = new Set<number>();
+    const resolve = (ids: number[] = []) =>
+      ids.flatMap((id) => {
+        const member = memberById.get(id);
+        if (!member || !MEMBER_KINDS.has(member.kind) || placed.has(id)) return [];
+        placed.add(id);
+        return [member];
+      });
+    const sections = (categories: TypedocCategory[]) =>
+      categories
+        .map((category) => ({ title: category.title, members: resolve(category.children) }))
+        .filter((section) => section.members.length > 0);
+
+    if (node.categories?.length) return sections(node.categories);
+    if (!hasCustomGroups(node)) return undefined;
+    return (node.groups ?? [])
+      .map((group): MemberSection => {
+        if (!group.categories?.length)
+          return { title: group.title, members: resolve(group.children) };
+        return { title: group.title, members: [], categories: sections(group.categories) };
+      })
+      .filter((section) => section.members.length > 0 || section.categories?.length);
+  }
+
+  private renderMembers(members: TypedocNode[], depth: number): string[] {
+    const lines: string[] = [];
+    for (const ctor of members.filter((member) => member.kind === KIND.constructor)) {
+      lines.push(...this.renderSignatures(ctor, depth));
+    }
+    for (const property of members.filter(
+      (member) => member.kind === KIND.property || member.kind === KIND.accessor
+    )) {
+      const type = property.kind === KIND.accessor ? property.getSignature?.type : property.type;
+      lines.push(
+        this.typeField(
+          property.name,
+          type,
+          property.flags?.isOptional === true,
+          this.comment(property.comment ?? property.getSignature?.comment)
+        )
+      );
+    }
+    for (const method of members.filter((member) => member.kind === KIND.method)) {
+      lines.push(`${'#'.repeat(depth)} ${headingText(method.name)}()`);
+      lines.push(...this.renderSignatures(method, depth + 1));
+    }
+    return lines;
+  }
+
   private renderEnum(node: TypedocNode): string[] {
     const lines: string[] = ['## Members'];
     for (const member of node.children ?? []) {
@@ -352,7 +513,7 @@ class TypedocRenderer {
       const comment = this.comment(signature.comment);
       if (comment) lines.push(comment);
       if (signature.parameters?.length) {
-        lines.push(`${'#'.repeat(depth)} Parameters`);
+        lines.push(detailHeading('Parameters', depth));
         for (const param of signature.parameters) {
           lines.push(
             this.typeField(
@@ -367,7 +528,7 @@ class TypedocRenderer {
       const returns = signature.comment?.blockTags?.find((tag) => tag.tag === '@returns');
       const returnType = this.type(signature.type);
       if (returnType && returnType !== 'void') {
-        lines.push(`${'#'.repeat(depth)} Returns`);
+        lines.push(detailHeading('Returns', depth));
         const linked = this.linkedType(signature.type) ?? inlineCode(returnType);
         lines.push(`${linked}${returns ? `\n\n${this.parts(returns.content)}` : ''}`);
       }
@@ -555,6 +716,16 @@ class TypedocRenderer {
   }
 }
 
+const MAX_HEADING_DEPTH = 4;
+
+function headingText(title: unknown): string {
+  return escapeMdxText(singleLine(title));
+}
+
+function detailHeading(title: string, depth: number): string {
+  return depth <= MAX_HEADING_DEPTH ? `${'#'.repeat(depth)} ${title}` : `**${title}**`;
+}
+
 function indent(text: string): string {
   return text
     .split('\n')

package/src/sdk/types.ts

@@ -21,7 +21,7 @@ export type SdkPage = {
 
 export type SdkNavGroup = {
   group: string;
-  pages: string[];
+  pages: (string | SdkNavGroup)[];
 };
 
 export type SdkReference = {