@mintlify/scraping 4.0.1211 → 4.0.1213

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.1211",
+  "version": "4.0.1213",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -43,7 +43,7 @@
     "format:check": "oxfmt --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.1333",
+    "@mintlify/common": "1.0.1334",
     "fast-xml-parser": "5.7.3",
     "fs-extra": "11.1.1",
     "graphql": "16.11.0",
@@ -62,9 +62,9 @@
     "zod": "3.24.0"
   },
   "devDependencies": {
-    "@mintlify/models": "0.0.384",
+    "@mintlify/models": "0.0.385",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.955",
+    "@mintlify/validation": "0.1.956",
     "@tsconfig/recommended": "1.0.2",
     "@types/hast": "3.0.4",
     "@types/mdast": "4.0.4",
@@ -87,5 +87,5 @@
       "optional": true
     }
   },
-  "gitHead": "9b36f4be014cc6561d5eaaefc733eac81ea78d1d"
+  "gitHead": "29387ac467c064ff32bc66c6a84dc41fc1abbfab"
 }

package/src/sdk/sphinx-navigation.ts

@@ -18,6 +18,7 @@ export function buildSphinxNavigation(docs: SphinxNavigationDoc[]): SdkNavGroup[
   const children = new Map<string, string[]>();
   const parents = new Map<string, string[]>();
   const guideContent = new Set<string>();
+  const generated = new Set<string>();
 
   function addChild(parent: string, child: string): void {
     if (parent === child) return;
@@ -31,7 +32,8 @@ export function buildSphinxNavigation(docs: SphinxNavigationDoc[]): SdkNavGroup[
 
   for (const doc of docs) {
     const body = htmlToHast(doc.body);
-    if (hasGuideContent(body)) guideContent.add(doc.slug);
+    if (isGeneratedReference(body)) generated.add(doc.slug);
+    else if (hasGuideContent(body)) guideContent.add(doc.slug);
     const wrappers = toctreeWrappers(body);
     const sources: (Root | Element)[] = wrappers;
     if (!sources.length && doc.toc) sources.push(htmlToHast(doc.toc));
@@ -87,7 +89,9 @@ export function buildSphinxNavigation(docs: SphinxNavigationDoc[]): SdkNavGroup[
     if (ancestor) addChild(ancestor, doc.slug);
   }
 
-  const reference = new Set(docs.filter((doc) => doc.tag).map((doc) => doc.slug));
+  const reference = new Set(
+    docs.filter((doc) => doc.tag || generated.has(doc.slug)).map((doc) => doc.slug)
+  );
   let changed = true;
   while (changed) {
     changed = false;
@@ -143,13 +147,21 @@ export function buildSphinxNavigation(docs: SphinxNavigationDoc[]): SdkNavGroup[
     );
     for (const doc of [...roots, ...categoryDocs])
       pages.push(...entries(doc.slug, category, emitted));
-    if (pages.length) {
-      groups.push({ group: category ? 'API Reference' : 'Getting Started', pages });
-    }
+    const group = category ? 'API Reference' : 'Getting Started';
+    const flattened = flattenGroup(pages, group);
+    if (flattened.length) groups.push({ group, pages: flattened });
   }
   return groups;
 }
 
+function flattenGroup(pages: (string | SdkNavGroup)[], title: string): (string | SdkNavGroup)[] {
+  return pages.flatMap((page) => {
+    if (typeof page === 'string') return [page];
+    const nested = flattenGroup(page.pages, title);
+    return page.group === title ? nested : [{ ...page, pages: nested }];
+  });
+}
+
 function ancestorPrefixes(slug: string): string[] {
   return [...slug.matchAll(/[/.]/g)].map((match) => slug.slice(0, match.index));
 }
@@ -212,6 +224,17 @@ function resolvePageHref(
   return;
 }
 
+function isGeneratedReference(node: Root | Element): boolean {
+  if (node.type === 'element') {
+    const { id, href } = node.properties;
+    if (typeof id === 'string' && id.startsWith('module-')) return true;
+    if (typeof href === 'string' && /github\.com\/readthedocs\/sphinx-autoapi/.test(href)) {
+      return true;
+    }
+  }
+  return node.children.some((child) => child.type === 'element' && isGeneratedReference(child));
+}
+
 function hasGuideContent(node: Root | Element): boolean {
   if (node.type === 'element') {
     if (classes(node).includes('toctree-wrapper')) return false;