@mintlify/scraping 4.0.1191 → 4.0.1193

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.1191",
+  "version": "4.0.1193",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -43,7 +43,7 @@
     "format:check": "oxfmt --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.1314",
+    "@mintlify/common": "1.0.1315",
     "fast-xml-parser": "5.7.3",
     "fs-extra": "11.1.1",
     "graphql": "16.11.0",
@@ -62,9 +62,9 @@
     "zod": "3.24.0"
   },
   "devDependencies": {
-    "@mintlify/models": "0.0.379",
+    "@mintlify/models": "0.0.380",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.945",
+    "@mintlify/validation": "0.1.946",
     "@tsconfig/recommended": "1.0.2",
     "@types/hast": "3.0.4",
     "@types/mdast": "4.0.4",
@@ -87,5 +87,5 @@
       "optional": true
     }
   },
-  "gitHead": "78b82add935dc4b66178be74d41b67a59712367c"
+  "gitHead": "ab2c294d7086f0c5b03babf75baee85a9d4ab608"
 }

package/src/sdk/converters/sphinx.ts

@@ -1,6 +1,7 @@
-import { hastToText } from '@mintlify/common';
+import { hastToText, safeCleanHeadingId } from '@mintlify/common';
 import fse from 'fs-extra';
 import type { Element, ElementContent, Root as HastRoot } from 'hast';
+import { randomUUID } from 'node:crypto';
 import path from 'node:path';
 import { visit, SKIP } from 'unist-util-visit';
 
@@ -8,7 +9,12 @@ import { htmlToHast } from '../../pipeline/root.js';
 import { turnChildrenIntoMdx } from '../../utils/children.js';
 import { removeHastComments } from '../../utils/hastComments.js';
 import { hastToMdx } from '../convert.js';
-import type { SdkNavGroup, SdkPage, SdkReference } from '../types.js';
+import { escapeInline, jsxString, safeHref } from '../mdx.js';
+import { rewriteSphinxImages } from '../sphinx-images.js';
+import { splitSphinxModules, type SphinxModuleDoc } from '../sphinx-modules.js';
+import { buildSphinxNavigation, type SphinxNavigationDoc } from '../sphinx-navigation.js';
+import { renderSphinxLinkedCode, renderSphinxSignature } from '../sphinx-signatures.js';
+import type { SdkPage, SdkReference } from '../types.js';
 
 const SKIP_PAGES = new Set(['genindex', 'search', 'py-modindex']);
 const LANG_MARKER = '@@mintlang:';
@@ -26,51 +32,71 @@ const ADMONITION_TAGS: Record<string, string> = {
   attention: 'Warning',
 };
 
-type SphinxDoc = { slug: string; title: string; body: string; tag?: string };
-
 export async function convertSphinx(sourcePath: string): Promise<SdkReference> {
-  const docs = await loadDocs(sourcePath);
+  const docs = splitSphinxModules(await loadDocs(sourcePath));
   const slugs = new Set(docs.map((doc) => doc.slug));
+  const anchors = collectAnchors(docs);
+  return {
+    pages: docs.map((doc) => renderPage(doc, slugs, anchors)),
+    groups: buildSphinxNavigation(docs),
+  };
+}
 
-  const guides = docs
-    .filter((doc) => !doc.tag)
-    .sort((left, right) =>
-      left.slug === 'index' ? -1 : right.slug === 'index' ? 1 : left.slug.localeCompare(right.slug)
-    );
-  const modules = docs
-    .filter((doc) => doc.tag)
-    .sort((left, right) => left.slug.localeCompare(right.slug));
-
-  const pages = [...guides, ...modules].map((doc) => renderPage(doc, slugs));
-  const groups: SdkNavGroup[] = [];
-  if (guides.length) groups.push({ group: 'Getting Started', pages: guides.map((d) => d.slug) });
-  if (modules.length) groups.push({ group: 'API Reference', pages: modules.map((d) => d.slug) });
+type SphinxAnchors = Map<string, Map<string, string>>;
 
-  return { pages, groups };
+function collectAnchors(docs: SphinxModuleDoc[]): SphinxAnchors {
+  const anchors: SphinxAnchors = new Map();
+  const ordered = [
+    ...docs.filter((doc) => !doc.sourceSlug),
+    ...docs.filter((doc) => doc.sourceSlug),
+  ];
+  for (const doc of ordered) {
+    const source = doc.sourceSlug ?? doc.slug;
+    const targets = anchors.get(source) ?? new Map<string, string>();
+    visit(htmlToHast(doc.body), 'element', (node: Element) => {
+      if (typeof node.properties.id === 'string')
+        targets.set(anchorId(node.properties.id), doc.slug);
+    });
+    anchors.set(source, targets);
+  }
+  return anchors;
 }
 
-async function loadDocs(sourcePath: string): Promise<SphinxDoc[]> {
-  const docs: SphinxDoc[] = [];
+async function loadDocs(sourcePath: string): Promise<SphinxNavigationDoc[]> {
+  const docs: SphinxNavigationDoc[] = [];
   for (const file of await collectFjsonFiles(sourcePath)) {
-    const raw = (await fse.readJson(file)) as {
-      body?: string;
-      title?: string;
-      current_page_name?: string;
-    };
+    const raw: unknown = await fse.readJson(file);
+    if (!raw || typeof raw !== 'object' || !('body' in raw)) continue;
+    if (typeof raw.body !== 'string' || !raw.body.trim()) continue;
     const slug =
-      raw.current_page_name ??
-      path
-        .relative(sourcePath, file)
-        .split(path.sep)
-        .join('/')
-        .replace(/\.fjson$/, '');
+      'current_page_name' in raw && typeof raw.current_page_name === 'string'
+        ? raw.current_page_name
+        : path
+            .relative(sourcePath, file)
+            .split(path.sep)
+            .join('/')
+            .replace(/\.fjson$/, '');
     if (SKIP_PAGES.has(slug) || slug.startsWith('_modules')) continue;
-    if (!raw.body?.trim()) continue;
+    const parents: { link: string }[] = [];
+    if ('parents' in raw && Array.isArray(raw.parents)) {
+      for (const parent of raw.parents) {
+        if (
+          parent &&
+          typeof parent === 'object' &&
+          'link' in parent &&
+          typeof parent.link === 'string'
+        ) {
+          parents.push({ link: parent.link });
+        }
+      }
+    }
     docs.push({
       slug,
-      title: cleanTitle(raw.title ?? slug),
+      title: cleanTitle('title' in raw && typeof raw.title === 'string' ? raw.title : slug),
       body: raw.body,
       tag: /<dl class="py[ "]/.test(raw.body) ? 'MODULE' : undefined,
+      toc: 'toc' in raw && typeof raw.toc === 'string' ? raw.toc : undefined,
+      parents,
     });
   }
   return docs;
@@ -87,18 +113,38 @@ async function collectFjsonFiles(dir: string): Promise<string[]> {
   return files.sort();
 }
 
-function renderPage(doc: SphinxDoc, slugs: Set<string>): SdkPage {
+function renderPage(doc: SphinxModuleDoc, slugs: Set<string>, anchors: SphinxAnchors): SdkPage {
   const hast = htmlToHast(doc.body);
+  const generated = new Map<string, string>();
+  const prefix = `MINTSPHINX${randomUUID().replaceAll('-', '')}`;
+  function marker(content: string): string {
+    const marker = `${prefix}BLOCK${generated.size}END`;
+    generated.set(marker, content);
+    return marker;
+  }
+  function block(content: string): Element {
+    return {
+      type: 'element',
+      tagName: 'p',
+      properties: {},
+      children: [{ type: 'text', value: marker(content) }],
+    };
+  }
   removeHastComments(hast);
-  removeSphinxChrome(hast);
+  removeSphinxChrome(hast, doc.title);
   const description = extractDescription(hast);
-  rewriteLinks(hast, doc.slug, slugs);
-  transformAdmonitions(hast);
+  rewriteLinks(hast, doc, slugs, anchors);
+  rewriteSphinxImages(hast, doc.sourceSlug ?? doc.slug, doc.slug);
   markHighlightLanguages(hast);
-  transformPyDls(hast, 0);
-  const content = hastToMdx(hast)
+  preserveAnchors(hast, block);
+  transformPyDls(hast, 0, block);
+  preserveLinkedCode(hast, marker);
+  transformAdmonitions(hast);
+  let content = hastToMdx(hast)
     .replace(FENCE_MARKER, '$1$2\n')
     .replace(/^[> ]*@@mintlang:[\w+-]+@@\n?/gm, '');
+  for (const [marker, replacement] of generated)
+    content = content.replaceAll(marker, () => replacement);
   return {
     slug: doc.slug,
     title: doc.title,
@@ -108,8 +154,7 @@ function renderPage(doc: SphinxDoc, slugs: Set<string>): SdkPage {
   };
 }
 
-function removeSphinxChrome(hast: HastRoot): void {
-  let removedH1 = false;
+function removeSphinxChrome(hast: HastRoot, title: string): void {
   visit(hast, 'element', (node: Element, index, parent) => {
     if (!parent || typeof index !== 'number') return;
     const isHeaderlink = node.tagName === 'a' && classList(node).includes('headerlink');
@@ -118,33 +163,120 @@ function removeSphinxChrome(hast: HastRoot): void {
       node.children.some(
         (child) => child.type === 'element' && classList(child).includes('viewcode-link')
       );
-    const isFirstH1 = node.tagName === 'h1' && !removedH1;
-    if (isFirstH1) removedH1 = true;
-    if (isHeaderlink || isViewcode || isFirstH1) {
+    if (isHeaderlink || isViewcode) {
       parent.children.splice(index, 1);
       return index;
     }
   });
+  let removedTitle = false;
+  visit(hast, 'element', (node: Element, index, parent) => {
+    if (removedTitle || !parent || typeof index !== 'number') return;
+    if (
+      node.tagName !== 'h1' &&
+      !(/^h[1-6]$/.test(node.tagName) && cleanTitle(hastToText(node)) === title)
+    )
+      return;
+    removedTitle = true;
+    parent.children.splice(index, 1);
+    return index;
+  });
 }
 
-function rewriteLinks(hast: HastRoot, fromSlug: string, slugs: Set<string>): void {
+function rewriteLinks(
+  hast: HastRoot,
+  doc: SphinxModuleDoc,
+  slugs: Set<string>,
+  anchors: SphinxAnchors
+): void {
   visit(hast, 'element', (node: Element) => {
     if (node.tagName !== 'a' || typeof node.properties.href !== 'string') return;
-    const rewritten = rewriteHref(node.properties.href, fromSlug, slugs);
+    const rewritten = rewriteHref(node.properties.href, doc, slugs, anchors);
     if (rewritten === undefined) delete node.properties.href;
     else node.properties.href = rewritten;
   });
 }
 
-function rewriteHref(href: string, fromSlug: string, slugs: Set<string>): string | undefined {
-  if (/^(https?:|mailto:|#)/.test(href)) return href;
-  const [target = ''] = href.split('#');
-  let resolved = path.posix
-    .normalize(path.posix.join(path.posix.dirname(fromSlug), target))
-    .replace(/\/$/, '')
-    .replace(/\.html?$/, '');
-  if (resolved === '.' || resolved === '') resolved = 'index';
-  return slugs.has(resolved) ? `/${resolved}` : undefined;
+function rewriteHref(
+  href: string,
+  doc: SphinxModuleDoc,
+  slugs: Set<string>,
+  anchors: SphinxAnchors
+): string | undefined {
+  const fromSlug = doc.sourceSlug ?? doc.slug;
+  const safe = safeHref(href);
+  if (safe === undefined || /^(https?:|mailto:)/i.test(safe)) return safe;
+  const hashIndex = safe.indexOf('#');
+  const hash = hashIndex < 0 ? '' : `#${anchorId(safe.slice(hashIndex + 1))}`;
+  const beforeHash = hashIndex < 0 ? safe : safe.slice(0, hashIndex);
+  const queryIndex = beforeHash.indexOf('?');
+  const query = queryIndex < 0 ? '' : beforeHash.slice(queryIndex);
+  let target = queryIndex < 0 ? beforeHash : beforeHash.slice(0, queryIndex);
+  function destination(slug: string): string {
+    let owner = slug;
+    if (hash.length > 1) owner = anchors.get(slug)?.get(hash.slice(1)) ?? slug;
+    if (!target && owner === doc.slug) return `${query}${hash}`;
+    return `/${owner}${query}${hash}`;
+  }
+  if (!target) return destination(fromSlug);
+  try {
+    target = decodeURIComponent(target);
+  } catch {
+    return undefined;
+  }
+  const base = path.posix.dirname(fromSlug);
+  let candidates = [path.posix.join(base, target), path.posix.join(fromSlug, target)];
+  if (target.endsWith('/') && path.posix.basename(fromSlug) !== 'index') candidates.reverse();
+  if (target.startsWith('/')) candidates = [target.slice(1)];
+  for (const candidate of candidates) {
+    let resolved = path.posix
+      .normalize(candidate)
+      .replace(/\.(?:html?|fjson)$/, '')
+      .replace(/\/$/, '');
+    if (resolved === '.' || resolved === '') resolved = 'index';
+    if (slugs.has(resolved)) return destination(resolved);
+    const index = path.posix.join(resolved, 'index');
+    if (slugs.has(index)) return destination(index);
+  }
+  return undefined;
+}
+
+function anchorId(value: string): string {
+  return safeCleanHeadingId(value)
+    .replace(/[<>"&]/g, '')
+    .replace(/\s+/g, '-');
+}
+
+function preserveAnchors(node: HastRoot | Element, block: (content: string) => Element): void {
+  const children: ElementContent[] = [];
+  for (const child of node.children) {
+    if (child.type === 'element') {
+      const id = child.properties.id;
+      if (typeof id === 'string' && !(child.tagName === 'dt' && classList(child).includes('sig'))) {
+        children.push(block(`<span id=${jsxString(anchorId(id))} />`));
+      }
+      preserveAnchors(child, block);
+    }
+    if (child.type !== 'doctype' && child.type !== 'mdxjsEsm') children.push(child);
+  }
+  node.children = children;
+}
+
+function preserveLinkedCode(node: HastRoot | Element, marker: (content: string) => string): void {
+  const children: ElementContent[] = [];
+  for (const child of node.children) {
+    if (child.type === 'element') {
+      if (child.tagName === 'code') {
+        const content = renderSphinxLinkedCode(child);
+        if (content) {
+          children.push({ type: 'text', value: marker(content) });
+          continue;
+        }
+      }
+      if (child.tagName !== 'pre') preserveLinkedCode(child, marker);
+    }
+    if (child.type !== 'doctype' && child.type !== 'mdxjsEsm') children.push(child);
+  }
+  node.children = children;
 }
 
 function transformAdmonitions(hast: HastRoot): void {
@@ -176,30 +308,48 @@ function markHighlightLanguages(hast: HastRoot): void {
   });
 }
 
-function transformPyDls(node: HastRoot | Element, depth: number): void {
+function transformPyDls(
+  node: HastRoot | Element,
+  depth: number,
+  block: (content: string) => Element
+): void {
   const children: ElementContent[] = [];
-  for (const child of node.children as ElementContent[]) {
+  for (const child of node.children) {
     if (child.type === 'element' && child.tagName === 'dl' && classList(child).includes('py')) {
-      children.push(...explodePyDl(child, depth));
+      children.push(...explodePyDl(child, depth, block));
     } else {
-      if (child.type === 'element') transformPyDls(child, depth);
-      children.push(child);
+      if (child.type === 'element') transformPyDls(child, depth, block);
+      if (child.type !== 'doctype' && child.type !== 'mdxjsEsm') children.push(child);
     }
   }
   node.children = children;
 }
 
-function explodePyDl(dl: Element, depth: number): ElementContent[] {
+function explodePyDl(
+  dl: Element,
+  depth: number,
+  block: (content: string) => Element
+): ElementContent[] {
   const callable = classList(dl).some((name) => ['method', 'function'].includes(name));
   const out: ElementContent[] = [];
-  for (const child of dl.children) {
+  for (const [index, child] of dl.children.entries()) {
     if (child.type !== 'element') continue;
     if (child.tagName === 'dt' && classList(child).includes('sig')) {
       const name = findDescName(child);
-      if (name) out.push(heading(Math.min(3 + depth, 6), callable ? `${name}()` : name));
-      out.push(pythonCodeBlock(hastToText(child).replace(/\s+/g, ' ').trim()));
+      if (name) {
+        const title = escapeInline(callable ? `${name}()` : name);
+        const id = child.properties.id;
+        const suffix = typeof id === 'string' ? ` {#${anchorId(id)}}` : '';
+        out.push(block(`${'#'.repeat(Math.min(3 + depth, 6))} ${title}${suffix}`));
+      }
+      const description = dl.children
+        .slice(index + 1)
+        .find(
+          (sibling): sibling is Element => sibling.type === 'element' && sibling.tagName === 'dd'
+        );
+      out.push(block(renderSphinxSignature(child, description)));
     } else if (child.tagName === 'dd') {
-      transformPyDls(child, depth + 1);
+      transformPyDls(child, depth + 1, block);
       out.push(...child.children);
     }
   }
@@ -216,24 +366,6 @@ function findDescName(node: Element): string | undefined {
   return name;
 }
 
-function heading(level: number, text: string): Element {
-  return {
-    type: 'element',
-    tagName: `h${level}`,
-    properties: {},
-    children: [{ type: 'text', value: text }],
-  };
-}
-
-function pythonCodeBlock(code: string): Element {
-  return {
-    type: 'element',
-    tagName: 'pre',
-    properties: {},
-    children: [{ type: 'text', value: `${LANG_MARKER}python@@\n${code}` }],
-  };
-}
-
 function extractDescription(hast: HastRoot): string | undefined {
   let description: string | undefined;
   visit(hast, 'element', (node: Element, _index, parent) => {
@@ -249,11 +381,9 @@ function extractDescription(hast: HastRoot): string | undefined {
 
 function classList(node: Element): string[] {
   const value = node.properties.className;
-  return Array.isArray(value)
-    ? value.map(String)
-    : typeof value === 'string'
-      ? value.split(' ')
-      : [];
+  if (Array.isArray(value)) return value.map(String);
+  if (typeof value === 'string') return value.split(' ');
+  return [];
 }
 
 function cleanTitle(title: string): string {

package/src/sdk/sphinx-images.ts

@@ -0,0 +1,25 @@
+import type { Element, Root } from 'hast';
+import path from 'node:path';
+import { visit } from 'unist-util-visit';
+
+import { safeHref } from './mdx.js';
+
+export function rewriteSphinxImages(hast: Root, sourceSlug: string, destinationSlug: string): void {
+  const base = path.posix.relative(
+    path.posix.dirname(destinationSlug),
+    path.posix.dirname(sourceSlug)
+  );
+  if (!base) return;
+  visit(hast, 'element', (node: Element) => {
+    if (node.tagName !== 'img' || typeof node.properties.src !== 'string') return;
+    const href = safeHref(node.properties.src);
+    if (!href || href.startsWith('/') || href.startsWith('#') || /^[a-z][a-z0-9+.-]*:/i.test(href))
+      return;
+    const suffixStart = href.search(/[?#]/);
+    const file = suffixStart === -1 ? href : href.slice(0, suffixStart);
+    if (!file) return;
+    const suffix = suffixStart === -1 ? '' : href.slice(suffixStart);
+    const relative = path.posix.join(base, file);
+    node.properties.src = `${relative.startsWith('../') ? relative : `./${relative}`}${suffix}`;
+  });
+}

package/src/sdk/sphinx-modules.ts

@@ -0,0 +1,124 @@
+import type { Element, ElementContent, Root } from 'hast';
+
+import { htmlToHast } from '../pipeline/root.js';
+import type { SphinxNavigationDoc } from './sphinx-navigation.js';
+
+export type SphinxModuleDoc = SphinxNavigationDoc & { sourceSlug?: string };
+
+type ModuleSection = { node: Element; name: string; start: number; end: number };
+
+export function splitSphinxModules(docs: SphinxNavigationDoc[]): SphinxModuleDoc[] {
+  const occupied = new Set(docs.map((doc) => doc.slug));
+  const result: SphinxModuleDoc[] = [];
+  for (const doc of docs) {
+    const sections = moduleSections(htmlToHast(doc.body)).sort(
+      (left, right) => left.start - right.start
+    );
+    const slugs = sections.map((section) => `${doc.slug}/${section.name}`);
+    if (
+      sections.length < 2 ||
+      new Set(slugs).size !== slugs.length ||
+      slugs.some((slug) => occupied.has(slug))
+    ) {
+      result.push(doc);
+      continue;
+    }
+
+    let body = '';
+    let cursor = 0;
+    const aliases = new Set<string>();
+    const modules: SphinxModuleDoc[] = [];
+    for (const section of sections) {
+      const slug = `${doc.slug}/${section.name}`;
+      const title = sectionTitle(section.node) || section.name;
+      body += doc.body.slice(cursor, section.start);
+      for (const id of sectionIds(section.node)) {
+        if (aliases.has(id)) continue;
+        aliases.add(id);
+        body += `<span id="${escapeHtml(id)}"></span>`;
+      }
+      body += `<div class="toctree-wrapper"><ul><li><a class="reference internal" href="/${escapeHtml(slug)}">${escapeHtml(title)}</a></li></ul></div>`;
+      cursor = section.end;
+      occupied.add(slug);
+      modules.push({
+        slug,
+        title,
+        body: doc.body.slice(section.start, section.end),
+        tag: 'MODULE',
+        sourceSlug: doc.slug,
+        parents: [{ link: `/${doc.slug}` }],
+      });
+    }
+    body += doc.body.slice(cursor);
+    result.push({ ...doc, body, tag: 'MODULE' }, ...modules);
+  }
+  return result;
+}
+
+function moduleSections(node: Root | Element): ModuleSection[] {
+  const peers: ModuleSection[] = [];
+  const containers: Element[] = [];
+  for (const child of node.children) {
+    if (child.type !== 'element') continue;
+    const name = moduleName(child);
+    if (name) {
+      const start = child.position?.start.offset;
+      const end = child.position?.end.offset;
+      if (typeof start === 'number' && typeof end === 'number') {
+        peers.push({ node: child, name, start, end });
+      }
+    } else containers.push(child);
+  }
+  const sections = containers.flatMap(moduleSections);
+  if (peers.length > 1) return [...peers, ...sections];
+  return sections;
+}
+
+function moduleName(node: Element): string | undefined {
+  const id = node.properties.id;
+  if (typeof id !== 'string' || !id.startsWith('module-')) return;
+  if (node.tagName !== 'section' && node.tagName !== 'div') return;
+  if (node.tagName === 'div' && !classes(node).includes('section')) return;
+  const name = id.slice('module-'.length);
+  if (!/^[\p{ID_Start}_][\p{ID_Continue}]*(?:\.[\p{ID_Start}_][\p{ID_Continue}]*)*$/u.test(name))
+    return;
+  return name;
+}
+
+function sectionTitle(node: Element): string | undefined {
+  for (const child of node.children) {
+    if (child.type !== 'element') continue;
+    if (/^h[1-6]$/.test(child.tagName)) return textContent(child).replace(/\s+/g, ' ').trim();
+  }
+  return;
+}
+
+function sectionIds(node: Element): string[] {
+  const ids: string[] = [];
+  if (typeof node.properties.id === 'string') ids.push(node.properties.id);
+  for (const child of node.children) {
+    if (child.type === 'element') ids.push(...sectionIds(child));
+  }
+  return ids;
+}
+
+function textContent(node: ElementContent): string {
+  if (node.type === 'text') return node.value;
+  if (node.type !== 'element' || classes(node).includes('headerlink')) return '';
+  return node.children.map(textContent).join('');
+}
+
+function classes(node: Element): string[] {
+  const value = node.properties.className;
+  if (Array.isArray(value)) return value.map(String);
+  if (typeof value === 'string') return value.split(/\s+/);
+  return [];
+}
+
+function escapeHtml(value: string): string {
+  return value
+    .replaceAll('&', '&amp;')
+    .replaceAll('<', '&lt;')
+    .replaceAll('>', '&gt;')
+    .replaceAll('"', '&quot;');
+}