@mintlify/scraping 4.0.1191 → 4.0.1193

package/src/sdk/sphinx-navigation.ts

@@ -0,0 +1,236 @@
+import type { Element, ElementContent, Root } from 'hast';
+import path from 'node:path';
+
+import { htmlToHast } from '../pipeline/root.js';
+import type { SdkNavGroup } from './types.js';
+
+export type SphinxNavigationDoc = {
+  slug: string;
+  title: string;
+  body: string;
+  tag?: string;
+  toc?: string;
+  parents?: { link: string }[];
+};
+
+export function buildSphinxNavigation(docs: SphinxNavigationDoc[]): SdkNavGroup[] {
+  const bySlug = new Map(docs.map((doc) => [doc.slug, doc]));
+  const children = new Map<string, string[]>();
+  const parents = new Map<string, string[]>();
+  const guideContent = new Set<string>();
+
+  function addChild(parent: string, child: string): void {
+    if (parent === child) return;
+    const siblings = children.get(parent) ?? [];
+    if (!siblings.includes(child)) siblings.push(child);
+    children.set(parent, siblings);
+    const ancestors = parents.get(child) ?? [];
+    if (!ancestors.includes(parent)) ancestors.push(parent);
+    parents.set(child, ancestors);
+  }
+
+  for (const doc of docs) {
+    const body = htmlToHast(doc.body);
+    if (hasGuideContent(body)) guideContent.add(doc.slug);
+    const wrappers = toctreeWrappers(body);
+    const sources: (Root | Element)[] = wrappers;
+    if (!sources.length && doc.toc) sources.push(htmlToHast(doc.toc));
+
+    function readTree(node: Root | Element, parentSlug: string): void {
+      let nextParent = parentSlug;
+      if (node.type === 'element' && node.tagName === 'li') {
+        const href = firstPageHref(node);
+        const child = href ? resolvePageHref(href, doc.slug, bySlug) : undefined;
+        if (child) {
+          addChild(parentSlug, child);
+          nextParent = child;
+        }
+      }
+      for (const child of node.children) {
+        if (child.type === 'element') readTree(child, nextParent);
+      }
+    }
+
+    for (const source of sources) readTree(source, doc.slug);
+  }
+
+  for (const doc of docs) {
+    if (parents.has(doc.slug)) continue;
+    const ancestor = doc.parents
+      ?.map((parent) => resolvePageHref(parent.link, doc.slug, bySlug))
+      .filter((slug): slug is string => slug !== undefined)
+      .pop();
+    if (ancestor) addChild(ancestor, doc.slug);
+  }
+
+  const prefixCounts = new Map<string, number>();
+  for (const doc of docs) {
+    if (parents.has(doc.slug)) continue;
+    for (const prefix of ancestorPrefixes(doc.slug)) {
+      prefixCounts.set(prefix, (prefixCounts.get(prefix) ?? 0) + 1);
+    }
+  }
+  const virtual = new Set<string>();
+  for (const [prefix, count] of prefixCounts) {
+    if (count < 2 || bySlug.has(prefix)) continue;
+    virtual.add(prefix);
+    bySlug.set(prefix, { slug: prefix, title: path.posix.basename(prefix), body: '' });
+  }
+  const navigationDocs = [...bySlug.values()].sort((left, right) =>
+    left.slug.localeCompare(right.slug)
+  );
+  for (const doc of navigationDocs) {
+    if (parents.has(doc.slug)) continue;
+    const ancestor = ancestorPrefixes(doc.slug)
+      .reverse()
+      .find((prefix) => bySlug.has(prefix));
+    if (ancestor) addChild(ancestor, doc.slug);
+  }
+
+  const reference = new Set(docs.filter((doc) => doc.tag).map((doc) => doc.slug));
+  let changed = true;
+  while (changed) {
+    changed = false;
+    for (const doc of navigationDocs) {
+      const descendants = children.get(doc.slug) ?? [];
+      if (
+        !reference.has(doc.slug) &&
+        descendants.length &&
+        descendants.every((slug) => reference.has(slug)) &&
+        !guideContent.has(doc.slug)
+      ) {
+        reference.add(doc.slug);
+        changed = true;
+      }
+    }
+  }
+
+  const orderedDocs = [...navigationDocs].sort((left, right) => {
+    if (left.slug === 'index') return -1;
+    if (right.slug === 'index') return 1;
+    return left.slug.localeCompare(right.slug);
+  });
+  function entries(
+    slug: string,
+    category: boolean,
+    emitted: Set<string>
+  ): (string | SdkNavGroup)[] {
+    if (emitted.has(slug) || (!virtual.has(slug) && reference.has(slug) !== category)) return [];
+    const doc = bySlug.get(slug);
+    if (!doc) return [];
+    emitted.add(slug);
+    const nested = (children.get(slug) ?? []).flatMap((child) => entries(child, category, emitted));
+    if (virtual.has(slug)) {
+      if (nested.length < 2) return nested;
+      return [{ group: doc.title, pages: nested }];
+    }
+    if (!nested.length) return [slug];
+    return [{ group: doc.title, pages: [slug, ...nested] }];
+  }
+
+  const groups: SdkNavGroup[] = [];
+  for (const category of [false, true]) {
+    const emitted = new Set<string>();
+    const pages: (string | SdkNavGroup)[] = [];
+    const categoryDocs = orderedDocs.filter(
+      (doc) => virtual.has(doc.slug) || reference.has(doc.slug) === category
+    );
+    const roots = categoryDocs.filter(
+      (doc) =>
+        !(parents.get(doc.slug) ?? []).some(
+          (slug) => virtual.has(slug) || reference.has(slug) === category
+        )
+    );
+    for (const doc of [...roots, ...categoryDocs])
+      pages.push(...entries(doc.slug, category, emitted));
+    if (pages.length) {
+      groups.push({ group: category ? 'API Reference' : 'Getting Started', pages });
+    }
+  }
+  return groups;
+}
+
+function ancestorPrefixes(slug: string): string[] {
+  return [...slug.matchAll(/[/.]/g)].map((match) => slug.slice(0, match.index));
+}
+
+function toctreeWrappers(node: Root | Element): Element[] {
+  const wrappers: Element[] = [];
+  for (const child of node.children) {
+    if (child.type !== 'element') continue;
+    if (classes(child).includes('toctree-wrapper')) wrappers.push(child);
+    else wrappers.push(...toctreeWrappers(child));
+  }
+  return wrappers;
+}
+
+function firstPageHref(node: Element): string | undefined {
+  for (const child of node.children) {
+    if (child.type !== 'element' || ['ul', 'ol'].includes(child.tagName)) continue;
+    if (child.tagName === 'a' && typeof child.properties.href === 'string') {
+      return child.properties.href;
+    }
+    const href = firstPageHref(child);
+    if (href) return href;
+  }
+  return undefined;
+}
+
+function resolvePageHref(
+  href: string,
+  fromSlug: string,
+  docs: Map<string, SphinxNavigationDoc>
+): string | undefined {
+  if (!href || href.includes('#') || /^(?:[a-z][\w+.-]*:|\/\/)/i.test(href)) return;
+  let target = href.split('?')[0] ?? '';
+  try {
+    target = decodeURIComponent(target);
+  } catch {
+    return;
+  }
+  let candidates: string[];
+  if (target.startsWith('/')) candidates = [target.slice(1)];
+  else {
+    const fileRelative = path.posix.join(path.posix.dirname(fromSlug), target);
+    const directoryRelative = path.posix.join(fromSlug, target);
+    if (target.endsWith('/') && fromSlug !== 'index' && !fromSlug.endsWith('/index')) {
+      candidates = [directoryRelative, fileRelative];
+    } else {
+      candidates = [fileRelative, directoryRelative];
+    }
+  }
+  for (const candidate of candidates) {
+    let slug = path.posix
+      .normalize(candidate)
+      .replace(/\.(?:html?|fjson)$/, '')
+      .replace(/\/$/, '');
+    if (!slug || slug === '.') slug = 'index';
+    if (docs.has(slug)) return slug;
+    const index = path.posix.join(slug, 'index');
+    if (docs.has(index)) return index;
+  }
+  return;
+}
+
+function hasGuideContent(node: Root | Element): boolean {
+  if (node.type === 'element') {
+    if (classes(node).includes('toctree-wrapper')) return false;
+    if (['p', 'pre', 'table', 'dl'].includes(node.tagName)) {
+      return node.children.some((child) => textContent(child).trim());
+    }
+  }
+  return node.children.some((child) => child.type === 'element' && hasGuideContent(child));
+}
+
+function textContent(node: ElementContent): string {
+  if (node.type === 'text') return node.value;
+  if (node.type !== 'element') return '';
+  return node.children.map(textContent).join('');
+}
+
+function classes(node: Element): string[] {
+  const value = node.properties.className;
+  if (Array.isArray(value)) return value.map(String);
+  if (typeof value === 'string') return value.split(/\s+/);
+  return [];
+}

package/src/sdk/sphinx-signatures.ts

@@ -0,0 +1,233 @@
+import { hastToText } from '@mintlify/common';
+import type { Element, ElementContent } from 'hast';
+
+import { inlineCode, linkedCode, responseField, safeHref, type LinkedToken } from './mdx.js';
+
+type DocumentedTypes = {
+  parameters: Map<string, LinkedToken[][]>;
+  returns: LinkedToken[][];
+};
+
+export function renderSphinxLinkedCode(element: Element): string | undefined {
+  return linkedCode(trimTokens(linkedTokens(element.children)));
+}
+
+export function renderSphinxSignature(signature: Element, description?: Element): string {
+  const code = hastToText(signature).replace(/\s+/g, ' ').trim();
+  const fence = '`'.repeat(Math.max(3, ...(code.match(/`+/g) ?? []).map((run) => run.length + 1)));
+  const sections = [`${fence}python\n${code}\n${fence}`];
+  const documented = documentedTypes(description);
+  const parameters = findElements(signature, 'sig-param')
+    .map((parameter) => renderParameter(parameter, documented))
+    .filter((field) => field !== undefined);
+  if (parameters.length) sections.push('**Parameters**', ...parameters);
+  const returns = findElements(signature, 'sig-return-typehint')[0];
+  if (returns) {
+    const tokens = trimTokens(linkedTokens(returns.children));
+    const text = tokens.map((token) => token.text).join('');
+    if (text && !documented.returns.some((type) => sameLinkedType(tokens, type))) {
+      sections.push(`**Returns:** ${linkedCode(tokens) ?? inlineCode(text)}`);
+    }
+  }
+  return sections.join('\n\n');
+}
+
+function renderParameter(parameter: Element, documented: DocumentedTypes): string | undefined {
+  const tokens = linkedTokens(parameter.children);
+  const text = tokens.map((token) => token.text).join('');
+  const colon = topLevelDelimiter(text, ':');
+  if (colon === -1) return undefined;
+  const name = text.slice(0, colon).trim();
+  if (!name) return undefined;
+  const defaultStart = topLevelDelimiter(text, '=');
+  const end = defaultStart === -1 ? text.length : defaultStart;
+  const typeTokens = trimTokens(sliceTokens(tokens, colon + 1, end));
+  const type = typeTokens.map((token) => token.text).join('');
+  if (!type) return undefined;
+  if (documented.parameters.get(name)?.some((type) => sameLinkedType(typeTokens, type))) {
+    return undefined;
+  }
+  const only = typeTokens[0];
+  const typeHref = typeTokens.length === 1 ? only?.href : undefined;
+  let body: string | undefined;
+  if (!typeHref || type.length > 80) {
+    const linked = linkedCode(typeTokens);
+    if (linked) body = `**Type:** ${linked}`;
+    else if (type.length > 80) body = `**Type:** ${inlineCode(type)}`;
+  }
+  return responseField(name, type, defaultStart !== -1 || name.startsWith('*'), body, typeHref);
+}
+
+function documentedTypes(description: Element | undefined): DocumentedTypes {
+  const documented: DocumentedTypes = { parameters: new Map(), returns: [] };
+  if (!description) return documented;
+  for (const fieldList of ownFieldLists(description)) {
+    let label = '';
+    for (const child of fieldList.children) {
+      if (child.type !== 'element') continue;
+      if (child.tagName === 'dt') {
+        label = hastToText(child).trim().replace(/:\s*$/, '').replace(/\s+/g, ' ').toLowerCase();
+      } else if (child.tagName === 'dd') {
+        if (label === 'return type' || label === 'rtype') {
+          documented.returns.push(trimTokens(linkedTokens(child.children)));
+        } else if (label === 'parameter' || label === 'parameters') {
+          addDocumentedParameters(child, documented.parameters);
+        }
+      }
+    }
+  }
+  return documented;
+}
+
+function ownFieldLists(description: Element): Element[] {
+  const lists: Element[] = [];
+  for (const child of description.children) {
+    if (child.type !== 'element') continue;
+    const classes = child.properties.className;
+    const names = Array.isArray(classes) ? classes.map(String) : String(classes ?? '').split(' ');
+    if (child.tagName === 'dl' && names.includes('py')) continue;
+    if (child.tagName === 'dl' && names.includes('field-list')) lists.push(child);
+    else lists.push(...ownFieldLists(child));
+  }
+  return lists;
+}
+
+function addDocumentedParameters(element: Element, parameters: Map<string, LinkedToken[][]>): void {
+  for (const child of element.children) {
+    if (child.type !== 'element') continue;
+    if (child.tagName !== 'p') {
+      addDocumentedParameters(child, parameters);
+      continue;
+    }
+    const first = child.children.find((node) => node.type !== 'text' || node.value.trim() !== '');
+    if (first?.type !== 'element' || first.tagName !== 'strong') continue;
+    const name = hastToText(first).trim();
+    const tokens = linkedTokens(child.children);
+    const text = tokens.map((token) => token.text).join('');
+    const nameEnd = text.indexOf(name) + name.length;
+    const start = text.indexOf('(', nameEnd);
+    if (start === -1 || text.slice(nameEnd, start).trim() !== '') continue;
+    const end = closingParenthesis(text, start);
+    if (end === -1) continue;
+    const types = parameters.get(name) ?? [];
+    types.push(trimTokens(sliceTokens(tokens, start + 1, end)));
+    parameters.set(name, types);
+  }
+}
+
+function closingParenthesis(text: string, start: number): number {
+  let depth = 0;
+  let quote: string | undefined;
+  let escaped = false;
+  for (let index = start; index < text.length; index += 1) {
+    const character = text[index];
+    if (escaped) escaped = false;
+    else if (quote) {
+      if (character === '\\') escaped = true;
+      else if (character === quote) quote = undefined;
+    } else if (character === '"' || character === "'") quote = character;
+    else if (character === '(') depth += 1;
+    else if (character === ')') {
+      depth -= 1;
+      if (depth === 0) return index;
+    }
+  }
+  return -1;
+}
+
+function sameLinkedType(left: LinkedToken[], right: LinkedToken[]): boolean {
+  const key = (tokens: LinkedToken[]) =>
+    JSON.stringify(
+      tokens
+        .map((token) =>
+          token.href ? ['href', token.href] : ['text', token.text.replace(/\s+/g, '')]
+        )
+        .filter(([, value]) => value)
+    );
+  return key(left) === key(right);
+}
+
+function linkedTokens(children: ElementContent[], href?: string): LinkedToken[] {
+  const tokens: LinkedToken[] = [];
+  for (const child of children) {
+    if (child.type === 'text') {
+      tokens.push({ text: child.value.replace(/\s+/g, ' '), href });
+    } else if (child.type === 'element') {
+      let nextHref = href;
+      if (child.tagName === 'a' && typeof child.properties.href === 'string') {
+        nextHref = safeHref(child.properties.href)?.replace(
+          /[\u0000-\u0020\u007f"'()[\]<>`{}\\]/g,
+          (character) => `%${character.charCodeAt(0).toString(16).toUpperCase().padStart(2, '0')}`
+        );
+      }
+      tokens.push(...linkedTokens(child.children, nextHref));
+    }
+  }
+  const merged: LinkedToken[] = [];
+  for (const token of tokens) {
+    const previous = merged.at(-1);
+    if (previous && previous.href === token.href) previous.text += token.text;
+    else merged.push({ ...token });
+  }
+  return merged;
+}
+
+function trimTokens(tokens: LinkedToken[]): LinkedToken[] {
+  const text = tokens.map((token) => token.text).join('');
+  return sliceTokens(tokens, text.length - text.trimStart().length, text.trimEnd().length);
+}
+
+function sliceTokens(tokens: LinkedToken[], start: number, end: number): LinkedToken[] {
+  const sliced: LinkedToken[] = [];
+  let offset = 0;
+  for (const token of tokens) {
+    const tokenEnd = offset + token.text.length;
+    if (tokenEnd > start && offset < end) {
+      sliced.push({
+        text: token.text.slice(Math.max(0, start - offset), end - offset),
+        href: token.href,
+      });
+    }
+    offset = tokenEnd;
+  }
+  return sliced;
+}
+
+function topLevelDelimiter(text: string, delimiter: string): number {
+  let depth = 0;
+  let quote: string | undefined;
+  let escaped = false;
+  for (let index = 0; index < text.length; index += 1) {
+    const character = text[index];
+    if (escaped) {
+      escaped = false;
+    } else if (quote) {
+      if (character === '\\') escaped = true;
+      else if (character === quote) quote = undefined;
+    } else if (character === '"' || character === "'") {
+      quote = character;
+    } else if (character === '(' || character === '[' || character === '{') {
+      depth += 1;
+    } else if (character === ')' || character === ']' || character === '}') {
+      depth = Math.max(0, depth - 1);
+    } else if (character === delimiter && depth === 0) {
+      return index;
+    }
+  }
+  return -1;
+}
+
+function findElements(node: Element, className: string): Element[] {
+  const elements: Element[] = [];
+  for (const child of node.children) {
+    if (child.type !== 'element') continue;
+    const classes = child.properties.className;
+    if (Array.isArray(classes) && classes.includes(className)) elements.push(child);
+    else if (typeof classes === 'string' && classes.split(' ').includes(className)) {
+      elements.push(child);
+    } else {
+      elements.push(...findElements(child, className));
+    }
+  }
+  return elements;
+}