@mintlify/scraping 4.0.1229 → 4.0.1230

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.1229",
+  "version": "4.0.1230",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -87,5 +87,5 @@
       "optional": true
     }
   },
-  "gitHead": "74e0ddf24ba0f846028330ceb844b40b44a1d12b"
+  "gitHead": "e929a72c0d7dc12e955ca24554b1a3d6e6171bfa"
 }

package/src/sdk/converters/javadoc.ts

@@ -1,10 +1,19 @@
 import fse from 'fs-extra';
 import type { Element, ElementContent, Root as HastRoot, RootContent } from 'hast';
+import { randomUUID } from 'node:crypto';
 import path from 'node:path';
 
 import { htmlToHast } from '../../pipeline/root.js';
 import { removeHastComments } from '../../utils/hastComments.js';
 import { hastToMdx, type RewriteLink } from '../convert.js';
+import {
+  linkedTokens,
+  linkedTypeField,
+  sliceTokens,
+  topLevelDelimiter,
+  trimTokens,
+} from '../linked-tokens.js';
+import { linkedCode, type LinkedToken } from '../mdx.js';
 import type { SdkNavGroup, SdkPage, SdkReference } from '../types.js';
 
 type JavadocKind = 'interface' | 'class' | 'enum' | 'record' | 'exception' | 'error' | 'annotation';
@@ -239,11 +248,13 @@ function renderPage(
   const anchors = collectDetailAnchors(kept);
   const resolveHref = makeHrefResolver(type, slugByFile, anchors);
   const signatures: string[] = [];
+  const types = new Map<string, string>();
+  const prefix = `MINTJAVADOC${randomUUID().replaceAll('-', '')}`;
   for (const node of kept) {
     pruneChrome(node);
     unwrapLists(node);
     resolveLinks(node, resolveHref);
-    signatures.push(...codifySignatures(node));
+    signatures.push(...codifySignatures(node, types, prefix));
   }
 
   const root: HastRoot = { type: 'root', children: kept as RootContent[] };
@@ -253,6 +264,8 @@ function renderPage(
       .split(`\`\`\`\n${signature}\n\`\`\``)
       .join(`\`\`\`java\n${signature}\n\`\`\``);
   }
+  for (const [marker, replacement] of types)
+    content = content.replaceAll(marker, () => replacement);
 
   return { slug: type.slug, title: type.name, description, tag, content, kind };
 }
@@ -342,14 +355,21 @@ function unwrapLists(node: Element): void {
   }
 }
 
-function codifySignatures(node: Element): string[] {
+function codifySignatures(node: Element, types: Map<string, string>, prefix: string): string[] {
   const signatures: string[] = [];
+  const markers = new Map<Element, string>();
   visitElements(node, (el) => {
     if (!hasClass(el, 'member-signature') && !hasClass(el, 'type-signature')) return;
     const signature = textOf(el)
       .replace(/\u00a0/g, ' ')
       .trim();
     signatures.push(signature);
+    const linked = signatureTypes(el);
+    if (linked) {
+      const marker = `${prefix}TYPES${types.size}END`;
+      types.set(marker, linked);
+      markers.set(el, marker);
+    }
     el.tagName = 'pre';
     el.properties = {};
     el.children = [
@@ -361,9 +381,72 @@ function codifySignatures(node: Element): string[] {
       },
     ];
   });
+  insertMarkers(node, markers);
   return signatures;
 }
 
+function insertMarkers(node: Element, markers: Map<Element, string>): void {
+  node.children = node.children.flatMap((child): ElementContent[] => {
+    if (child.type !== 'element') return [child];
+    insertMarkers(child, markers);
+    const marker = markers.get(child);
+    if (!marker) return [child];
+    return [
+      child,
+      {
+        type: 'element',
+        tagName: 'p',
+        properties: {},
+        children: [{ type: 'text', value: marker }],
+      },
+    ];
+  });
+}
+
+function signatureTypes(signature: Element): string | undefined {
+  const sections: string[] = [];
+  const parameters = findElement(signature, (el) => hasClass(el, 'parameters'));
+  const fields = parameters ? splitParameters(linkedTokens(parameters.children)) : [];
+  if (fields.some((field) => field.tokens.some((token) => token.href))) {
+    sections.push(
+      '**Parameters**',
+      ...fields.map(({ name, tokens }) => linkedTypeField(name, tokens, false))
+    );
+  }
+  const returns = findElement(signature, (el) => hasClass(el, 'return-type'));
+  const returnLink = returns ? linkedCode(trimTokens(linkedTokens(returns.children))) : undefined;
+  if (returnLink) sections.push(`**${parameters ? 'Returns' : 'Type'}:** ${returnLink}`);
+  return sections.length ? sections.join('\n\n') : undefined;
+}
+
+type JavadocParameter = { name: string; tokens: LinkedToken[] };
+
+function splitParameters(tokens: LinkedToken[]): JavadocParameter[] {
+  const text = tokens.map((token) => token.text).join('');
+  const open = text.indexOf('(');
+  const close = text.lastIndexOf(')');
+  if (open === -1 || close <= open) return [];
+  const inner = sliceTokens(tokens, open + 1, close);
+  const list = inner.map((token) => token.text).join('');
+  const parameters: JavadocParameter[] = [];
+  let start = 0;
+  while (start <= list.length) {
+    const comma = topLevelDelimiter(list, ',', start, '<([{', '>)]}');
+    const end = comma === -1 ? list.length : comma;
+    const parameter = trimTokens(sliceTokens(inner, start, end));
+    const value = parameter.map((token) => token.text).join('');
+    const name = /([A-Za-z_$][\w$]*)$/.exec(value)?.[1];
+    if (name && value.length > name.length) {
+      parameters.push({
+        name,
+        tokens: trimTokens(sliceTokens(parameter, 0, value.length - name.length)),
+      });
+    }
+    start = end + 1;
+  }
+  return parameters;
+}
+
 function kindFromHeading(scope: Element | HastRoot): JavadocKind {
   const heading = findElement(scope, (el) => el.tagName === 'h1');
   const title = heading ? textOf(heading).trim() : '';

package/src/sdk/linked-tokens.ts

@@ -0,0 +1,91 @@
+import type { ElementContent } from 'hast';
+
+import { inlineCode, linkedCode, responseField, safeHref, type LinkedToken } from './mdx.js';
+
+export function linkedTokens(children: ElementContent[], href?: string): LinkedToken[] {
+  const tokens: LinkedToken[] = [];
+  for (const child of children) {
+    if (child.type === 'text') {
+      tokens.push({ text: child.value.replace(/\s+/g, ' '), href });
+    } else if (child.type === 'element') {
+      let nextHref = href;
+      if (child.tagName === 'a' && typeof child.properties.href === 'string') {
+        nextHref = safeHref(child.properties.href)?.replace(
+          /[\u0000-\u0020\u007f"'()[\]<>`{}\\]/g,
+          (character) => `%${character.charCodeAt(0).toString(16).toUpperCase().padStart(2, '0')}`
+        );
+      }
+      tokens.push(...linkedTokens(child.children, nextHref));
+    }
+  }
+  const merged: LinkedToken[] = [];
+  for (const token of tokens) {
+    const previous = merged.at(-1);
+    if (previous && previous.href === token.href) previous.text += token.text;
+    else merged.push({ ...token });
+  }
+  return merged;
+}
+
+export function trimTokens(tokens: LinkedToken[]): LinkedToken[] {
+  const text = tokens.map((token) => token.text).join('');
+  return sliceTokens(tokens, text.length - text.trimStart().length, text.trimEnd().length);
+}
+
+export function sliceTokens(tokens: LinkedToken[], start: number, end: number): LinkedToken[] {
+  const sliced: LinkedToken[] = [];
+  let offset = 0;
+  for (const token of tokens) {
+    const tokenEnd = offset + token.text.length;
+    if (tokenEnd > start && offset < end) {
+      sliced.push({
+        text: token.text.slice(Math.max(0, start - offset), end - offset),
+        href: token.href,
+      });
+    }
+    offset = tokenEnd;
+  }
+  return sliced;
+}
+
+export function topLevelDelimiter(
+  text: string,
+  delimiter: string,
+  start = 0,
+  opening = '([{',
+  closing = ')]}'
+): number {
+  let depth = 0;
+  let quote: string | undefined;
+  let escaped = false;
+  for (let index = start; index < text.length; index += 1) {
+    const character = text[index] ?? '';
+    if (escaped) {
+      escaped = false;
+    } else if (quote) {
+      if (character === '\\') escaped = true;
+      else if (character === quote) quote = undefined;
+    } else if (character === '"' || character === "'") {
+      quote = character;
+    } else if (opening.includes(character)) {
+      depth += 1;
+    } else if (closing.includes(character)) {
+      depth = Math.max(0, depth - 1);
+    } else if (character === delimiter && depth === 0) {
+      return index;
+    }
+  }
+  return -1;
+}
+
+export function linkedTypeField(name: string, tokens: LinkedToken[], optional: boolean): string {
+  const type = tokens.map((token) => token.text).join('');
+  const typeHref = tokens.length === 1 ? tokens[0]?.href : undefined;
+  let body: string | undefined;
+  if (!typeHref || type.length > 80) {
+    const linked = linkedCode(tokens);
+    if (linked) body = `**Type:** ${linked}`;
+    else if (type.length > 80) body = `**Type:** ${inlineCode(type)}`;
+  }
+  return responseField(name, type, optional, body, typeHref);
+}

package/src/sdk/sphinx-signatures.ts

@@ -1,7 +1,14 @@
 import { hastToText } from '@mintlify/common';
-import type { Element, ElementContent } from 'hast';
+import type { Element } from 'hast';
 
-import { inlineCode, linkedCode, responseField, safeHref, type LinkedToken } from './mdx.js';
+import {
+  linkedTokens,
+  linkedTypeField,
+  sliceTokens,
+  topLevelDelimiter,
+  trimTokens,
+} from './linked-tokens.js';
+import { inlineCode, linkedCode, type LinkedToken } from './mdx.js';
 
 type DocumentedTypes = {
   parameters: Map<string, LinkedToken[][]>;
@@ -47,15 +54,7 @@ function renderParameter(parameter: Element, documented: DocumentedTypes): strin
   if (documented.parameters.get(name)?.some((type) => sameLinkedType(typeTokens, type))) {
     return undefined;
   }
-  const only = typeTokens[0];
-  const typeHref = typeTokens.length === 1 ? only?.href : undefined;
-  let body: string | undefined;
-  if (!typeHref || type.length > 80) {
-    const linked = linkedCode(typeTokens);
-    if (linked) body = `**Type:** ${linked}`;
-    else if (type.length > 80) body = `**Type:** ${inlineCode(type)}`;
-  }
-  return responseField(name, type, defaultStart !== -1 || name.startsWith('*'), body, typeHref);
+  return linkedTypeField(name, typeTokens, defaultStart !== -1 || name.startsWith('*'));
 }
 
 function documentedTypes(description: Element | undefined): DocumentedTypes {
@@ -147,76 +146,6 @@ function sameLinkedType(left: LinkedToken[], right: LinkedToken[]): boolean {
   return key(left) === key(right);
 }
 
-function linkedTokens(children: ElementContent[], href?: string): LinkedToken[] {
-  const tokens: LinkedToken[] = [];
-  for (const child of children) {
-    if (child.type === 'text') {
-      tokens.push({ text: child.value.replace(/\s+/g, ' '), href });
-    } else if (child.type === 'element') {
-      let nextHref = href;
-      if (child.tagName === 'a' && typeof child.properties.href === 'string') {
-        nextHref = safeHref(child.properties.href)?.replace(
-          /[\u0000-\u0020\u007f"'()[\]<>`{}\\]/g,
-          (character) => `%${character.charCodeAt(0).toString(16).toUpperCase().padStart(2, '0')}`
-        );
-      }
-      tokens.push(...linkedTokens(child.children, nextHref));
-    }
-  }
-  const merged: LinkedToken[] = [];
-  for (const token of tokens) {
-    const previous = merged.at(-1);
-    if (previous && previous.href === token.href) previous.text += token.text;
-    else merged.push({ ...token });
-  }
-  return merged;
-}
-
-function trimTokens(tokens: LinkedToken[]): LinkedToken[] {
-  const text = tokens.map((token) => token.text).join('');
-  return sliceTokens(tokens, text.length - text.trimStart().length, text.trimEnd().length);
-}
-
-function sliceTokens(tokens: LinkedToken[], start: number, end: number): LinkedToken[] {
-  const sliced: LinkedToken[] = [];
-  let offset = 0;
-  for (const token of tokens) {
-    const tokenEnd = offset + token.text.length;
-    if (tokenEnd > start && offset < end) {
-      sliced.push({
-        text: token.text.slice(Math.max(0, start - offset), end - offset),
-        href: token.href,
-      });
-    }
-    offset = tokenEnd;
-  }
-  return sliced;
-}
-
-function topLevelDelimiter(text: string, delimiter: string): number {
-  let depth = 0;
-  let quote: string | undefined;
-  let escaped = false;
-  for (let index = 0; index < text.length; index += 1) {
-    const character = text[index];
-    if (escaped) {
-      escaped = false;
-    } else if (quote) {
-      if (character === '\\') escaped = true;
-      else if (character === quote) quote = undefined;
-    } else if (character === '"' || character === "'") {
-      quote = character;
-    } else if (character === '(' || character === '[' || character === '{') {
-      depth += 1;
-    } else if (character === ')' || character === ']' || character === '}') {
-      depth = Math.max(0, depth - 1);
-    } else if (character === delimiter && depth === 0) {
-      return index;
-    }
-  }
-  return -1;
-}
-
 function findElements(node: Element, className: string): Element[] {
   const elements: Element[] = [];
   for (const child of node.children) {