@mintlify/scraping 4.0.1120 → 4.0.1122

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.1120",
+  "version": "4.0.1122",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -43,7 +43,7 @@
     "format:check": "oxfmt --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.1248",
+    "@mintlify/common": "1.0.1249",
     "fast-xml-parser": "5.7.3",
     "fs-extra": "11.1.1",
     "graphql": "16.11.0",
@@ -64,7 +64,7 @@
   "devDependencies": {
     "@mintlify/models": "0.0.374",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.916",
+    "@mintlify/validation": "0.1.917",
     "@tsconfig/recommended": "1.0.2",
     "@types/hast": "3.0.4",
     "@types/mdast": "4.0.4",
@@ -87,5 +87,5 @@
       "optional": true
     }
   },
-  "gitHead": "8d6693e80052163587c46d12cb0e0b419dd149ad"
+  "gitHead": "a881933511e4a6a6a1b4e3cd0539361571d6491a"
 }

package/src/sdk/converters/javadoc.ts

@@ -7,7 +7,18 @@ import { removeHastComments } from '../../utils/hastComments.js';
 import { hastToMdx, type RewriteLink } from '../convert.js';
 import type { SdkNavGroup, SdkPage, SdkReference } from '../types.js';
 
-type JavadocType = { pkg: string; name: string; file: string; slug: string };
+type JavadocKind = 'interface' | 'class' | 'enum' | 'record' | 'exception' | 'error' | 'annotation';
+type JavadocType = { pkg: string; name: string; file: string; slug: string; kind: JavadocKind };
+
+const KINDS: { kind: JavadocKind; group: string; tag: string }[] = [
+  { kind: 'interface', group: 'Interfaces', tag: 'INTERFACE' },
+  { kind: 'class', group: 'Classes', tag: 'CLASS' },
+  { kind: 'enum', group: 'Enums', tag: 'ENUM' },
+  { kind: 'record', group: 'Records', tag: 'RECORD' },
+  { kind: 'exception', group: 'Exceptions', tag: 'EXCEPTION' },
+  { kind: 'error', group: 'Errors', tag: 'ERROR' },
+  { kind: 'annotation', group: 'Annotations', tag: 'ANNOTATION' },
+];
 
 const TYPE_NAME_PATTERN = /^[A-Za-z_$][\w$]*(\.[A-Za-z_$][\w$]*)*$/;
 const DROPPED_CLASSES = new Set([
@@ -24,12 +35,20 @@ export async function convertJavadoc(sourcePath: string): Promise<SdkReference>
     throw new Error(`No javadoc types found in ${sourcePath}`);
   }
   const slugByFile = new Map(types.map((type) => [type.file, type.slug]));
+  const kindsByPackage = new Map<string, Map<string, JavadocKind>>();
 
   const pages: SdkPage[] = [];
   const byPackage = new Map<string, JavadocType[]>();
   for (const type of types) {
     const html = await fse.readFile(path.join(sourcePath, type.file), 'utf8');
-    pages.push(renderPage(type, html, slugByFile));
+    let packageKinds = kindsByPackage.get(type.pkg);
+    if (!packageKinds) {
+      packageKinds = await readPackageKinds(sourcePath, type.pkg);
+      kindsByPackage.set(type.pkg, packageKinds);
+    }
+    const page = renderPage(type, html, slugByFile, packageKinds.get(type.name));
+    type.kind = page.kind;
+    pages.push(page);
     const siblings = byPackage.get(type.pkg) ?? [];
     siblings.push(type);
     byPackage.set(type.pkg, siblings);
@@ -37,16 +56,85 @@ export async function convertJavadoc(sourcePath: string): Promise<SdkReference>
 
   const groups: SdkNavGroup[] = [...byPackage.entries()]
     .sort(([left], [right]) => left.localeCompare(right))
-    .map(([pkg, members]) => ({
-      group: pkg,
-      pages: members
-        .sort((left, right) => left.name.localeCompare(right.name))
-        .map((member) => member.slug),
-    }));
+    .map(([pkg, members]) => ({ group: pkg, pages: packageEntries(members) }));
 
   return { pages, groups };
 }
 
+function packageEntries(members: JavadocType[]): (string | SdkNavGroup)[] {
+  const sorted = [...members].sort((left, right) => left.name.localeCompare(right.name));
+  const buckets = KINDS.map((entry) => ({
+    group: entry.group,
+    pages: sorted.filter((member) => member.kind === entry.kind).map((member) => member.slug),
+  })).filter((bucket) => bucket.pages.length > 0);
+  if (buckets.length <= 1) return sorted.map((member) => member.slug);
+  return buckets;
+}
+
+async function readPackageKinds(root: string, pkg: string): Promise<Map<string, JavadocKind>> {
+  const summaryPath = path.join(root, ...pkg.split('.'), 'package-summary.html');
+  if (!(await fse.pathExists(summaryPath))) return new Map();
+  const hast = htmlToHast(await fse.readFile(summaryPath, 'utf8'));
+  const kinds = new Map<string, JavadocKind>();
+  const record = (container: Element, kind: JavadocKind) => {
+    const cells = hasClass(container, 'col-first') || hasClass(container, 'colFirst');
+    visitElements(container, (el) => {
+      if (!cells && !hasClass(el, 'col-first') && !hasClass(el, 'colFirst')) return;
+      visitElements(el, (anchor) => recordAnchor(anchor, kind));
+    });
+    if (cells) visitElements(container, (anchor) => recordAnchor(anchor, kind));
+  };
+  const recordAnchor = (el: Element, kind: JavadocKind) => {
+    if (el.tagName !== 'a' || typeof el.properties.href !== 'string') return;
+    const href = el.properties.href.split(/[?#]/)[0] ?? '';
+    if (!href.endsWith('.html') || /^[a-z][a-z0-9+.-]*:/i.test(href)) return;
+    const name = path.posix.basename(href, '.html');
+    if (TYPE_NAME_PATTERN.test(name) && !kinds.has(name)) kinds.set(name, kind);
+  };
+
+  const kindByTab = new Map<string, JavadocKind>();
+  visitElements(hast, (el) => {
+    const id = el.properties?.id;
+    if (el.tagName !== 'button' || typeof id !== 'string' || !/^class-summary-tab\d+$/.test(id)) {
+      return;
+    }
+    const kind = kindFromLabel(textOf(el));
+    if (kind) kindByTab.set(id, kind);
+  });
+  if (kindByTab.size > 0) {
+    visitElements(hast, (el) => {
+      const tab = elementClasses(el).find((name) => kindByTab.has(name));
+      if (tab) record(el, kindByTab.get(tab) as JavadocKind);
+    });
+    return kinds;
+  }
+
+  visitElements(hast, (el) => {
+    if (el.tagName !== 'table') return;
+    const caption = findElement(el, (child) => child.tagName === 'caption');
+    const kind = caption ? kindFromLabel(textOf(caption)) : undefined;
+    if (kind) record(el, kind);
+  });
+  return kinds;
+}
+
+function kindFromLabel(label: string): JavadocKind | undefined {
+  const text = label
+    .replace(/\u00a0/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .toLowerCase();
+  if (text.startsWith('all ')) return undefined;
+  if (text.startsWith('interface')) return 'interface';
+  if (text.startsWith('class')) return 'class';
+  if (text.startsWith('enum')) return 'enum';
+  if (text.startsWith('record')) return 'record';
+  if (text.startsWith('exception')) return 'exception';
+  if (text.startsWith('error')) return 'error';
+  if (text.startsWith('annotation')) return 'annotation';
+  return undefined;
+}
+
 async function discoverTypes(root: string): Promise<JavadocType[]> {
   const entries = (await typesFromSearchIndex(root)) ?? (await typesFromPackageDirs(root));
   const seen = new Set<string>();
@@ -120,10 +208,16 @@ function makeType(pkg: string, name: string): JavadocType {
     name,
     file: `${pkg.split('.').join('/')}/${name}.html`,
     slug: `${pkg.replace(/\./g, '-')}/${name}`,
+    kind: 'class',
   };
 }
 
-function renderPage(type: JavadocType, html: string, slugByFile: Map<string, string>): SdkPage {
+function renderPage(
+  type: JavadocType,
+  html: string,
+  slugByFile: Map<string, string>,
+  summaryKind: JavadocKind | undefined
+): SdkPage & { kind: JavadocKind } {
   const hast = htmlToHast(html);
   removeHastComments(hast);
   const main =
@@ -138,7 +232,8 @@ function renderPage(type: JavadocType, html: string, slugByFile: Map<string, str
   const details = findElement(main, (el) => hasClass(el, 'details'));
 
   const description = extractDescription(declaration);
-  const tag = extractTag(main);
+  const kind = summaryKind ?? kindFromHeading(main);
+  const tag = KINDS.find((entry) => entry.kind === kind)?.tag ?? 'CLASS';
 
   const kept = [declaration, details].filter((node): node is Element => node !== undefined);
   const anchors = collectDetailAnchors(kept);
@@ -159,7 +254,7 @@ function renderPage(type: JavadocType, html: string, slugByFile: Map<string, str
       .join(`\`\`\`java\n${signature}\n\`\`\``);
   }
 
-  return { slug: type.slug, title: type.name, description, tag, content };
+  return { slug: type.slug, title: type.name, description, tag, content, kind };
 }
 
 function makeHrefResolver(
@@ -269,13 +364,14 @@ function codifySignatures(node: Element): string[] {
   return signatures;
 }
 
-function extractTag(scope: Element | HastRoot): string {
+function kindFromHeading(scope: Element | HastRoot): JavadocKind {
   const heading = findElement(scope, (el) => el.tagName === 'h1');
   const title = heading ? textOf(heading).trim() : '';
-  if (title.startsWith('Annotation')) return 'ANNOTATION';
-  if (title.startsWith('Enum')) return 'ENUM';
-  if (title.startsWith('Interface')) return 'INTERFACE';
-  return 'CLASS';
+  if (title.startsWith('Annotation')) return 'annotation';
+  if (title.startsWith('Enum')) return 'enum';
+  if (title.startsWith('Interface')) return 'interface';
+  if (title.startsWith('Record')) return 'record';
+  return 'class';
 }
 
 function extractDescription(declaration: Element | undefined): string | undefined {