@mintlify/scraping 4.0.907 → 4.0.909

package/src/sdk/converters/javadoc.ts

@@ -0,0 +1,330 @@
+import fse from 'fs-extra';
+import type { Element, ElementContent, Root as HastRoot, RootContent } from 'hast';
+import path from 'node:path';
+
+import { htmlToHast } from '../../pipeline/root.js';
+import { removeHastComments } from '../../utils/hastComments.js';
+import { hastToMdx, type RewriteLink } from '../convert.js';
+import type { SdkNavGroup, SdkPage, SdkReference } from '../types.js';
+
+type JavadocType = { pkg: string; name: string; file: string; slug: string };
+
+const TYPE_NAME_PATTERN = /^[A-Za-z_$][\w$]*(\.[A-Za-z_$][\w$]*)*$/;
+const DROPPED_CLASSES = new Set([
+  'summary',
+  'inherited-list',
+  'sub-title',
+  'header',
+  'inheritance',
+]);
+
+export async function convertJavadoc(sourcePath: string): Promise<SdkReference> {
+  const types = await discoverTypes(sourcePath);
+  if (types.length === 0) {
+    throw new Error(`No javadoc types found in ${sourcePath}`);
+  }
+  const slugByFile = new Map(types.map((type) => [type.file, type.slug]));
+
+  const pages: SdkPage[] = [];
+  const byPackage = new Map<string, JavadocType[]>();
+  for (const type of types) {
+    const html = await fse.readFile(path.join(sourcePath, type.file), 'utf8');
+    pages.push(renderPage(type, html, slugByFile));
+    const siblings = byPackage.get(type.pkg) ?? [];
+    siblings.push(type);
+    byPackage.set(type.pkg, siblings);
+  }
+
+  const groups: SdkNavGroup[] = [...byPackage.entries()]
+    .sort(([left], [right]) => left.localeCompare(right))
+    .map(([pkg, members]) => ({
+      group: pkg,
+      pages: members
+        .sort((left, right) => left.name.localeCompare(right.name))
+        .map((member) => member.slug),
+    }));
+
+  return { pages, groups };
+}
+
+async function discoverTypes(root: string): Promise<JavadocType[]> {
+  const entries = (await typesFromSearchIndex(root)) ?? (await typesFromPackageDirs(root));
+  const seen = new Set<string>();
+  const types: JavadocType[] = [];
+  for (const entry of entries) {
+    if (seen.has(entry.file)) continue;
+    seen.add(entry.file);
+    if (await fse.pathExists(path.join(root, entry.file))) types.push(entry);
+  }
+  return types;
+}
+
+async function typesFromSearchIndex(root: string): Promise<JavadocType[] | undefined> {
+  const indexPath = path.join(root, 'type-search-index.js');
+  if (!(await fse.pathExists(indexPath))) return undefined;
+  const raw = await fse.readFile(indexPath, 'utf8');
+  const json = raw
+    .slice(raw.indexOf('=') + 1)
+    .trim()
+    .replace(/;$/, '');
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(json);
+  } catch {
+    return undefined;
+  }
+  if (!Array.isArray(parsed)) return undefined;
+  const types: JavadocType[] = [];
+  for (const item of parsed as { p?: string; l?: string }[]) {
+    const pkg = item.p;
+    const name = item.l?.replace(/<.*$/, '');
+    if (!pkg || !name || !TYPE_NAME_PATTERN.test(pkg) || !TYPE_NAME_PATTERN.test(name)) continue;
+    types.push(makeType(pkg, name));
+  }
+  return types.length > 0 ? types : undefined;
+}
+
+async function typesFromPackageDirs(root: string): Promise<JavadocType[]> {
+  const packages = await readPackageList(root);
+  const types: JavadocType[] = [];
+  for (const pkg of packages) {
+    if (!TYPE_NAME_PATTERN.test(pkg)) continue;
+    const dir = path.join(root, ...pkg.split('.'));
+    if (!(await fse.pathExists(dir))) continue;
+    for (const file of await fse.readdir(dir)) {
+      if (!file.endsWith('.html')) continue;
+      const name = file.slice(0, -'.html'.length);
+      if (name.startsWith('package-') || !TYPE_NAME_PATTERN.test(name)) continue;
+      types.push(makeType(pkg, name));
+    }
+  }
+  return types;
+}
+
+async function readPackageList(root: string): Promise<string[]> {
+  for (const listFile of ['element-list', 'package-list']) {
+    const listPath = path.join(root, listFile);
+    if (!(await fse.pathExists(listPath))) continue;
+    const raw = await fse.readFile(listPath, 'utf8');
+    return raw
+      .split('\n')
+      .map((line) => line.trim())
+      .filter((line) => line.length > 0 && !line.startsWith('module:'));
+  }
+  throw new Error(`No element-list or package-list found in ${root}`);
+}
+
+function makeType(pkg: string, name: string): JavadocType {
+  return {
+    pkg,
+    name,
+    file: `${pkg.split('.').join('/')}/${name}.html`,
+    slug: `${pkg.replace(/\./g, '-')}/${name}`,
+  };
+}
+
+function renderPage(type: JavadocType, html: string, slugByFile: Map<string, string>): SdkPage {
+  const hast = htmlToHast(html);
+  removeHastComments(hast);
+  const main =
+    findElement(hast, (el) => el.tagName === 'main') ??
+    findElement(hast, (el) => el.properties?.id === 'main-content') ??
+    findElement(hast, (el) => hasClass(el, 'contentContainer')) ??
+    hast;
+
+  const declaration =
+    findElement(main, (el) => hasClass(el, 'class-description')) ??
+    findElement(main, (el) => hasClass(el, 'description'));
+  const details = findElement(main, (el) => hasClass(el, 'details'));
+
+  const description = extractDescription(declaration);
+  const tag = extractTag(main);
+
+  const kept = [declaration, details].filter((node): node is Element => node !== undefined);
+  const anchors = collectDetailAnchors(kept);
+  const resolveHref = makeHrefResolver(type, slugByFile, anchors);
+  const signatures: string[] = [];
+  for (const node of kept) {
+    pruneChrome(node);
+    unwrapLists(node);
+    resolveLinks(node, resolveHref);
+    signatures.push(...codifySignatures(node));
+  }
+
+  const root: HastRoot = { type: 'root', children: kept as RootContent[] };
+  let content = hastToMdx(root);
+  for (const signature of signatures) {
+    content = content
+      .split(`\`\`\`\n${signature}\n\`\`\``)
+      .join(`\`\`\`java\n${signature}\n\`\`\``);
+  }
+
+  return { slug: type.slug, title: type.name, description, tag, content };
+}
+
+function makeHrefResolver(
+  type: JavadocType,
+  slugByFile: Map<string, string>,
+  anchors: Set<string>
+): RewriteLink {
+  const pkgDir = type.pkg.split('.').join('/');
+  return (href) => {
+    if (/^[a-z][a-z0-9+.-]*:/i.test(href) || href.startsWith('//')) return href;
+    if (href.startsWith('#')) {
+      const name = decodeFragment(href.slice(1)).split('(')[0] ?? '';
+      return anchors.has(name) ? `#${name.toLowerCase()}` : undefined;
+    }
+    const target = href.split('#')[0] ?? '';
+    if (!target.endsWith('.html')) return undefined;
+    const resolved = path.posix.normalize(path.posix.join(pkgDir, target));
+    const slug = slugByFile.get(resolved);
+    return slug === undefined ? undefined : `/${slug}`;
+  };
+}
+
+function decodeFragment(fragment: string): string {
+  try {
+    return decodeURIComponent(fragment);
+  } catch {
+    return fragment;
+  }
+}
+
+function collectDetailAnchors(nodes: Element[]): Set<string> {
+  const anchors = new Set<string>();
+  for (const node of nodes) {
+    visitElements(node, (el) => {
+      if (!hasClass(el, 'detail')) return;
+      const id = el.properties.id;
+      if (typeof id === 'string') anchors.add(id.split('(')[0] ?? id);
+    });
+  }
+  return anchors;
+}
+
+function resolveLinks(node: Element, resolveHref: RewriteLink): void {
+  node.children = node.children.flatMap((child): ElementContent[] => {
+    if (child.type !== 'element') return [child];
+    resolveLinks(child, resolveHref);
+    if (child.tagName !== 'a' || typeof child.properties.href !== 'string') return [child];
+    const resolved = resolveHref(child.properties.href);
+    if (resolved === undefined) return child.children;
+    child.properties.href = resolved;
+    return [child];
+  });
+}
+
+function pruneChrome(node: Element): void {
+  node.children = node.children.filter((child) => {
+    if (child.type !== 'element') return true;
+    if (child.tagName === 'hr') return false;
+    return !elementClasses(child).some((name) => DROPPED_CLASSES.has(name));
+  });
+  for (const child of node.children) {
+    if (child.type !== 'element') continue;
+    if (child.tagName === 'a') delete child.properties.title;
+    pruneChrome(child);
+  }
+}
+
+const UNWRAPPED_LISTS = new Set(['details-list', 'member-list', 'detail-list']);
+
+function unwrapLists(node: Element): void {
+  node.children = node.children.flatMap((child): ElementContent[] => {
+    if (
+      child.type !== 'element' ||
+      child.tagName !== 'ul' ||
+      !elementClasses(child).some((name) => UNWRAPPED_LISTS.has(name))
+    ) {
+      return [child];
+    }
+    return child.children.flatMap((item) =>
+      item.type === 'element' && item.tagName === 'li' ? item.children : []
+    );
+  });
+  for (const child of node.children) {
+    if (child.type === 'element') unwrapLists(child);
+  }
+}
+
+function codifySignatures(node: Element): string[] {
+  const signatures: string[] = [];
+  visitElements(node, (el) => {
+    if (!hasClass(el, 'member-signature') && !hasClass(el, 'type-signature')) return;
+    const signature = textOf(el)
+      .replace(/\u00a0/g, ' ')
+      .trim();
+    signatures.push(signature);
+    el.tagName = 'pre';
+    el.properties = {};
+    el.children = [
+      {
+        type: 'element',
+        tagName: 'code',
+        properties: {},
+        children: [{ type: 'text', value: signature }],
+      },
+    ];
+  });
+  return signatures;
+}
+
+function extractTag(scope: Element | HastRoot): string {
+  const heading = findElement(scope, (el) => el.tagName === 'h1');
+  const title = heading ? textOf(heading).trim() : '';
+  if (title.startsWith('Annotation')) return 'ANNOTATION';
+  if (title.startsWith('Enum')) return 'ENUM';
+  if (title.startsWith('Interface')) return 'INTERFACE';
+  return 'CLASS';
+}
+
+function extractDescription(declaration: Element | undefined): string | undefined {
+  if (!declaration) return undefined;
+  const block = findElement(declaration, (el) => hasClass(el, 'block'));
+  if (!block) return undefined;
+  const text = textOf(block).replace(/\s+/g, ' ').trim();
+  if (!text) return undefined;
+  const sentence = text.split(/(?<=\.)\s/)[0] ?? text;
+  return sentence.length > 160 ? `${sentence.slice(0, 159)}…` : sentence;
+}
+
+function elementClasses(el: Element): string[] {
+  const className = el.properties?.className;
+  return Array.isArray(className) ? className.map(String) : [];
+}
+
+function hasClass(el: Element, name: string): boolean {
+  return elementClasses(el).includes(name);
+}
+
+function findElement(
+  scope: Element | HastRoot,
+  predicate: (el: Element) => boolean
+): Element | undefined {
+  for (const child of scope.children as (RootContent | ElementContent)[]) {
+    if (child.type !== 'element') continue;
+    if (predicate(child)) return child;
+    const found = findElement(child, predicate);
+    if (found) return found;
+  }
+  return undefined;
+}
+
+function visitElements(scope: Element | HastRoot, visitor: (el: Element) => void): void {
+  for (const child of scope.children as (RootContent | ElementContent)[]) {
+    if (child.type !== 'element') continue;
+    visitor(child);
+    visitElements(child, visitor);
+  }
+}
+
+function textOf(node: Element): string {
+  let text = '';
+  for (const child of node.children) {
+    if (child.type === 'text') text += child.value;
+    else if (child.type === 'element' && child.tagName !== 'script' && child.tagName !== 'style') {
+      text += textOf(child);
+    }
+  }
+  return text;
+}