@mintlify/scraping 4.0.1160 → 4.0.1162

package/src/sdk/converters/typedoc.ts

@@ -3,6 +3,7 @@ import fse from 'fs-extra';
 import path from 'node:path';
 
 import { escapeMdxText, markdownToMdx } from '../convert.js';
+import { inlineCode, linkedCode, responseField, truncate } from '../mdx.js';
 import type { SdkNavGroup, SdkPage, SdkPageTarget, SdkReference } from '../types.js';
 
 const KIND = {
@@ -489,7 +490,7 @@ class TypedocRenderer {
     const lines: string[] = ['## Members'];
     for (const member of node.children ?? []) {
       lines.push(
-        this.responseField(
+        responseField(
           member.name,
           member.type?.type === 'literal' ? JSON.stringify(member.type.value) : '',
           false,
@@ -547,24 +548,7 @@ class TypedocRenderer {
     const typeHref = tokens.length === 1 && only?.href ? `/${only.href}` : undefined;
     const linked = typeHref ? undefined : this.linkedType(type);
     const content = [linked && `**Type:** ${linked}`, body].filter(Boolean).join('\n\n');
-    return this.responseField(name, this.type(type), optional, content, typeHref);
-  }
-
-  private responseField(
-    name: string,
-    type: string,
-    optional: boolean,
-    body?: string,
-    typeHref?: string
-  ): string {
-    const typeAttr = type ? ` type=${this.jsxString(truncate(type, 80))}` : '';
-    const typeHrefAttr = typeHref ? ` typeHref=${this.jsxString(typeHref)}` : '';
-    const requiredAttr = optional ? '' : ' required';
-    return `<ResponseField name=${this.jsxString(name)}${typeAttr}${typeHrefAttr}${requiredAttr}>\n${indent(body || '')}\n</ResponseField>`;
-  }
-
-  private jsxString(value: string): string {
-    return `{${JSON.stringify(value)}}`;
+    return responseField(name, this.type(type), optional, content, typeHref);
   }
 
   private codeBlock(code: string): string {
@@ -623,18 +607,12 @@ class TypedocRenderer {
   }
 
   private linkedType(type?: TypedocType): string | undefined {
-    const tokens = this.typeTokens(type);
-    if (!tokens.some((token) => token.href)) return undefined;
-    const merged: TypeToken[] = [];
-    for (const token of tokens) {
-      const last = merged.at(-1);
-      if (last && !last.href && !token.href) last.text += token.text;
-      else merged.push({ ...token });
-    }
-    const children = merged
-      .map((token) => (token.href ? `[${token.text}](/${token.href})` : this.jsxString(token.text)))
-      .join('');
-    return `<code>${children}</code>`;
+    return linkedCode(
+      this.typeTokens(type).map((token) => ({
+        text: token.text,
+        ...(token.href ? { href: `/${token.href}` } : {}),
+      }))
+    );
   }
 
   private typeTokens(type?: TypedocType): TypeToken[] {
@@ -725,24 +703,3 @@ function headingText(title: unknown): string {
 function detailHeading(title: string, depth: number): string {
   return depth <= MAX_HEADING_DEPTH ? `${'#'.repeat(depth)} ${title}` : `**${title}**`;
 }
-
-function indent(text: string): string {
-  return text
-    .split('\n')
-    .map((line) => (line ? `  ${line}` : line))
-    .join('\n');
-}
-
-function truncate(text: string, max: number): string {
-  return text.length > max ? `${text.slice(0, max - 1)}…` : text;
-}
-
-function inlineCode(text: string): string {
-  const code = text.trim();
-  if (!code) return text;
-  const start = text.indexOf(code);
-  const longestRun = Math.max(0, ...(code.match(/`+/g) ?? []).map((run) => run.length));
-  const fence = '`'.repeat(longestRun + 1);
-  const pad = code.startsWith('`') || code.endsWith('`') ? ' ' : '';
-  return `${text.slice(0, start)}${fence}${pad}${code}${pad}${fence}${text.slice(start + code.length)}`;
-}

package/src/sdk/findSdkPage.ts

@@ -3,6 +3,7 @@ import type { ResolvedSdkTarget, SdkSymbolKind } from '@mintlify/models';
 import type { SdkPage, SdkPageTarget, SdkReference } from './types.js';
 
 const KIND_DIRS: Record<SdkSymbolKind, string> = {
+  package: 'packages',
   class: 'classes',
   interface: 'interfaces',
   enum: 'enums',
@@ -13,20 +14,41 @@ const KIND_DIRS: Record<SdkSymbolKind, string> = {
   property: 'properties',
 };
 
+type SdkTargetRequest = Pick<ResolvedSdkTarget, 'kind' | 'name' | 'parent' | 'namespace'>;
+
+// An explicit namespace (object-form frontmatter) must match the page's exactly;
+// a dotted shorthand such as `audio.client.Foo` matches any trailing part of
+// the page's namespace.
 export function matchesSdkPageTarget(
   pageTarget: SdkPageTarget | undefined,
-  target: Pick<ResolvedSdkTarget, 'kind' | 'name' | 'parent'>
+  target: SdkTargetRequest
 ): boolean {
   if (!pageTarget) return false;
-  if (pageTarget.kind !== target.kind || pageTarget.name !== target.name) return false;
-  if (target.parent) return pageTarget.parent === target.parent;
+  if (pageTarget.kind !== target.kind) return false;
+  if (target.namespace !== undefined && pageTarget.namespace !== target.namespace) return false;
+  const exact = target.namespace !== undefined;
+  if (!matchesSdkName(pageTarget.name, pageTarget.namespace, target.name, exact)) return false;
+  if (target.parent) {
+    return (
+      pageTarget.parent !== undefined &&
+      matchesSdkName(pageTarget.parent, pageTarget.namespace, target.parent, exact)
+    );
+  }
   return pageTarget.parent === undefined;
 }
 
-function matchesSdkPageHeuristic(
-  page: SdkPage,
-  target: Pick<ResolvedSdkTarget, 'kind' | 'name' | 'parent'>
+function matchesSdkName(
+  name: string,
+  namespace: string | undefined,
+  requested: string,
+  exact: boolean
 ): boolean {
+  if (requested === name) return true;
+  if (exact || namespace === undefined) return false;
+  return `.${namespace}.${name}`.endsWith(`.${requested}`);
+}
+
+function matchesSdkPageHeuristic(page: SdkPage, target: SdkTargetRequest): boolean {
   if (target.parent) {
     return (
       page.title === `${target.parent}.${target.name}` ||
@@ -46,20 +68,54 @@ function slugify(name: string): string {
   return name.replace(/[^a-zA-Z0-9-_.]+/g, '-').replace(/^-+|-+$/g, '') || 'item';
 }
 
+function qualifiedName(target: SdkPageTarget): string {
+  const name = target.parent ? `${target.parent}.${target.name}` : target.name;
+  return target.namespace ? `${target.namespace}.${name}` : name;
+}
+
+// Several pages can match a bare name. When their targets carry namespaces the
+// request can be qualified, so that is an error listing the candidates; when
+// they do not (TypeDoc has no namespaces), the first match stands as before.
 export function findSdkPage(
   reference: SdkReference,
-  target: Pick<ResolvedSdkTarget, 'kind' | 'name' | 'parent'>
+  target: SdkTargetRequest
 ): SdkPage | undefined {
-  return (
-    reference.pages.find((page) => matchesSdkPageTarget(page.target, target)) ??
-    reference.pages.find((page) => matchesSdkPageHeuristic(page, target))
+  const matches = reference.pages.filter((page) => matchesSdkPageTarget(page.target, target));
+  if (matches.length > 1) {
+    // `package client` names the root package `client` outright even when a
+    // `client/` directory also ends in that name; only the shorthand that fits
+    // several pages equally is ambiguous.
+    const requested = target.parent ? `${target.parent}.${target.name}` : target.name;
+    const exact = matches.filter(
+      (page) => qualifiedName(page.target as SdkPageTarget) === requested
+    );
+    if (exact.length === 1) return exact[0];
+  }
+  if (matches.length > 1 && matches.some((page) => page.target?.namespace)) {
+    const candidates = matches.map((page) => qualifiedName(page.target as SdkPageTarget));
+    const requested = target.parent ? `${target.parent}.${target.name}` : target.name;
+    throw new Error(
+      `SDK symbol is ambiguous: ${target.kind} ${requested} matches ${candidates.join(', ')}; qualify it with its package`
+    );
+  }
+  if (matches[0] || target.namespace !== undefined) return matches[0];
+  // The slug/title heuristic covers converters that emit no target metadata
+  // and TypeDoc members addressed through their namespace (`Auth.Client.login`
+  // is the title, while the target only knows `parent: Client`). A page whose
+  // target names a different symbol is never picked up by title.
+  return reference.pages.find(
+    (page) => agreesWithTarget(page.target, target) && matchesSdkPageHeuristic(page, target)
   );
 }
 
-export function renderSdkTarget(
-  reference: SdkReference,
-  target: Pick<ResolvedSdkTarget, 'kind' | 'name' | 'parent'>
-): SdkPage {
+function agreesWithTarget(pageTarget: SdkPageTarget | undefined, target: SdkTargetRequest) {
+  if (!pageTarget) return true;
+  if (pageTarget.kind !== target.kind || pageTarget.name !== target.name) return false;
+  if (!target.parent) return pageTarget.parent === undefined;
+  return pageTarget.parent !== undefined && `.${target.parent}`.endsWith(`.${pageTarget.parent}`);
+}
+
+export function renderSdkTarget(reference: SdkReference, target: SdkTargetRequest): SdkPage {
   const page = findSdkPage(reference, target);
   if (!page) {
     const qualified = target.parent ? `${target.parent}.${target.name}` : target.name;

package/src/sdk/generateSdkReference.ts

@@ -1,4 +1,5 @@
 import { convertDocfx } from './converters/docfx.js';
+import { convertGo } from './converters/go.js';
 import { convertJavadoc } from './converters/javadoc.js';
 import { convertPhpdoc } from './converters/phpdoc.js';
 import { convertSphinx } from './converters/sphinx.js';
@@ -11,6 +12,7 @@ const converters: Record<SdkFormat, SdkConverter> = {
   javadoc: convertJavadoc,
   sphinx: convertSphinx,
   phpdoc: convertPhpdoc,
+  go: convertGo,
 };
 
 export async function generateSdkReference(

package/src/sdk/mdx.ts

@@ -0,0 +1,123 @@
+export type LinkedToken = { text: string; href?: string };
+
+export function jsxString(value: string): string {
+  return `{${JSON.stringify(value)}}`;
+}
+
+export function responseField(
+  name: string,
+  type: string,
+  optional: boolean,
+  body?: string,
+  typeHref?: string,
+  id?: string
+): string {
+  const idAttr = id ? ` id=${jsxString(id)}` : '';
+  const typeAttr = type ? ` type=${jsxString(truncate(type, 80))}` : '';
+  const typeHrefAttr = typeHref ? ` typeHref=${jsxString(typeHref)}` : '';
+  const requiredAttr = optional ? '' : ' required';
+  return `<ResponseField name=${jsxString(name)}${idAttr}${typeAttr}${typeHrefAttr}${requiredAttr}>\n${indent(body || '')}\n</ResponseField>`;
+}
+
+export function escapeInline(text: string): string {
+  return collapseWhitespace(text).replace(/[\\`*_{}[\]<>]/g, '\\$&');
+}
+
+// Code spans and link text are inline; a blank line inside them would end the
+// paragraph and let the remainder parse as MDX. Line endings (\n, \r, and
+// \r\n, which CommonMark treats alike) become one space along with the
+// indentation that follows them; other whitespace is kept, so a string literal
+// such as "a  b" keeps its spacing. The pattern cannot backtrack, so a long
+// run of spaces costs linear time.
+export function collapseWhitespace(text: string): string {
+  return text.replace(/[\r\n]+[ \t]*/g, ' ');
+}
+
+const SAFE_SCHEMES = /^(?:https?|mailto):/i;
+
+// Keeps http(s), mailto, fragments and relative paths; drops javascript:,
+// data:, and every other scheme so artifact text cannot produce an active link.
+// Browsers drop C0 controls, spaces, and DEL while parsing a URL and read a
+// backslash as a slash, so the check runs on the URL as a browser would see it.
+// The returned URL keeps its interior spaces (a file can be named "Getting
+// Started.md") and loses only what a browser drops: surrounding controls and
+// spaces, and tabs or newlines anywhere.
+//
+// A character reference before the first `/`, `?`, or `#` (after any leading
+// slashes, so `/&sol;evil.com` cannot become `//evil.com`) is refused: Markdown
+// leaves `&colon;` and `&#58;` in reference definitions for MDX to decode, so
+// `javascript&colon;x` would become `javascript:x` after this check ran.
+export function safeHref(url: string): string | undefined {
+  const cleaned = url.replace(/[\u0000-\u0020\u007f]/g, '');
+  const kept = trimControls(url).replace(/[\t\n\r]/g, '');
+  if (cleaned === '' || cleaned.startsWith('#')) return kept;
+  if (/&[#a-z]/i.test(cleaned.replace(/^[\\/]+/, '').split(/[/?#]/, 1)[0] ?? '')) return undefined;
+  if (
+    /^[a-z][a-z0-9+.-]*:/i.test(cleaned) ||
+    /^[\\/][\\/]/.test(cleaned) ||
+    cleaned.startsWith('\\')
+  ) {
+    return SAFE_SCHEMES.test(cleaned) ? kept : undefined;
+  }
+  return kept;
+}
+
+// Trims C0 controls and spaces from both ends with two index scans; a regex
+// with a `$` alternative retries from every position of a long interior run.
+function trimControls(url: string): string {
+  let start = 0;
+  let end = url.length;
+  while (start < end && url.charCodeAt(start) <= 0x20) start += 1;
+  while (end > start && url.charCodeAt(end - 1) <= 0x20) end -= 1;
+  return url.slice(start, end);
+}
+
+export function encodeHref(text: string): string {
+  return text
+    .split('/')
+    .map((segment) =>
+      encodeURIComponent(segment).replace(
+        /[()!'*]/g,
+        (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`
+      )
+    )
+    .join('/');
+}
+
+export function linkedCode(tokens: LinkedToken[]): string | undefined {
+  if (!tokens.some((token) => token.href)) return undefined;
+  const merged: LinkedToken[] = [];
+  for (const token of tokens) {
+    const last = merged.at(-1);
+    if (last && !last.href && !token.href) last.text += token.text;
+    else merged.push({ ...token });
+  }
+  const children = merged
+    .map((token) =>
+      token.href
+        ? `[${escapeInline(token.text)}](${token.href})`
+        : jsxString(collapseWhitespace(token.text))
+    )
+    .join('');
+  return `<code>${children}</code>`;
+}
+
+export function indent(text: string): string {
+  return text
+    .split('\n')
+    .map((line) => (line ? `  ${line}` : line))
+    .join('\n');
+}
+
+export function truncate(text: string, max: number): string {
+  return text.length > max ? `${text.slice(0, max - 1)}…` : text;
+}
+
+export function inlineCode(text: string): string {
+  const code = collapseWhitespace(text).trim();
+  if (!code) return text;
+  const longestRun = Math.max(0, ...(code.match(/`+/g) ?? []).map((run) => run.length));
+  const fence = '`'.repeat(longestRun + 1);
+  const pad = code.startsWith('`') || code.endsWith('`') ? ' ' : '';
+  return `${fence}${pad}${code}${pad}${fence}`;
+}

package/src/sdk/types.ts

@@ -7,6 +7,7 @@ export type SdkPageTarget = {
   kind: SdkSymbolKind;
   name: string;
   parent?: string;
+  namespace?: string;
 };
 
 export type SdkPage = {
@@ -15,6 +16,7 @@ export type SdkPage = {
   description?: string;
   tag?: string;
   icon?: string;
+  intro?: string;
   content: string;
   target?: SdkPageTarget;
 };