@mintlify/scraping 4.0.907 → 4.0.908

package/__test__/sdk-docfx.test.ts

@@ -0,0 +1,204 @@
+import fse from 'fs-extra';
+import os from 'node:os';
+import path from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+import { convertDocfx } from '../src/sdk/converters/docfx.js';
+import type { SdkReference } from '../src/sdk/types.js';
+
+const clientYml = `### YamlMime:ManagedReference
+items:
+- uid: Acme.Widgets.Client
+  id: Client
+  parent: Acme.Widgets
+  children:
+  - Acme.Widgets.Client.Name
+  - Acme.Widgets.Client.Timeout
+  - Acme.Widgets.Client.PaintAsync(Acme.Widgets.Models.Color,System.Int32)
+  name: Client
+  fullName: Acme.Widgets.Client
+  type: Class
+  namespace: Acme.Widgets
+  summary: A widget client. Prefer <xref href="Acme.Widgets.Models.Color" data-throw-if-not-resolved="false"></xref> over raw values.
+  syntax:
+    content: public class Client
+- uid: Acme.Widgets.Client.Name
+  id: Name
+  parent: Acme.Widgets.Client
+  name: Name
+  type: Property
+  namespace: Acme.Widgets
+  summary: The client name.
+  syntax:
+    content: public string Name { get; set; }
+    return:
+      type: System.String
+- uid: Acme.Widgets.Client.Timeout
+  id: Timeout
+  parent: Acme.Widgets.Client
+  name: Timeout
+  type: Property
+  namespace: Acme.Widgets
+  summary: Timeout backed by <xref href="System.TimeSpan" data-throw-if-not-resolved="false"></xref>.
+  syntax:
+    content: public TimeSpan? Timeout { get; set; }
+    return:
+      type: System.TimeSpan
+- uid: Acme.Widgets.Client.PaintAsync(Acme.Widgets.Models.Color,System.Int32)
+  id: PaintAsync(Acme.Widgets.Models.Color,System.Int32)
+  parent: Acme.Widgets.Client
+  name: PaintAsync(Color, int)
+  type: Method
+  namespace: Acme.Widgets
+  summary: Paints using a <code>Color</code>.
+  syntax:
+    content: public Task<bool> PaintAsync(Color color, int coats = 1)
+    parameters:
+    - id: color
+      type: Acme.Widgets.Models.Color
+      description: The <xref href="Acme.Widgets.Models.Color" data-throw-if-not-resolved="false"></xref> to apply.
+    - id: coats
+      type: System.Int32
+      description: Number of coats.
+    return:
+      type: System.Threading.Tasks.Task{System.Boolean}
+references:
+- uid: System.String
+  name: string
+- uid: System.TimeSpan
+  name: TimeSpan
+- uid: System.Int32
+  name: int
+- uid: Acme.Widgets.Models.Color
+  name: Color
+- uid: System.Threading.Tasks.Task{System.Boolean}
+  name: Task<bool>
+`;
+
+const colorYml = `### YamlMime:ManagedReference
+items:
+- uid: Acme.Widgets.Models.Color
+  id: Color
+  parent: Acme.Widgets.Models
+  children:
+  - Acme.Widgets.Models.Color.Red
+  name: Color
+  fullName: Acme.Widgets.Models.Color
+  type: Enum
+  namespace: Acme.Widgets.Models
+  summary: Available colors for <code>IDictionary&lt;string, string&gt;</code> lookups.
+  syntax:
+    content: public enum Color
+- uid: Acme.Widgets.Models.Color.Red
+  id: Red
+  parent: Acme.Widgets.Models.Color
+  name: Red
+  type: Field
+  namespace: Acme.Widgets.Models
+  summary: The color red.
+  syntax:
+    content: Red = 0
+`;
+
+const tocYml = `### YamlMime:TableOfContent
+items:
+- uid: Acme.Widgets
+  name: Acme.Widgets
+  type: Namespace
+  items:
+  - uid: Acme.Widgets.Client
+    name: Client
+    type: Class
+- uid: Acme.Widgets.Models
+  name: Acme.Widgets.Models
+  type: Namespace
+  items:
+  - uid: Acme.Widgets.Models.Color
+    name: Color
+    type: Enum
+`;
+
+describe('convertDocfx', () => {
+  let sourcePath: string;
+  let reference: SdkReference;
+
+  beforeAll(async () => {
+    sourcePath = await fse.mkdtemp(path.join(os.tmpdir(), 'docfx-test-'));
+    await fse.writeFile(path.join(sourcePath, 'Acme.Widgets.Client.yml'), clientYml);
+    await fse.writeFile(path.join(sourcePath, 'Acme.Widgets.Models.Color.yml'), colorYml);
+    await fse.writeFile(path.join(sourcePath, 'toc.yml'), tocYml);
+    reference = await convertDocfx(sourcePath);
+  });
+
+  afterAll(async () => {
+    await fse.remove(sourcePath);
+  });
+
+  it('creates one page per top-level type with namespace-scoped slugs', () => {
+    expect(reference.pages.map((page) => page.slug)).toEqual(['widgets/client', 'models/color']);
+    expect(reference.pages.map((page) => page.tag)).toEqual(['CLASS', 'ENUM']);
+    expect(reference.pages.map((page) => page.title)).toEqual(['Client', 'Color']);
+  });
+
+  it('groups pages by namespace in toc order', () => {
+    expect(reference.groups).toEqual([
+      { group: 'Acme.Widgets', pages: ['widgets/client'] },
+      { group: 'Acme.Widgets.Models', pages: ['models/color'] },
+    ]);
+  });
+
+  it('renders properties as ResponseFields with nullability-driven required', () => {
+    const content = reference.pages[0]?.content ?? '';
+    expect(content).toContain('<ResponseField name={"Name"} type={"string"} required>');
+    expect(content).toContain('<ResponseField name={"Timeout"} type={"TimeSpan"}>');
+  });
+
+  it('renders method signatures, parameters, and returns', () => {
+    const content = reference.pages[0]?.content ?? '';
+    expect(content).toContain('### PaintAsync()');
+    expect(content).toContain(
+      '```csharp\npublic Task<bool> PaintAsync(Color color, int coats = 1)\n```'
+    );
+    expect(content).toContain('<ResponseField name={"color"} type={"Color"} required>');
+    expect(content).toContain('<ResponseField name={"coats"} type={"int"}>');
+    expect(content).toContain('`Task<bool>`');
+  });
+
+  it('resolves xrefs to page links or inline code', () => {
+    const content = reference.pages[0]?.content ?? '';
+    expect(content).toContain('Prefer [Color](/models/color) over raw values.');
+    expect(content).toContain('Timeout backed by `TimeSpan`.');
+    expect(content).toContain('The [Color](/models/color) to apply.');
+  });
+
+  it('keeps namespace slugs unique when names differ only by separator', async () => {
+    const dir = await fse.mkdtemp(path.join(os.tmpdir(), 'docfx-collide-'));
+    const typeYml = (namespace: string) => `### YamlMime:ManagedReference
+items:
+- uid: ${namespace}.Thing
+  id: Thing
+  name: Thing
+  fullName: ${namespace}.Thing
+  type: Class
+  namespace: ${namespace}
+  syntax:
+    content: public class Thing
+`;
+    await fse.writeFile(path.join(dir, 'a.yml'), typeYml('Acme.Sub.Zone'));
+    await fse.writeFile(path.join(dir, 'b.yml'), typeYml('Acme.Sub-Zone'));
+    const result = await convertDocfx(dir);
+    const slugs = result.pages.map((page) => page.slug);
+    expect(new Set(slugs).size).toBe(slugs.length);
+    expect(result.pages).toHaveLength(2);
+    await fse.rm(dir, { recursive: true, force: true });
+  });
+
+  it('escapes generic angle brackets in prose', () => {
+    const content = reference.pages[1]?.content ?? '';
+    expect(content).toContain('`IDictionary<string, string>`');
+    expect(content).toContain('<ResponseField name={"Red"} type={"0"} required>');
+    expect(reference.pages[1]?.description).toBe(
+      'Available colors for IDictionary<string, string> lookups.'
+    );
+  });
+});

package/__test__/sdk-javadoc.test.ts

@@ -0,0 +1,150 @@
+import fse from 'fs-extra';
+import os from 'node:os';
+import path from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+import { convertJavadoc } from '../src/sdk/converters/javadoc.js';
+
+const TRAVERSAL_INDEX = 'typeSearchIndex = [{"p":"com/../../../etc","l":"Passwd"}];';
+import type { SdkReference } from '../src/sdk/types.js';
+
+const page = (title: string, body: string) => `<!DOCTYPE HTML>
+<html lang="en">
+<head><title>${title} (fake API)</title></head>
+<body class="class-declaration-page">
+<main role="main">
+${body}
+</main>
+</body>
+</html>`;
+
+const greeterHtml = page(
+  'Greeter',
+  `<div class="header">
+<h1 title="Class Greeter" class="title">Class Greeter</h1>
+</div>
+<section class="class-description" id="class-description">
+<hr>
+<div class="type-signature"><span class="modifiers">public class </span><span class="element-name type-name-label">Greeter</span></div>
+<div class="block">Greets people politely. Supports many languages.</div>
+</section>
+<section class="summary">
+<ul class="summary-list">
+<li>
+<section class="method-summary" id="method-summary">
+<h2>Method Summary</h2>
+<div class="summary-table three-column-summary">
+<div class="col-first even-row-color"><code>String</code></div>
+<div class="col-second even-row-color"><code><a href="#greet(java.lang.String)" class="member-name-link">greet</a>(String name)</code></div>
+<div class="col-last even-row-color"><div class="block">SUMMARY_ROW_MARKER greet a person.</div></div>
+</div>
+<div class="inherited-list">
+<h3>Methods inherited from class java.lang.Object</h3>
+<code>equals, hashCode, toString</code>
+</div>
+</section>
+</li>
+</ul>
+</section>
+<section class="details">
+<ul class="details-list">
+<li>
+<section class="method-details" id="method-detail">
+<h2>Method Details</h2>
+<ul class="member-list">
+<li>
+<section class="detail" id="greet(java.lang.String)">
+<h3>greet</h3>
+<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">String</span>&nbsp;<span class="element-name">greet</span><span class="parameters">(String&nbsp;name)</span></div>
+<div class="block">Builds a greeting for the given <a href="Person.html" title="interface in com.example">Person</a> name.</div>
+<dl class="notes">
+<dt>Parameters:</dt>
+<dd><code>name</code> - who to greet.</dd>
+</dl>
+</section>
+</li>
+</ul>
+</section>
+</li>
+</ul>
+</section>`
+);
+
+const personHtml = page(
+  'Person',
+  `<div class="header">
+<h1 title="Interface Person" class="title">Interface Person</h1>
+</div>
+<section class="class-description" id="class-description">
+<hr>
+<div class="type-signature"><span class="modifiers">public interface </span><span class="element-name type-name-label">Person</span></div>
+<div class="block">A person that can be greeted.</div>
+</section>`
+);
+
+let dir: string;
+let ref: SdkReference;
+
+beforeAll(async () => {
+  dir = await fse.mkdtemp(path.join(os.tmpdir(), 'javadoc-test-'));
+  await fse.writeFile(path.join(dir, 'element-list'), 'com.example\n');
+  const pkgDir = path.join(dir, 'com', 'example');
+  await fse.mkdirp(pkgDir);
+  await fse.writeFile(path.join(pkgDir, 'Greeter.html'), greeterHtml);
+  await fse.writeFile(path.join(pkgDir, 'Person.html'), personHtml);
+  await fse.writeFile(
+    path.join(pkgDir, 'package-summary.html'),
+    page('com.example', '<h1>Package com.example</h1>')
+  );
+  ref = await convertJavadoc(dir);
+});
+
+afterAll(async () => {
+  await fse.remove(dir);
+});
+
+describe('convertJavadoc', () => {
+  it('emits one page per type with package-based slugs', () => {
+    expect(ref.pages.map((p) => p.slug).sort()).toEqual([
+      'com-example/Greeter',
+      'com-example/Person',
+    ]);
+  });
+
+  it('groups types by package', () => {
+    expect(ref.groups).toEqual([
+      { group: 'com.example', pages: ['com-example/Greeter', 'com-example/Person'] },
+    ]);
+  });
+
+  it('extracts title, tag, and first-sentence description', () => {
+    const greeter = ref.pages.find((p) => p.slug === 'com-example/Greeter');
+    expect(greeter?.title).toBe('Greeter');
+    expect(greeter?.tag).toBe('CLASS');
+    expect(greeter?.description).toBe('Greets people politely.');
+    const person = ref.pages.find((p) => p.slug === 'com-example/Person');
+    expect(person?.tag).toBe('INTERFACE');
+  });
+
+  it('keeps method detail sections and rewrites type links', () => {
+    const content = ref.pages.find((p) => p.slug === 'com-example/Greeter')?.content ?? '';
+    expect(content).toContain('### greet');
+    expect(content).toContain('public String greet(String name)');
+    expect(content).toContain('`name` - who to greet.');
+    expect(content).toContain('[Person](/com-example/Person)');
+  });
+
+  it('drops summary tables and inherited-method boilerplate', () => {
+    const content = ref.pages.find((p) => p.slug === 'com-example/Greeter')?.content ?? '';
+    expect(content).not.toContain('SUMMARY_ROW_MARKER');
+    expect(content).not.toContain('Method Summary');
+    expect(content).not.toContain('Methods inherited from');
+  });
+
+  it('ignores search-index entries with invalid package names', async () => {
+    const evil = await fse.mkdtemp(path.join(os.tmpdir(), 'javadoc-evil-'));
+    await fse.writeFile(path.join(evil, 'type-search-index.js'), TRAVERSAL_INDEX);
+    await expect(convertJavadoc(evil)).rejects.toThrow(/No element-list or package-list found/);
+    await fse.rm(evil, { recursive: true, force: true });
+  });
+});

package/__test__/sdk-phpdoc.test.ts

@@ -0,0 +1,134 @@
+import fse from 'fs-extra';
+import os from 'node:os';
+import path from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+import { convertPhpdoc } from '../src/sdk/converters/phpdoc.js';
+import type { SdkReference } from '../src/sdk/types.js';
+
+const STRUCTURE_XML = `<?xml version="1.0"?>
+<project name="Acme-PHP">
+  <file path="Widget/Greeter.php" hash="abc">
+    <namespace-alias name="\\Acme\\SDK\\Widget"/>
+    <class final="true" abstract="false" namespace="\\Acme\\SDK\\Widget" line="10">
+      <name>Greeter</name>
+      <full_name>\\Acme\\SDK\\Widget\\Greeter</full_name>
+      <docblock line="10">
+        <description>Greets people politely.</description>
+        <long-description>Supports optional shouting.</long-description>
+      </docblock>
+      <implements>\\Acme\\SDK\\Contract\\GreeterInterface</implements>
+      <property namespace="\\Acme\\SDK\\Widget\\Greeter" line="12" visibility="public">
+        <name>salutation</name>
+        <default>'Hello'</default>
+        <docblock line="12">
+          <description></description>
+          <long-description></long-description>
+          <tag name="var" description="The salutation to use." type="string"/>
+        </docblock>
+      </property>
+      <property namespace="\\Acme\\SDK\\Widget\\Greeter" line="13" visibility="private">
+        <name>secret</name>
+        <default></default>
+      </property>
+      <method final="false" abstract="false" static="false" namespace="\\Acme\\SDK\\Widget" line="20" visibility="public" returnByReference="false">
+        <name>greet</name>
+        <full_name>\\Acme\\SDK\\Widget\\Greeter::greet()</full_name>
+        <argument line="20" by_reference="false">
+          <name>name</name>
+          <default></default>
+          <type>string</type>
+        </argument>
+        <argument line="20" by_reference="false">
+          <name>shout</name>
+          <default>false</default>
+          <type>?bool</type>
+        </argument>
+        <docblock line="20">
+          <description>Greet a person by name.</description>
+          <long-description></long-description>
+          <tag name="param" description="Name of the person." variable="name" type="string"/>
+          <tag name="param" description="Whether to shout." variable="shout" type="?bool"/>
+          <tag name="return" description="The greeting." type="string"/>
+          <tag name="throws" description="when name is empty" type="\\Acme\\SDK\\Exception\\ArgumentException"/>
+        </docblock>
+      </method>
+      <method final="false" abstract="false" static="false" namespace="\\Acme\\SDK\\Widget" line="30" visibility="private" returnByReference="false">
+        <name>hidden</name>
+        <full_name>\\Acme\\SDK\\Widget\\Greeter::hidden()</full_name>
+      </method>
+    </class>
+  </file>
+</project>
+`;
+
+describe('convertPhpdoc', () => {
+  let dir: string;
+  let reference: SdkReference;
+
+  beforeAll(async () => {
+    dir = await fse.mkdtemp(path.join(os.tmpdir(), 'phpdoc-test-'));
+    await fse.writeFile(path.join(dir, 'structure.xml'), STRUCTURE_XML);
+    reference = await convertPhpdoc(dir);
+  });
+
+  afterAll(async () => {
+    await fse.remove(dir);
+  });
+
+  it('creates one page per class with a namespace-derived slug', () => {
+    expect(reference.pages).toHaveLength(1);
+    const page = reference.pages[0]!;
+    expect(page.slug).toBe('greeter');
+    expect(page.title).toBe('Greeter');
+    expect(page.tag).toBe('CLASS');
+    expect(page.description).toBe('Greets people politely.');
+  });
+
+  it('groups pages by namespace', () => {
+    expect(reference.groups).toEqual([{ group: 'Acme\\SDK\\Widget', pages: ['greeter'] }]);
+  });
+
+  it('reconstructs the method signature', () => {
+    const content = reference.pages[0]!.content;
+    expect(content).toContain(
+      '```php\npublic function greet(string $name, ?bool $shout = false): string\n```'
+    );
+  });
+
+  it('renders class heritage and sections', () => {
+    const content = reference.pages[0]!.content;
+    expect(content).toContain(
+      '```php\nfinal class Greeter implements \\Acme\\SDK\\Contract\\GreeterInterface\n```'
+    );
+    expect(content).toContain('## Properties');
+    expect(content).toContain('## Methods');
+    expect(content).toContain('### greet()');
+    expect(content).toContain('#### Returns');
+    expect(content).toContain('#### Throws');
+  });
+
+  it('emits ResponseField entries for properties and parameters', () => {
+    const content = reference.pages[0]!.content;
+    expect(content).toContain(
+      '<ResponseField name={"salutation"} type={"string"} required>\n  The salutation to use.\n\n  Default: `\'Hello\'`\n</ResponseField>'
+    );
+    expect(content).toContain(
+      '<ResponseField name={"name"} type={"string"} required>\n  Name of the person.\n</ResponseField>'
+    );
+    expect(content).toContain(
+      '<ResponseField name={"shout"} type={"?bool"}>\n  Whether to shout.\n</ResponseField>'
+    );
+  });
+
+  it('excludes private members', () => {
+    const content = reference.pages[0]!.content;
+    expect(content).not.toContain('secret');
+    expect(content).not.toContain('hidden');
+  });
+
+  it('accepts the structure.xml file path directly', async () => {
+    const direct = await convertPhpdoc(path.join(dir, 'structure.xml'));
+    expect(direct.pages[0]?.slug).toBe('greeter');
+  });
+});

package/__test__/sdk-sphinx.test.ts

@@ -0,0 +1,84 @@
+import fse from 'fs-extra';
+import os from 'node:os';
+import path from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+import { convertSphinx } from '../src/sdk/converters/sphinx.js';
+import type { SdkReference } from '../src/sdk/types.js';
+
+const indexPage = {
+  current_page_name: 'index',
+  title: 'Example SDK documentation',
+  body: '<section><h1>Example SDK documentation<a class="headerlink" href="#example" title="Link to this heading"></a></h1><p>Welcome to the example SDK. It does things.</p><p>See the <a class="reference internal" href="widgets.html#widgets.Widget">Widget docs</a> and the <a class="reference external" href="https://example.com/docs">docs site</a>.</p><div class="admonition warning"><p class="admonition-title">Warning</p><p>Handle with care.</p></div></section>',
+};
+
+const widgetsPage = {
+  current_page_name: 'widgets',
+  title: 'widgets module',
+  body: '<section><h1>widgets module<a class="headerlink" href="#widgets" title="Link to this heading"></a></h1><dl class="py class"><dt class="sig sig-object py" id="widgets.Widget"><em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">widgets.</span></span><span class="sig-name descname"><span class="pre">Widget</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/widgets/#Widget"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#widgets.Widget" title="Link to this definition"></a></dt><dd><p>A widget.</p><dl class="py method"><dt class="sig sig-object py" id="widgets.Widget.spin"><span class="sig-name descname"><span class="pre">spin</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">speed</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">1</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#widgets.Widget.spin" title="Link to this definition"></a></dt><dd><p>Spins the widget.</p></dd></dl></dd></dl></section>',
+};
+
+const skippedPage = { current_page_name: 'genindex', title: 'Index', body: '<h1>Index</h1>' };
+
+describe('convertSphinx', () => {
+  let dir: string;
+  let reference: SdkReference;
+
+  beforeAll(async () => {
+    dir = await fse.mkdtemp(path.join(os.tmpdir(), 'sphinx-test-'));
+    await fse.writeJson(path.join(dir, 'index.fjson'), indexPage);
+    await fse.writeJson(path.join(dir, 'widgets.fjson'), widgetsPage);
+    await fse.writeJson(path.join(dir, 'genindex.fjson'), skippedPage);
+    reference = await convertSphinx(dir);
+  });
+
+  afterAll(async () => {
+    await fse.remove(dir);
+  });
+
+  it('emits one page per content file with slugs from current_page_name', () => {
+    expect(reference.pages.map((page) => page.slug).sort()).toEqual(['index', 'widgets']);
+  });
+
+  it('groups guide pages and autodoc module pages separately', () => {
+    expect(reference.groups).toEqual([
+      { group: 'Getting Started', pages: ['index'] },
+      { group: 'API Reference', pages: ['widgets'] },
+    ]);
+  });
+
+  it('tags autodoc pages as MODULE', () => {
+    const widgets = reference.pages.find((page) => page.slug === 'widgets');
+    expect(widgets?.tag).toBe('MODULE');
+    expect(reference.pages.find((page) => page.slug === 'index')?.tag).toBeUndefined();
+  });
+
+  it('converts signatures to python code blocks with headings', () => {
+    const content = reference.pages.find((page) => page.slug === 'widgets')?.content ?? '';
+    expect(content).toContain('### Widget');
+    expect(content).toContain('```python\nclass widgets.Widget(name)\n```');
+    expect(content).toContain('#### spin()');
+    expect(content).toContain('```python\nspin(speed=1)\n```');
+  });
+
+  it('strips headerlinks and viewcode links', () => {
+    for (const page of reference.pages) {
+      expect(page.content).not.toContain('');
+      expect(page.content).not.toContain('[source]');
+      expect(page.content).not.toContain('_modules');
+    }
+  });
+
+  it('rewrites internal links and keeps external ones', () => {
+    const index = reference.pages.find((page) => page.slug === 'index');
+    expect(index?.content).toContain('[Widget docs](/widgets)');
+    expect(index?.content).toContain('https://example.com/docs');
+  });
+
+  it('converts admonitions to callouts and extracts descriptions', () => {
+    const index = reference.pages.find((page) => page.slug === 'index');
+    expect(index?.content).toContain('<Warning>');
+    expect(index?.content).toContain('Handle with care.');
+    expect(index?.description).toBe('Welcome to the example SDK.');
+  });
+});