@relayfile/adapter-core 0.1.0

package/dist/src/docs/change-detector.js

@@ -0,0 +1,126 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+export class ChangeDetector {
+    fetchImpl;
+    stateFile;
+    constructor(options) {
+        this.fetchImpl = options?.fetchImpl ?? fetch;
+        this.stateFile = options?.stateFile ?? ".adapter-core-state.json";
+    }
+    async check(input) {
+        const stateKey = this.getStateKey(input);
+        const state = await this.loadState(input.stateFile);
+        const previous = state[stateKey]?.lastValue;
+        const current = await this.readCurrentValue(input);
+        return {
+            changed: previous === undefined ? true : previous !== current,
+            reason: previous === undefined
+                ? `No baseline recorded for ${input.trigger}`
+                : previous !== current
+                    ? `${input.trigger} changed`
+                    : `${input.trigger} unchanged`,
+            previousHash: previous,
+            currentHash: current,
+            stateKey,
+        };
+    }
+    async record(input, result) {
+        const state = await this.loadState(input.stateFile);
+        state[result.stateKey] = { lastValue: result.currentHash };
+        const file = resolve(input.stateFile ?? this.stateFile);
+        await mkdir(dirname(file), { recursive: true });
+        await writeFile(file, `${JSON.stringify(state, null, 2)}\n`);
+    }
+    getStateKey(input) {
+        if (input.trigger === "changelog-rss") {
+            return `${input.trigger}:${input.feedUrl ?? input.url}`;
+        }
+        if (input.trigger === "github-release") {
+            return `${input.trigger}:${input.repo ?? input.url}`;
+        }
+        return `${input.trigger}:${input.url}`;
+    }
+    async readCurrentValue(input) {
+        if (input.trigger === "changelog-rss") {
+            return this.readFeedValue(input.feedUrl ?? input.url);
+        }
+        if (input.trigger === "github-release") {
+            if (!input.repo) {
+                throw new Error("github-release trigger requires sync.repo");
+            }
+            return this.readGitHubReleaseValue(input.repo, input.githubToken);
+        }
+        return this.readContentHash(input.url);
+    }
+    async readContentHash(url) {
+        const response = await this.fetchImpl(url);
+        if (!response.ok) {
+            throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+        }
+        return createHash("sha256")
+            .update(await response.text())
+            .digest("hex");
+    }
+    async readFeedValue(feedUrl) {
+        const response = await this.fetchImpl(feedUrl);
+        if (!response.ok) {
+            throw new Error(`Failed to fetch ${feedUrl}: ${response.status} ${response.statusText}`);
+        }
+        const xml = await response.text();
+        const entries = Array.from(xml.matchAll(/<(entry|item)\b[\s\S]*?<\/\1>/gi), (match) => match[0]);
+        const first = entries[0] ?? xml;
+        return (first.match(/<(guid|id)>([\s\S]*?)<\/\1>/i)?.[2]?.trim() ??
+            first.match(/<(updated|pubDate)>([\s\S]*?)<\/\1>/i)?.[2]?.trim() ??
+            first.match(/<link[^>]*href="([^"]+)"/i)?.[1]?.trim() ??
+            createHash("sha256").update(first).digest("hex"));
+    }
+    async readGitHubReleaseValue(repo, token) {
+        const headers = {
+            accept: "application/vnd.github+json",
+        };
+        if (token || process.env.GITHUB_TOKEN) {
+            headers.authorization = `Bearer ${token ?? process.env.GITHUB_TOKEN}`;
+        }
+        const releaseResponse = await this.fetchImpl(`https://api.github.com/repos/${repo}/releases/latest`, { headers });
+        if (releaseResponse.ok) {
+            const release = (await releaseResponse.json());
+            const value = (typeof release.tag_name === "string" && release.tag_name) ||
+                (typeof release.name === "string" && release.name);
+            if (value) {
+                return value;
+            }
+        }
+        const tagsResponse = await this.fetchImpl(`https://api.github.com/repos/${repo}/tags?per_page=1`, { headers });
+        if (!tagsResponse.ok) {
+            throw new Error(`Failed to fetch GitHub release info for ${repo}: ${tagsResponse.status} ${tagsResponse.statusText}`);
+        }
+        const tags = (await tagsResponse.json());
+        const first = tags[0];
+        const name = first && typeof first.name === "string" ? first.name : undefined;
+        if (!name) {
+            throw new Error(`No GitHub releases or tags found for ${repo}`);
+        }
+        return name;
+    }
+    async loadState(stateFile) {
+        const file = resolve(stateFile ?? this.stateFile);
+        try {
+            const text = await readFile(file, "utf8");
+            return JSON.parse(text);
+        }
+        catch {
+            return {};
+        }
+    }
+}
+export function defaultSyncConfig(url, sync) {
+    return {
+        trigger: (sync?.trigger ?? "content-hash"),
+        feedUrl: sync?.feedUrl,
+        repo: sync?.repo,
+        stateFile: sync?.stateFile,
+        url,
+    };
+}
+//# sourceMappingURL=change-detector.js.map

package/dist/src/docs/change-detector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-detector.js","sourceRoot":"","sources":["../../../src/docs/change-detector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAkB7C,MAAM,OAAO,cAAc;IACR,SAAS,CAAe;IACxB,SAAS,CAAS;IAEnC,YAAY,OAA0D;QACpE,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,KAAK,CAAC;QAC7C,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,0BAA0B,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAA+B;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,SAAS,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAEnD,OAAO;YACL,OAAO,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO;YAC7D,MAAM,EACJ,QAAQ,KAAK,SAAS;gBACpB,CAAC,CAAC,4BAA4B,KAAK,CAAC,OAAO,EAAE;gBAC7C,CAAC,CAAC,QAAQ,KAAK,OAAO;oBACpB,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,UAAU;oBAC5B,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,YAAY;YACpC,YAAY,EAAE,QAAQ;YACtB,WAAW,EAAE,OAAO;YACpB,QAAQ;SACT,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAA+B,EAC/B,MAA+D;QAE/D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACpD,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC;QACxD,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/D,CAAC;IAEO,WAAW,CAAC,KAA+B;QACjD,IAAI,KAAK,CAAC,OAAO,KAAK,eAAe,EAAE,CAAC;YACtC,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;QAC1D,CAAC;QACD,IAAI,KAAK,CAAC,OAAO,KAAK,gBAAgB,EAAE,CAAC;YACvC,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;QACvD,CAAC;QACD,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,KAA+B;QAC5D,IAAI,KAAK,CAAC,OAAO,KAAK,eAAe,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,KAAK,CAAC,OAAO,KAAK,gBAAgB,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,OAAO,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,GAAW;QACvC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,mBAAmB,GAAG,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CACpE,CAAC;QACJ,CAAC;QACD,OAAO,UAAU,CAAC,QAAQ,CAAC;aACxB,MAAM,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;aAC7B,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,mBAAmB,OAAO,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CACxE,CAAC;QACJ,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CACxB,GAAG,CAAC,QAAQ,CAAC,iCAAiC,CAAC,EAC/C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CACpB,CAAC;QACF,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;QAChC,OAAO,CACL,KAAK,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE;YACxD,KAAK,CAAC,KAAK,CAAC,sCAAsC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE;YAChE,KAAK,CAAC,KAAK,CAAC,2BAA2B,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE;YACrD,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACjD,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAClC,IAAY,EACZ,KAAc;QAEd,MAAM,OAAO,GAA2B;YACtC,MAAM,EAAE,6BAA6B;SACtC,CAAC;QACF,IAAI,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;YACtC,OAAO,CAAC,aAAa,GAAG,UAAU,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QACxE,CAAC;QAED,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,SAAS,CAC1C,gCAAgC,IAAI,kBAAkB,EACtD,EAAE,OAAO,EAAE,CACZ,CAAC;QACF,IAAI,eAAe,CAAC,EAAE,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,CAAC,MAAM,eAAe,CAAC,IAAI,EAAE,CAA4B,CAAC;YAC1E,MAAM,KAAK,GACT,CAAC,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC;gBAC1D,CAAC,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;YACrD,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,CACvC,gCAAgC,IAAI,kBAAkB,EACtD,EAAE,OAAO,EAAE,CACZ,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CACb,2CAA2C,IAAI,KAAK,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,EAAE,CACrG,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,YAAY,CAAC,IAAI,EAAE,CAAmC,CAAC;QAC3E,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,IAAI,GAAG,KAAK,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;QAC9E,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,SAAkB;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC;QAClD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC1C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAc,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AAED,MAAM,UAAU,iBAAiB,CAC/B,GAAW,EACX,IAAqB;IAErB,OAAO;QACL,OAAO,EAAE,CAAC,IAAI,EAAE,OAAO,IAAI,cAAc,CAAoB;QAC7D,OAAO,EAAE,IAAI,EAAE,OAAO;QACtB,IAAI,EAAE,IAAI,EAAE,IAAI;QAChB,SAAS,EAAE,IAAI,EAAE,SAAS;QAC1B,GAAG;KACJ,CAAC;AACJ,CAAC"}

package/dist/src/docs/crawler.d.ts

@@ -0,0 +1,22 @@
+import type { DocPage, DocsSourceConfig } from "./types.js";
+export interface DocsCrawlerOptions extends DocsSourceConfig {
+    fetchImpl?: typeof fetch;
+}
+export declare class DocsCrawler {
+    private readonly options;
+    private readonly fetchImpl;
+    private readonly maxPages;
+    private readonly rateLimitMs;
+    private lastRequestAt;
+    private robotsRules?;
+    constructor(options: DocsCrawlerOptions);
+    crawl(): Promise<DocPage[]>;
+    private getSeedUrls;
+    private isAllowedByRobots;
+    private getRobotsRules;
+    private fetchText;
+    private extractPage;
+    private extractLinks;
+    private isInScope;
+}
+//# sourceMappingURL=crawler.d.ts.map

package/dist/src/docs/crawler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../../src/docs/crawler.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE5D,MAAM,WAAW,kBAAmB,SAAQ,gBAAgB;IAC1D,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B;AAoBD,qBAAa,WAAW;IAOV,OAAO,CAAC,QAAQ,CAAC,OAAO;IANpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,WAAW,CAAC,CAAc;gBAEL,OAAO,EAAE,kBAAkB;IAMlD,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAgCjC,OAAO,CAAC,WAAW;YAQL,iBAAiB;YAQjB,cAAc;YAoBd,SAAS;IAevB,OAAO,CAAC,WAAW;IAsBnB,OAAO,CAAC,YAAY;IA2CpB,OAAO,CAAC,SAAS;CAclB"}

package/dist/src/docs/crawler.js

@@ -0,0 +1,296 @@
+import { load } from "cheerio";
+const DEFAULT_REMOVE_SELECTORS = [
+    "footer",
+    "header",
+    "nav",
+    "script",
+    "style",
+    "noscript",
+    "svg",
+    "iframe",
+    "aside",
+    "[role='navigation']",
+    ".sidebar",
+    ".toc",
+    ".breadcrumbs",
+    ".pagination",
+    ".ads",
+];
+export class DocsCrawler {
+    options;
+    fetchImpl;
+    maxPages;
+    rateLimitMs;
+    lastRequestAt = 0;
+    robotsRules;
+    constructor(options) {
+        this.options = options;
+        this.fetchImpl = options.fetchImpl ?? fetch;
+        this.maxPages = options.maxPages ?? 25;
+        this.rateLimitMs = options.rateLimitMs ?? 1_000;
+    }
+    async crawl() {
+        const queue = this.getSeedUrls();
+        const pages = [];
+        const visited = new Set();
+        while (queue.length > 0 && pages.length < this.maxPages) {
+            const currentUrl = queue.shift();
+            if (!currentUrl || visited.has(currentUrl)) {
+                continue;
+            }
+            visited.add(currentUrl);
+            if (!(await this.isAllowedByRobots(currentUrl))) {
+                continue;
+            }
+            const html = await this.fetchText(currentUrl);
+            const page = this.extractPage(currentUrl, html);
+            if (page.content) {
+                pages.push(page);
+            }
+            for (const nextUrl of this.extractLinks(currentUrl, html)) {
+                if (!visited.has(nextUrl) && !queue.includes(nextUrl)) {
+                    queue.push(nextUrl);
+                }
+            }
+        }
+        return pages;
+    }
+    getSeedUrls() {
+        const seeds = [this.options.url];
+        for (const path of this.options.crawlPaths ?? []) {
+            seeds.push(new URL(path, this.options.url).toString());
+        }
+        return Array.from(new Set(seeds.map((url) => normalizeUrl(url))));
+    }
+    async isAllowedByRobots(url) {
+        const rules = await this.getRobotsRules();
+        if (!rules) {
+            return true;
+        }
+        return rules.isAllowed(new URL(url).pathname);
+    }
+    async getRobotsRules() {
+        if (this.robotsRules !== undefined) {
+            return this.robotsRules;
+        }
+        const robotsUrl = new URL("/robots.txt", this.options.url).toString();
+        try {
+            const response = await this.fetchImpl(robotsUrl);
+            if (!response.ok) {
+                this.robotsRules = undefined;
+                return this.robotsRules;
+            }
+            this.robotsRules = new RobotsRules(await response.text());
+            return this.robotsRules;
+        }
+        catch {
+            this.robotsRules = undefined;
+            return this.robotsRules;
+        }
+    }
+    async fetchText(url) {
+        const now = Date.now();
+        const delay = this.rateLimitMs - (now - this.lastRequestAt);
+        if (delay > 0) {
+            await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+        const response = await this.fetchImpl(url);
+        this.lastRequestAt = Date.now();
+        if (!response.ok) {
+            throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+        }
+        return response.text();
+    }
+    extractPage(url, html) {
+        const $ = load(html);
+        for (const selector of DEFAULT_REMOVE_SELECTORS) {
+            $(selector).remove();
+        }
+        const root = this.options.selectors?.content
+            ? $(this.options.selectors.content).first()
+            : $("main, article, .content, .docs, .markdown-body, body").first();
+        const title = $("title").first().text().trim() ||
+            root.find("h1").first().text().trim() ||
+            new URL(url).pathname;
+        return {
+            url,
+            title,
+            content: renderMarkdown($, root.length > 0 ? root : $("body").first()),
+        };
+    }
+    extractLinks(url, html) {
+        const $ = load(html);
+        const links = [];
+        const seen = new Set();
+        const paginationSelector = this.options.selectors?.pagination ?? "a[rel='next'], a:contains('Next')";
+        const push = (href) => {
+            if (!href) {
+                return;
+            }
+            try {
+                const resolved = new URL(href, url);
+                if (!this.isInScope(resolved)) {
+                    return;
+                }
+                const value = normalizeUrl(resolved.toString());
+                if (!seen.has(value)) {
+                    seen.add(value);
+                    links.push(value);
+                }
+            }
+            catch {
+                // Ignore malformed URLs.
+            }
+        };
+        $(paginationSelector).each((_, element) => {
+            push($(element).attr("href"));
+        });
+        $("a[href]").each((_, element) => {
+            const text = $(element).text().trim().toLowerCase();
+            const rel = ($(element).attr("rel") ?? "").toLowerCase();
+            if (rel.includes("next") || text.includes("next")) {
+                push($(element).attr("href"));
+                return;
+            }
+            push($(element).attr("href"));
+        });
+        return links;
+    }
+    isInScope(url) {
+        const base = new URL(this.options.url);
+        if (url.origin !== base.origin) {
+            return false;
+        }
+        const allowedPaths = [base.pathname, ...(this.options.crawlPaths ?? [])]
+            .map((path) => new URL(path, this.options.url).pathname)
+            .filter(Boolean);
+        return allowedPaths.length === 0
+            ? true
+            : allowedPaths.some((path) => url.pathname.startsWith(path));
+    }
+}
+function normalizeUrl(url) {
+    const parsed = new URL(url);
+    parsed.hash = "";
+    return parsed.toString();
+}
+class RobotsRules {
+    allow = [];
+    disallow = [];
+    constructor(text) {
+        let active = false;
+        for (const rawLine of text.split(/\r?\n/)) {
+            const line = rawLine.split("#")[0]?.trim();
+            if (!line) {
+                continue;
+            }
+            const [directiveRaw, ...rest] = line.split(":");
+            const directive = directiveRaw.trim().toLowerCase();
+            const value = rest.join(":").trim();
+            if (directive === "user-agent") {
+                active = value === "*" || value.toLowerCase() === "adapter-core";
+                continue;
+            }
+            if (!active) {
+                continue;
+            }
+            if (directive === "allow") {
+                this.allow.push(value);
+            }
+            if (directive === "disallow") {
+                this.disallow.push(value);
+            }
+        }
+    }
+    isAllowed(pathname) {
+        const matchedAllow = this.matchLongest(pathname, this.allow);
+        const matchedDisallow = this.matchLongest(pathname, this.disallow);
+        return matchedAllow.length >= matchedDisallow.length;
+    }
+    matchLongest(pathname, candidates) {
+        let winner = "";
+        for (const candidate of candidates) {
+            if (!candidate || !pathname.startsWith(candidate)) {
+                continue;
+            }
+            if (candidate.length > winner.length) {
+                winner = candidate;
+            }
+        }
+        return winner;
+    }
+}
+function renderMarkdown($, root) {
+    const blocks = [];
+    root.contents().each((_, node) => {
+        const rendered = renderNode($, node).trim();
+        if (rendered) {
+            blocks.push(rendered);
+        }
+    });
+    return blocks.join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+function renderNode($, node) {
+    if ("data" in node && typeof node.data === "string") {
+        return normalizeWhitespace(node.data ?? "");
+    }
+    if (node.type !== "tag" && node.type !== "script" && node.type !== "style") {
+        return "";
+    }
+    const name = node.name?.toLowerCase() ?? "";
+    const element = $(node);
+    if (name === "pre") {
+        const code = normalizeWhitespace(element.text(), false);
+        return code ? `\`\`\`\n${code}\n\`\`\`` : "";
+    }
+    if (name === "code") {
+        const code = normalizeWhitespace(element.text(), false);
+        return code ? `\`${code}\`` : "";
+    }
+    if (/^h[1-6]$/.test(name)) {
+        const level = Number(name.slice(1));
+        const text = renderChildren($, node).trim();
+        return text ? `${"#".repeat(level)} ${text}` : "";
+    }
+    if (name === "li") {
+        const text = renderChildren($, node).trim();
+        return text ? `- ${text}` : "";
+    }
+    if (name === "a") {
+        const text = renderChildren($, node).trim();
+        const href = element.attr("href");
+        return href && text ? `[${text}](${href})` : text;
+    }
+    if (name === "br") {
+        return "\n";
+    }
+    if (name === "table") {
+        return element
+            .find("tr")
+            .toArray()
+            .map((row) => $(row)
+            .find("th, td")
+            .toArray()
+            .map((cell) => normalizeWhitespace($(cell).text()))
+            .filter(Boolean)
+            .join(" | "))
+            .filter(Boolean)
+            .join("\n");
+    }
+    return renderChildren($, node);
+}
+function renderChildren($, node) {
+    return (node.children ?? [])
+        .map((child) => renderNode($, child))
+        .filter(Boolean)
+        .join(" ")
+        .replace(/\s+\n/g, "\n")
+        .replace(/\n\s+/g, "\n")
+        .trim();
+}
+function normalizeWhitespace(value, collapse = true) {
+    return collapse
+        ? value.replace(/\s+/g, " ").trim()
+        : value.replace(/\r/g, "").trim();
+}
+//# sourceMappingURL=crawler.js.map

package/dist/src/docs/crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../../src/docs/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAQ/B,MAAM,wBAAwB,GAAG;IAC/B,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,QAAQ;IACR,OAAO;IACP,UAAU;IACV,KAAK;IACL,QAAQ;IACR,OAAO;IACP,qBAAqB;IACrB,UAAU;IACV,MAAM;IACN,cAAc;IACd,aAAa;IACb,MAAM;CACP,CAAC;AAEF,MAAM,OAAO,WAAW;IAOO;IANZ,SAAS,CAAe;IACxB,QAAQ,CAAS;IACjB,WAAW,CAAS;IAC7B,aAAa,GAAG,CAAC,CAAC;IAClB,WAAW,CAAe;IAElC,YAA6B,OAA2B;QAA3B,YAAO,GAAP,OAAO,CAAoB;QACtD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;QAC5C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,KAAK,GAAc,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAElC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACxD,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC3C,SAAS;YACX,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAExB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;gBAChD,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YAChD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;YAED,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC;gBAC1D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBACtD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,WAAW;QACjB,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC;YACjD,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,GAAW;QACzC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC,WAAW,CAAC;QAC1B,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;QACtE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YACjD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;gBAC7B,OAAO,IAAI,CAAC,WAAW,CAAC;YAC1B,CAAC;YACD,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,OAAO,IAAI,CAAC,WAAW,CAAC;QAC1B,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;YAC7B,OAAO,IAAI,CAAC,WAAW,CAAC;QAC1B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAW;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;QAC5D,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAChC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACvF,CAAC;QACD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,WAAW,CAAC,GAAW,EAAE,IAAY;QAC3C,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,KAAK,MAAM,QAAQ,IAAI,wBAAwB,EAAE,CAAC;YAChD,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;QACvB,CAAC;QAED,MAAM,IAAI,GACR,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO;YAC7B,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE;YAC3C,CAAC,CAAC,CAAC,CAAC,sDAAsD,CAAC,CAAC,KAAK,EAAE,CAAC;QACxE,MAAM,KAAK,GACT,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;YAChC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;YACrC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAExB,OAAO;YACL,GAAG;YACH,KAAK;YACL,OAAO,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC;SACvE,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,GAAW,EAAE,IAAY;QAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,kBAAkB,GACtB,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,UAAU,IAAI,mCAAmC,CAAC;QAE5E,MAAM,IAAI,GAAG,CAAC,IAAwB,EAAE,EAAE;YACxC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO;YACT,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBACpC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC9B,OAAO;gBACT,CAAC;gBACD,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAChD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACrB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAChB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,yBAAyB;YAC3B,CAAC;QACH,CAAC,CAAC;QAEF,CAAC,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACxC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YAC/B,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACpD,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YACzD,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC9B,OAAO;YACT,CAAC;YACD,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,SAAS,CAAC,GAAQ;QACxB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;aACrE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;aACvD,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,OAAO,YAAY,CAAC,MAAM,KAAK,CAAC;YAC9B,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,CAAC;CACF;AAED,SAAS,YAAY,CAAC,GAAW;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;IACjB,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,WAAW;IACE,KAAK,GAAa,EAAE,CAAC;IACrB,QAAQ,GAAa,EAAE,CAAC;IAEzC,YAAY,IAAY;QACtB,IAAI,MAAM,GAAG,KAAK,CAAC;QAEnB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;YAC3C,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,SAAS;YACX,CAAC;YACD,MAAM,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAChD,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAEpC,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;gBAC/B,MAAM,GAAG,KAAK,KAAK,GAAG,IAAI,KAAK,CAAC,WAAW,EAAE,KAAK,cAAc,CAAC;gBACjE,SAAS;YACX,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YAED,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;YACD,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;gBAC7B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IAED,SAAS,CAAC,QAAgB;QACxB,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7D,MAAM,eAAe,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnE,OAAO,YAAY,CAAC,MAAM,IAAI,eAAe,CAAC,MAAM,CAAC;IACvD,CAAC;IAEO,YAAY,CAAC,QAAgB,EAAE,UAAoB;QACzD,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBAClD,SAAS;YACX,CAAC;YACD,IAAI,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;gBACrC,MAAM,GAAG,SAAS,CAAC;YACrB,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,SAAS,cAAc,CACrB,CAA0B,EAC1B,IAA0B;IAE1B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QAC/B,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,EAAE,IAA4B,CAAC,CAAC,IAAI,EAAE,CAAC;QACpE,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;AAC/D,CAAC;AAED,SAAS,UAAU,CAAC,CAA0B,EAAE,IAA0B;IACxE,IAAI,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACpD,OAAO,mBAAmB,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC3E,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAC5C,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAExB,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnB,MAAM,IAAI,GAAG,mBAAmB,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/C,CAAC;IAED,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACpB,MAAM,IAAI,GAAG,mBAAmB,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IACnC,CAAC;IAED,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACpD,CAAC;IAED,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACjC,CAAC;IAED,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IACpD,CAAC;IAED,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACrB,OAAO,OAAO;aACX,IAAI,CAAC,IAAI,CAAC;aACV,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACX,CAAC,CAAC,GAAG,CAAC;aACH,IAAI,CAAC,QAAQ,CAAC;aACd,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;aAClD,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,KAAK,CAAC,CACf;aACA,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,OAAO,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,cAAc,CAAC,CAA0B,EAAE,IAA0B;IAC5E,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;SACzB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAA6B,CAAC,CAAC;SAC5D,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,GAAG,CAAC;SACT,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;SACvB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa,EAAE,QAAQ,GAAG,IAAI;IACzD,OAAO,QAAQ;QACb,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE;QACnC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AACtC,CAAC"}

package/dist/src/docs/extractor.d.ts

@@ -0,0 +1,12 @@
+import type { DocPage, DocsLlmConfig, ExtractedAPI } from "./types.js";
+export interface APIExtractorOptions extends DocsLlmConfig {
+    fetchImpl?: typeof fetch;
+}
+export declare class APIExtractor {
+    private readonly options;
+    private readonly fetchImpl;
+    constructor(options?: APIExtractorOptions);
+    extract(pages: DocPage[]): Promise<ExtractedAPI>;
+    private extractChunk;
+}
+//# sourceMappingURL=extractor.d.ts.map

package/dist/src/docs/extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../../src/docs/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EACb,YAAY,EAKb,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,mBAAoB,SAAQ,aAAa;IACxD,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B;AAoFD,qBAAa,YAAY;IAGX,OAAO,CAAC,QAAQ,CAAC,OAAO;IAFpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;gBAEZ,OAAO,GAAE,mBAAwB;IAIxD,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC;YAcxC,YAAY;CAkB3B"}