@taurgis/bonsai 1.0.0

package/dist/lib/research/docs/markdown-source.js

@@ -0,0 +1,194 @@
+import { dropEmptyLinks, htmlToMarkdown } from '../markdown.js';
+// Parses a leading YAML frontmatter block (--- ... ---). Only flat scalar keys are read; nested
+// structures are ignored (we only need title/description). The block is stripped from the body.
+export function parseFrontmatter(md) {
+    const normalized = md.replace(/^/, '');
+    if (!normalized.startsWith('---'))
+        return { frontmatter: {}, body: normalized };
+    const end = normalized.indexOf('\n---', 3);
+    if (end === -1)
+        return { frontmatter: {}, body: normalized };
+    const block = normalized.slice(3, end).trim();
+    const rest = normalized.slice(end + 4).replace(/^[\r\n]+/, '');
+    const frontmatter = {};
+    for (const line of block.split('\n')) {
+        const colon = line.indexOf(':');
+        if (colon === -1 || line.startsWith(' ') || line.startsWith('-'))
+            continue;
+        const key = line.slice(0, colon).trim();
+        const value = line
+            .slice(colon + 1)
+            .trim()
+            .replace(/^["']|["']$/g, '');
+        if (key)
+            frontmatter[key] = value;
+    }
+    return { frontmatter, body: rest };
+}
+/**
+ * Removes dangerous embedded HTML from untrusted Markdown: <script>/<style> blocks, inline event
+ * handler attributes, and javascript: URLs. Leaves ordinary Markdown and benign inline HTML intact.
+ */
+export function sanitizeSourceMarkdown(md) {
+    return (md
+        .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')
+        .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '')
+        .replace(/<iframe\b[^>]*>[\s\S]*?<\/iframe>/gi, '')
+        // Interactive / icon / media widgets authored into docs source (e.g. VitePress's pronunciation
+        // <audio>+<button><svg> control) are non-content chrome that just wastes agent tokens.
+        .replace(/<svg\b[^>]*>[\s\S]*?<\/svg>/gi, '')
+        .replace(/<audio\b[^>]*>[\s\S]*?<\/audio>/gi, '')
+        .replace(/<video\b[^>]*>[\s\S]*?<\/video>/gi, '')
+        .replace(/<button\b[^>]*>[\s\S]*?<\/button>/gi, '')
+        .replace(/<use\b[^>]*\/?>/gi, '')
+        .replace(/\son[a-z]+\s*=\s*"[^"]*"/gi, '')
+        .replace(/\son[a-z]+\s*=\s*'[^']*'/gi, '')
+        .replace(/javascript:/gi, ''));
+}
+// MDN raw source (mdn/content) is KumaScript: macros like {{jsxref("Promise.all()")}} and
+// {{Compat}} are placeholders that don't render to content. Reduce cross-reference macros to their
+// display text and drop section/embed/sidebar macros, so MDN source is as clean as its rendered
+// page without the macro noise. Gated to MDN source URLs (see extractFromSource) — other ecosystems
+// legitimately use {{ }} (Vue/Angular interpolation), so this must never run on them.
+const KUMASCRIPT_DROP = new Set([
+    'compat',
+    'specifications',
+    'embedinteractiveexample',
+    'interactiveexample',
+    'embedlivesample',
+    'livesample',
+    'previousnext',
+    'previousmenunext',
+    'page',
+    'jssidebar',
+    'defaultapisidebar',
+    'htmlsidebar',
+    'cssref',
+    'jsref',
+    'glossarysidebar',
+    'learnsidebar',
+    'apiref',
+    'seecompattable',
+    'quicklinkswithsubpages',
+]);
+const KUMASCRIPT_BADGES = {
+    optional_inline: 'optional',
+    deprecated_inline: 'deprecated',
+    experimental_inline: 'experimental',
+    'non-standard_inline': 'non-standard',
+    readonlyinline: 'read-only',
+};
+function replaceMacro(_match, inner) {
+    const name = (inner.match(/^\s*([\w-]+)/)?.[1] ?? '').toLowerCase();
+    if (KUMASCRIPT_DROP.has(name))
+        return '';
+    if (KUMASCRIPT_BADGES[name])
+        return `(${KUMASCRIPT_BADGES[name]})`;
+    // Cross-reference macros (jsxref/domxref/cssxref/htmlelement…): use the last quoted argument,
+    // which is the human display text (e.g. {{jsxref("Promise/then", "then()")}} -> then()).
+    const quoted = [...inner.matchAll(/"([^"]*)"|'([^']*)'/g)].map((m) => m[1] ?? m[2]);
+    if (quoted.length)
+        return quoted[quoted.length - 1];
+    return '';
+}
+// MDN live-sample sources flagged `hidden` (e.g. <pre class="brush: html hidden">) are plumbing for
+// the rendered {{EmbedLiveSample}} widget (which we already drop), not reader-facing content. Remove
+// them so they don't survive — e.g. flattened into a converted table cell — as raw HTML.
+function stripMdnHiddenSamples(md) {
+    return md.replace(/<pre\b[^>]*\bhidden\b[^>]*>[\s\S]*?<\/pre>/gi, '');
+}
+function cleanKumaScript(md) {
+    // Never touch fenced code blocks.
+    return md
+        .split(/(```[\s\S]*?```)/)
+        .map((segment, i) => (i % 2 === 1 ? segment : segment.replace(/\{\{(.+?)\}\}/g, replaceMacro)))
+        .join('');
+}
+// MDN (and occasionally other docs) author tables as raw HTML inside their Markdown source, which
+// otherwise leaks verbatim into the artifact and wastes agent tokens. Turndown already emits clean
+// GFM tables, so convert any literal <table> region to Markdown. Code fences are skipped so an HTML
+// table shown as a *code example* (```html …<table>…) is preserved, not rewritten.
+function convertHtmlTables(md) {
+    return md
+        .split(/(```[\s\S]*?```)/)
+        .map((segment, i) => i % 2 === 1
+        ? segment
+        : // ponytail: non-greedy per-table match. A <table> nested in a cell would stop the match at
+            // the inner </table>, leaving the outer wrapper's closing tags as inert residue (no attrs,
+            // so it can't trip the leak gate). Doc sources use flat tables; upgrade to a DOM walk over
+            // the whole segment via htmlToMarkdown if a nested-table source ever appears.
+            segment.replace(/<table\b[\s\S]*?<\/table>/gi, (table) => `\n\n${htmlToMarkdown(table).trim()}\n\n`))
+        .join('');
+}
+const HEADING = /^(#{1,6})\s/;
+function headingLevel(line) {
+    return line?.match(HEADING)?.[1]?.length ?? 0;
+}
+// Drops a heading whose section body is empty after macro/widget removal — e.g. MDN's
+// `## Specifications` / `## Browser compatibility`, generated from external compat data that only
+// exists in the rendered page, leaving a bare heading once `{{Specifications}}`/`{{Compat}}` are
+// stripped. A heading is empty when only blank lines separate it from the next heading at the same
+// or higher level (or end of document); parents that still own subsections are kept. Iterates to a
+// fixed point so a parent left empty by dropping its only (empty) child is removed too.
+function dropEmptySections(md) {
+    let prev;
+    let current = md;
+    do {
+        prev = current;
+        current = dropEmptySectionsOnce(prev);
+    } while (current !== prev);
+    return current;
+}
+function dropEmptySectionsOnce(md) {
+    const lines = md.split('\n');
+    const keep = lines.map(() => true);
+    for (let i = 0; i < lines.length; i++) {
+        const level = headingLevel(lines[i]);
+        if (!level)
+            continue;
+        let next = i + 1;
+        while (next < lines.length && lines[next].trim() === '')
+            next++;
+        const nextLevel = headingLevel(lines[next]);
+        const sectionIsEmpty = next >= lines.length || (nextLevel > 0 && nextLevel <= level);
+        if (sectionIsEmpty)
+            for (let k = i; k < next; k++)
+                keep[k] = false;
+    }
+    return lines.filter((_, i) => keep[i]).join('\n');
+}
+function titleFrom(frontmatter, body) {
+    if (frontmatter.title)
+        return frontmatter.title;
+    const h1 = body.match(/^#\s+(.+)$/m);
+    return h1 ? h1[1].trim() : 'Untitled';
+}
+function confidenceFor(length) {
+    if (length < 500)
+        return 'low';
+    if (length < 2000)
+        return 'medium';
+    return 'high';
+}
+/**
+ * Builds an ExtractionResult from public Markdown/MDX source. Frontmatter is parsed for the title
+ * and the body is sanitized. The detailed Markdown is the source itself, preserving code fences,
+ * tabs, and admonitions better than HTML conversion.
+ */
+export function extractFromSource(md, sourceUrl) {
+    const { frontmatter, body } = parseFrontmatter(md);
+    // MDN content is KumaScript; strip its macros. Gated by host so {{ }} in other ecosystems' source
+    // (Vue/Angular interpolation) is left untouched.
+    const deMacroed = /mdn\/content/.test(sourceUrl)
+        ? cleanKumaScript(stripMdnHiddenSamples(body))
+        : body;
+    const tablesAsMarkdown = convertHtmlTables(sanitizeSourceMarkdown(deMacroed));
+    const cleaned = dropEmptySections(dropEmptyLinks(tablesAsMarkdown)).trim();
+    const notes = [`captured from public Markdown/MDX source: ${sourceUrl}`];
+    return {
+        title: titleFrom(frontmatter, cleaned),
+        detailedMarkdown: cleaned,
+        confidence: confidenceFor(cleaned.length),
+        qualityNotes: notes,
+    };
+}

package/dist/lib/research/docs/quality-gates.d.ts

@@ -0,0 +1,9 @@
+export interface QualitySignals {
+    codes: string[];
+    isIndexHub: boolean;
+}
+/**
+ * Analyzes extracted Markdown and returns stable quality codes plus an index/hub flag. `markdown`
+ * is the detailed representation; `title` helps the error-page heuristic on short pages.
+ */
+export declare function analyzeMarkdownQuality(markdown: string, title?: string): QualitySignals;

package/dist/lib/research/docs/quality-gates.js

@@ -0,0 +1,65 @@
+import { estimateTokens } from '../token-estimate.js';
+import { looksLikeErrorPage } from './validate.js';
+const OVERSIZED_TOKEN_THRESHOLD = 8000;
+const IMAGE_RE = /!\[[^\]]*\]\([^)]*\)/g;
+const LINK_RE = /\[[^\]]*\]\([^)]*\)/g;
+const HEADING_RE = /^#{1,6}\s+/;
+// A lowercase letter, digit, or close-paren/quote butted directly against a shell/import keyword —
+// the signature of two commands collapsed onto one line (e.g. "my-projectcd my-project").
+const COLLAPSED_CODE_RE = /[a-z0-9)"'](?:cd|npm|yarn|pnpm|npx|import|export|git)\s/;
+function countMatches(text, re) {
+    return (text.match(re) || []).length;
+}
+function proseLineCount(markdown) {
+    return markdown.split('\n').filter((line) => {
+        const t = line.trim();
+        if (!t || HEADING_RE.test(t))
+            return false;
+        // A line that is essentially just a link/image is navigation, not prose.
+        const withoutLinks = t
+            .replace(IMAGE_RE, '')
+            .replace(LINK_RE, '')
+            .replace(/[-*>!\s]/g, '');
+        return withoutLinks.length > 0;
+    }).length;
+}
+// Scans fenced code blocks for a line that looks like two commands were merged (T-17 warning).
+function hasCollapsedCode(markdown) {
+    const lines = markdown.split('\n');
+    let inFence = false;
+    for (const line of lines) {
+        if (/^\s*```/.test(line)) {
+            inFence = !inFence;
+            continue;
+        }
+        if (inFence && COLLAPSED_CODE_RE.test(line))
+            return true;
+    }
+    return false;
+}
+/**
+ * Analyzes extracted Markdown and returns stable quality codes plus an index/hub flag. `markdown`
+ * is the detailed representation; `title` helps the error-page heuristic on short pages.
+ */
+export function analyzeMarkdownQuality(markdown, title = '') {
+    const codes = [];
+    const images = countMatches(markdown, IMAGE_RE);
+    const links = countMatches(markdown, LINK_RE);
+    const headings = markdown.split('\n').filter((l) => HEADING_RE.test(l.trim())).length;
+    const prose = proseLineCount(markdown);
+    if (/!\[[^\]]*\]\(\s*data:/.test(markdown))
+        codes.push('quality:base64-image');
+    if (images > 0 && images > headings + prose)
+        codes.push('quality:high-image-ratio');
+    if (hasCollapsedCode(markdown))
+        codes.push('quality:collapsed-code');
+    if (estimateTokens(markdown) > OVERSIZED_TOKEN_THRESHOLD)
+        codes.push('quality:oversized');
+    if (looksLikeErrorPage(`${title}\n${markdown}`))
+        codes.push('quality:error-page');
+    // Hub page: many outgoing links and little prose to wrap them.
+    const isIndexHub = links >= 10 && links > prose * 2;
+    if (isIndexHub)
+        codes.push('quality:index-hub');
+    return { codes, isIndexHub };
+}

package/dist/lib/research/docs/remote-search-runner.d.ts

@@ -0,0 +1,21 @@
+import type { DocsSearchResult } from './search-index.js';
+export interface RemoteSearchDeps {
+    fetchStatic: (url: string) => Promise<{
+        content: string;
+        finalUrl: string;
+    }>;
+    fetchText: (url: string) => Promise<{
+        content: string;
+        status: number;
+    }>;
+    postJson: (url: string, body: unknown, headers: Record<string, string>) => Promise<string>;
+}
+export interface RemoteSearchOutcome {
+    provider: string;
+    results: DocsSearchResult[];
+}
+/**
+ * Runs remote docs search for a documentation site. Detects the provider from the page, queries the
+ * matching connector, and returns normalized results. Throws when no verified connector applies.
+ */
+export declare function runRemoteDocsSearch(docsUrl: string, query: string, deps: RemoteSearchDeps): Promise<RemoteSearchOutcome>;

package/dist/lib/research/docs/remote-search-runner.js

@@ -0,0 +1,54 @@
+import { detectDocsEngine } from './detect.js';
+import { searchMkDocsIndex, searchJekyllIndex, searchSphinxIndex } from './search-index.js';
+import { extractAlgoliaConfig, algoliaQueryUrl, parseAlgoliaResponse } from './remote-search.js';
+async function getText(url, deps) {
+    const res = await deps.fetchText(url);
+    if (res.status === 304 || !res.content)
+        throw new Error(`empty search index at ${url}`);
+    return res.content;
+}
+// Conventional static index path + parser per framework (verified by T-25 fixtures).
+async function staticIndexSearch(framework, origin, query, deps) {
+    if (framework === 'mkdocs' || framework === 'material-mkdocs') {
+        const indexUrl = `${origin}/search/search_index.json`;
+        return {
+            provider: 'mkdocs-local',
+            results: searchMkDocsIndex(await getText(indexUrl, deps), indexUrl, query),
+        };
+    }
+    if (framework === 'sphinx') {
+        const indexUrl = `${origin}/searchindex.js`;
+        return {
+            provider: 'sphinx-searchindex',
+            results: searchSphinxIndex(await getText(indexUrl, deps), indexUrl, query),
+        };
+    }
+    if (framework === 'just-the-docs') {
+        const indexUrl = `${origin}/assets/js/search-data.json`;
+        return {
+            provider: 'jekyll-json',
+            results: searchJekyllIndex(await getText(indexUrl, deps), indexUrl, query),
+        };
+    }
+    return null;
+}
+/**
+ * Runs remote docs search for a documentation site. Detects the provider from the page, queries the
+ * matching connector, and returns normalized results. Throws when no verified connector applies.
+ */
+export async function runRemoteDocsSearch(docsUrl, query, deps) {
+    const page = await deps.fetchStatic(docsUrl);
+    const origin = new URL(page.finalUrl).origin;
+    const algolia = extractAlgoliaConfig(page.content);
+    if (algolia) {
+        const url = algoliaQueryUrl(algolia);
+        const body = await deps.postJson(url, { params: `query=${encodeURIComponent(query)}` }, { 'X-Algolia-API-Key': algolia.apiKey, 'X-Algolia-Application-Id': algolia.appId });
+        return { provider: 'algolia-docsearch', results: parseAlgoliaResponse(body, url) };
+    }
+    const caps = detectDocsEngine(page.content, page.finalUrl);
+    const staticResult = await staticIndexSearch(caps.framework, origin, query, deps);
+    if (staticResult)
+        return staticResult;
+    throw new Error(`No verified remote search connector for this site (detected: ${caps.framework ?? caps.docsEngine ?? 'unknown'}). ` +
+        'Supported: Algolia DocSearch, MkDocs/Material, Sphinx, Just the Docs.');
+}

package/dist/lib/research/docs/remote-search.d.ts

@@ -0,0 +1,14 @@
+import type { DocsSearchResult } from './search-index.js';
+export interface AlgoliaConfig {
+    appId: string;
+    apiKey: string;
+    indexName: string;
+}
+export declare function extractAlgoliaConfig(html: string): AlgoliaConfig | null;
+export declare function algoliaQueryUrl(config: AlgoliaConfig): string;
+export declare function parseAlgoliaResponse(body: string, indexUrl: string): DocsSearchResult[];
+/**
+ * Parses a VitePress local search index (MiniSearch documents) and ranks docs by query term hits.
+ * `base` is the docs site origin/base used to resolve the relative `id` routes.
+ */
+export declare function searchVitePressLocalIndex(body: string, indexUrl: string, base: string, query: string): DocsSearchResult[];

package/dist/lib/research/docs/remote-search.js

@@ -0,0 +1,101 @@
+// Algolia identifiers extracted from untrusted page HTML are interpolated into the request URL, so
+// they MUST be format-validated to prevent SSRF (an unvalidated appId like "evil.com/x" would point
+// the request at an attacker host). App IDs are uppercase alphanumeric; public search keys are hex;
+// the index name must contain no slash (it is path-encoded but we still reject separators).
+const ALGOLIA_APP_ID_RE = /^[A-Z0-9]{6,24}$/;
+const ALGOLIA_API_KEY_RE = /^[a-f0-9]{16,64}$/;
+const ALGOLIA_INDEX_RE = /^[A-Za-z0-9._~-]{1,128}$/;
+// Pulls a DocSearch config out of embedded site data. Returns null unless all three fields are
+// present AND well-formed — a partial or malformed signal is not enough to query.
+export function extractAlgoliaConfig(html) {
+    const text = html.replace(/\\"/g, '"');
+    const appId = text.match(/"appId"\s*:\s*"([^"]+)"/)?.[1];
+    const apiKey = text.match(/"apiKey"\s*:\s*"([^"]+)"/)?.[1];
+    const indexName = text.match(/"indexName"\s*:\s*"([^"]+)"/)?.[1];
+    if (!appId || !apiKey || !indexName)
+        return null;
+    if (!ALGOLIA_APP_ID_RE.test(appId) ||
+        !ALGOLIA_API_KEY_RE.test(apiKey) ||
+        !ALGOLIA_INDEX_RE.test(indexName)) {
+        return null;
+    }
+    return { appId, apiKey, indexName };
+}
+export function algoliaQueryUrl(config) {
+    return `https://${config.appId}-dsn.algolia.net/1/indexes/${encodeURIComponent(config.indexName)}/query`;
+}
+// DocSearch records nest the page hierarchy under `hierarchy` and the page URL under `url`.
+function algoliaTitle(hit) {
+    const h = hit?.hierarchy ?? {};
+    // Prefer the deepest (most specific) hierarchy level present.
+    for (const level of ['lvl6', 'lvl5', 'lvl4', 'lvl3', 'lvl2', 'lvl1', 'lvl0']) {
+        if (h[level])
+            return String(h[level]);
+    }
+    return hit?.url ?? 'Untitled';
+}
+export function parseAlgoliaResponse(body, indexUrl) {
+    const data = JSON.parse(body);
+    if (!Array.isArray(data.hits))
+        throw new Error('Algolia response missing "hits"');
+    return data.hits
+        .filter((hit) => typeof hit?.url === 'string')
+        .map((hit, i) => ({
+        title: algoliaTitle(hit),
+        url: hit.url,
+        snippet: hit?.content ?? undefined,
+        provider: 'algolia-docsearch',
+        score: data.hits.length - i, // preserve Algolia's relevance order
+        indexUrl,
+    }));
+}
+function vitepressUrl(base, id) {
+    try {
+        return new URL(id, base).toString();
+    }
+    catch {
+        return null;
+    }
+}
+// Ranks one VitePress doc by query term hits: a title match weighs 10×, body occurrences 1× each
+// (capped at 5). Kept separate so the search loop stays a flat filter → score → format.
+function scoreVitePressDoc(doc, terms) {
+    const haystack = `${doc.title ?? ''} ${(doc.titles ?? []).join(' ')} ${doc.text ?? ''}`.toLowerCase();
+    let score = 0;
+    for (const term of terms) {
+        if ((doc.title ?? '').toLowerCase().includes(term))
+            score += 10;
+        score += Math.min(haystack.split(term).length - 1, 5);
+    }
+    return score;
+}
+/**
+ * Parses a VitePress local search index (MiniSearch documents) and ranks docs by query term hits.
+ * `base` is the docs site origin/base used to resolve the relative `id` routes.
+ */
+export function searchVitePressLocalIndex(body, indexUrl, base, query) {
+    const docs = JSON.parse(body);
+    if (!Array.isArray(docs))
+        throw new Error('VitePress local index is not an array');
+    const terms = query.toLowerCase().split(/\s+/).filter(Boolean);
+    const results = [];
+    for (const doc of docs) {
+        if (!doc.id)
+            continue;
+        const score = scoreVitePressDoc(doc, terms);
+        if (score <= 0)
+            continue;
+        const url = vitepressUrl(base, doc.id);
+        if (!url)
+            continue;
+        results.push({
+            title: doc.title ?? doc.id,
+            url,
+            snippet: (doc.text ?? '').slice(0, 150),
+            provider: 'vitepress-local',
+            score,
+            indexUrl,
+        });
+    }
+    return results.sort((a, b) => b.score - a.score);
+}

package/dist/lib/research/docs/rsc.d.ts

@@ -0,0 +1,10 @@
+import type { PageMapEntry } from './capabilities.js';
+export interface NextRscData {
+    pageMap: PageMapEntry[];
+    sourcePathHints: string[];
+}
+/**
+ * Scans Next/RSC HTML for a page map (route + title, with optional source path) and any
+ * `filePath` source hints. Returns empty arrays when no payload is present.
+ */
+export declare function extractNextRscData(html: string): NextRscData;

package/dist/lib/research/docs/rsc.js

@@ -0,0 +1,30 @@
+// Flight payloads embed JSON as a JS string literal, so quotes/backslashes are escaped. Undo the
+// two escapes we care about to expose the inner JSON to scanning. Still inert text.
+function unescapeFlight(html) {
+    return html.replace(/\\"/g, '"').replace(/\\\\/g, '\\');
+}
+function dedupe(items) {
+    return [...new Set(items.filter(Boolean))];
+}
+/**
+ * Scans Next/RSC HTML for a page map (route + title, with optional source path) and any
+ * `filePath` source hints. Returns empty arrays when no payload is present.
+ */
+export function extractNextRscData(html) {
+    const text = unescapeFlight(html);
+    const sourcePathHints = dedupe([...text.matchAll(/"filePath"\s*:\s*"([^"]+)"/g)].map((m) => m[1]));
+    const pageMap = [];
+    const seen = new Set();
+    for (const match of text.matchAll(/"route"\s*:\s*"([^"]+)"/g)) {
+        const route = match[1];
+        if (seen.has(route))
+            continue;
+        seen.add(route);
+        // Look just past the route for a title / filePath belonging to the same object.
+        const window = text.slice(match.index, match.index + 400);
+        const title = window.match(/"title"\s*:\s*"([^"]+)"/)?.[1];
+        const sourcePath = window.match(/"filePath"\s*:\s*"([^"]+)"/)?.[1];
+        pageMap.push({ title: title ?? route, url: route, sourcePath });
+    }
+    return { pageMap, sourcePathHints };
+}

package/dist/lib/research/docs/search-index.d.ts

@@ -0,0 +1,13 @@
+import type { SearchProvider } from './capabilities.js';
+export interface DocsSearchResult {
+    title: string;
+    url: string;
+    anchor?: string;
+    snippet?: string;
+    provider: SearchProvider;
+    score: number;
+    indexUrl: string;
+}
+export declare function searchMkDocsIndex(body: string, indexUrl: string, query: string): DocsSearchResult[];
+export declare function searchJekyllIndex(body: string, indexUrl: string, query: string): DocsSearchResult[];
+export declare function searchSphinxIndex(js: string, indexUrl: string, query: string): DocsSearchResult[];