@taurgis/bonsai 1.0.0

package/dist/lib/research/docs/search-index.js

@@ -0,0 +1,176 @@
+function queryTerms(query) {
+    return query.toLowerCase().split(/\s+/).filter(Boolean);
+}
+// Score = title term hits (heavy) + body term hits (light). Zero means no match (dropped).
+function scoreText(title, body, terms) {
+    const t = title.toLowerCase();
+    const b = body.toLowerCase();
+    let score = 0;
+    for (const term of terms) {
+        if (t.includes(term))
+            score += 10;
+        const bodyHits = b.split(term).length - 1;
+        score += Math.min(bodyHits, 5);
+    }
+    return score;
+}
+function snippetFrom(text, terms) {
+    const lower = text.toLowerCase();
+    let idx = 0;
+    for (const term of terms) {
+        const found = lower.indexOf(term);
+        if (found !== -1) {
+            idx = found;
+            break;
+        }
+    }
+    const start = Math.max(0, idx - 40);
+    return text
+        .slice(start, start + 150)
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+function splitAnchor(location) {
+    const hashIdx = location.indexOf('#');
+    if (hashIdx === -1)
+        return { path: location };
+    return { path: location.slice(0, hashIdx), anchor: location.slice(hashIdx + 1) || undefined };
+}
+function resolve(base, relative) {
+    try {
+        return new URL(relative, base).toString();
+    }
+    catch {
+        return null;
+    }
+}
+// MkDocs / Material for MkDocs: { docs: [ { location: "page/#anchor", title, text } ] }.
+// Locations are relative to the site root (the index sits at <root>/search/search_index.json).
+export function searchMkDocsIndex(body, indexUrl, query) {
+    const data = JSON.parse(body);
+    if (!Array.isArray(data.docs))
+        throw new Error('MkDocs index missing "docs" array');
+    const root = resolve(indexUrl, '../') ?? indexUrl;
+    const terms = queryTerms(query);
+    const results = [];
+    for (const doc of data.docs) {
+        if (!doc.location && doc.location !== '')
+            continue;
+        const { path, anchor } = splitAnchor(doc.location ?? '');
+        const url = resolve(root, path);
+        if (!url)
+            continue;
+        const score = scoreText(doc.title ?? '', doc.text ?? '', terms);
+        if (score <= 0)
+            continue;
+        results.push({
+            title: doc.title ?? path,
+            url,
+            anchor,
+            snippet: snippetFrom(doc.text ?? '', terms),
+            provider: 'mkdocs-local',
+            score,
+            indexUrl,
+        });
+    }
+    return results.sort((a, b) => b.score - a.score);
+}
+// Just the Docs: { "0": { title, content, url|relUrl }, "1": {...} } keyed by stringified ints.
+export function searchJekyllIndex(body, indexUrl, query) {
+    const data = JSON.parse(body);
+    if (typeof data !== 'object' || data === null)
+        throw new Error('Jekyll index is not an object');
+    const terms = queryTerms(query);
+    const results = [];
+    for (const entry of Object.values(data)) {
+        const rel = entry.url ?? entry.relUrl;
+        if (!rel)
+            continue;
+        const { path, anchor } = splitAnchor(rel);
+        const url = resolve(indexUrl, path);
+        if (!url)
+            continue;
+        const score = scoreText(entry.title ?? '', entry.content ?? '', terms);
+        if (score <= 0)
+            continue;
+        results.push({
+            title: entry.title ?? path,
+            url,
+            anchor,
+            snippet: snippetFrom(entry.content ?? '', terms),
+            provider: 'jekyll-json',
+            score,
+            indexUrl,
+        });
+    }
+    return results.sort((a, b) => b.score - a.score);
+}
+// Extracts the object literal from `Search.setIndex({...})` as inert data and JSON.parses it —
+// never eval. Anchors on the `setIndex(` call so leading comments/whitespace can't shift the slice;
+// falls back to first-`{`/last-`}` for index variants that assign the object differently.
+function parseSphinxObject(js) {
+    const call = js.indexOf('setIndex(');
+    const start = call !== -1 ? js.indexOf('{', call) : js.indexOf('{');
+    const end = js.lastIndexOf('}');
+    if (start === -1 || end <= start)
+        throw new Error('Sphinx index has no object literal');
+    return JSON.parse(js.slice(start, end + 1));
+}
+function collectDocIndices(map, terms) {
+    const hits = new Map();
+    if (!map)
+        return hits;
+    for (const [word, posting] of Object.entries(map)) {
+        const lower = word.toLowerCase();
+        if (!terms.some((t) => lower.includes(t)))
+            continue;
+        const docs = Array.isArray(posting) ? posting : [posting];
+        for (const d of docs)
+            hits.set(d, (hits.get(d) ?? 0) + 1);
+    }
+    return hits;
+}
+// Scores each Sphinx doc index by term hits: title-term hits and exact title matches weigh 10×,
+// body-term hits 1×. Kept separate so the search function stays a flat parse → score → format.
+function scoreSphinxDocs(index, terms, titles) {
+    const scores = new Map();
+    for (const [doc, count] of collectDocIndices(index.titleterms, terms)) {
+        scores.set(doc, (scores.get(doc) ?? 0) + count * 10);
+    }
+    for (const [doc, count] of collectDocIndices(index.terms, terms)) {
+        scores.set(doc, (scores.get(doc) ?? 0) + count);
+    }
+    for (let i = 0; i < titles.length; i++) {
+        if (terms.some((t) => (titles[i] ?? '').toLowerCase().includes(t))) {
+            scores.set(i, (scores.get(i) ?? 0) + 10);
+        }
+    }
+    return scores;
+}
+// Sphinx searchindex.js. Page URLs are docname + ".html" relative to the index's directory.
+export function searchSphinxIndex(js, indexUrl, query) {
+    const index = parseSphinxObject(js);
+    const docnames = index.docnames ?? [];
+    const titles = index.titles ?? [];
+    if (docnames.length === 0)
+        throw new Error('Sphinx index has no docnames');
+    const terms = queryTerms(query);
+    const scores = scoreSphinxDocs(index, terms, titles);
+    const results = [];
+    for (const [doc, score] of scores) {
+        const name = docnames[doc];
+        if (!name || score <= 0)
+            continue;
+        const url = resolve(indexUrl, `${name}.html`);
+        if (!url)
+            continue;
+        results.push({
+            title: titles[doc] ?? name,
+            url,
+            provider: 'sphinx-searchindex',
+            score,
+            indexUrl,
+        });
+    }
+    return results.sort((a, b) => b.score - a.score);
+}

package/dist/lib/research/docs/section-artifacts.d.ts

@@ -0,0 +1,12 @@
+import type { ResearchArtifact } from '../schema.js';
+/**
+ * Derives section child artifacts from a parent page artifact. Returns [] when the page is short
+ * or has too few headings to chunk.
+ */
+export declare function buildSectionArtifacts(parent: ResearchArtifact, currentTime: Date): ResearchArtifact[];
+/**
+ * Regenerates a page's section artifacts: writes fresh children and archives any prior child whose
+ * heading no longer exists, so a revalidated parent never leaves orphaned sections active. Returns
+ * the number of active section children written.
+ */
+export declare function persistSectionArtifacts(dataDir: string, parent: ResearchArtifact, currentTime: Date): number;

package/dist/lib/research/docs/section-artifacts.js

@@ -0,0 +1,75 @@
+import { createHash } from 'node:crypto';
+import { join } from 'node:path';
+import { compressMarkdown } from '../compress.js';
+import { estimateTokens } from '../token-estimate.js';
+import { scanCacheDir, writeArtifact } from '../storage.js';
+import { splitMarkdownSections } from './sections.js';
+// Builds and persists section-level child artifacts for a long page (T-22). Children are keyed by
+// sha256(parentKey#anchor) so the key stays hex (storage requires it), link back via
+// parent_cache_key, and carry the parent's freshness policy. The page artifact is untouched and
+// remains the provenance/revalidation anchor.
+// Only split pages large enough to be awkward as one blob (~Node url.html territory).
+const SECTION_TOKEN_THRESHOLD = 6000;
+function childKey(parentKey, anchor) {
+    return createHash('sha256').update(`${parentKey}#${anchor}`).digest('hex');
+}
+/**
+ * Derives section child artifacts from a parent page artifact. Returns [] when the page is short
+ * or has too few headings to chunk.
+ */
+export function buildSectionArtifacts(parent, currentTime) {
+    if (estimateTokens(parent.detailed) <= SECTION_TOKEN_THRESHOLD)
+        return [];
+    const sections = splitMarkdownSections(parent.detailed);
+    const pm = parent.metadata;
+    return sections.map((section) => {
+        const key = childKey(pm.cache_key, section.anchor);
+        const detailed = section.content;
+        const compressed = compressMarkdown(detailed);
+        return {
+            metadata: {
+                ...pm,
+                artifact_type: 'section',
+                source_url: `${pm.source_url}#${section.anchor}`,
+                source_urls: [`${pm.source_url}#${section.anchor}`],
+                cache_key: key,
+                content_hash: createHash('sha256').update(detailed).digest('hex'),
+                token_estimate: {
+                    compressed: estimateTokens(compressed),
+                    detailed: estimateTokens(detailed),
+                },
+                parent_cache_key: pm.cache_key,
+                section_anchor: section.anchor,
+                section_heading_path: section.headingPath,
+            },
+            summary: section.headingPath,
+            compressed,
+            detailed,
+            provenance: `Section "${section.headingPath}" of ${pm.source_url} (parent ${pm.cache_key})`,
+        };
+    });
+}
+function activeChildrenOf(dataDir, parentKey) {
+    return scanCacheDir(join(dataDir, 'research'), (artifact) => artifact.metadata.parent_cache_key === parentKey && artifact.metadata.status === 'active'
+        ? artifact
+        : null);
+}
+/**
+ * Regenerates a page's section artifacts: writes fresh children and archives any prior child whose
+ * heading no longer exists, so a revalidated parent never leaves orphaned sections active. Returns
+ * the number of active section children written.
+ */
+export function persistSectionArtifacts(dataDir, parent, currentTime) {
+    const children = buildSectionArtifacts(parent, currentTime);
+    const freshKeys = new Set(children.map((c) => c.metadata.cache_key));
+    for (const orphan of activeChildrenOf(dataDir, parent.metadata.cache_key)) {
+        if (!freshKeys.has(orphan.metadata.cache_key)) {
+            orphan.metadata.status = 'archived';
+            writeArtifact(dataDir, orphan.metadata.cache_key, orphan);
+        }
+    }
+    for (const child of children) {
+        writeArtifact(dataDir, child.metadata.cache_key, child);
+    }
+    return children.length;
+}

package/dist/lib/research/docs/sections.d.ts

@@ -0,0 +1,12 @@
+export interface SectionChunk {
+    anchor: string;
+    headingPath: string;
+    level: number;
+    content: string;
+}
+export declare function slugifyHeading(text: string): string;
+/**
+ * Splits Markdown into H2/H3 section chunks. Returns an empty array when the document has fewer
+ * than two such headings (nothing meaningful to chunk).
+ */
+export declare function splitMarkdownSections(markdown: string): SectionChunk[];

package/dist/lib/research/docs/sections.js

@@ -0,0 +1,67 @@
+// Splits a long detailed-Markdown document into section chunks at H2/H3 boundaries (T-22), so a
+// big API reference (e.g. Node `url.html`, ~17.5k tokens) becomes addressable section artifacts.
+// Pure string work: no I/O, no artifact construction. H4+ headings stay within their parent chunk.
+const HEADING_RE = /^(#{1,6})\s+(.+?)\s*#*\s*$/;
+// GitHub-style anchor slug: lowercase, drop punctuation, spaces -> hyphens.
+export function slugifyHeading(text) {
+    return text
+        .toLowerCase()
+        .replace(/`/g, '')
+        .replace(/[^\w\s-]/g, '')
+        .trim()
+        .replace(/\s+/g, '-')
+        .replace(/-+/g, '-');
+}
+function findHeadings(lines) {
+    const out = [];
+    let inFence = false;
+    for (let i = 0; i < lines.length; i++) {
+        if (/^\s*```/.test(lines[i]))
+            inFence = !inFence;
+        if (inFence)
+            continue;
+        const match = lines[i].match(HEADING_RE);
+        if (match)
+            out.push({ index: i, level: match[1].length, text: match[2].trim() });
+    }
+    return out;
+}
+// Builds the breadcrumb for a section from the nearest H1/H2 ancestors preceding it.
+function breadcrumb(headings, current) {
+    const path = [];
+    const currentLevel = headings[current].level;
+    let needed = currentLevel - 1;
+    for (let i = current - 1; i >= 0 && needed >= 1; i--) {
+        if (headings[i].level === needed) {
+            path.unshift(headings[i].text);
+            needed--;
+        }
+    }
+    path.push(headings[current].text);
+    return path.join(' > ');
+}
+/**
+ * Splits Markdown into H2/H3 section chunks. Returns an empty array when the document has fewer
+ * than two such headings (nothing meaningful to chunk).
+ */
+export function splitMarkdownSections(markdown) {
+    const lines = markdown.split('\n');
+    const headings = findHeadings(lines);
+    const chunkStarts = headings.filter((h) => h.level === 2 || h.level === 3);
+    if (chunkStarts.length < 2)
+        return [];
+    const chunks = [];
+    for (let i = 0; i < chunkStarts.length; i++) {
+        const start = chunkStarts[i];
+        const end = chunkStarts[i + 1]?.index ?? lines.length;
+        const content = lines.slice(start.index, end).join('\n').trim();
+        const headingIdx = headings.indexOf(start);
+        chunks.push({
+            anchor: slugifyHeading(start.text),
+            headingPath: breadcrumb(headings, headingIdx),
+            level: start.level,
+            content,
+        });
+    }
+    return chunks;
+}

package/dist/lib/research/docs/source-map.d.ts

@@ -0,0 +1,19 @@
+export interface GithubSource {
+    repository: string;
+    branch: string;
+    path: string;
+    rawUrl: string;
+}
+/**
+ * Converts a GitHub `edit`/`blob` URL into a raw.githubusercontent.com URL. Returns null for any
+ * URL that is not a recognized GitHub edit/blob link, so callers never fabricate a raw URL.
+ */
+export declare function parseGithubSourceLink(href: string): GithubSource | null;
+export declare function mapNodeApiSource(url: string): GithubSource | null;
+export declare function mapMdnSource(url: string): GithubSource | null;
+export declare function vitepressRouteMarkdown(url: string): string | null;
+/**
+ * Tries every known URL->source mapper in turn. Returns the first candidate, or null. The result
+ * is a CANDIDATE; it must be validated by fetching before being recorded as a verified source.
+ */
+export declare function mapUrlToSource(url: string): GithubSource | null;

package/dist/lib/research/docs/source-map.js

@@ -0,0 +1,87 @@
+// Maps a rendered documentation page to its public Markdown/MDX source. Every mapper returns a
+// candidate URL only; callers MUST validate it with a real fetch before trusting it (T-19). A
+// repository link alone is never "supported source mapping". All inputs are untrusted.
+const GITHUB_EDIT_BLOB = /^https?:\/\/github\.com\/([^/]+)\/([^/]+)\/(?:edit|blob)\/([^/]+)\/(.+)$/i;
+/**
+ * Converts a GitHub `edit`/`blob` URL into a raw.githubusercontent.com URL. Returns null for any
+ * URL that is not a recognized GitHub edit/blob link, so callers never fabricate a raw URL.
+ */
+export function parseGithubSourceLink(href) {
+    const match = href.match(GITHUB_EDIT_BLOB);
+    if (!match)
+        return null;
+    const [, owner, repo, branch, rawPath] = match;
+    // Strip query/hash; a source path never legitimately contains them.
+    const path = rawPath.split(/[?#]/)[0];
+    if (!path || path.includes('..'))
+        return null;
+    return {
+        repository: `${owner}/${repo}`,
+        branch: branch,
+        path,
+        rawUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${path}`,
+    };
+}
+// Node.js API docs: https://nodejs.org/api/url.html -> nodejs/node/doc/api/url.md on the release
+// line. Deterministic and fixture-proven; mapping is advertised only after a successful fetch.
+const NODE_API = /^https?:\/\/nodejs\.org\/(?:[a-z-]+\/)?(?:docs\/)?(?:latest(?:-v\d+\.x)?\/)?api\/([a-z_]+)\.html/i;
+export function mapNodeApiSource(url) {
+    const match = url.match(NODE_API);
+    if (!match)
+        return null;
+    const name = match[1];
+    const path = `doc/api/${name}.md`;
+    return {
+        repository: 'nodejs/node',
+        branch: 'main',
+        path,
+        rawUrl: `https://raw.githubusercontent.com/nodejs/node/main/${path}`,
+    };
+}
+// MDN: https://developer.mozilla.org/en-US/docs/Web/JavaScript/...  ->  mdn/content
+// files/en-us/web/javascript/.../index.md (content paths are lowercased).
+const MDN_DOCS = /^https?:\/\/developer\.mozilla\.org\/([a-zA-Z-]+)\/docs\/(.+?)\/?$/;
+export function mapMdnSource(url) {
+    const match = url.match(MDN_DOCS);
+    if (!match)
+        return null;
+    const locale = match[1].toLowerCase();
+    const slug = match[2].split(/[?#]/)[0].toLowerCase();
+    if (slug.includes('..'))
+        return null;
+    const path = `files/${locale}/${slug}/index.md`;
+    return {
+        repository: 'mdn/content',
+        branch: 'main',
+        path,
+        rawUrl: `https://raw.githubusercontent.com/mdn/content/main/${path}`,
+    };
+}
+// VitePress route Markdown: a docs page at /guide/intro.html (or /guide/intro) also serves
+// /guide/intro.md. Returns the candidate `.md` route URL; validate before use (T-24/T-19).
+export function vitepressRouteMarkdown(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return null;
+    }
+    let path = parsed.pathname;
+    if (path.endsWith('/'))
+        path += 'index';
+    path = path.replace(/\.html$/, '');
+    if (path.endsWith('.md'))
+        return null;
+    parsed.pathname = `${path}.md`;
+    parsed.search = '';
+    parsed.hash = '';
+    return parsed.toString();
+}
+/**
+ * Tries every known URL->source mapper in turn. Returns the first candidate, or null. The result
+ * is a CANDIDATE; it must be validated by fetching before being recorded as a verified source.
+ */
+export function mapUrlToSource(url) {
+    return mapNodeApiSource(url) ?? mapMdnSource(url);
+}

package/dist/lib/research/docs/validate.d.ts

@@ -0,0 +1,15 @@
+export interface ArtifactValidation {
+    ok: boolean;
+    reason?: string;
+}
+/** True when short text reads like an error/soft-404 page rather than real content. */
+export declare function looksLikeErrorPage(text: string): boolean;
+/** Same docs origin guard: a probed artifact must live on the same hostname as the page. */
+export declare function isSameDocsOrigin(candidateUrl: string, baseUrl: string): boolean;
+/**
+ * Validates that an `llms.txt` or route-Markdown body is usable: non-empty, not an HTML page, and
+ * not a soft error page. HTML returned for a `.md`/`.txt` probe is the most common false positive.
+ */
+export declare function validateTextArtifact(body: string): ArtifactValidation;
+/** Validates a JSON search-index body parses and is a non-empty object/array. */
+export declare function validateJsonArtifact(body: string): ArtifactValidation;

package/dist/lib/research/docs/validate.js

@@ -0,0 +1,63 @@
+import { looksLikeHtml } from '../fetcher.js';
+const ERROR_PAGE_PATTERNS = [
+    /\b404\b/,
+    /page\s*(?:not\s*found|doesn'?t\s*exist|can'?t\s*be\s*found)/i,
+    /does(?:n'?t| not)\s*exist/i,
+    /\bnot\s*found\b/i,
+    /there\s*isn'?t\s*a\s*github\s*pages\s*site\s*here/i,
+    /site\s*not\s*found/i,
+];
+/** True when short text reads like an error/soft-404 page rather than real content. */
+export function looksLikeErrorPage(text) {
+    const trimmed = text.trim();
+    // Only treat short bodies as errors; a long article mentioning "404" once is fine.
+    if (trimmed.length > 1500)
+        return false;
+    return ERROR_PAGE_PATTERNS.some((re) => re.test(trimmed));
+}
+/** Same docs origin guard: a probed artifact must live on the same hostname as the page. */
+export function isSameDocsOrigin(candidateUrl, baseUrl) {
+    try {
+        return new URL(candidateUrl).hostname === new URL(baseUrl).hostname;
+    }
+    catch {
+        return false;
+    }
+}
+const MIN_BODY_CHARS = 10;
+/**
+ * Validates that an `llms.txt` or route-Markdown body is usable: non-empty, not an HTML page, and
+ * not a soft error page. HTML returned for a `.md`/`.txt` probe is the most common false positive.
+ */
+export function validateTextArtifact(body) {
+    const trimmed = body.trim();
+    if (trimmed.length < MIN_BODY_CHARS)
+        return { ok: false, reason: 'empty or near-empty body' };
+    if (looksLikeHtml(trimmed))
+        return { ok: false, reason: 'body is HTML, not text/markdown' };
+    if (looksLikeErrorPage(trimmed))
+        return { ok: false, reason: 'body looks like an error page' };
+    return { ok: true };
+}
+/** Validates a JSON search-index body parses and is a non-empty object/array. */
+export function validateJsonArtifact(body) {
+    const trimmed = body.trim();
+    if (trimmed.length < MIN_BODY_CHARS)
+        return { ok: false, reason: 'empty body' };
+    if (looksLikeHtml(trimmed))
+        return { ok: false, reason: 'body is HTML, not JSON' };
+    let parsed;
+    try {
+        parsed = JSON.parse(trimmed);
+    }
+    catch (err) {
+        return { ok: false, reason: `invalid JSON: ${err.message}` };
+    }
+    if (parsed === null || typeof parsed !== 'object') {
+        return { ok: false, reason: 'JSON is not an object or array' };
+    }
+    if (Array.isArray(parsed) ? parsed.length === 0 : Object.keys(parsed).length === 0) {
+        return { ok: false, reason: 'JSON is empty' };
+    }
+    return { ok: true };
+}

package/dist/lib/research/extract.d.ts

@@ -0,0 +1,11 @@
+export interface ExtractionResult {
+    title: string;
+    detailedMarkdown: string;
+    confidence: 'high' | 'medium' | 'low';
+    qualityNotes: string[];
+    isIndexHub?: boolean;
+}
+/**
+ * Extracts readerable content from an HTML document, sanitizes it, and converts it to detailed Markdown.
+ */
+export declare function extractHtmlContent(html: string, finalUrl: string): ExtractionResult;

package/dist/lib/research/extract.js

@@ -0,0 +1,83 @@
+import { parseHTML } from 'linkedom';
+import { Readability } from '@mozilla/readability';
+import { htmlToMarkdown } from './markdown.js';
+import { normalizeCodeBlocks } from './docs/code-blocks.js';
+import { cleanDocsChrome } from './docs/clean-dom.js';
+import { analyzeMarkdownQuality } from './docs/quality-gates.js';
+function resolveRelativeLinks(document, finalUrl) {
+    const base = new URL(finalUrl);
+    const links = document.querySelectorAll('a');
+    for (const link of links) {
+        const href = link.getAttribute('href');
+        if (!href)
+            continue;
+        try {
+            const resolved = new URL(href, base).toString();
+            if (resolved.startsWith('javascript:')) {
+                link.removeAttribute('href');
+            }
+            else {
+                link.setAttribute('href', resolved);
+            }
+        }
+        catch {
+            // Ignore resolution errors for invalid link formats
+        }
+    }
+}
+function cleanUnsafeElements(document) {
+    const scripts = document.querySelectorAll('script, style, iframe, noscript');
+    for (const script of scripts) {
+        script.remove();
+    }
+    const allElements = document.querySelectorAll('*');
+    for (const el of allElements) {
+        for (const attr of Array.from(el.attributes)) {
+            if (attr.name.startsWith('on')) {
+                el.removeAttribute(attr.name);
+            }
+        }
+    }
+}
+function determineConfidence(textLength) {
+    const notes = ['readability extracted main article'];
+    let confidence = 'high';
+    if (textLength < 500) {
+        confidence = 'low';
+        notes.push('warning: extracted content is very short (less than 500 characters)');
+    }
+    else if (textLength < 2000) {
+        confidence = 'medium';
+    }
+    return { confidence, notes };
+}
+/**
+ * Extracts readerable content from an HTML document, sanitizes it, and converts it to detailed Markdown.
+ */
+export function extractHtmlContent(html, finalUrl) {
+    const { document } = parseHTML(html);
+    resolveRelativeLinks(document, finalUrl);
+    cleanUnsafeElements(document);
+    // Strip decorative docs chrome and base64 images, then rebuild highlighted code blocks, before
+    // Readability runs so per-line spans don't collapse (T-21, T-17).
+    cleanDocsChrome(document);
+    normalizeCodeBlocks(document);
+    // keepClasses preserves the `language-x` class on <code> so Turndown emits a fenced language tag.
+    const reader = new Readability(document, { keepClasses: true });
+    const article = reader.parse();
+    if (!article || !article.content) {
+        throw new Error('Content extraction failed: The target page could not be parsed by the readability engine. ' +
+            'Hint: The "import" command lets you import pre-cleaned, agent-supplied research notes directly.');
+    }
+    const detailedMarkdown = htmlToMarkdown(article.content);
+    const textLength = article.textContent ? article.textContent.trim().length : 0;
+    const { confidence, notes } = determineConfidence(textLength);
+    const quality = analyzeMarkdownQuality(detailedMarkdown, article.title || '');
+    return {
+        title: article.title || 'Untitled',
+        detailedMarkdown,
+        confidence,
+        qualityNotes: [...notes, ...quality.codes],
+        isIndexHub: quality.isIndexHub,
+    };
+}

package/dist/lib/research/fetcher.d.ts

@@ -0,0 +1,29 @@
+export interface FetchOptions {
+    timeoutMs?: number;
+    bodyLimitBytes?: number;
+    maxRedirects?: number;
+    headers?: Record<string, string>;
+}
+export interface FetchResult {
+    status: number;
+    contentType: string | null;
+    etag: string | null;
+    lastModified: string | null;
+    finalUrl: string;
+    responseSize: number;
+    content: string;
+}
+export declare function looksLikeHtml(text: string): boolean;
+export declare function checkDnsSafety(hostname: string): Promise<void>;
+export declare function fetchStaticHtml(url: string, options?: FetchOptions): Promise<FetchResult>;
+/**
+ * Fetches a non-HTML text/data resource (llms.txt, route Markdown, search index) with the same
+ * DNS, redirect, timeout, and body-size safety as fetchStaticHtml. Does not enforce a content
+ * type; the caller must validate the returned body is the artifact kind it expected.
+ */
+export declare function fetchText(url: string, options?: FetchOptions): Promise<FetchResult>;
+/**
+ * Sends a JSON POST and returns the response body text. Used by remote search connectors (e.g.
+ * Algolia DocSearch). Enforces the same DNS safety, timeout, and body-size limits as GET fetches.
+ */
+export declare function postJson(url: string, body: unknown, headers?: Record<string, string>, options?: FetchOptions): Promise<string>;