@taurgis/bonsai 3.0.3 → 3.1.0

package/dist/commands/fetch.js

@@ -15,6 +15,7 @@ import { capturePage } from '../lib/research/capture.js';
 import { persistSectionArtifacts } from '../lib/research/docs/section-artifacts.js';
 import { applyAutoTags } from '../lib/research/keywords.js';
 import { detectSite } from '../sites/index.js';
+import { applySiteFetchProvenance } from '../sites/types.js';
 const CAPTURE_DEPS = {
     fetchStatic: (url) => fetchStaticHtml(url),
     fetchRendered: (url) => fetchRenderedHtml(url),
@@ -124,9 +125,11 @@ export default class FetchCommand extends BaseCommand {
         let fetchResult;
         let extraction;
         let capture = null;
+        let siteFetch = null;
         if (siteModule?.fetchPage) {
             // Custom site modules own their fetch/extract strategy; the generic capture path is skipped.
-            ({ fetchResult, extraction } = await siteModule.fetchPage(normalizedUrl));
+            siteFetch = await siteModule.fetchPage(normalizedUrl);
+            ({ fetchResult, extraction } = siteFetch);
         }
         else {
             capture = await capturePage(normalizedUrl, { forceRendered: useRendered }, CAPTURE_DEPS);
@@ -140,6 +143,9 @@ export default class FetchCommand extends BaseCommand {
         if (capture) {
             applyCaptureMetadata(artifact, capture);
         }
+        else if (siteFetch?.captureMethod) {
+            applySiteFetchProvenance(artifact.metadata, siteFetch);
+        }
         else if (useRendered) {
             artifact.metadata.capture_method = 'browser_fallback';
         }

package/dist/lib/research/browser.js

@@ -127,6 +127,9 @@ export function buildChromeArgs() {
         ...getChromeTlsCompatibilityArgs(),
     ];
 }
+// CI runners occasionally starve headless Chrome's startup (disk/CPU contention from concurrent
+// jobs), so this is generous enough to absorb that rather than a "typical" local startup time.
+const CHROME_STARTUP_TIMEOUT_MS = 20_000;
 async function spawnChrome(chromePath) {
     const chromeProcess = spawn(chromePath, buildChromeArgs());
     try {
@@ -135,7 +138,7 @@ async function spawnChrome(chromePath) {
             const timeoutId = setTimeout(() => {
                 chromeProcess.kill('SIGKILL');
                 reject(new Error('Timed out waiting for Chrome to start.'));
-            }, 10000);
+            }, CHROME_STARTUP_TIMEOUT_MS);
             const onData = (data) => {
                 output += data.toString();
                 const match = output.match(/ws:\/\/127\.0\.0\.1:\d+\/devtools\/browser\/[a-zA-Z0-9-]+/);
@@ -363,9 +366,17 @@ export function describeNavigationFailure(errorText, url) {
 }
 // Chrome can emit these on first navigation in headless CI when the cert verifier reloads mid-run.
 const TRANSIENT_NAV_ERROR_CODES = ['ERR_CERT_VERIFIER_CHANGED'];
-function isTransientBrowserNavigationError(err) {
+// Resource contention in CI (concurrent jobs competing for CPU/disk) can make Chrome itself fail
+// to come up in time, or die before its devtools websocket URL is printed. Neither reflects a
+// problem with the target page, so it's worth a fresh attempt rather than failing the whole fetch.
+const TRANSIENT_CHROME_STARTUP_MESSAGES = [
+    'Timed out waiting for Chrome to start.',
+    'Chrome exited prematurely',
+];
+function isTransientBrowserError(err) {
     const message = err instanceof Error ? err.message : String(err);
-    return TRANSIENT_NAV_ERROR_CODES.some((code) => message.includes(code));
+    return (TRANSIENT_NAV_ERROR_CODES.some((code) => message.includes(code)) ||
+        TRANSIENT_CHROME_STARTUP_MESSAGES.some((text) => message.includes(text)));
 }
 async function fetchRenderedHtmlOnce(url, options = {}) {
     const timeout = options.timeoutMs ?? 15_000;
@@ -432,7 +443,7 @@ export async function fetchRenderedHtml(url, options = {}) {
             return await fetchRenderedHtmlOnce(url, options);
         }
         catch (err) {
-            if (attempt === maxAttempts || !isTransientBrowserNavigationError(err))
+            if (attempt === maxAttempts || !isTransientBrowserError(err))
                 throw err;
         }
     }

package/dist/lib/research/docs/markdown-source.d.ts

@@ -9,6 +9,13 @@ export declare function parseFrontmatter(md: string): ParsedFrontmatter;
  * handler attributes, and javascript: URLs. Leaves ordinary Markdown and benign inline HTML intact.
  */
 export declare function sanitizeSourceMarkdown(md: string): string;
+/**
+ * Applies a cleanup transform to the parts of a document OUTSIDE fenced code blocks, passing the
+ * fences through verbatim. Every text-level cleanup in this pipeline must go through this guard:
+ * fenced code is quoted material, and "cleaning" it destroys the samples readers came for
+ * (a `<script>` line in an example, a bash `# comment`, a literal `javascript:` URL).
+ */
+export declare function outsideFences(md: string, transform: (segment: string) => string): string;
 /**
  * Builds an ExtractionResult from public Markdown/MDX source. Frontmatter is parsed for the title
  * and the body is sanitized. The detailed Markdown is the source itself, preserving code fences,

package/dist/lib/research/docs/markdown-source.js

@@ -98,28 +98,31 @@ function replaceMacro(_match, inner) {
 function stripMdnHiddenSamples(md) {
     return md.replace(/<pre\b[^>]*\bhidden\b[^>]*>[\s\S]*?<\/pre>/gi, '');
 }
-function cleanKumaScript(md) {
-    // Never touch fenced code blocks.
+/**
+ * Applies a cleanup transform to the parts of a document OUTSIDE fenced code blocks, passing the
+ * fences through verbatim. Every text-level cleanup in this pipeline must go through this guard:
+ * fenced code is quoted material, and "cleaning" it destroys the samples readers came for
+ * (a `<script>` line in an example, a bash `# comment`, a literal `javascript:` URL).
+ */
+export function outsideFences(md, transform) {
     return md
         .split(/(```[\s\S]*?```)/)
-        .map((segment, i) => (i % 2 === 1 ? segment : segment.replace(/\{\{(.+?)\}\}/g, replaceMacro)))
+        .map((segment, i) => (i % 2 === 1 ? segment : transform(segment)))
         .join('');
 }
+function cleanKumaScript(md) {
+    return outsideFences(md, (segment) => segment.replace(/\{\{(.+?)\}\}/g, replaceMacro));
+}
 // MDN (and occasionally other docs) author tables as raw HTML inside their Markdown source, which
 // otherwise leaks verbatim into the artifact and wastes agent tokens. Turndown already emits clean
 // GFM tables, so convert any literal <table> region to Markdown. Code fences are skipped so an HTML
 // table shown as a *code example* (```html …<table>…) is preserved, not rewritten.
 function convertHtmlTables(md) {
-    return md
-        .split(/(```[\s\S]*?```)/)
-        .map((segment, i) => i % 2 === 1
-        ? segment
-        : // ponytail: non-greedy per-table match. A <table> nested in a cell would stop the match at
-            // the inner </table>, leaving the outer wrapper's closing tags as inert residue (no attrs,
-            // so it can't trip the leak gate). Doc sources use flat tables; upgrade to a DOM walk over
-            // the whole segment via htmlToMarkdown if a nested-table source ever appears.
-            segment.replace(/<table\b[\s\S]*?<\/table>/gi, (table) => `\n\n${htmlToMarkdown(table).trim()}\n\n`))
-        .join('');
+    // ponytail: non-greedy per-table match. A <table> nested in a cell would stop the match at
+    // the inner </table>, leaving the outer wrapper's closing tags as inert residue (no attrs,
+    // so it can't trip the leak gate). Doc sources use flat tables; upgrade to a DOM walk over
+    // the whole segment via htmlToMarkdown if a nested-table source ever appears.
+    return outsideFences(md, (segment) => segment.replace(/<table\b[\s\S]*?<\/table>/gi, (table) => `\n\n${htmlToMarkdown(table).trim()}\n\n`));
 }
 const HEADING = /^(#{1,6})\s/;
 function headingLevel(line) {
@@ -140,17 +143,37 @@ function dropEmptySections(md) {
     } while (current !== prev);
     return current;
 }
+const FENCE_DELIMITER = /^\s*```/;
+// Marks each line that belongs to fenced code, delimiters included. Fenced lines are content,
+// never headings: a bash `# comment` in an example must not be classified — and deleted — as an
+// empty section.
+function fencedLineMask(lines) {
+    let fenced = false;
+    return lines.map((line) => {
+        if (!FENCE_DELIMITER.test(line))
+            return fenced;
+        fenced = !fenced;
+        return true;
+    });
+}
+/** Index of the first non-blank line after `start`, or lines.length. */
+function nextContentLine(lines, start) {
+    let next = start + 1;
+    while (next < lines.length && lines[next].trim() === '')
+        next++;
+    return next;
+}
 function dropEmptySectionsOnce(md) {
     const lines = md.split('\n');
+    const inFence = fencedLineMask(lines);
     const keep = lines.map(() => true);
     for (let i = 0; i < lines.length; i++) {
-        const level = headingLevel(lines[i]);
+        const level = inFence[i] ? 0 : headingLevel(lines[i]);
         if (!level)
             continue;
-        let next = i + 1;
-        while (next < lines.length && lines[next].trim() === '')
-            next++;
-        const nextLevel = headingLevel(lines[next]);
+        const next = nextContentLine(lines, i);
+        // A fenced line after the heading means the section has content (nextLevel 0 → not empty).
+        const nextLevel = next < lines.length && !inFence[next] ? headingLevel(lines[next]) : 0;
         const sectionIsEmpty = next >= lines.length || (nextLevel > 0 && nextLevel <= level);
         if (sectionIsEmpty)
             for (let k = i; k < next; k++)
@@ -183,8 +206,11 @@ export function extractFromSource(md, sourceUrl) {
     const deMacroed = /mdn\/content/.test(sourceUrl)
         ? cleanKumaScript(stripMdnHiddenSamples(body))
         : body;
-    const tablesAsMarkdown = convertHtmlTables(sanitizeSourceMarkdown(deMacroed));
-    const cleaned = sanitizePromptInjection(dropEmptySections(dropEmptyLinks(tablesAsMarkdown))).trim();
+    // Sanitization and link/section cleanup are fence-guarded: code samples legitimately contain
+    // <script>/<button>/on*=/javascript: and bare [](…) lines. sanitizePromptInjection deliberately
+    // is NOT guarded — injection text is just as live when the agent reads it inside a code block.
+    const tablesAsMarkdown = convertHtmlTables(outsideFences(deMacroed, sanitizeSourceMarkdown));
+    const cleaned = sanitizePromptInjection(dropEmptySections(outsideFences(tablesAsMarkdown, dropEmptyLinks))).trim();
     const notes = [`captured from public Markdown/MDX source: ${sourceUrl}`];
     return {
         title: titleFrom(frontmatter, cleaned),

package/dist/lib/research/regression.js

@@ -93,7 +93,10 @@ export function contentMetrics(markdown) {
     return {
         chars: markdown.length,
         codeBlocks: Math.floor((markdown.match(/```/g) || []).length / 2),
-        tables: (markdown.match(/^\|(?:\s*---\s*\|)+\s*$/gm) || []).length,
+        // Separator rows come in every authored width and alignment (| --- |, | ------ |, | :---: |).
+        // Converted HTML always emits exactly ---, but route/source Markdown keeps the author's
+        // padding, so a width-sensitive count would misreport source captures as losing tables.
+        tables: (markdown.match(/^\|(?:\s*:?-{3,}:?\s*\|)+\s*$/gm) || []).length,
         images: (markdown.match(/!\[[^\]]*\]\(/g) || []).length,
         orderedSteps: (markdown.match(/^\d+\.\s+/gm) || []).length,
         bullets: (markdown.match(/^[-*]\s+/gm) || []).length,

package/dist/lib/research/revalidate.js

@@ -1,5 +1,6 @@
 import { createHash } from 'node:crypto';
 import { getSiteModuleById } from '../../sites/index.js';
+import { applySiteFetchProvenance } from '../../sites/types.js';
 import { fetchStaticHtml } from './fetcher.js';
 import { extractHtmlContent } from './extract.js';
 import { writeArtifact } from './storage.js';
@@ -111,9 +112,13 @@ function preserveUserMetadata(source, target, rendered) {
 // Builds a refreshed artifact from a fresh fetch, carries over user metadata, persists it, and
 // returns the "refreshed" result. Shared by both refresh paths (conditional-request fallthrough
 // and site-module re-fetch), which differ only in how fetchResult/extraction are obtained.
-function persistRefreshedArtifact(dataDir, meta, fetchResult, extraction, currentTime, options) {
+function persistRefreshedArtifact(dataDir, meta, fetchResult, extraction, currentTime, options, siteFetch) {
     const refreshed = createArtifactFromFetch(meta.source_url, meta.normalized_url, meta.cache_key, fetchResult, extraction, meta.tier, options.ttlOverride || meta.ttl, currentTime, options.summaryLevel);
     preserveUserMetadata(meta, refreshed, options.rendered);
+    // A site-module refresh reports how it actually captured this time; that overrides the
+    // carried-over provenance, so a withdrawn .md twin can't leave a stale source_doc_url behind.
+    if (siteFetch)
+        applySiteFetchProvenance(refreshed.metadata, siteFetch);
     // Back-fill keyword tags when the carried-over set is empty (e.g. an artifact first cached before
     // auto-tagging, or one originally stored without tags), so refreshing keeps it searchable.
     applyAutoTags(refreshed);
@@ -162,8 +167,8 @@ export async function revalidateCache(dataDir, existing, currentTime, options) {
         if (siteModule?.fetchPage) {
             // Site modules use custom fetch strategies that don't speak HTTP conditional requests
             // (no ETag/If-Modified-Since), so a full re-fetch is the only correct revalidation here.
-            const { fetchResult, extraction } = await siteModule.fetchPage(meta.source_url);
-            return persistRefreshedArtifact(dataDir, meta, fetchResult, extraction, currentTime, options);
+            const siteFetch = await siteModule.fetchPage(meta.source_url);
+            return persistRefreshedArtifact(dataDir, meta, siteFetch.fetchResult, siteFetch.extraction, currentTime, options, siteFetch);
         }
         const fetchResult = options.rendered
             ? await fetchRenderedHtml(meta.source_url)

package/dist/sites/salesforce-developer/fetch-page.d.ts

@@ -1,3 +1,8 @@
 import type { SiteFetchResult } from '../types.js';
-/** Fetches a developer.salesforce.com doc page (guides + API reference) via the shared LWR fetcher. */
+/**
+ * Fetches a developer.salesforce.com doc page. Supported articles publish a Markdown twin at the
+ * derived `.md` route ("View as Markdown"), which is preferred: one static request instead of a
+ * browser render, and it returns the article's source. Pages without a validated twin (API
+ * reference, atlas.* books, landing pages) fall back to the shared LWR shadow-DOM fetcher.
+ */
 export declare function fetchDeveloperPage(url: string): Promise<SiteFetchResult>;

package/dist/sites/salesforce-developer/fetch-page.js

@@ -1,4 +1,5 @@
 import { fetchSalesforceDoc } from '../salesforce-doc-fetch.js';
+import { fetchDeveloperRouteMarkdown } from './route-markdown.js';
 // Containers a developer doc renders into. doc-content-layout / doc-amf-reference are the
 // web-component hosts for guide pages and API-reference (AMF) pages respectively.
 const CONTENT_SELECTORS = [
@@ -11,8 +12,16 @@ const CONTENT_SELECTORS = [
     'article',
     'main',
 ];
-/** Fetches a developer.salesforce.com doc page (guides + API reference) via the shared LWR fetcher. */
+/**
+ * Fetches a developer.salesforce.com doc page. Supported articles publish a Markdown twin at the
+ * derived `.md` route ("View as Markdown"), which is preferred: one static request instead of a
+ * browser render, and it returns the article's source. Pages without a validated twin (API
+ * reference, atlas.* books, landing pages) fall back to the shared LWR shadow-DOM fetcher.
+ */
 export async function fetchDeveloperPage(url) {
+    const fromMarkdownRoute = await fetchDeveloperRouteMarkdown(url);
+    if (fromMarkdownRoute)
+        return fromMarkdownRoute;
     return fetchSalesforceDoc(url, {
         allowedHost: 'developer.salesforce.com',
         contentSelectors: CONTENT_SELECTORS,

package/dist/sites/salesforce-developer/route-markdown.d.ts

@@ -0,0 +1,23 @@
+import { type FetchResult } from '../../lib/research/fetcher.js';
+import type { SiteFetchResult } from '../types.js';
+/**
+ * Derives the candidate `.md` route for a developer.salesforce.com doc URL, or null when the URL
+ * can't serve one (wrong host, not under /docs/, or an atlas.* book — verified to answer `.md`
+ * with an HTML shell, never Markdown). Query/hash are view state and are dropped.
+ */
+export declare function deriveMarkdownUrl(url: string): string | null;
+/** Removes unresolvable `::include{…}` directive lines, reporting how many were dropped. */
+export declare function stripIncludeDirectives(md: string): {
+    body: string;
+    dropped: number;
+};
+export type RouteMarkdownFetcher = (url: string, options?: {
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+}) => Promise<FetchResult>;
+/**
+ * Probes the derived `.md` route for a developer doc URL. Returns a complete SiteFetchResult
+ * (with route_markdown provenance) when the twin validates, or null so the caller falls back to
+ * the browser capture. Never throws: any network/validation failure is just "no twin".
+ */
+export declare function fetchDeveloperRouteMarkdown(url: string, fetcher?: RouteMarkdownFetcher): Promise<SiteFetchResult | null>;

package/dist/sites/salesforce-developer/route-markdown.js

@@ -0,0 +1,142 @@
+import { fetchText } from '../../lib/research/fetcher.js';
+import { validateTextArtifact } from '../../lib/research/docs/validate.js';
+import { extractFromSource } from '../../lib/research/docs/markdown-source.js';
+// developer.salesforce.com now publishes a Markdown twin for supported doc articles (the
+// "View as Markdown" link): /docs/<cloud>/<book>/guide/<article>[.html] also serves
+// <article>.md with Content-Type text/markdown. The URL mapping is deterministic, so we derive
+// the .md URL and probe it BEFORE the browser path — one static request replaces a ~45s
+// shadow-DOM render and returns the article's actual source (code fences, admonitions intact).
+//
+// Coverage is rolling out per doc set, so the probe result is only a candidate (T-19/T-24):
+// unsupported articles 404, and atlas.* books answer the .md route with an HTTP 200 HTML SPA
+// shell — which makes the strict content-type + body validation below load-bearing, not
+// defensive garnish.
+const DEVELOPER_HOST = 'developer.salesforce.com';
+const MARKDOWN_CONTENT_TYPE = 'text/markdown';
+// Salesforce's WAF answers 403 unless the User-Agent LEADS with a recognized HTTP-tool token
+// (curl/wget/python-requests pass; bare product names and browser UAs on a non-browser TLS stack
+// are blocked). Lead with the compatibility token — the same convention as browsers' Mozilla/5.0
+// prefix — and carry Bonsai's honest identity after it.
+const PROBE_HEADERS = {
+    'user-agent': 'curl/8.7.1 bonsai-research-cache (+https://bonsai.rhino-inquisitor.com)',
+};
+/**
+ * Derives the candidate `.md` route for a developer.salesforce.com doc URL, or null when the URL
+ * can't serve one (wrong host, not under /docs/, or an atlas.* book — verified to answer `.md`
+ * with an HTML shell, never Markdown). Query/hash are view state and are dropped.
+ */
+export function deriveMarkdownUrl(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return null;
+    }
+    if (parsed.hostname.toLowerCase() !== DEVELOPER_HOST)
+        return null;
+    let path = parsed.pathname.replace(/\/+$/, '');
+    if (!path.startsWith('/docs/') || path.startsWith('/docs/atlas.'))
+        return null;
+    if (path.endsWith('.md'))
+        return `https://${DEVELOPER_HOST}${path}`;
+    path = path.replace(/\.html?$/i, '');
+    return `https://${DEVELOPER_HOST}${path}.md`;
+}
+// `::include{src="../../shared/….md"}` directives reference build-time snippet files that are
+// not published on the .md route (they 404), so they'd survive as inert noise in the artifact.
+const INCLUDE_DIRECTIVE = /^\s*::include\{[^}]*\}\s*$/;
+/** Removes unresolvable `::include{…}` directive lines, reporting how many were dropped. */
+export function stripIncludeDirectives(md) {
+    let dropped = 0;
+    const body = md
+        .split(/(```[\s\S]*?```)/)
+        .map((segment, i) => {
+        // Fenced code is untouched: an ::include shown as a syntax example is content.
+        if (i % 2 === 1)
+            return segment;
+        return segment
+            .split('\n')
+            .filter((line) => {
+            if (!INCLUDE_DIRECTIVE.test(line))
+                return true;
+            dropped++;
+            return false;
+        })
+            .join('\n');
+    })
+        .join('');
+    return { body, dropped };
+}
+// A probe response counts as the Markdown twin only when the server labels it text/markdown AND
+// the redirect chain ended on https at the developer host AND the body validates as non-HTML,
+// non-error text. Atlas shells fail the content-type check; non-2xx statuses already threw in
+// the fetcher (the probe sends no conditional headers, so a 304 can't occur either).
+function isMarkdownResponse(res) {
+    if (!res.content)
+        return false;
+    const mediaType = (res.contentType?.split(';')[0] ?? '').trim().toLowerCase();
+    if (mediaType !== MARKDOWN_CONTENT_TYPE)
+        return false;
+    let finalUrl;
+    try {
+        finalUrl = new URL(res.finalUrl);
+    }
+    catch {
+        return false;
+    }
+    // Scheme matters, not just host: a redirect hop downgrading to plain http would let an on-path
+    // response be cached as the trusted twin.
+    if (finalUrl.protocol !== 'https:' || finalUrl.hostname.toLowerCase() !== DEVELOPER_HOST) {
+        return false;
+    }
+    return validateTextArtifact(res.content).ok;
+}
+// A real twin is a small static file that answers fast; anything slower should mean "no twin,
+// fall back now" instead of stacking the fetcher's default 10s on top of the browser render.
+const PROBE_TIMEOUT_MS = 4_000;
+// The browser path refuses to cache captures under this floor (MIN_CONTAINER_CHARS); a twin that
+// thin is a rollout stub, and falling back gives the rendered page a chance to do better.
+const MIN_TWIN_CHARS = 100;
+/**
+ * Probes the derived `.md` route for a developer doc URL. Returns a complete SiteFetchResult
+ * (with route_markdown provenance) when the twin validates, or null so the caller falls back to
+ * the browser capture. Never throws: any network/validation failure is just "no twin".
+ */
+export async function fetchDeveloperRouteMarkdown(url, fetcher = fetchText) {
+    const candidate = deriveMarkdownUrl(url);
+    if (!candidate)
+        return null;
+    let res;
+    try {
+        res = await fetcher(candidate, { headers: PROBE_HEADERS, timeoutMs: PROBE_TIMEOUT_MS });
+    }
+    catch {
+        return null;
+    }
+    if (!isMarkdownResponse(res))
+        return null;
+    const { body, dropped } = stripIncludeDirectives(res.content);
+    const extraction = extractFromSource(body, res.finalUrl);
+    if (extraction.detailedMarkdown.length < MIN_TWIN_CHARS)
+        return null;
+    if (dropped > 0) {
+        extraction.qualityNotes = [
+            ...(extraction.qualityNotes ?? []),
+            `${dropped} ::include directive(s) removed (shared snippets are not published on the .md route)`,
+        ];
+    }
+    return {
+        fetchResult: {
+            contentType: res.contentType,
+            etag: res.etag,
+            lastModified: res.lastModified,
+            finalUrl: res.finalUrl,
+            responseSize: res.responseSize,
+            content: res.content,
+        },
+        extraction,
+        captureMethod: 'route_markdown',
+        sourceDocUrl: res.finalUrl,
+    };
+}

package/dist/sites/salesforce-doc-fetch.js

@@ -64,13 +64,19 @@ function confidenceFor(length) {
 // page. Help wraps it in a removable component (see fetch-page.ts), but Developer renders it
 // as bare divs, so we also strip its lines from the converted Markdown for both sites.
 const BOILERPLATE_LINE = /(did this article solve your issue|let us know so we can improve|share your feedback)/i;
+// Toolbar of the "View as Markdown" feature on Developer articles. When the .md route probe
+// succeeds the browser never runs, but rollout/network asymmetries can still land such a page
+// here, and the toolbar labels serialize as bare paragraph lines.
+const MARKDOWN_TOOLBAR_LINE = /^(copy as markdown|view as markdown|copy url to markdown)$/i;
 // A link with empty anchor text — e.g. an icon-only "home" logo link. It carries no readable
 // content and renders as a bare `[](url)`. (Images `![](url)` are intentionally not matched.)
 const EMPTY_LINK_LINE = /^\[\]\([^)]*\)$/;
 export function stripBoilerplate(markdown) {
     return markdown
         .split('\n')
-        .filter((line) => !BOILERPLATE_LINE.test(line) && !EMPTY_LINK_LINE.test(line.trim()))
+        .filter((line) => !BOILERPLATE_LINE.test(line) &&
+        !MARKDOWN_TOOLBAR_LINE.test(line.trim()) &&
+        !EMPTY_LINK_LINE.test(line.trim()))
         .join('\n')
         .replace(/\n{3,}/g, '\n\n')
         .trim();
@@ -336,6 +342,9 @@ function buildSalesforceFetchResult(url, html, title, detailedMarkdown, qualityN
             confidence: confidenceFor(detailedMarkdown.length),
             qualityNotes,
         },
+        // Rendered capture is this fetcher's only strategy; reporting it lets a refresh overwrite
+        // route_markdown provenance when an article's .md twin has since vanished.
+        captureMethod: 'browser_fallback',
     };
 }
 /**

package/dist/sites/types.d.ts

@@ -1,4 +1,5 @@
 import type { ExtractionResult } from '../lib/research/extract.js';
+import type { CaptureMethod } from '../lib/research/schema.js';
 export interface SiteFetchResult {
     fetchResult: {
         contentType: string | null;
@@ -9,7 +10,19 @@ export interface SiteFetchResult {
         content: string;
     };
     extraction: ExtractionResult;
+    captureMethod?: CaptureMethod;
+    sourceDocUrl?: string;
 }
+/**
+ * Stamps a site module's reported provenance onto artifact metadata. Shared by the cache-miss
+ * fetch and revalidation so the two write paths can't drift. A result without captureMethod (a
+ * module predating provenance reporting) leaves the metadata untouched.
+ */
+export declare function applySiteFetchProvenance(metadata: {
+    capture_method: CaptureMethod | null;
+    source_doc_url: string | null;
+    source_urls: string[];
+}, siteFetch: SiteFetchResult): void;
 export interface SiteModule {
     id: string;
     name: string;

package/dist/sites/types.js

@@ -1 +1,14 @@
-export {};
+/**
+ * Stamps a site module's reported provenance onto artifact metadata. Shared by the cache-miss
+ * fetch and revalidation so the two write paths can't drift. A result without captureMethod (a
+ * module predating provenance reporting) leaves the metadata untouched.
+ */
+export function applySiteFetchProvenance(metadata, siteFetch) {
+    if (!siteFetch.captureMethod)
+        return;
+    metadata.capture_method = siteFetch.captureMethod;
+    metadata.source_doc_url = siteFetch.sourceDocUrl ?? null;
+    if (siteFetch.sourceDocUrl && !metadata.source_urls.includes(siteFetch.sourceDocUrl)) {
+        metadata.source_urls.push(siteFetch.sourceDocUrl);
+    }
+}

package/oclif.manifest.json

@@ -843,5 +843,5 @@
       "stdoutIsPrimaryData": true
     }
   },
-  "version": "3.0.3"
+  "version": "3.1.0"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@taurgis/bonsai",
-  "version": "3.0.3",
+  "version": "3.1.0",
   "description": "Bonsai is a standalone local research cache CLI for AI agents",
   "repository": {
     "type": "git",