@taurgis/bonsai 1.0.4 → 1.0.6

package/dist/lib/research/browser.d.ts

@@ -39,6 +39,18 @@ export interface CdpPage {
     sessionId: string;
     close: () => Promise<void>;
 }
+/** Status line of the main-frame document response, captured from Network.responseReceived. */
+export interface MainDocumentResponse {
+    status: number;
+    statusText: string;
+}
+/**
+ * Rejects a rendered page whose main document returned an HTTP error, mirroring the static
+ * fetcher's `assertOk`. Without this, `--rendered` on a 404/500 would capture the server's (or
+ * the browser's) error page as if it were real content and cache it. A missing response (no
+ * Document event seen — e.g. data: or about: targets) is left to the caller's content checks.
+ */
+export declare function assertRenderedHttpOk(mainDoc: MainDocumentResponse | undefined): void;
 /**
  * Spawns a headless Chrome, attaches a flat CDP session, and enables the Page, Network,
  * and Runtime domains. The caller drives navigation and must call close() when finished.

package/dist/lib/research/browser.js

@@ -90,7 +90,8 @@ export function findChromePath() {
         }
     }
     catch { }
-    throw new Error('Chrome executable not found. Please set CHROME_PATH environment variable.');
+    throw new Error('No Chrome or Chromium browser found for browser-based extraction. Install Chrome/Chromium or ' +
+        'set CHROME_PATH to the browser executable. Static (non-rendered) fetching does not require a browser.');
 }
 async function spawnChrome(chromePath) {
     const chromeProcess = spawn(chromePath, [
@@ -220,6 +221,17 @@ export const BLOCKED_ASSET_URLS = [
     '*.ico',
 ];
 const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+/**
+ * Rejects a rendered page whose main document returned an HTTP error, mirroring the static
+ * fetcher's `assertOk`. Without this, `--rendered` on a 404/500 would capture the server's (or
+ * the browser's) error page as if it were real content and cache it. A missing response (no
+ * Document event seen — e.g. data: or about: targets) is left to the caller's content checks.
+ */
+export function assertRenderedHttpOk(mainDoc) {
+    if (mainDoc && mainDoc.status >= 400) {
+        throw new Error(`Fetch failed with status ${mainDoc.status} ${mainDoc.statusText}`.trimEnd());
+    }
+}
 /**
  * Spawns a headless Chrome, attaches a flat CDP session, and enables the Page, Network,
  * and Runtime domains. The caller drives navigation and must call close() when finished.
@@ -332,8 +344,33 @@ export async function fetchRenderedHtml(url, options = {}) {
     const page = await openCdpPage();
     try {
         await page.client.send('Network.setBlockedURLs', { urls: BLOCKED_ASSET_URLS }, page.sessionId);
-        await page.client.send('Page.navigate', { url: currentUrl }, page.sessionId);
+        // Record the status of each frame's first Document response so we can reject HTTP errors after
+        // the page settles. Keyed by frameId; the navigated top frame is matched via the navigate result.
+        const documentResponses = new Map();
+        page.client.on(`${page.sessionId}:Network.responseReceived`, (params) => {
+            const { frameId } = params;
+            const status = params.response?.status;
+            // Only record real numeric statuses: a Document event without a usable status would
+            // otherwise store NaN, which silently slips past the `>= 400` guard and the `?? 200`
+            // fallback alike, leaking a NaN into BrowserFetchResult.status (declared `number`).
+            if (params.type !== 'Document' || !frameId || typeof status !== 'number')
+                return;
+            if (documentResponses.has(frameId))
+                return;
+            documentResponses.set(frameId, {
+                status,
+                statusText: params.response?.statusText ?? '',
+            });
+        });
+        const nav = await page.client.send('Page.navigate', { url: currentUrl }, page.sessionId);
+        // A network-level failure (DNS, refused/closed connection, TLS) never fires a load event, so
+        // fail fast here instead of waiting out the full navigation timeout on a dead page.
+        if (nav.errorText) {
+            throw new Error(`Navigation failed: ${nav.errorText}`);
+        }
         await waitForLoad(page.client, page.sessionId, timeout, options.settleMs ?? 1000);
+        const mainDoc = nav.frameId ? documentResponses.get(nav.frameId) : undefined;
+        assertRenderedHttpOk(mainDoc);
         const evalResult = await page.client.send('Runtime.evaluate', { expression: 'document.documentElement.outerHTML', returnByValue: true }, page.sessionId);
         const html = evalResult.result?.value;
         if (typeof html !== 'string') {
@@ -344,7 +381,7 @@ export async function fetchRenderedHtml(url, options = {}) {
             throw new Error(`Response body size limit exceeded. Rendered HTML is ${byteLength} bytes.`);
         }
         return {
-            status: 200,
+            status: mainDoc?.status ?? 200,
             contentType: 'text/html',
             etag: null,
             lastModified: null,

package/dist/lib/research/docs/capabilities.d.ts

@@ -15,7 +15,7 @@ export interface SourceHint {
     path?: string;
     evidence: Evidence;
 }
-export type SearchProvider = 'algolia-docsearch' | 'vitepress-local' | 'pagefind' | 'docusaurus-local' | 'mkdocs-local' | 'sphinx-searchindex' | 'jekyll-json' | 'readme-algolia' | 'custom';
+export type SearchProvider = 'algolia-docsearch' | 'vitepress-local' | 'pagefind' | 'docusaurus-local' | 'mkdocs-local' | 'sphinx-searchindex' | 'jekyll-json' | 'sitemap' | 'llms.txt' | 'readme-algolia' | 'custom';
 export interface SearchCapability {
     provider: SearchProvider;
     evidence: Evidence;

package/dist/lib/research/docs/discovery-index.d.ts

@@ -0,0 +1,14 @@
+import type { DocsSearchResult } from './search-index.js';
+interface DiscoveryEntry {
+    title: string;
+    url: string;
+    body?: string;
+}
+export interface SitemapParseResult {
+    entries: DiscoveryEntry[];
+    sitemapUrls: string[];
+}
+export declare function parseSitemap(body: string): SitemapParseResult;
+export declare function searchSitemap(body: string, indexUrl: string, query: string): DocsSearchResult[];
+export declare function searchLlmsTxt(body: string, indexUrl: string, query: string): DocsSearchResult[];
+export {};

package/dist/lib/research/docs/discovery-index.js

@@ -0,0 +1,112 @@
+import { DOMParser } from 'linkedom';
+function queryTerms(query) {
+    return query.toLowerCase().split(/\s+/).filter(Boolean);
+}
+function scoreEntry(entry, terms) {
+    const haystack = `${entry.title} ${entry.url} ${entry.body ?? ''}`.toLowerCase();
+    let score = 0;
+    for (const term of terms) {
+        if (entry.title.toLowerCase().includes(term))
+            score += 10;
+        if (entry.url.toLowerCase().includes(term))
+            score += 5;
+        score += Math.min(haystack.split(term).length - 1, 5);
+    }
+    return score;
+}
+function snippetFrom(entry) {
+    return (entry.body || entry.url).replace(/\s+/g, ' ').trim().slice(0, 150);
+}
+function titleFromUrl(url) {
+    try {
+        const parsed = new URL(url);
+        const last = parsed.pathname.split('/').filter(Boolean).pop();
+        return last ? last.replace(/\.[a-z0-9]+$/i, '').replace(/[-_]+/g, ' ') : parsed.hostname;
+    }
+    catch {
+        return url;
+    }
+}
+function toResults(entries, indexUrl, provider, query) {
+    const terms = queryTerms(query);
+    return entries
+        .map((entry) => ({ entry, score: scoreEntry(entry, terms) }))
+        .filter(({ score }) => score > 0)
+        .map(({ entry, score }) => ({
+        title: entry.title,
+        url: entry.url,
+        snippet: snippetFrom(entry),
+        provider,
+        score,
+        indexUrl,
+    }))
+        .sort((a, b) => b.score - a.score);
+}
+function textContent(element) {
+    const text = element?.textContent?.trim();
+    return text || undefined;
+}
+export function parseSitemap(body) {
+    const trimmed = body.trim();
+    if (!trimmed)
+        return { entries: [], sitemapUrls: [] };
+    if (!trimmed.startsWith('<')) {
+        const entries = trimmed
+            .split(/\r?\n/)
+            .map((line) => line.trim())
+            .filter((line) => /^https?:\/\//i.test(line))
+            .map((url) => ({ title: titleFromUrl(url), url }));
+        return { entries, sitemapUrls: [] };
+    }
+    const doc = new DOMParser().parseFromString(trimmed, 'text/xml');
+    const sitemapUrls = Array.from(doc.querySelectorAll('sitemap > loc'))
+        .map((loc) => textContent(loc))
+        .filter((url) => !!url);
+    const entries = Array.from(doc.querySelectorAll('url'))
+        .map((urlElement) => {
+        const url = textContent(urlElement.querySelector('loc'));
+        if (!url)
+            return null;
+        const lastmod = textContent(urlElement.querySelector('lastmod'));
+        return { title: titleFromUrl(url), url, body: lastmod ? `Last modified ${lastmod}` : '' };
+    })
+        .filter((entry) => !!entry);
+    return { entries, sitemapUrls };
+}
+export function searchSitemap(body, indexUrl, query) {
+    return toResults(parseSitemap(body).entries, indexUrl, 'sitemap', query);
+}
+function referenceDefinitions(body) {
+    const refs = new Map();
+    for (const match of body.matchAll(/^\s*\[([^\]]+)\]:\s*(\S+)/gm)) {
+        refs.set(match[1].toLowerCase(), match[2]);
+    }
+    return refs;
+}
+export function searchLlmsTxt(body, indexUrl, query) {
+    const refs = referenceDefinitions(body);
+    const entries = [];
+    let section = '';
+    for (const line of body.split(/\r?\n/)) {
+        const heading = line.match(/^##\s+(.+?)\s*$/);
+        if (heading)
+            section = heading[1];
+        const inline = line.match(/^\s*[-*]\s+\[([^\]]+)\]\(([^)]+)\)\s*:?\s*(.*)$/);
+        const ref = line.match(/^\s*[-*]\s+\[([^\]]+)\]\s*:?\s*(.*)$/);
+        const title = inline?.[1] ?? ref?.[1];
+        const url = inline?.[2] ?? (title ? refs.get(title.toLowerCase()) : undefined);
+        if (!title || !url)
+            continue;
+        try {
+            entries.push({
+                title,
+                url: new URL(url, indexUrl).toString(),
+                body: [section, inline?.[3] ?? ref?.[2]].filter(Boolean).join(' - '),
+            });
+        }
+        catch {
+            // Ignore malformed links in untrusted machine-readable text.
+        }
+    }
+    return toResults(entries, indexUrl, 'llms.txt', query);
+}

package/dist/lib/research/docs/machine-readable.d.ts

@@ -15,7 +15,7 @@ export type TextFetcher = (url: string) => Promise<{
  */
 export declare function probeLlmsTxt(pageUrl: string, caps: SiteCapabilities, fetcher?: TextFetcher): Promise<MachineReadableResult | null>;
 /**
- * Probes a page-level `.md` route, but only when the engine/framework is known to serve one or the
- * page advertises one. Returns validated Markdown source for the page, or null.
+ * Probes a page-level `.md` route. The derived route is only a candidate: it must be same-origin,
+ * fetch successfully, and validate as non-HTML/non-error Markdown before being trusted.
  */
 export declare function probeRouteMarkdown(pageUrl: string, caps: SiteCapabilities, fetcher?: TextFetcher): Promise<MachineReadableResult | null>;

package/dist/lib/research/docs/machine-readable.js

@@ -2,10 +2,6 @@ import { fetchText } from '../fetcher.js';
 import { isSameDocsOrigin, validateTextArtifact } from './validate.js';
 import { vitepressRouteMarkdown } from './source-map.js';
 const defaultFetcher = (url) => fetchText(url);
-// Frameworks/engines that the research observed serving page-level `.md` routes (T-24 evidence).
-function advertisesRouteMarkdown(caps) {
-    return (caps.docsEngine === 'vitepress' || caps.framework === 'rspress' || caps.framework === 'gitbook');
-}
 function llmsTxtCandidates(pageUrl, caps) {
     const advertised = caps.machineReadable
         .filter((m) => m.type === 'llms.txt' || m.type === 'llms-full.txt')
@@ -52,14 +48,14 @@ export async function probeLlmsTxt(pageUrl, caps, fetcher = defaultFetcher) {
     return null;
 }
 /**
- * Probes a page-level `.md` route, but only when the engine/framework is known to serve one or the
- * page advertises one. Returns validated Markdown source for the page, or null.
+ * Probes a page-level `.md` route. The derived route is only a candidate: it must be same-origin,
+ * fetch successfully, and validate as non-HTML/non-error Markdown before being trusted.
  */
 export async function probeRouteMarkdown(pageUrl, caps, fetcher = defaultFetcher) {
     const advertised = caps.machineReadable
         .filter((m) => m.type === 'route-markdown')
         .map((m) => m.url);
-    const derived = advertisesRouteMarkdown(caps) ? [vitepressRouteMarkdown(pageUrl)] : [];
+    const derived = [vitepressRouteMarkdown(pageUrl)];
     const candidates = [...new Set([...advertised, ...derived].filter((u) => !!u))];
     for (const candidate of candidates) {
         if (!isSameDocsOrigin(candidate, pageUrl))

package/dist/lib/research/docs/remote-search-runner.js

@@ -1,5 +1,7 @@
 import { detectDocsEngine } from './detect.js';
+import { probeLlmsTxt } from './machine-readable.js';
 import { searchMkDocsIndex, searchJekyllIndex, searchSphinxIndex } from './search-index.js';
+import { parseSitemap, searchLlmsTxt, searchSitemap } from './discovery-index.js';
 import { extractAlgoliaConfig, algoliaQueryUrl, parseAlgoliaResponse } from './remote-search.js';
 async function getText(url, deps) {
     const res = await deps.fetchText(url);
@@ -7,6 +9,14 @@ async function getText(url, deps) {
         throw new Error(`empty search index at ${url}`);
     return res.content;
 }
+function sameHost(url, base) {
+    try {
+        return new URL(url).hostname === new URL(base).hostname;
+    }
+    catch {
+        return false;
+    }
+}
 // Conventional static index path + parser per framework (verified by T-25 fixtures).
 async function staticIndexSearch(framework, origin, query, deps) {
     if (framework === 'mkdocs' || framework === 'material-mkdocs') {
@@ -32,6 +42,50 @@ async function staticIndexSearch(framework, origin, query, deps) {
     }
     return null;
 }
+async function llmsTxtSearch(docsUrl, caps, query, deps) {
+    const llms = await probeLlmsTxt(docsUrl, caps, async (url) => {
+        const res = await deps.fetchText(url);
+        return { ...res, finalUrl: url, contentType: 'text/plain' };
+    });
+    if (!llms)
+        return null;
+    return {
+        provider: llms.artifact.type,
+        results: searchLlmsTxt(llms.body, llms.artifact.url, query),
+    };
+}
+async function sitemapSearch(pageUrl, query, deps) {
+    const origin = new URL(pageUrl).origin;
+    const firstSegment = new URL(pageUrl).pathname.split('/').filter(Boolean)[0];
+    const candidates = [
+        `${origin}/sitemap.xml`,
+        `${origin}/sitemap_index.xml`,
+        firstSegment ? `${origin}/${firstSegment}/sitemap.xml` : undefined,
+    ].filter((url) => !!url);
+    for (const sitemapUrl of [...new Set(candidates)]) {
+        try {
+            const body = await getText(sitemapUrl, deps);
+            const results = searchSitemap(body, sitemapUrl, query);
+            if (results.length > 0)
+                return { provider: 'sitemap', results };
+            const index = parseSitemap(body);
+            const childResults = [];
+            for (const childUrl of index.sitemapUrls.slice(0, 10)) {
+                if (!sameHost(childUrl, pageUrl))
+                    continue;
+                const childBody = await getText(childUrl, deps);
+                childResults.push(...searchSitemap(childBody, childUrl, query));
+            }
+            if (childResults.length > 0) {
+                return { provider: 'sitemap', results: childResults.sort((a, b) => b.score - a.score) };
+            }
+        }
+        catch {
+            // Try the next conventional sitemap location.
+        }
+    }
+    return null;
+}
 /**
  * Runs remote docs search for a documentation site. Detects the provider from the page, queries the
  * matching connector, and returns normalized results. Throws when no verified connector applies.
@@ -49,6 +103,12 @@ export async function runRemoteDocsSearch(docsUrl, query, deps) {
     const staticResult = await staticIndexSearch(caps.framework, origin, query, deps);
     if (staticResult)
         return staticResult;
+    const llmsResult = await llmsTxtSearch(page.finalUrl, caps, query, deps);
+    if (llmsResult)
+        return llmsResult;
+    const sitemapResult = await sitemapSearch(page.finalUrl, query, deps);
+    if (sitemapResult)
+        return sitemapResult;
     throw new Error(`No verified remote search connector for this site (detected: ${caps.framework ?? caps.docsEngine ?? 'unknown'}). ` +
-        'Supported: Algolia DocSearch, MkDocs/Material, Sphinx, Just the Docs.');
+        'Supported: Algolia DocSearch, MkDocs/Material, Sphinx, Just the Docs, llms.txt, sitemap.xml.');
 }

package/dist/lib/research/docs/sections.d.ts

@@ -4,6 +4,7 @@ export interface SectionChunk {
     level: number;
     content: string;
 }
+export declare function headingPlainText(raw: string): string;
 export declare function slugifyHeading(text: string): string;
 /**
  * Splits Markdown into H2/H3 section chunks. Returns an empty array when the document has fewer

package/dist/lib/research/docs/sections.js

@@ -2,6 +2,24 @@
 // big API reference (e.g. Node `url.html`, ~17.5k tokens) becomes addressable section artifacts.
 // Pure string work: no I/O, no artifact construction. H4+ headings stay within their parent chunk.
 const HEADING_RE = /^(#{1,6})\s+(.+?)\s*#*\s*$/;
+// Reduce a heading's inline Markdown to the plain text GitHub anchors and breadcrumbs are built from.
+// GitHub strips formatting (links, emphasis, code) and slugs the *rendered* text, so a heading like
+// `## [API documentation](https://prismjs.com/extending.html#api-documentation)` must collapse to
+// "API documentation" — not the raw link syntax, which otherwise leaks the URL into the anchor and
+// breadcrumb. Confirmed against github-slugger, which documents that it operates on plain heading
+// text rather than parsing Markdown: https://github.com/Flet/github-slugger
+export function headingPlainText(raw) {
+    // The `\(…\)` part tolerates one level of balanced parens so a real-world target like
+    // `URL_(disambiguation)` doesn't leave a dangling ")" in the rendered breadcrumb text.
+    return raw
+        .replace(/!\[([^\]]*)\]\([^()]*(?:\([^()]*\)[^()]*)*\)/g, '$1') // images: ![alt](url) -> alt
+        .replace(/\[([^\]]*)\]\([^()]*(?:\([^()]*\)[^()]*)*\)/g, '$1') // inline links: [text](url) -> text
+        .replace(/\[([^\]]*)\]\[[^\]]*\]/g, '$1') // reference links: [text][ref] -> text
+        .replace(/`([^`]*)`/g, '$1') // inline code: `code` -> code
+        .replace(/[*_~]/g, '') // emphasis / strikethrough markers
+        .replace(/\s+/g, ' ')
+        .trim();
+}
 // GitHub-style anchor slug: lowercase, drop punctuation, spaces -> hyphens.
 export function slugifyHeading(text) {
     return text
@@ -22,7 +40,7 @@ function findHeadings(lines) {
             continue;
         const match = lines[i].match(HEADING_RE);
         if (match)
-            out.push({ index: i, level: match[1].length, text: match[2].trim() });
+            out.push({ index: i, level: match[1].length, text: headingPlainText(match[2]) });
     }
     return out;
 }

package/dist/lib/research/freshness.d.ts

@@ -4,9 +4,17 @@ export interface FreshnessPolicy {
     graceWindowMs: number;
 }
 /**
- * Parses simple TTL duration strings (e.g. '24h', '7d', '30d') to milliseconds.
+ * Parses simple TTL duration strings to milliseconds. The unit `m` is months (not minutes); the
+ * smallest unit is hours, since sub-hour cache lifespans are not meaningful for research artifacts.
  */
 export declare function parseTtlToMs(ttl: string): number;
+/**
+ * Validate a duration-valued flag. Returns an actionable error message that names the
+ * offending flag when the value is malformed, or null when the flag is absent or valid.
+ * Callers map a non-null message onto `this.error(msg, { exit: 2 })`, so every command
+ * reports the same parse failure while naming the exact flag the user passed.
+ */
+export declare function durationFlagError(flag: string, value: string | undefined): string | null;
 /**
  * Derives the active freshness policy based on the tier and optional TTL override.
  */

package/dist/lib/research/freshness.js

@@ -1,12 +1,17 @@
 const HOUR_MS = 3600 * 1000;
 const DAY_MS = 24 * HOUR_MS;
 /**
- * Parses simple TTL duration strings (e.g. '24h', '7d', '30d') to milliseconds.
+ * Parses simple TTL duration strings to milliseconds. The unit `m` is months (not minutes); the
+ * smallest unit is hours, since sub-hour cache lifespans are not meaningful for research artifacts.
  */
 export function parseTtlToMs(ttl) {
     const match = ttl.match(/^(\d+)([hdwmy])$/);
     if (!match) {
-        throw new Error(`Invalid TTL format "${ttl}". Use formats like '24h', '7d', '30d'.`);
+        // Deliberately does not start with "Invalid": durationFlagError wraps this as
+        // "Invalid <flag>: <message>", and a leading "Invalid" here produced a "Invalid --ttl:
+        // Invalid TTL format" stutter.
+        throw new Error(`Duration "${ttl}" is not a valid format. Use a whole number plus a unit: ` +
+            `h (hours), d (days), w (weeks), m (months), or y (years), e.g. '24h', '7d', '6m'.`);
     }
     const val = parseInt(match[1] || '', 10);
     const unit = match[2] || '';
@@ -25,6 +30,23 @@ export function parseTtlToMs(ttl) {
             return 0;
     }
 }
+/**
+ * Validate a duration-valued flag. Returns an actionable error message that names the
+ * offending flag when the value is malformed, or null when the flag is absent or valid.
+ * Callers map a non-null message onto `this.error(msg, { exit: 2 })`, so every command
+ * reports the same parse failure while naming the exact flag the user passed.
+ */
+export function durationFlagError(flag, value) {
+    if (!value)
+        return null;
+    try {
+        parseTtlToMs(value);
+        return null;
+    }
+    catch (err) {
+        return `Invalid ${flag}: ${err.message}`;
+    }
+}
 /**
  * Derives the active freshness policy based on the tier and optional TTL override.
  */

package/dist/lib/research/keywords.d.ts

@@ -8,10 +8,17 @@ export declare function tokenize(text: string): string[];
  * deterministic (frequency desc, then alphabetical) so the same input always yields the same tags.
  */
 export declare function extractKeywords(text: string, max?: number): string[];
+/**
+ * Removes duplicate and blank tags, comparing case-insensitively (search/list filtering already
+ * matches tags case-insensitively, so "node" and "Node" are the same tag) while preserving
+ * first-seen order and the original casing of the kept tag. Whitespace is trimmed.
+ */
+export declare function dedupeTags(tags: string[]): string[];
 /**
  * Back-fills keyword tags for an artifact that has none, deriving them from its detailed content and
- * recording a quality note. No-op when tags are already present, so user- or caller-supplied tags
- * always win. Mutates and returns the same artifact. This is the single entry point every ingestion
- * path (import, fetch, revalidation) uses, so the auto-tag policy lives in exactly one place.
+ * recording a quality note. When tags are already present they win, but are still deduped/trimmed so
+ * caller-supplied tags do not persist blank or duplicate entries. Mutates and returns the same
+ * artifact. This is the single entry point every ingestion path (import, fetch, revalidation) uses,
+ * so tag hygiene lives in exactly one place.
  */
 export declare function applyAutoTags(artifact: ResearchArtifact): ResearchArtifact;

package/dist/lib/research/keywords.js

@@ -202,15 +202,42 @@ export function extractKeywords(text, max = 5) {
         .slice(0, max)
         .map(([term]) => term);
 }
+/**
+ * Removes duplicate and blank tags, comparing case-insensitively (search/list filtering already
+ * matches tags case-insensitively, so "node" and "Node" are the same tag) while preserving
+ * first-seen order and the original casing of the kept tag. Whitespace is trimmed.
+ */
+export function dedupeTags(tags) {
+    const seen = new Set();
+    const result = [];
+    for (const tag of tags) {
+        const trimmed = tag.trim();
+        const key = trimmed.toLowerCase();
+        if (!key || seen.has(key))
+            continue;
+        seen.add(key);
+        result.push(trimmed);
+    }
+    return result;
+}
 /**
  * Back-fills keyword tags for an artifact that has none, deriving them from its detailed content and
- * recording a quality note. No-op when tags are already present, so user- or caller-supplied tags
- * always win. Mutates and returns the same artifact. This is the single entry point every ingestion
- * path (import, fetch, revalidation) uses, so the auto-tag policy lives in exactly one place.
+ * recording a quality note. When tags are already present they win, but are still deduped/trimmed so
+ * caller-supplied tags do not persist blank or duplicate entries. Mutates and returns the same
+ * artifact. This is the single entry point every ingestion path (import, fetch, revalidation) uses,
+ * so tag hygiene lives in exactly one place.
  */
 export function applyAutoTags(artifact) {
-    if (artifact.metadata.tags.length > 0)
-        return artifact;
+    if (artifact.metadata.tags.length > 0) {
+        const deduped = dedupeTags(artifact.metadata.tags);
+        // If the caller supplied only blank entries, deduped is empty — treat that as "no tags" and fall
+        // through to auto-tagging rather than persisting a tagless artifact.
+        if (deduped.length > 0) {
+            artifact.metadata.tags = deduped;
+            return artifact;
+        }
+        artifact.metadata.tags = [];
+    }
     const tags = extractKeywords(artifact.detailed);
     if (tags.length === 0)
         return artifact;

package/dist/lib/research/revalidate.js

@@ -178,7 +178,9 @@ export async function revalidateCache(dataDir, existing, currentTime, options) {
                 status: 'stale',
                 artifact: existing,
                 allowed: options.allowStale,
-                error: `Revalidation failed: ${err.message}. Serving stale content within grace period.`,
+                // Carry only the underlying cause; the command layer owns the user-facing framing
+                // (so the warning reads cleanly instead of stuttering "stale … stale", "failed … failed").
+                error: err.message,
             };
         }
         throw new Error(`Revalidation failed and cache has expired beyond the grace period: ${err.message}`);

package/dist/lib/research/schema.d.ts

@@ -2,10 +2,13 @@ export interface TokenEstimate {
     compressed: number | null;
     detailed: number | null;
 }
-export type CaptureMethod = 'static_fetch' | 'browser_fallback' | 'agent_supplied' | 'route_markdown' | 'github_source';
+export declare const CAPTURE_METHODS: readonly ["static_fetch", "browser_fallback", "agent_supplied", "route_markdown", "github_source"];
+export type CaptureMethod = (typeof CAPTURE_METHODS)[number];
+export declare const ARTIFACT_TYPES: readonly ["source", "research_note", "index", "section"];
+export type ArtifactType = (typeof ARTIFACT_TYPES)[number];
 export interface ResearchArtifactMetadata {
     schema_version: number;
-    artifact_type: 'source' | 'research_note' | 'index' | 'section';
+    artifact_type: ArtifactType;
     source_url: string;
     source_urls: string[];
     normalized_url: string;

package/dist/lib/research/schema.js

@@ -1 +1,12 @@
-export {};
+// Single source of truth for the capture-method enum. Command flag `options` lists derive from this
+// so a new method can never drift out of sync with the filters that are supposed to expose it.
+export const CAPTURE_METHODS = [
+    'static_fetch',
+    'browser_fallback',
+    'agent_supplied',
+    'route_markdown',
+    'github_source',
+];
+// Single source of truth for the artifact-type enum. `index` = navigation/hub or llms.txt site
+// index; `section` = a heading-level child of a page. Command flag `options` derive from this.
+export const ARTIFACT_TYPES = ['source', 'research_note', 'index', 'section'];

package/dist/lib/research/secret-scan.js

@@ -15,9 +15,11 @@ const SECRET_PATTERNS = [
         re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/,
     },
     {
-        // Generic `api_key = "…"` / `secret: …` style assignments with a non-trivial value.
+        // Generic credential assignments with a non-trivial value: `api_key = "…"`, `secret: …`, and
+        // UPPER_SNAKE env forms like `AWS_SECRET_ACCESS_KEY=…` / `DB_PASSWORD=…`. The keyword may be a
+        // segment inside a longer identifier (so `_SECRET_` matches), which `\b…\b` boundaries missed.
         label: 'credential assignment',
-        re: /\b(?:api[_-]?key|secret|token|password|passwd|client[_-]?secret|access[_-]?token)\b\s*[:=]\s*['"]?[A-Za-z0-9._\-/+]{16,}/i,
+        re: /(?:^|[^A-Za-z])(?:api[_-]?key|secret|token|password|passwd|client[_-]?secret|access[_-]?token)[A-Za-z0-9_-]*\s*[:=]\s*['"]?[A-Za-z0-9._\-/+]{16,}/i,
     },
 ];
 /** Return the label of the first secret pattern found in `text`, or null if none match. */

package/dist/lib/research/url.js

@@ -92,8 +92,11 @@ export function normalizeUrl(input) {
     try {
         url = new URL(input);
     }
-    catch (err) {
-        throw new Error(`Invalid URL "${input}": ${err.message}`);
+    catch {
+        // new URL() only ever throws the constant "Invalid URL" (no structured cause worth keeping),
+        // and every caller prefixes its own "Invalid URL:" — forwarding the native text would stutter.
+        // Drop the binding and emit an actionable, self-contained reason instead.
+        throw new Error(`Could not parse "${input}" as a URL; provide a full http(s) URL like https://example.com/page.`);
     }
     if (url.protocol !== 'http:' && url.protocol !== 'https:') {
         throw new Error(`Unsupported protocol "${url.protocol}". Only http: and https: are supported.`);