@taurgis/bonsai 3.2.0 → 3.2.2

package/dist/lib/research/markdown.js

@@ -47,15 +47,23 @@ function addGfmTableRule(service) {
     });
 }
 /**
- * Converts HTML content to clean, deterministic Markdown.
- * Configured to use ATX-style headings (#) and fenced code blocks (```), with GFM tables.
+ * Drops empty anchor links (e.g. `[](url)`) from Markdown. These arise from icon-only heading
+ * permalinks and ReadMe query-param chips that carry no text content, wasting tokens.
+ * Images (`![](url)`) are intentionally not matched.
+ *
+ * @param markdown - Markdown text to clean.
+ * @returns Markdown with text-less links removed.
  */
-// Icon-only/empty anchors (e.g. a heading's "#" permalink, a ReadMe query-param chip) convert to a
-// text-less link `[](url)` that carries no information and just wastes tokens. Drop them, but never
-// match images `![](url)` (empty alt is still a real image).
 export function dropEmptyLinks(markdown) {
     return markdown.replace(/(?<!!)\[\s*\]\([^)]*\)/g, '');
 }
+/**
+ * Converts HTML content to clean, deterministic Markdown.
+ * Configured to use ATX-style headings (#) and fenced code blocks (```), with GFM tables.
+ *
+ * @param html - HTML string to convert.
+ * @returns Clean Markdown with empty links stripped.
+ */
 export function htmlToMarkdown(html) {
     const turndownService = new TurndownService({
         headingStyle: 'atx',

package/dist/lib/research/persist-artifact.d.ts

@@ -0,0 +1,23 @@
+import { type SecureWriteResult } from './secure-write.js';
+import type { StoreRoots } from './store-roots.js';
+import type { ResearchArtifact } from './schema.js';
+export type ArtifactWriteKind = 'import' | 'fetch';
+export interface PersistArtifactResult extends SecureWriteResult {
+    dryRun: boolean;
+    /** Secret-redirect warning for stderr, or null when no redirect. */
+    redirectWarning: string | null;
+}
+/**
+ * Persist (or dry-run preview) an artifact with secret-safe routing. Owns redirect warning copy
+ * so fetch/import cannot drift. When `scratchDir` is set (fetch dry-run), also writes a throwaway
+ * copy there for downstream extraction while reporting the real would-be `dataDir`.
+ */
+export declare function persistArtifact(opts: {
+    roots: StoreRoots;
+    cacheKey: string;
+    artifact: ResearchArtifact;
+    dryRun: boolean;
+    kind: ArtifactWriteKind;
+    /** Fetch dry-run throwaway directory; real destination still comes from the secure write. */
+    scratchDir?: string | null;
+}): PersistArtifactResult;

package/dist/lib/research/persist-artifact.js

@@ -0,0 +1,27 @@
+import { writeArtifact } from './storage.js';
+import { writeArtifactSecurely } from './secure-write.js';
+function redirectWarningFor(kind, dryRun, secretLabel) {
+    const content = kind === 'import' ? 'imported content' : 'page content';
+    const verb = dryRun ? 'would store' : 'stored';
+    return `Detected ${secretLabel} in the ${content}; ${verb} in the global cache instead of the project to avoid committing secrets.`;
+}
+/**
+ * Persist (or dry-run preview) an artifact with secret-safe routing. Owns redirect warning copy
+ * so fetch/import cannot drift. When `scratchDir` is set (fetch dry-run), also writes a throwaway
+ * copy there for downstream extraction while reporting the real would-be `dataDir`.
+ */
+export function persistArtifact(opts) {
+    const result = writeArtifactSecurely(opts.roots, opts.cacheKey, opts.artifact, {
+        dryRun: opts.dryRun,
+    });
+    if (opts.scratchDir) {
+        writeArtifact(opts.scratchDir, opts.cacheKey, opts.artifact);
+    }
+    return {
+        ...result,
+        dryRun: opts.dryRun,
+        redirectWarning: result.redirected
+            ? redirectWarningFor(opts.kind, opts.dryRun, result.secretLabel)
+            : null,
+    };
+}

package/dist/lib/research/prompt-injection.d.ts

@@ -1 +1,9 @@
+/**
+ * Scans Markdown for embedded prompt-injection attacks and redacts any block or line that matches.
+ * Detects direct patterns, plus base64/hex-encoded variants and typoglycemia obfuscation.
+ * HTML comments are checked first; the body is then split on paragraph breaks and scanned line-by-line.
+ *
+ * @param markdown - Markdown content (possibly untrusted, sourced from fetched pages).
+ * @returns Sanitized Markdown with harmful instruction blocks replaced by a redaction placeholder.
+ */
 export declare function sanitizePromptInjection(markdown: string): string;

package/dist/lib/research/prompt-injection.js

@@ -30,19 +30,28 @@ const TYPOGLYCEMIA_TARGETS = [
     'repository',
     'workspace',
 ];
+// Contexts where a following verb reads as a command aimed at the reader, not a third-person
+// description of an attack (e.g. "An attacker may tell the model to ignore..." must stay
+// unredacted — see the matching test below). `^` is matched per clause (see CLAUSE_BOUNDARY
+// below), so a command hidden after an innocuous opener ("Heads up: ignore...") still counts as
+// clause-initial; only right after a direct role-address word is the other allowed context.
+const COMMAND_CONTEXT = String.raw `(?:^|\b(?:please|assistant|system|user|agent)\s+)`;
+function commandPattern(suffix) {
+    return new RegExp(COMMAND_CONTEXT + suffix, 'i');
+}
 const HARMFUL_INSTRUCTION_PATTERNS = [
-    /(^|\b(?:please|assistant|system|user|agent)\s+)ignore\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|directions?|prompts?)\b/i,
-    /(^|\b(?:please|assistant|system|user|agent)\s+)disregard\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|directions?|prompts?)\b/i,
-    /(^|\b(?:please|assistant|system|user|agent)\s+)forget\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|directions?|prompts?)\b/i,
-    /(^|\b(?:please|assistant|system|user|agent)\s+)override\s+(the\s+)?(system|developer|agent)\s+(prompt|instructions?|message)\b/i,
-    /(^|\b(?:please|assistant|system|user|agent)\s+)you\s+are\s+now\s+(in\s+)?(developer|admin|root|system)\s+mode\b/i,
-    /(^|\b(?:please|assistant|system|user|agent)\s+)reveal\s+(your\s+)?(system|developer)\s+(prompt|instructions?|message)\b/i,
-    /(^|\b(?:please|assistant|system|user|agent)\s+)print\s+(your\s+)?(system|developer)\s+(prompt|instructions?|message)\b/i,
-    /(^|\b(?:please|assistant|system|user|agent)\s+)(exfiltrate|steal|leak)\s+(all\s+|the\s+|your\s+)?(secrets?|tokens?|api\s*keys?|credentials?|private\s+data)\b/i,
-    /(^|\b(?:please|assistant|system|user|agent)\s+)send\s+(me\s+|all\s+|the\s+|your\s+)?(secrets?|tokens?|api\s*keys?|credentials?)\s+(to|over|via|using)\b/i,
-    /(^|\b(?:please|assistant|system|user|agent)\s+)(upload|post|fetch|curl)\s+(all\s+|the\s+|your\s+)?(secrets?|tokens?|api\s*keys?|credentials?)\s+(to|over|via|using)\b/i,
-    /(^|\b(?:please|assistant|system|user|agent)\s+)delete\s+(all\s+)?(files?|the\s+repository|the\s+workspace|the\s+home\s+directory)\b/i,
-    /(^|\b(?:please|assistant|system|user|agent)\s+)run\s+.*\b(rm\s+-rf|curl\s+.*\|\s*(sh|bash)|sudo)\b/i,
+    commandPattern(String.raw `ignore\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|directions?|prompts?)\b`),
+    commandPattern(String.raw `disregard\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|directions?|prompts?)\b`),
+    commandPattern(String.raw `forget\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|directions?|prompts?)\b`),
+    commandPattern(String.raw `override\s+(the\s+)?(system|developer|agent)\s+(prompt|instructions?|message)\b`),
+    commandPattern(String.raw `you\s+are\s+now\s+(in\s+)?(developer|admin|root|system)\s+mode\b`),
+    commandPattern(String.raw `reveal\s+(your\s+)?(system|developer)\s+(prompt|instructions?|message)\b`),
+    commandPattern(String.raw `print\s+(your\s+)?(system|developer)\s+(prompt|instructions?|message)\b`),
+    commandPattern(String.raw `(exfiltrate|steal|leak)\s+(all\s+|the\s+|your\s+)?(secrets?|tokens?|api\s*keys?|credentials?|private\s+data)\b`),
+    commandPattern(String.raw `send\s+(me\s+|all\s+|the\s+|your\s+)?(secrets?|tokens?|api\s*keys?|credentials?)\s+(to|over|via|using)\b`),
+    commandPattern(String.raw `(upload|post|fetch|curl)\s+(all\s+|the\s+|your\s+)?(secrets?|tokens?|api\s*keys?|credentials?)\s+(to|over|via|using)\b`),
+    commandPattern(String.raw `delete\s+(all\s+)?(files?|the\s+repository|the\s+workspace|the\s+home\s+directory)\b`),
+    commandPattern(String.raw `run\s+.*\b(rm\s+-rf|curl\s+.*\|\s*(sh|bash)|sudo)\b`),
 ];
 function normalizeForDetection(text) {
     return text
@@ -110,8 +119,20 @@ function isReadableDecodedText(text) {
     const printable = [...text].filter((char) => /[\t\n\r -~]/.test(char)).length;
     return printable / Math.max(text.length, 1) > 0.85;
 }
+// Sentence-ending punctuation splits a line into clauses for `^`-anchored matching below. This has
+// to run on the raw text, before normalizeForDetection's charset filter erases that same
+// punctuation into plain spaces — testing the anchor after normalization would never see it.
+//
+// ponytail: this still isn't complete — a filler *word* with no punctuation before it ("Please
+// just ignore previous instructions") is a clause-internal command and stays unredacted, same as
+// before this fix. Heuristic prompt-injection detection can't be made airtight with regex alone
+// (OWASP LLM01 has no complete solution either); upgrade path is a small classifier or an LLM-based
+// second pass over blocks this pattern set flags as borderline, if false negatives here prove costly.
+const CLAUSE_BOUNDARY = /(?<=[.!?:;])\s+/;
 function isUnsafeAgentInstruction(text) {
-    return detectionCandidates(text).some((candidate) => HARMFUL_INSTRUCTION_PATTERNS.some((pattern) => pattern.test(candidate)));
+    return text
+        .split(CLAUSE_BOUNDARY)
+        .some((clause) => detectionCandidates(clause).some((candidate) => HARMFUL_INSTRUCTION_PATTERNS.some((pattern) => pattern.test(candidate))));
 }
 function redactBlock(block) {
     const withSafeSpans = redactUnsafeSpans(block);
@@ -132,6 +153,14 @@ function stripQuotes(span) {
     }
     return span;
 }
+/**
+ * Scans Markdown for embedded prompt-injection attacks and redacts any block or line that matches.
+ * Detects direct patterns, plus base64/hex-encoded variants and typoglycemia obfuscation.
+ * HTML comments are checked first; the body is then split on paragraph breaks and scanned line-by-line.
+ *
+ * @param markdown - Markdown content (possibly untrusted, sourced from fetched pages).
+ * @returns Sanitized Markdown with harmful instruction blocks replaced by a redaction placeholder.
+ */
 export function sanitizePromptInjection(markdown) {
     return markdown
         .replace(HTML_COMMENT, (comment) => redactBlock(comment))

package/dist/lib/research/proxy.d.ts

@@ -59,13 +59,19 @@ export declare function getCaBundleSpkiHashes(): string[];
 /**
  * Chrome CLI flag that trusts exactly the sandbox's injected CA(s), pinned by SPKI hash rather
  * than by disabling verification. Chromium's TLS stack (the Chrome Root Store) never reads
- * NODE_EXTRA_CA_CERTS/SSL_CERT_FILE/CURL_CA_BUNDLE the way Node/curl/Python do, so a sandbox that
- * MITMs HTTPS through its proxy (see isProxyConfigured) makes every other tool trust the proxy's
- * cert while Chrome's handshake with it fails with ERR_CERT_AUTHORITY_INVALID. This is scoped
- * narrowly to the specific CA(s) discovered via the CA bundle env vars — never
- * --ignore-certificate-errors, which would disable verification for every host, including ones
- * that have nothing to do with the sandbox's proxy. A no-op (empty array) when no proxy is
- * configured or no CA bundle is discoverable, so default Chrome trust is untouched on ordinary
- * developer machines.
+ * NODE_EXTRA_CA_CERTS/SSL_CERT_FILE/CURL_CA_BUNDLE the way Node/curl/Python do, so an environment
+ * that MITMs HTTPS makes every other tool trust that CA while Chrome's handshake with it fails
+ * with ERR_CERT_AUTHORITY_INVALID. This is scoped narrowly to the specific CA(s) discovered via
+ * the CA bundle env vars — never --ignore-certificate-errors, which would disable verification
+ * for every host, including ones that have nothing to do with the sandbox.
+ *
+ * Deliberately independent of isProxyConfigured(): some sandboxes intercept HTTPS transparently
+ * (or via a proxy Chrome reaches without an explicit --proxy-server flag) while still exporting a
+ * CA bundle env var for CLI tools to trust — HTTPS_PROXY/HTTP_PROXY and the CA bundle vars are
+ * two separate signals that usually co-occur but aren't guaranteed to. Gating on proxy detection
+ * left Chrome untrusting a CA bundle that was in fact discoverable, so it rejected every
+ * browser-rendered fetch with ERR_CERT_AUTHORITY_INVALID in that shape of environment. A no-op
+ * (empty array) when no CA bundle is discoverable, so default Chrome trust is untouched on
+ * ordinary developer machines.
  */
 export declare function getChromeSpkiArgs(): string[];

package/dist/lib/research/proxy.js

@@ -169,18 +169,22 @@ export function getCaBundleSpkiHashes() {
 /**
  * Chrome CLI flag that trusts exactly the sandbox's injected CA(s), pinned by SPKI hash rather
  * than by disabling verification. Chromium's TLS stack (the Chrome Root Store) never reads
- * NODE_EXTRA_CA_CERTS/SSL_CERT_FILE/CURL_CA_BUNDLE the way Node/curl/Python do, so a sandbox that
- * MITMs HTTPS through its proxy (see isProxyConfigured) makes every other tool trust the proxy's
- * cert while Chrome's handshake with it fails with ERR_CERT_AUTHORITY_INVALID. This is scoped
- * narrowly to the specific CA(s) discovered via the CA bundle env vars — never
- * --ignore-certificate-errors, which would disable verification for every host, including ones
- * that have nothing to do with the sandbox's proxy. A no-op (empty array) when no proxy is
- * configured or no CA bundle is discoverable, so default Chrome trust is untouched on ordinary
- * developer machines.
+ * NODE_EXTRA_CA_CERTS/SSL_CERT_FILE/CURL_CA_BUNDLE the way Node/curl/Python do, so an environment
+ * that MITMs HTTPS makes every other tool trust that CA while Chrome's handshake with it fails
+ * with ERR_CERT_AUTHORITY_INVALID. This is scoped narrowly to the specific CA(s) discovered via
+ * the CA bundle env vars — never --ignore-certificate-errors, which would disable verification
+ * for every host, including ones that have nothing to do with the sandbox.
+ *
+ * Deliberately independent of isProxyConfigured(): some sandboxes intercept HTTPS transparently
+ * (or via a proxy Chrome reaches without an explicit --proxy-server flag) while still exporting a
+ * CA bundle env var for CLI tools to trust — HTTPS_PROXY/HTTP_PROXY and the CA bundle vars are
+ * two separate signals that usually co-occur but aren't guaranteed to. Gating on proxy detection
+ * left Chrome untrusting a CA bundle that was in fact discoverable, so it rejected every
+ * browser-rendered fetch with ERR_CERT_AUTHORITY_INVALID in that shape of environment. A no-op
+ * (empty array) when no CA bundle is discoverable, so default Chrome trust is untouched on
+ * ordinary developer machines.
  */
 export function getChromeSpkiArgs() {
-    if (!isProxyConfigured())
-        return [];
     const hashes = getCaBundleSpkiHashes();
     if (!hashes.length)
         return [];

package/dist/lib/research/revalidate.d.ts

@@ -1,34 +1,63 @@
 import type { ResearchArtifact } from './schema.js';
+import { type FetchedContent } from './fetcher.js';
+import { type ExtractionResult } from './extract.js';
+import type { StoreRoots } from './store-roots.js';
 import type { SummaryLevel } from '../config/schema.js';
+/**
+ * When set, project-located revalidation writes reuse {@link persistArtifact} (secret-scan +
+ * redirect) so a refresh cannot leave credentials in a committable project cache (#90).
+ */
+export interface RevalidateSecureWrite {
+    roots: StoreRoots;
+    dryRun?: boolean;
+    scratchDir?: string | null;
+}
+/** Result of conditional revalidation or a full refresh when validators miss. */
 export interface RevalidationResult {
+    /** Outcome of the revalidation attempt. */
     status: 'revalidated' | 'refreshed' | 'stale';
+    /** Artifact to serve (updated, refreshed, or the prior stale entry). */
     artifact: ResearchArtifact;
+    /** When status is `stale`, whether serving stale content is allowed. */
     allowed?: boolean;
+    /** Revalidation failure message when status is `stale`. */
     error?: string;
+    /** Data dir the write landed in (or would land in under dry-run), when a write ran. */
+    storageDir?: string;
+    /** True when a project refresh was redirected to global due to a detected secret. */
+    redirectedToGlobal?: boolean;
+    /** Secret-redirect warning for stderr, or null/undefined when none. */
+    redirectWarning?: string | null;
+}
+type RevalidateOptions = {
+    allowStale: boolean;
+    ttlOverride?: string | null;
+    rendered?: boolean;
+    summaryLevel: SummaryLevel;
+    /** Present only when the live entry is in the project cache. */
+    secure?: RevalidateSecureWrite;
+};
+interface CreateArtifactFromFetchInput {
+    url: string;
+    normalizedUrl: string;
+    cacheKey: string;
+    fetchResult: FetchedContent;
+    extraction: ExtractionResult;
+    tier: 'stable' | 'standard' | 'volatile';
+    ttl: string | null;
+    currentTime: Date;
+    summaryLevel: SummaryLevel;
 }
 /**
- * Helper to construct a ResearchArtifact from a fresh HTML fetch.
+ * Construct a ResearchArtifact from a fresh HTML fetch (or error-page marker).
+ *
+ * @param input - URL, fetch result, extraction, tier/TTL, clock, and summary level.
+ * @returns A ready-to-persist research artifact.
  */
-export declare function createArtifactFromFetch(url: string, normalizedUrl: string, cacheKey: string, fetchResult: {
-    contentType: string | null;
-    etag: string | null;
-    lastModified: string | null;
-    finalUrl: string;
-    responseSize: number;
-    content: string;
-}, extraction: {
-    title: string;
-    detailedMarkdown: string;
-    confidence: 'high' | 'medium' | 'low';
-    qualityNotes: string[];
-}, tier: 'stable' | 'standard' | 'volatile', ttl: string | null, currentTime: Date, summaryLevel: SummaryLevel): ResearchArtifact;
+export declare function createArtifactFromFetch(input: CreateArtifactFromFetchInput): ResearchArtifact;
 /**
  * Revalidates a stale cache artifact using conditional request headers,
  * falling back to stale serving within the grace period if the remote host is offline.
  */
-export declare function revalidateCache(dataDir: string, existing: ResearchArtifact, currentTime: Date, options: {
-    allowStale: boolean;
-    ttlOverride?: string | null;
-    rendered?: boolean;
-    summaryLevel: SummaryLevel;
-}): Promise<RevalidationResult>;
+export declare function revalidateCache(dataDir: string, existing: ResearchArtifact, currentTime: Date, options: RevalidateOptions): Promise<RevalidationResult>;
+export {};

package/dist/lib/research/revalidate.js

@@ -4,16 +4,37 @@ import { applySiteFetchProvenance } from '../../sites/types.js';
 import { fetchStaticHtml } from './fetcher.js';
 import { extractHtmlContent } from './extract.js';
 import { writeArtifact } from './storage.js';
-import { evaluateFreshness, getPolicy } from './freshness.js';
+import { persistArtifact } from './persist-artifact.js';
+import { evaluateFreshness, resolveFreshnessPolicy } from './freshness.js';
 import { buildCompressed } from './compress.js';
 import { applyAutoTags } from './keywords.js';
 import { estimateTokens } from './token-estimate.js';
 import { fetchRenderedHtml } from './browser.js';
 import { looksLikeErrorPage } from './docs/validate.js';
-function buildMetadata(url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, compressed) {
+function persistRevalidated(dataDir, key, artifact, secure) {
+    if (secure) {
+        const result = persistArtifact({
+            roots: secure.roots,
+            cacheKey: key,
+            artifact,
+            dryRun: Boolean(secure.dryRun),
+            kind: 'fetch',
+            scratchDir: secure.scratchDir,
+        });
+        return {
+            storageDir: result.dataDir,
+            redirectedToGlobal: result.redirected,
+            redirectWarning: result.redirectWarning,
+        };
+    }
+    writeArtifact(dataDir, key, artifact);
+    return { storageDir: dataDir, redirectedToGlobal: false, redirectWarning: null };
+}
+function buildMetadata(input) {
+    const { url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, compressed, } = input;
     const contentHash = createHash('sha256').update(extraction.detailedMarkdown).digest('hex');
     const staleAfterTime = new Date(currentTime);
-    const { freshWindowMs } = getPolicy(tier, ttl);
+    const { freshWindowMs } = resolveFreshnessPolicy(tier, ttl);
     staleAfterTime.setTime(staleAfterTime.getTime() + freshWindowMs);
     return {
         schema_version: 1,
@@ -57,7 +78,8 @@ function buildMetadata(url, normalizedUrl, cacheKey, fetchResult, extraction, ti
 // Builds a compact error marker instead of caching a full error page. Subsequent lookups serve this
 // tiny marker (a handful of tokens) instead of re-fetching, and revalidation re-checks it when stale
 // so a transient failure recovers into a real artifact.
-function buildErrorArtifact(url, normalizedUrl, cacheKey, fetchResult, tier, ttl, currentTime, reason) {
+function buildErrorArtifact(input) {
+    const { url, normalizedUrl, cacheKey, fetchResult, tier, ttl, currentTime, reason } = input;
     const marker = `Error: ${url} could not be cached — ${reason}. ` +
         'The page returned an error, so its content was not stored; it will be re-fetched when this entry goes stale.';
     const extraction = {
@@ -66,7 +88,17 @@ function buildErrorArtifact(url, normalizedUrl, cacheKey, fetchResult, tier, ttl
         confidence: 'low',
         qualityNotes: [`error: ${reason}`],
     };
-    const metadata = buildMetadata(url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, marker);
+    const metadata = buildMetadata({
+        url,
+        normalizedUrl,
+        cacheKey,
+        fetchResult,
+        extraction,
+        tier,
+        ttl,
+        currentTime,
+        compressed: marker,
+    });
     metadata.extraction_status = 'failed';
     metadata.extraction_confidence = null;
     return {
@@ -78,17 +110,40 @@ function buildErrorArtifact(url, normalizedUrl, cacheKey, fetchResult, tier, ttl
     };
 }
 /**
- * Helper to construct a ResearchArtifact from a fresh HTML fetch.
+ * Construct a ResearchArtifact from a fresh HTML fetch (or error-page marker).
+ *
+ * @param input - URL, fetch result, extraction, tier/TTL, clock, and summary level.
+ * @returns A ready-to-persist research artifact.
  */
-export function createArtifactFromFetch(url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, summaryLevel) {
+export function createArtifactFromFetch(input) {
+    const { url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, summaryLevel, } = input;
     // A managed platform / SPA can return HTTP 200 but render only a "not found" / "something went
     // wrong" shell. Cache a compact marker for those instead of the full error markdown, so repeat
     // lookups cost a few tokens and revalidation still re-checks the page when the entry goes stale.
     if (looksLikeErrorPage(extraction.detailedMarkdown)) {
-        return buildErrorArtifact(url, normalizedUrl, cacheKey, fetchResult, tier, ttl, currentTime, 'page reported an error or was not found');
+        return buildErrorArtifact({
+            url,
+            normalizedUrl,
+            cacheKey,
+            fetchResult,
+            tier,
+            ttl,
+            currentTime,
+            reason: 'page reported an error or was not found',
+        });
     }
     const compressed = buildCompressed(extraction.detailedMarkdown, summaryLevel);
-    const metadata = buildMetadata(url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, compressed);
+    const metadata = buildMetadata({
+        url,
+        normalizedUrl,
+        cacheKey,
+        fetchResult,
+        extraction,
+        tier,
+        ttl,
+        currentTime,
+        compressed,
+    });
     return {
         metadata,
         summary: extraction.title,
@@ -113,7 +168,17 @@ function preserveUserMetadata(source, target, rendered) {
 // returns the "refreshed" result. Shared by both refresh paths (conditional-request fallthrough
 // and site-module re-fetch), which differ only in how fetchResult/extraction are obtained.
 function persistRefreshedArtifact(dataDir, meta, fetchResult, extraction, currentTime, options, siteFetch) {
-    const refreshed = createArtifactFromFetch(meta.source_url, meta.normalized_url, meta.cache_key, fetchResult, extraction, meta.tier, options.ttlOverride || meta.ttl, currentTime, options.summaryLevel);
+    const refreshed = createArtifactFromFetch({
+        url: meta.source_url,
+        normalizedUrl: meta.normalized_url,
+        cacheKey: meta.cache_key,
+        fetchResult,
+        extraction,
+        tier: meta.tier,
+        ttl: options.ttlOverride || meta.ttl,
+        currentTime,
+        summaryLevel: options.summaryLevel,
+    });
     preserveUserMetadata(meta, refreshed, options.rendered);
     // A site-module refresh reports how it actually captured this time; that overrides the
     // carried-over provenance, so a withdrawn .md twin can't leave a stale source_doc_url behind.
@@ -122,8 +187,8 @@ function persistRefreshedArtifact(dataDir, meta, fetchResult, extraction, curren
     // Back-fill keyword tags when the carried-over set is empty (e.g. an artifact first cached before
     // auto-tagging, or one originally stored without tags), so refreshing keeps it searchable.
     applyAutoTags(refreshed);
-    writeArtifact(dataDir, meta.cache_key, refreshed);
-    return { status: 'refreshed', artifact: refreshed };
+    const write = persistRevalidated(dataDir, meta.cache_key, refreshed, options.secure);
+    return { status: 'refreshed', artifact: refreshed, ...write };
 }
 async function handleRevalidateResponse(dataDir, existing, fetchResult, currentTime, options) {
     const meta = existing.metadata;
@@ -136,11 +201,11 @@ async function handleRevalidateResponse(dataDir, existing, fetchResult, currentT
             },
         };
         const staleAfterTime = new Date(currentTime);
-        const { freshWindowMs } = getPolicy(meta.tier, options.ttlOverride || meta.ttl);
+        const { freshWindowMs } = resolveFreshnessPolicy(meta.tier, options.ttlOverride || meta.ttl);
         staleAfterTime.setTime(staleAfterTime.getTime() + freshWindowMs);
         updated.metadata.stale_after = staleAfterTime.toISOString();
-        writeArtifact(dataDir, meta.cache_key, updated);
-        return { status: 'revalidated', artifact: updated };
+        const write = persistRevalidated(dataDir, meta.cache_key, updated, options.secure);
+        return { status: 'revalidated', artifact: updated, ...write };
     }
     const extraction = extractHtmlContent(fetchResult.content, fetchResult.finalUrl);
     return persistRefreshedArtifact(dataDir, meta, fetchResult, extraction, currentTime, options);

package/dist/lib/research/schema.d.ts

@@ -1,11 +1,17 @@
+/** Token counts for the two stored representations of a research artifact. */
 export interface TokenEstimate {
     compressed: number | null;
     detailed: number | null;
 }
+/** All recognized capture methods. Command flag option lists derive from this constant. */
 export declare const CAPTURE_METHODS: readonly ["static_fetch", "browser_fallback", "agent_supplied", "route_markdown", "github_source"];
+/** How a research artifact's content was obtained. */
 export type CaptureMethod = (typeof CAPTURE_METHODS)[number];
+/** All recognized artifact types. `index` = navigation hub or llms.txt; `section` = heading-level child of a page. */
 export declare const ARTIFACT_TYPES: readonly ["source", "research_note", "index", "section"];
+/** Classification of a research artifact's role. */
 export type ArtifactType = (typeof ARTIFACT_TYPES)[number];
+/** Full metadata block stored alongside every cached research artifact. */
 export interface ResearchArtifactMetadata {
     schema_version: number;
     artifact_type: ArtifactType;
@@ -41,6 +47,7 @@ export interface ResearchArtifactMetadata {
     section_anchor: string | null;
     section_heading_path: string | null;
 }
+/** A complete research artifact: metadata, summary, compressed and detailed Markdown, and provenance. */
 export interface ResearchArtifact {
     metadata: ResearchArtifactMetadata;
     summary: string;

package/dist/lib/research/schema.js

@@ -1,5 +1,6 @@
 // Single source of truth for the capture-method enum. Command flag `options` lists derive from this
 // so a new method can never drift out of sync with the filters that are supposed to expose it.
+/** All recognized capture methods. Command flag option lists derive from this constant. */
 export const CAPTURE_METHODS = [
     'static_fetch',
     'browser_fallback',
@@ -9,4 +10,5 @@ export const CAPTURE_METHODS = [
 ];
 // Single source of truth for the artifact-type enum. `index` = navigation/hub or llms.txt site
 // index; `section` = a heading-level child of a page. Command flag `options` derive from this.
+/** All recognized artifact types. `index` = navigation hub or llms.txt; `section` = heading-level child of a page. */
 export const ARTIFACT_TYPES = ['source', 'research_note', 'index', 'section'];

package/dist/lib/research/secret-scan.d.ts

@@ -1,5 +1,11 @@
 import type { ResearchArtifact } from './schema.js';
 /** Return the label of the first secret pattern found in `text`, or null if none match. */
 export declare function detectSecret(text: string): string | null;
+/**
+ * Strip every secret-pattern match from `text`, replacing each with a single space. Matches the
+ * original (case-preserved, unsplit) text, so callers must run this before any lowercasing or
+ * tokenization step that would otherwise hide a pattern like `AKIA…` or `ghp_…` from detection.
+ */
+export declare function stripSecrets(text: string): string;
 /** Scan every content field of an artifact. Returns the detected secret's label, or null. */
 export declare function scanArtifactForSecret(artifact: ResearchArtifact): string | null;

package/dist/lib/research/secret-scan.js

@@ -30,6 +30,14 @@ export function detectSecret(text) {
     }
     return null;
 }
+/**
+ * Strip every secret-pattern match from `text`, replacing each with a single space. Matches the
+ * original (case-preserved, unsplit) text, so callers must run this before any lowercasing or
+ * tokenization step that would otherwise hide a pattern like `AKIA…` or `ghp_…` from detection.
+ */
+export function stripSecrets(text) {
+    return SECRET_PATTERNS.reduce((result, { re }) => result.replace(new RegExp(re.source, `${re.flags}g`), ' '), text);
+}
 /** Scan every content field of an artifact. Returns the detected secret's label, or null. */
 export function scanArtifactForSecret(artifact) {
     const haystack = [artifact.summary, artifact.compressed, artifact.detailed, artifact.provenance]

package/dist/lib/research/secure-write.d.ts

@@ -16,6 +16,10 @@ export interface SecureWriteResult {
  * `dryRun` still runs the secret scan and reports the real would-be destination, but skips the
  * actual `writeArtifact` persist — so a read-only preview never gives a falsely reassuring answer
  * about where secret-bearing content would land.
+ *
+ * On redirect, any existing project copy is archived and removed so lookup cannot keep serving the
+ * project shadow after the secret-bearing content moved to global (revalidation parity with
+ * first-time project writes — see #90 / AUDIT_71_FINAL).
  */
 export declare function writeArtifactSecurely(roots: StoreRoots, key: string, artifact: ResearchArtifact, options?: {
     dryRun?: boolean;

package/dist/lib/research/secure-write.js

@@ -1,5 +1,27 @@
-import { writeArtifact } from './storage.js';
+import { existsSync, readFileSync, writeFileSync, unlinkSync } from 'node:fs';
+import { join } from 'node:path';
+import { writeArtifact, getArtifactPath } from './storage.js';
 import { scanArtifactForSecret } from './secret-scan.js';
+/**
+ * Archive then remove an active project artifact so a redirected global copy is not shadowed by
+ * project→global lookup order. Best-effort: a failed archive/unlink never undoes the global write.
+ *
+ * @see https://nodejs.org/api/fs.html#fsunlinksyncpath
+ */
+function clearProjectArtifactAfterRedirect(projectRoot, key) {
+    const path = getArtifactPath(projectRoot, key);
+    if (!existsSync(path))
+        return;
+    try {
+        const content = readFileSync(path, 'utf-8');
+        const dir = join(projectRoot, 'research');
+        writeFileSync(join(dir, `${key}.superseded.${Date.now()}.md`), content, 'utf-8');
+        unlinkSync(path);
+    }
+    catch (err) {
+        console.warn(`Warning: failed to clear project artifact after secret redirect: ${err.message}`);
+    }
+}
 /**
  * Write an artifact to the configured write root, except when it contains a secret and the
  * target is a (potentially committed) project cache — those are redirected to the global cache.
@@ -8,12 +30,19 @@ import { scanArtifactForSecret } from './secret-scan.js';
  * `dryRun` still runs the secret scan and reports the real would-be destination, but skips the
  * actual `writeArtifact` persist — so a read-only preview never gives a falsely reassuring answer
  * about where secret-bearing content would land.
+ *
+ * On redirect, any existing project copy is archived and removed so lookup cannot keep serving the
+ * project shadow after the secret-bearing content moved to global (revalidation parity with
+ * first-time project writes — see #90 / AUDIT_71_FINAL).
  */
 export function writeArtifactSecurely(roots, key, artifact, options = {}) {
     const isProjectWrite = roots.writeRoot !== roots.globalRoot;
     const secretLabel = isProjectWrite ? scanArtifactForSecret(artifact) : null;
     const dataDir = secretLabel ? roots.globalRoot : roots.writeRoot;
-    if (!options.dryRun)
+    if (!options.dryRun) {
         writeArtifact(dataDir, key, artifact);
+        if (secretLabel)
+            clearProjectArtifactAfterRedirect(roots.writeRoot, key);
+    }
     return { dataDir, redirected: Boolean(secretLabel), secretLabel };
 }

package/dist/lib/research/storage.d.ts

@@ -48,5 +48,9 @@ export declare function locateArtifact(dataDirs: string[], key: string): Located
  * order and deduplicates by cache key so an entry present in an earlier root (project) shadows
  * the same key in a later root (global) — even when the callback filters it out. Composes
  * scanCacheDir so the file-format and read/parse rules stay defined in exactly one place.
+ *
+ * Pass `persistIndex: false` under read-only/plan mode so listing never creates `.search-index.json`.
  */
-export declare function scanCacheDirs<T>(dataDirs: string[], fn: (artifact: ResearchArtifact, filePath: string) => T | null): T[];
+export declare function scanCacheDirs<T>(dataDirs: string[], fn: (artifact: ResearchArtifact, filePath: string) => T | null, options?: {
+    persistIndex?: boolean;
+}): T[];