@taurgis/bonsai 3.2.0 → 4.0.0

package/dist/lib/research/revalidate.js

@@ -4,16 +4,37 @@ import { applySiteFetchProvenance } from '../../sites/types.js';
 import { fetchStaticHtml } from './fetcher.js';
 import { extractHtmlContent } from './extract.js';
 import { writeArtifact } from './storage.js';
-import { evaluateFreshness, getPolicy } from './freshness.js';
+import { persistArtifact } from './persist-artifact.js';
+import { evaluateFreshness, resolveFreshnessPolicy } from './freshness.js';
 import { buildCompressed } from './compress.js';
 import { applyAutoTags } from './keywords.js';
 import { estimateTokens } from './token-estimate.js';
 import { fetchRenderedHtml } from './browser.js';
 import { looksLikeErrorPage } from './docs/validate.js';
-function buildMetadata(url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, compressed) {
+function persistRevalidated(dataDir, key, artifact, secure) {
+    if (secure) {
+        const result = persistArtifact({
+            roots: secure.roots,
+            cacheKey: key,
+            artifact,
+            dryRun: Boolean(secure.dryRun),
+            kind: 'fetch',
+            scratchDir: secure.scratchDir,
+        });
+        return {
+            storageDir: result.dataDir,
+            redirectedToGlobal: result.redirected,
+            redirectWarning: result.redirectWarning,
+        };
+    }
+    writeArtifact(dataDir, key, artifact);
+    return { storageDir: dataDir, redirectedToGlobal: false, redirectWarning: null };
+}
+function buildMetadata(input) {
+    const { url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, compressed, } = input;
     const contentHash = createHash('sha256').update(extraction.detailedMarkdown).digest('hex');
     const staleAfterTime = new Date(currentTime);
-    const { freshWindowMs } = getPolicy(tier, ttl);
+    const { freshWindowMs } = resolveFreshnessPolicy(tier, ttl);
     staleAfterTime.setTime(staleAfterTime.getTime() + freshWindowMs);
     return {
         schema_version: 1,
@@ -57,7 +78,8 @@ function buildMetadata(url, normalizedUrl, cacheKey, fetchResult, extraction, ti
 // Builds a compact error marker instead of caching a full error page. Subsequent lookups serve this
 // tiny marker (a handful of tokens) instead of re-fetching, and revalidation re-checks it when stale
 // so a transient failure recovers into a real artifact.
-function buildErrorArtifact(url, normalizedUrl, cacheKey, fetchResult, tier, ttl, currentTime, reason) {
+function buildErrorArtifact(input) {
+    const { url, normalizedUrl, cacheKey, fetchResult, tier, ttl, currentTime, reason } = input;
     const marker = `Error: ${url} could not be cached — ${reason}. ` +
         'The page returned an error, so its content was not stored; it will be re-fetched when this entry goes stale.';
     const extraction = {
@@ -66,7 +88,17 @@ function buildErrorArtifact(url, normalizedUrl, cacheKey, fetchResult, tier, ttl
         confidence: 'low',
         qualityNotes: [`error: ${reason}`],
     };
-    const metadata = buildMetadata(url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, marker);
+    const metadata = buildMetadata({
+        url,
+        normalizedUrl,
+        cacheKey,
+        fetchResult,
+        extraction,
+        tier,
+        ttl,
+        currentTime,
+        compressed: marker,
+    });
     metadata.extraction_status = 'failed';
     metadata.extraction_confidence = null;
     return {
@@ -78,17 +110,40 @@ function buildErrorArtifact(url, normalizedUrl, cacheKey, fetchResult, tier, ttl
     };
 }
 /**
- * Helper to construct a ResearchArtifact from a fresh HTML fetch.
+ * Construct a ResearchArtifact from a fresh HTML fetch (or error-page marker).
+ *
+ * @param input - URL, fetch result, extraction, tier/TTL, clock, and summary level.
+ * @returns A ready-to-persist research artifact.
  */
-export function createArtifactFromFetch(url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, summaryLevel) {
+export function createArtifactFromFetch(input) {
+    const { url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, summaryLevel, } = input;
     // A managed platform / SPA can return HTTP 200 but render only a "not found" / "something went
     // wrong" shell. Cache a compact marker for those instead of the full error markdown, so repeat
     // lookups cost a few tokens and revalidation still re-checks the page when the entry goes stale.
     if (looksLikeErrorPage(extraction.detailedMarkdown)) {
-        return buildErrorArtifact(url, normalizedUrl, cacheKey, fetchResult, tier, ttl, currentTime, 'page reported an error or was not found');
+        return buildErrorArtifact({
+            url,
+            normalizedUrl,
+            cacheKey,
+            fetchResult,
+            tier,
+            ttl,
+            currentTime,
+            reason: 'page reported an error or was not found',
+        });
     }
     const compressed = buildCompressed(extraction.detailedMarkdown, summaryLevel);
-    const metadata = buildMetadata(url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, compressed);
+    const metadata = buildMetadata({
+        url,
+        normalizedUrl,
+        cacheKey,
+        fetchResult,
+        extraction,
+        tier,
+        ttl,
+        currentTime,
+        compressed,
+    });
     return {
         metadata,
         summary: extraction.title,
@@ -113,7 +168,17 @@ function preserveUserMetadata(source, target, rendered) {
 // returns the "refreshed" result. Shared by both refresh paths (conditional-request fallthrough
 // and site-module re-fetch), which differ only in how fetchResult/extraction are obtained.
 function persistRefreshedArtifact(dataDir, meta, fetchResult, extraction, currentTime, options, siteFetch) {
-    const refreshed = createArtifactFromFetch(meta.source_url, meta.normalized_url, meta.cache_key, fetchResult, extraction, meta.tier, options.ttlOverride || meta.ttl, currentTime, options.summaryLevel);
+    const refreshed = createArtifactFromFetch({
+        url: meta.source_url,
+        normalizedUrl: meta.normalized_url,
+        cacheKey: meta.cache_key,
+        fetchResult,
+        extraction,
+        tier: meta.tier,
+        ttl: options.ttlOverride || meta.ttl,
+        currentTime,
+        summaryLevel: options.summaryLevel,
+    });
     preserveUserMetadata(meta, refreshed, options.rendered);
     // A site-module refresh reports how it actually captured this time; that overrides the
     // carried-over provenance, so a withdrawn .md twin can't leave a stale source_doc_url behind.
@@ -122,8 +187,8 @@ function persistRefreshedArtifact(dataDir, meta, fetchResult, extraction, curren
     // Back-fill keyword tags when the carried-over set is empty (e.g. an artifact first cached before
     // auto-tagging, or one originally stored without tags), so refreshing keeps it searchable.
     applyAutoTags(refreshed);
-    writeArtifact(dataDir, meta.cache_key, refreshed);
-    return { status: 'refreshed', artifact: refreshed };
+    const write = persistRevalidated(dataDir, meta.cache_key, refreshed, options.secure);
+    return { status: 'refreshed', artifact: refreshed, ...write };
 }
 async function handleRevalidateResponse(dataDir, existing, fetchResult, currentTime, options) {
     const meta = existing.metadata;
@@ -136,11 +201,11 @@ async function handleRevalidateResponse(dataDir, existing, fetchResult, currentT
             },
         };
         const staleAfterTime = new Date(currentTime);
-        const { freshWindowMs } = getPolicy(meta.tier, options.ttlOverride || meta.ttl);
+        const { freshWindowMs } = resolveFreshnessPolicy(meta.tier, options.ttlOverride || meta.ttl);
         staleAfterTime.setTime(staleAfterTime.getTime() + freshWindowMs);
         updated.metadata.stale_after = staleAfterTime.toISOString();
-        writeArtifact(dataDir, meta.cache_key, updated);
-        return { status: 'revalidated', artifact: updated };
+        const write = persistRevalidated(dataDir, meta.cache_key, updated, options.secure);
+        return { status: 'revalidated', artifact: updated, ...write };
     }
     const extraction = extractHtmlContent(fetchResult.content, fetchResult.finalUrl);
     return persistRefreshedArtifact(dataDir, meta, fetchResult, extraction, currentTime, options);

package/dist/lib/research/schema.d.ts

@@ -1,11 +1,17 @@
+/** Token counts for the two stored representations of a research artifact. */
 export interface TokenEstimate {
     compressed: number | null;
     detailed: number | null;
 }
+/** All recognized capture methods. Command flag option lists derive from this constant. */
 export declare const CAPTURE_METHODS: readonly ["static_fetch", "browser_fallback", "agent_supplied", "route_markdown", "github_source"];
+/** How a research artifact's content was obtained. */
 export type CaptureMethod = (typeof CAPTURE_METHODS)[number];
+/** All recognized artifact types. `index` = navigation hub or llms.txt; `section` = heading-level child of a page. */
 export declare const ARTIFACT_TYPES: readonly ["source", "research_note", "index", "section"];
+/** Classification of a research artifact's role. */
 export type ArtifactType = (typeof ARTIFACT_TYPES)[number];
+/** Full metadata block stored alongside every cached research artifact. */
 export interface ResearchArtifactMetadata {
     schema_version: number;
     artifact_type: ArtifactType;
@@ -41,6 +47,7 @@ export interface ResearchArtifactMetadata {
     section_anchor: string | null;
     section_heading_path: string | null;
 }
+/** A complete research artifact: metadata, summary, compressed and detailed Markdown, and provenance. */
 export interface ResearchArtifact {
     metadata: ResearchArtifactMetadata;
     summary: string;

package/dist/lib/research/schema.js

@@ -1,5 +1,6 @@
 // Single source of truth for the capture-method enum. Command flag `options` lists derive from this
 // so a new method can never drift out of sync with the filters that are supposed to expose it.
+/** All recognized capture methods. Command flag option lists derive from this constant. */
 export const CAPTURE_METHODS = [
     'static_fetch',
     'browser_fallback',
@@ -9,4 +10,5 @@ export const CAPTURE_METHODS = [
 ];
 // Single source of truth for the artifact-type enum. `index` = navigation/hub or llms.txt site
 // index; `section` = a heading-level child of a page. Command flag `options` derive from this.
+/** All recognized artifact types. `index` = navigation hub or llms.txt; `section` = heading-level child of a page. */
 export const ARTIFACT_TYPES = ['source', 'research_note', 'index', 'section'];

package/dist/lib/research/secret-scan.d.ts

@@ -1,5 +1,11 @@
 import type { ResearchArtifact } from './schema.js';
 /** Return the label of the first secret pattern found in `text`, or null if none match. */
 export declare function detectSecret(text: string): string | null;
+/**
+ * Strip every secret-pattern match from `text`, replacing each with a single space. Matches the
+ * original (case-preserved, unsplit) text, so callers must run this before any lowercasing or
+ * tokenization step that would otherwise hide a pattern like `AKIA…` or `ghp_…` from detection.
+ */
+export declare function stripSecrets(text: string): string;
 /** Scan every content field of an artifact. Returns the detected secret's label, or null. */
 export declare function scanArtifactForSecret(artifact: ResearchArtifact): string | null;

package/dist/lib/research/secret-scan.js

@@ -30,6 +30,14 @@ export function detectSecret(text) {
     }
     return null;
 }
+/**
+ * Strip every secret-pattern match from `text`, replacing each with a single space. Matches the
+ * original (case-preserved, unsplit) text, so callers must run this before any lowercasing or
+ * tokenization step that would otherwise hide a pattern like `AKIA…` or `ghp_…` from detection.
+ */
+export function stripSecrets(text) {
+    return SECRET_PATTERNS.reduce((result, { re }) => result.replace(new RegExp(re.source, `${re.flags}g`), ' '), text);
+}
 /** Scan every content field of an artifact. Returns the detected secret's label, or null. */
 export function scanArtifactForSecret(artifact) {
     const haystack = [artifact.summary, artifact.compressed, artifact.detailed, artifact.provenance]

package/dist/lib/research/secure-write.d.ts

@@ -16,6 +16,10 @@ export interface SecureWriteResult {
  * `dryRun` still runs the secret scan and reports the real would-be destination, but skips the
  * actual `writeArtifact` persist — so a read-only preview never gives a falsely reassuring answer
  * about where secret-bearing content would land.
+ *
+ * On redirect, any existing project copy is archived and removed so lookup cannot keep serving the
+ * project shadow after the secret-bearing content moved to global (revalidation parity with
+ * first-time project writes — see #90 / AUDIT_71_FINAL).
  */
 export declare function writeArtifactSecurely(roots: StoreRoots, key: string, artifact: ResearchArtifact, options?: {
     dryRun?: boolean;

package/dist/lib/research/secure-write.js

@@ -1,5 +1,27 @@
-import { writeArtifact } from './storage.js';
+import { existsSync, readFileSync, writeFileSync, unlinkSync } from 'node:fs';
+import { join } from 'node:path';
+import { writeArtifact, getArtifactPath } from './storage.js';
 import { scanArtifactForSecret } from './secret-scan.js';
+/**
+ * Archive then remove an active project artifact so a redirected global copy is not shadowed by
+ * project→global lookup order. Best-effort: a failed archive/unlink never undoes the global write.
+ *
+ * @see https://nodejs.org/api/fs.html#fsunlinksyncpath
+ */
+function clearProjectArtifactAfterRedirect(projectRoot, key) {
+    const path = getArtifactPath(projectRoot, key);
+    if (!existsSync(path))
+        return;
+    try {
+        const content = readFileSync(path, 'utf-8');
+        const dir = join(projectRoot, 'research');
+        writeFileSync(join(dir, `${key}.superseded.${Date.now()}.md`), content, 'utf-8');
+        unlinkSync(path);
+    }
+    catch (err) {
+        console.warn(`Warning: failed to clear project artifact after secret redirect: ${err.message}`);
+    }
+}
 /**
  * Write an artifact to the configured write root, except when it contains a secret and the
  * target is a (potentially committed) project cache — those are redirected to the global cache.
@@ -8,12 +30,19 @@ import { scanArtifactForSecret } from './secret-scan.js';
  * `dryRun` still runs the secret scan and reports the real would-be destination, but skips the
  * actual `writeArtifact` persist — so a read-only preview never gives a falsely reassuring answer
  * about where secret-bearing content would land.
+ *
+ * On redirect, any existing project copy is archived and removed so lookup cannot keep serving the
+ * project shadow after the secret-bearing content moved to global (revalidation parity with
+ * first-time project writes — see #90 / AUDIT_71_FINAL).
  */
 export function writeArtifactSecurely(roots, key, artifact, options = {}) {
     const isProjectWrite = roots.writeRoot !== roots.globalRoot;
     const secretLabel = isProjectWrite ? scanArtifactForSecret(artifact) : null;
     const dataDir = secretLabel ? roots.globalRoot : roots.writeRoot;
-    if (!options.dryRun)
+    if (!options.dryRun) {
         writeArtifact(dataDir, key, artifact);
+        if (secretLabel)
+            clearProjectArtifactAfterRedirect(roots.writeRoot, key);
+    }
     return { dataDir, redirected: Boolean(secretLabel), secretLabel };
 }

package/dist/lib/research/storage.d.ts

@@ -13,13 +13,10 @@ export declare function getArtifactPath(dataDir: string, key: string): string;
 /**
  * Scans the research storage directory, reading metadata of all valid markdown files
  * to find the active artifact for the given cache key with the newest validated_at timestamp.
- * If a corrupt artifact is found, it is skipped and archived as corrupt.
+ * If a corrupt artifact is found, it is skipped and archived as corrupt — unless `readOnly` is set,
+ * in which case it is only reported, never renamed on disk.
  */
-export declare function findArtifact(dataDir: string, key: string): ResearchArtifact | null;
-/**
- * Checks if a cached artifact exists on disk for a given cache key.
- */
-export declare function hasArtifact(dataDir: string, key: string): boolean;
+export declare function findArtifact(dataDir: string, key: string, readOnly?: boolean): ResearchArtifact | null;
 /**
  * Writes a ResearchArtifact to disk, creating the research storage folder if needed.
  * Writes to a temporary file first, then atomically renames it.
@@ -42,11 +39,15 @@ export interface LocatedArtifact {
  * returning the first match along with where it lives. Implements the project→global
  * read fallback: a missing project entry still resolves against the global cache.
  */
-export declare function locateArtifact(dataDirs: string[], key: string): LocatedArtifact | null;
+export declare function locateArtifact(dataDirs: string[], key: string, readOnly?: boolean): LocatedArtifact | null;
 /**
  * Multi-root variant of {@link scanCacheDir}. Scans the `research/` dir of each data dir in
  * order and deduplicates by cache key so an entry present in an earlier root (project) shadows
  * the same key in a later root (global) — even when the callback filters it out. Composes
  * scanCacheDir so the file-format and read/parse rules stay defined in exactly one place.
+ *
+ * Pass `persistIndex: false` under read-only/plan mode so listing never creates `.search-index.json`.
  */
-export declare function scanCacheDirs<T>(dataDirs: string[], fn: (artifact: ResearchArtifact, filePath: string) => T | null): T[];
+export declare function scanCacheDirs<T>(dataDirs: string[], fn: (artifact: ResearchArtifact, filePath: string) => T | null, options?: {
+    persistIndex?: boolean;
+}): T[];

package/dist/lib/research/storage.js

@@ -18,7 +18,14 @@ export function scanCacheDir(dir, fn) {
             continue;
         try {
             const content = readFileSync(join(dir, file), 'utf-8');
-            const result = fn(parseArtifact(content), join(dir, file));
+            const artifact = parseArtifact(content);
+            // parseArtifact defaults unrecognized frontmatter to a valid-looking shell (cache_key: '',
+            // status: 'active'), so a foreign or otherwise-garbled *.md file with `---`/`---` fences never
+            // throws here — it would otherwise surface as a phantom entry with no usable cache key. Every
+            // artifact Bonsai itself writes always has one, so an empty cache_key means "not ours."
+            if (!artifact.metadata.cache_key)
+                continue;
+            const result = fn(artifact, join(dir, file));
             if (result !== null)
                 results.push(result);
         }
@@ -36,8 +43,9 @@ export function getArtifactPath(dataDir, key) {
     }
     return join(dataDir, 'research', `${key}.md`);
 }
-function processFileForLookup(file, dir, key) {
-    if (!file.endsWith('.md') || file.includes('.tmp')) {
+function processFileForLookup(file, dir, key, readOnly) {
+    // Same file filter as scanCacheDir — superseded/corrupt/tmp archives must not resolve as active.
+    if (!isResearchFile(file)) {
         return null;
     }
     const filePath = join(dir, file);
@@ -50,13 +58,17 @@ function processFileForLookup(file, dir, key) {
     }
     catch (err) {
         console.warn(`Warning: Corrupt research artifact found at "${filePath}": ${err.message}`);
-        try {
-            const corruptPath = `${filePath}.corrupt.${Date.now()}`;
-            renameSync(filePath, corruptPath);
-            console.warn(`Archived corrupt artifact to "${corruptPath}"`);
-        }
-        catch (archiveErr) {
-            console.error(`Failed to archive corrupt artifact: ${archiveErr.message}`);
+        // Archiving renames the file on disk — a write a read-only/plan-mode lookup (e.g. `status`,
+        // `inspect --read-only`) must never perform, even incidentally while diagnosing a corrupt entry.
+        if (!readOnly) {
+            try {
+                const corruptPath = `${filePath}.corrupt.${Date.now()}`;
+                renameSync(filePath, corruptPath);
+                console.warn(`Archived corrupt artifact to "${corruptPath}"`);
+            }
+            catch (archiveErr) {
+                console.error(`Failed to archive corrupt artifact: ${archiveErr.message}`);
+            }
         }
     }
     return null;
@@ -64,9 +76,10 @@ function processFileForLookup(file, dir, key) {
 /**
  * Scans the research storage directory, reading metadata of all valid markdown files
  * to find the active artifact for the given cache key with the newest validated_at timestamp.
- * If a corrupt artifact is found, it is skipped and archived as corrupt.
+ * If a corrupt artifact is found, it is skipped and archived as corrupt — unless `readOnly` is set,
+ * in which case it is only reported, never renamed on disk.
  */
-export function findArtifact(dataDir, key) {
+export function findArtifact(dataDir, key, readOnly = false) {
     const dir = join(dataDir, 'research');
     if (!existsSync(dir)) {
         return null;
@@ -75,7 +88,7 @@ export function findArtifact(dataDir, key) {
     let bestValidatedAt = -1;
     const files = readdirSync(dir);
     for (const file of files) {
-        const artifact = processFileForLookup(file, dir, key);
+        const artifact = processFileForLookup(file, dir, key, readOnly);
         if (!artifact) {
             continue;
         }
@@ -89,17 +102,6 @@ export function findArtifact(dataDir, key) {
     }
     return bestArtifact;
 }
-/**
- * Checks if a cached artifact exists on disk for a given cache key.
- */
-export function hasArtifact(dataDir, key) {
-    try {
-        return findArtifact(dataDir, key) !== null;
-    }
-    catch {
-        return false;
-    }
-}
 /**
  * Writes a ResearchArtifact to disk, creating the research storage folder if needed.
  * Writes to a temporary file first, then atomically renames it.
@@ -148,9 +150,9 @@ export function readArtifact(dataDir, key) {
  * returning the first match along with where it lives. Implements the project→global
  * read fallback: a missing project entry still resolves against the global cache.
  */
-export function locateArtifact(dataDirs, key) {
+export function locateArtifact(dataDirs, key, readOnly = false) {
     for (const dataDir of dataDirs) {
-        const artifact = findArtifact(dataDir, key);
+        const artifact = findArtifact(dataDir, key, readOnly);
         if (artifact) {
             return { artifact, dataDir, path: getArtifactPath(dataDir, key) };
         }
@@ -162,12 +164,16 @@ export function locateArtifact(dataDirs, key) {
  * order and deduplicates by cache key so an entry present in an earlier root (project) shadows
  * the same key in a later root (global) — even when the callback filters it out. Composes
  * scanCacheDir so the file-format and read/parse rules stay defined in exactly one place.
+ *
+ * Pass `persistIndex: false` under read-only/plan mode so listing never creates `.search-index.json`.
  */
-export function scanCacheDirs(dataDirs, fn) {
+export function scanCacheDirs(dataDirs, fn, options = {}) {
     const seen = new Set();
     return dataDirs.flatMap((dataDir) => {
         const researchDir = join(dataDir, 'research');
-        const searchable = loadIndexedArtifactsForDir(researchDir);
+        const searchable = loadIndexedArtifactsForDir(researchDir, {
+            persist: options.persistIndex,
+        });
         return searchable
             .map(({ artifact, filePath }) => {
             const key = artifact.metadata.cache_key;

package/dist/lib/research/summarize.d.ts

@@ -1,5 +1,9 @@
 import type { SummaryLevel } from '../config/schema.js';
+/** Re-export of the configured summary aggressiveness level (`conservative` | `balanced` | `aggressive`). */
 export type { SummaryLevel };
+/**
+ * Options for `summarizeMarkdown`. All fields are optional; omitted values use the documented defaults.
+ */
 export interface SummarizeOptions {
     /** Aggressiveness of prose reduction. Default 'conservative'. */
     level?: SummaryLevel;
@@ -8,10 +12,21 @@ export interface SummarizeOptions {
     /** Prose paragraphs with fewer sentences than this are kept whole. Default 2. */
     minSentences?: number;
 }
+/**
+ * Splits a prose paragraph into individual sentences. Avoids false boundaries inside inline code,
+ * after known abbreviations, and before digits.
+ *
+ * @param paragraph - A single prose paragraph (no Markdown structure like headings or lists).
+ * @returns Array of sentence strings in document order.
+ */
 export declare function splitSentences(paragraph: string): string[];
 /**
  * Extractively shortens the prose paragraphs of a Markdown document while preserving headings,
  * fenced code blocks, tables, and lists verbatim. Returns the input unchanged when it is below
  * `minTokens` or contains no prose to condense. Deterministic for identical input + options.
+ *
+ * @param markdown - Markdown document to summarize.
+ * @param options - Summarization options (level, minTokens, minSentences).
+ * @returns Summarized Markdown, or the original when no summarization is needed.
  */
 export declare function summarizeMarkdown(markdown: string, options?: SummarizeOptions): string;

package/dist/lib/research/summarize.js

@@ -166,6 +166,13 @@ function shouldSkipBoundary(masked, i, end) {
 // Splits a single prose paragraph into sentences, avoiding splits inside inline code, after known
 // abbreviations or single-letter initials, and before digits. Exported for unit testing because
 // sentence boundaries are not observable once kept sentences are rejoined.
+/**
+ * Splits a prose paragraph into individual sentences. Avoids false boundaries inside inline code,
+ * after known abbreviations, and before digits.
+ *
+ * @param paragraph - A single prose paragraph (no Markdown structure like headings or lists).
+ * @returns Array of sentence strings in document order.
+ */
 export function splitSentences(paragraph) {
     const { masked, restore } = maskInlineCode(paragraph);
     const sentences = [];
@@ -381,6 +388,10 @@ function reassemble(segments, sentences, kept) {
  * Extractively shortens the prose paragraphs of a Markdown document while preserving headings,
  * fenced code blocks, tables, and lists verbatim. Returns the input unchanged when it is below
  * `minTokens` or contains no prose to condense. Deterministic for identical input + options.
+ *
+ * @param markdown - Markdown document to summarize.
+ * @param options - Summarization options (level, minTokens, minSentences).
+ * @returns Summarized Markdown, or the original when no summarization is needed.
  */
 export function summarizeMarkdown(markdown, options = {}) {
     const level = options.level ?? 'conservative';

package/dist/lib/research/url.d.ts

@@ -1,11 +1,13 @@
 /**
  * Checks if a literal IPv4 address is in a safe range.
- * Rejects loopback, private RFC1918, and link-local ranges.
+ * Rejects "this network", loopback, private RFC1918, shared address space (CGNAT), and link-local
+ * ranges — the non-globally-reachable blocks in the IANA IPv4 special-purpose registry
+ * (https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml).
  */
 export declare function isSafeIpv4(ip: string): boolean;
 /**
  * Checks if a literal IPv6 address is in a safe range.
- * Rejects loopback, unspecified, link-local, and mapped unsafe IPv4 ranges.
+ * Rejects loopback, unspecified, link-local, unique-local, and mapped unsafe IPv4 ranges.
  */
 export declare function isSafeIpv6(ip: string): boolean;
 /**
@@ -13,10 +15,6 @@ export declare function isSafeIpv6(ip: string): boolean;
  * Returns false for loopback, private, link-local, or unspecified ranges.
  */
 export declare function isSafeIp(ip: string): boolean;
-/**
- * Validates a hostname to ensure it is not blocked (e.g. localhost or literal private IP).
- */
-export declare function validateHostname(hostname: string): void;
 /**
  * Whether the input looks like a URL that simply forgot its `http(s)://` scheme
  * (`example.com`, `docs.nestjs.com/guide`, `192.168.1.1`) rather than unparseable junk.
@@ -40,6 +38,11 @@ export declare function artifactMatchesUrlFilter(meta: {
     source_urls?: string[];
     source_url?: string;
 }, filterUrl: string | undefined): boolean;
+/**
+ * Reject empty/whitespace `--url` globs on list/prune. Almost always a shell-quoting mistake;
+ * matching the empty string silently would look like "no results" instead of bad input.
+ */
+export declare function emptyUrlFilterError(url: string | undefined): string | undefined;
 /**
  * Checks if a filename matches the standard research note naming pattern.
  */

package/dist/lib/research/url.js

@@ -1,7 +1,9 @@
 import { isIP } from 'node:net';
 /**
  * Checks if a literal IPv4 address is in a safe range.
- * Rejects loopback, private RFC1918, and link-local ranges.
+ * Rejects "this network", loopback, private RFC1918, shared address space (CGNAT), and link-local
+ * ranges — the non-globally-reachable blocks in the IANA IPv4 special-purpose registry
+ * (https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml).
  */
 export function isSafeIpv4(ip) {
     const parts = ip.split('.').map(Number);
@@ -10,6 +12,10 @@ export function isSafeIpv4(ip) {
     const [a, b, c] = parts;
     if (a === undefined || b === undefined || c === undefined)
         return false;
+    // "This network" / unspecified: 0.0.0.0/8 (includes 0.0.0.0 itself, a well-known SSRF-filter
+    // bypass — many stacks route it to the loopback interface, same as 127.0.0.1).
+    if (a === 0)
+        return false;
     // Loopback: 127.0.0.0/8
     if (a === 127)
         return false;
@@ -20,6 +26,9 @@ export function isSafeIpv4(ip) {
         return false;
     if (a === 192 && b === 168)
         return false;
+    // Shared Address Space / CGNAT (RFC6598): 100.64.0.0/10
+    if (a === 100 && b >= 64 && b <= 127)
+        return false;
     // Link-local: 169.254.0.0/16
     if (a === 169 && b === 254)
         return false;
@@ -27,7 +36,7 @@ export function isSafeIpv4(ip) {
 }
 /**
  * Checks if a literal IPv6 address is in a safe range.
- * Rejects loopback, unspecified, link-local, and mapped unsafe IPv4 ranges.
+ * Rejects loopback, unspecified, link-local, unique-local, and mapped unsafe IPv4 ranges.
  */
 export function isSafeIpv6(ip) {
     const cleanIp = ip.toLowerCase().trim();
@@ -44,6 +53,13 @@ export function isSafeIpv6(ip) {
         cleanIp.startsWith('feb')) {
         return false;
     }
+    // Unique Local Address (RFC4193): fc00::/7 (fc00 to fdff) — IPv6's RFC1918 equivalent, commonly
+    // used for internal/container networks. Only a full 4-digit first hextet can reach 0xfc00; a
+    // shorter one (leading zeros suppressed) is always < 0x1000 and safely out of range.
+    const firstGroup = cleanIp.split(':')[0] ?? '';
+    if (firstGroup.length === 4 && (firstGroup.startsWith('fc') || firstGroup.startsWith('fd'))) {
+        return false;
+    }
     // IPv4-mapped IPv6: ::ffff:127.0.0.1
     if (cleanIp.startsWith('::ffff:')) {
         const ipv4Part = ip.slice(7);
@@ -65,24 +81,6 @@ export function isSafeIp(ip) {
         return isSafeIpv6(ip);
     return false;
 }
-/**
- * Validates a hostname to ensure it is not blocked (e.g. localhost or literal private IP).
- */
-export function validateHostname(hostname) {
-    const lower = hostname.toLowerCase().trim();
-    if (lower === 'localhost' || lower.endsWith('.localhost')) {
-        throw new Error(`Hostname "${hostname}" is a blocked local target.`);
-    }
-    let ipToCheck = lower;
-    if (ipToCheck.startsWith('[') && ipToCheck.endsWith(']')) {
-        ipToCheck = ipToCheck.slice(1, -1);
-    }
-    if (isIP(ipToCheck) !== 0) {
-        if (!isSafeIp(ipToCheck)) {
-            throw new Error(`IP address "${ipToCheck}" is a blocked local or private target.`);
-        }
-    }
-}
 // Dot-separated, non-empty labels: a real domain (`docs.nestjs.com`) or IPv4 (`192.168.1.1`). The
 // URL parser also accepts a leading-dot filename (`.env`) or relative path (`./x`) as a "host", whose
 // suggestion would be a nonsense `https://.env`; requiring well-formed labels rejects those. Anchored
@@ -133,7 +131,9 @@ export function normalizeUrl(input) {
     if (url.username || url.password) {
         throw new Error('URLs containing usernames or passwords are rejected.');
     }
-    validateHostname(url.hostname);
+    // Localhost / private hosts are valid cache keys (import + cache-hit lookup). SSRF protection
+    // lives in the network layer (fetcher/browser DNS checks), so a later `bonsai http://localhost/…`
+    // can serve an imported note without ever opening a socket — while a cache miss still fails safely.
     // Clear default ports
     if (url.protocol === 'https:' && url.port === '443') {
         url.port = '';
@@ -175,6 +175,18 @@ export function artifactMatchesUrlFilter(meta, filterUrl) {
     const urls = meta.source_urls || (meta.source_url ? [meta.source_url] : []);
     return urls.some((u) => matchesUrlGlob(u, filterUrl));
 }
+/**
+ * Reject empty/whitespace `--url` globs on list/prune. Almost always a shell-quoting mistake;
+ * matching the empty string silently would look like "no results" instead of bad input.
+ */
+export function emptyUrlFilterError(url) {
+    if (url === undefined)
+        return undefined;
+    if (url.trim() === '') {
+        return '--url must be a non-empty glob pattern (e.g. "https://react.dev/*").';
+    }
+    return undefined;
+}
 /**
  * Checks if a filename matches the standard research note naming pattern.
  */