@taurgis/bonsai 1.0.3 → 1.0.4

package/dist/lib/research/docs/section-artifacts.js

@@ -1,6 +1,6 @@
 import { createHash } from 'node:crypto';
 import { join } from 'node:path';
-import { compressMarkdown } from '../compress.js';
+import { buildCompressed } from '../compress.js';
 import { estimateTokens } from '../token-estimate.js';
 import { scanCacheDir, writeArtifact } from '../storage.js';
 import { splitMarkdownSections } from './sections.js';
@@ -17,7 +17,7 @@ function childKey(parentKey, anchor) {
  * Derives section child artifacts from a parent page artifact. Returns [] when the page is short
  * or has too few headings to chunk.
  */
-export function buildSectionArtifacts(parent, currentTime) {
+export function buildSectionArtifacts(parent, currentTime, summaryLevel) {
     if (estimateTokens(parent.detailed) <= SECTION_TOKEN_THRESHOLD)
         return [];
     const sections = splitMarkdownSections(parent.detailed);
@@ -25,7 +25,7 @@ export function buildSectionArtifacts(parent, currentTime) {
     return sections.map((section) => {
         const key = childKey(pm.cache_key, section.anchor);
         const detailed = section.content;
-        const compressed = compressMarkdown(detailed);
+        const compressed = buildCompressed(detailed, summaryLevel);
         return {
             metadata: {
                 ...pm,
@@ -59,8 +59,8 @@ function activeChildrenOf(dataDir, parentKey) {
  * heading no longer exists, so a revalidated parent never leaves orphaned sections active. Returns
  * the number of active section children written.
  */
-export function persistSectionArtifacts(dataDir, parent, currentTime) {
-    const children = buildSectionArtifacts(parent, currentTime);
+export function persistSectionArtifacts(dataDir, parent, currentTime, summaryLevel) {
+    const children = buildSectionArtifacts(parent, currentTime, summaryLevel);
     const freshKeys = new Set(children.map((c) => c.metadata.cache_key));
     for (const orphan of activeChildrenOf(dataDir, parent.metadata.cache_key)) {
         if (!freshKeys.has(orphan.metadata.cache_key)) {

package/dist/lib/research/keywords.d.ts

@@ -0,0 +1,17 @@
+import type { ResearchArtifact } from './schema.js';
+export declare const STOPWORDS: Set<string>;
+export declare function tokenize(text: string): string[];
+/**
+ * Derives up to `max` keyword tags from free-form Markdown/text by term frequency.
+ * Code blocks, inline code, and URLs are stripped first; stopwords, pure numbers, and very short
+ * tokens are dropped; terms appearing in Markdown headings get a frequency boost. Ranking is
+ * deterministic (frequency desc, then alphabetical) so the same input always yields the same tags.
+ */
+export declare function extractKeywords(text: string, max?: number): string[];
+/**
+ * Back-fills keyword tags for an artifact that has none, deriving them from its detailed content and
+ * recording a quality note. No-op when tags are already present, so user- or caller-supplied tags
+ * always win. Mutates and returns the same artifact. This is the single entry point every ingestion
+ * path (import, fetch, revalidation) uses, so the auto-tag policy lives in exactly one place.
+ */
+export declare function applyAutoTags(artifact: ResearchArtifact): ResearchArtifact;

package/dist/lib/research/keywords.js

@@ -0,0 +1,220 @@
+// Dependency-free keyword extraction used to auto-tag imported/fetched research when the caller
+// supplies no tags. This is a deliberately simple term-frequency heuristic.
+// ponytail: TF + stopwords + heading boost. Ceiling: no phrase/entity detection, English-only
+// stopwords. Upgrade path: swap in RAKE/YAKE or an LLM pass behind the same extractKeywords() API.
+const AUTO_TAG_NOTE = 'auto-generated tags via keyword extraction';
+// Common English function words plus a few markdown/doc-noise words. Kept inline (no dependency)
+// because the list is small and stable.
+export const STOPWORDS = new Set([
+    'the',
+    'a',
+    'an',
+    'and',
+    'or',
+    'but',
+    'if',
+    'then',
+    'else',
+    'when',
+    'while',
+    'for',
+    'of',
+    'to',
+    'in',
+    'on',
+    'at',
+    'by',
+    'with',
+    'from',
+    'into',
+    'onto',
+    'as',
+    'is',
+    'are',
+    'was',
+    'were',
+    'be',
+    'been',
+    'being',
+    'am',
+    'do',
+    'does',
+    'did',
+    'doing',
+    'have',
+    'has',
+    'had',
+    'having',
+    'this',
+    'that',
+    'these',
+    'those',
+    'it',
+    'its',
+    'they',
+    'them',
+    'their',
+    'there',
+    'here',
+    'we',
+    'you',
+    'your',
+    'our',
+    'us',
+    'i',
+    'me',
+    'my',
+    'he',
+    'she',
+    'his',
+    'her',
+    'not',
+    'no',
+    'nor',
+    'so',
+    'than',
+    'too',
+    'very',
+    'can',
+    'will',
+    'just',
+    'should',
+    'would',
+    'could',
+    'may',
+    'might',
+    'must',
+    'shall',
+    'about',
+    'above',
+    'after',
+    'again',
+    'against',
+    'all',
+    'any',
+    'because',
+    'before',
+    'below',
+    'between',
+    'both',
+    'down',
+    'during',
+    'each',
+    'few',
+    'further',
+    'how',
+    'more',
+    'most',
+    'other',
+    'over',
+    'own',
+    'same',
+    'some',
+    'such',
+    'only',
+    'out',
+    'up',
+    'off',
+    'where',
+    'which',
+    'who',
+    'whom',
+    'why',
+    'what',
+    'whose',
+    'also',
+    'use',
+    'used',
+    'using',
+    'via',
+    'per',
+    'e.g',
+    'i.e',
+    'etc',
+    'get',
+    'set',
+    'see',
+    'note',
+    'example',
+    'one',
+    'two',
+    'three',
+    'new',
+    'like',
+    'within',
+    'without',
+]);
+// Splits text into candidate tokens, keeping tech-flavoured shapes intact (e.g. "node.js", "c#",
+// "asp.net") while trimming leading/trailing separators left over from sentence punctuation.
+export function tokenize(text) {
+    const raw = text.toLowerCase().match(/[a-z0-9][a-z0-9+#.-]*/g) ?? [];
+    return raw.map((w) => w.replace(/^[.\-]+|[.\-]+$/g, '')).filter(Boolean);
+}
+function cleanNoise(text) {
+    return text
+        .replace(/```[\s\S]*?```/g, ' ')
+        .replace(/`[^`]*`/g, ' ')
+        .replace(/https?:\/\/\S+/g, ' ');
+}
+function getHeadingTerms(cleaned) {
+    const headingTerms = new Set();
+    for (const line of cleaned.split('\n')) {
+        if (/^\s{0,3}#{1,6}\s/.test(line)) {
+            for (const term of tokenize(line)) {
+                headingTerms.add(term);
+            }
+        }
+    }
+    return headingTerms;
+}
+function calculateFrequencies(cleaned, headingTerms) {
+    const counts = new Map();
+    for (const token of tokenize(cleaned)) {
+        if (token.length < 3 || STOPWORDS.has(token) || /^\d+$/.test(token)) {
+            continue;
+        }
+        counts.set(token, (counts.get(token) ?? 0) + 1);
+    }
+    for (const term of headingTerms) {
+        const val = counts.get(term);
+        if (val !== undefined) {
+            counts.set(term, val + 3);
+        }
+    }
+    return counts;
+}
+/**
+ * Derives up to `max` keyword tags from free-form Markdown/text by term frequency.
+ * Code blocks, inline code, and URLs are stripped first; stopwords, pure numbers, and very short
+ * tokens are dropped; terms appearing in Markdown headings get a frequency boost. Ranking is
+ * deterministic (frequency desc, then alphabetical) so the same input always yields the same tags.
+ */
+export function extractKeywords(text, max = 5) {
+    if (!text || !text.trim() || max <= 0)
+        return [];
+    const cleaned = cleanNoise(text);
+    const headingTerms = getHeadingTerms(cleaned);
+    const counts = calculateFrequencies(cleaned, headingTerms);
+    if (counts.size === 0)
+        return [];
+    return [...counts.entries()]
+        .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+        .slice(0, max)
+        .map(([term]) => term);
+}
+/**
+ * Back-fills keyword tags for an artifact that has none, deriving them from its detailed content and
+ * recording a quality note. No-op when tags are already present, so user- or caller-supplied tags
+ * always win. Mutates and returns the same artifact. This is the single entry point every ingestion
+ * path (import, fetch, revalidation) uses, so the auto-tag policy lives in exactly one place.
+ */
+export function applyAutoTags(artifact) {
+    if (artifact.metadata.tags.length > 0)
+        return artifact;
+    const tags = extractKeywords(artifact.detailed);
+    if (tags.length === 0)
+        return artifact;
+    artifact.metadata.tags = tags;
+    artifact.metadata.quality_notes = [...artifact.metadata.quality_notes, AUTO_TAG_NOTE];
+    return artifact;
+}

package/dist/lib/research/revalidate.d.ts

@@ -1,4 +1,5 @@
 import type { ResearchArtifact } from './schema.js';
+import type { SummaryLevel } from '../config/schema.js';
 export interface RevalidationResult {
     status: 'revalidated' | 'refreshed' | 'stale';
     artifact: ResearchArtifact;
@@ -20,7 +21,7 @@ export declare function createArtifactFromFetch(url: string, normalizedUrl: stri
     detailedMarkdown: string;
     confidence: 'high' | 'medium' | 'low';
     qualityNotes: string[];
-}, tier: 'stable' | 'standard' | 'volatile', ttl: string | null, currentTime: Date): ResearchArtifact;
+}, tier: 'stable' | 'standard' | 'volatile', ttl: string | null, currentTime: Date, summaryLevel: SummaryLevel): ResearchArtifact;
 /**
  * Revalidates a stale cache artifact using conditional request headers,
  * falling back to stale serving within the grace period if the remote host is offline.
@@ -29,4 +30,5 @@ export declare function revalidateCache(dataDir: string, existing: ResearchArtif
     allowStale: boolean;
     ttlOverride?: string | null;
     rendered?: boolean;
+    summaryLevel: SummaryLevel;
 }): Promise<RevalidationResult>;

package/dist/lib/research/revalidate.js

@@ -4,7 +4,8 @@ import { fetchStaticHtml } from './fetcher.js';
 import { extractHtmlContent } from './extract.js';
 import { writeArtifact } from './storage.js';
 import { evaluateFreshness, getPolicy } from './freshness.js';
-import { compressMarkdown } from './compress.js';
+import { buildCompressed } from './compress.js';
+import { applyAutoTags } from './keywords.js';
 import { estimateTokens } from './token-estimate.js';
 import { fetchRenderedHtml } from './browser.js';
 import { looksLikeErrorPage } from './docs/validate.js';
@@ -78,14 +79,14 @@ function buildErrorArtifact(url, normalizedUrl, cacheKey, fetchResult, tier, ttl
 /**
  * Helper to construct a ResearchArtifact from a fresh HTML fetch.
  */
-export function createArtifactFromFetch(url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime) {
+export function createArtifactFromFetch(url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, summaryLevel) {
     // A managed platform / SPA can return HTTP 200 but render only a "not found" / "something went
     // wrong" shell. Cache a compact marker for those instead of the full error markdown, so repeat
     // lookups cost a few tokens and revalidation still re-checks the page when the entry goes stale.
     if (looksLikeErrorPage(extraction.detailedMarkdown)) {
         return buildErrorArtifact(url, normalizedUrl, cacheKey, fetchResult, tier, ttl, currentTime, 'page reported an error or was not found');
     }
-    const compressed = compressMarkdown(extraction.detailedMarkdown);
+    const compressed = buildCompressed(extraction.detailedMarkdown, summaryLevel);
     const metadata = buildMetadata(url, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl, currentTime, compressed);
     return {
         metadata,
@@ -111,8 +112,11 @@ function preserveUserMetadata(source, target, rendered) {
 // returns the "refreshed" result. Shared by both refresh paths (conditional-request fallthrough
 // and site-module re-fetch), which differ only in how fetchResult/extraction are obtained.
 function persistRefreshedArtifact(dataDir, meta, fetchResult, extraction, currentTime, options) {
-    const refreshed = createArtifactFromFetch(meta.source_url, meta.normalized_url, meta.cache_key, fetchResult, extraction, meta.tier, options.ttlOverride || meta.ttl, currentTime);
+    const refreshed = createArtifactFromFetch(meta.source_url, meta.normalized_url, meta.cache_key, fetchResult, extraction, meta.tier, options.ttlOverride || meta.ttl, currentTime, options.summaryLevel);
     preserveUserMetadata(meta, refreshed, options.rendered);
+    // Back-fill keyword tags when the carried-over set is empty (e.g. an artifact first cached before
+    // auto-tagging, or one originally stored without tags), so refreshing keeps it searchable.
+    applyAutoTags(refreshed);
     writeArtifact(dataDir, meta.cache_key, refreshed);
     return { status: 'refreshed', artifact: refreshed };
 }

package/dist/lib/research/search-index.d.ts

@@ -0,0 +1,18 @@
+import type { ResearchArtifact } from './schema.js';
+export interface SearchableArtifact {
+    artifact: ResearchArtifact;
+    filePath: string;
+}
+/**
+ * Returns the searchable view of every active artifact in one research dir, using the sidecar index
+ * to skip unchanged files. Re-reads only files whose signature changed, then persists the refreshed
+ * index (best-effort; a write failure never breaks search). Corrupt/unreadable files are skipped —
+ * the real read paths in storage.ts handle archiving them.
+ */
+export declare function loadSearchableArtifactsForDir(researchDir: string): SearchableArtifact[];
+/**
+ * Multi-root variant: scans the `research/` dir of each data dir in order and deduplicates by cache
+ * key so a project entry shadows the same key in the global cache. Mirrors the dedup semantics of
+ * {@link scanCacheDirs} but reads through the per-root sidecar index.
+ */
+export declare function loadSearchableArtifacts(dataDirs: string[]): SearchableArtifact[];

package/dist/lib/research/search-index.js

@@ -0,0 +1,109 @@
+import { join } from 'node:path';
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { atomicWriteFile } from '../atomic-write.js';
+import { parseArtifactShallow } from './artifact.js';
+import { isResearchFile } from './storage.js';
+// Sidecar index that caches the lightweight (metadata + summary + compressed) view of each artifact
+// so repeat searches avoid re-reading and re-parsing every file. It is a derived cache only: a stale
+// or missing index never changes results, because each entry is validated against the file's current
+// signature and re-read on any mismatch.
+const INDEX_FILE = '.search-index.json';
+const INDEX_VERSION = 1;
+// The repo writes artifacts via temp-file + rename, so every rewrite yields a new inode. Combining
+// mtime + size + ino makes any change a guaranteed cache miss even when filesystem mtime resolution
+// is coarse (HFS+/FAT) and a same-size edit lands within the same timestamp tick.
+function fileSignature(stat) {
+    return `${stat.mtimeMs}:${stat.size}:${stat.ino}`;
+}
+function loadIndex(indexPath) {
+    try {
+        const parsed = JSON.parse(readFileSync(indexPath, 'utf-8'));
+        if (parsed && parsed.version === INDEX_VERSION && parsed.entries) {
+            return parsed;
+        }
+    }
+    catch {
+        // Missing or corrupt index: start fresh and rebuild.
+    }
+    return { version: INDEX_VERSION, entries: {} };
+}
+function getOrUpdateIndexEntry(filePath, cached, onChanged) {
+    let sig;
+    try {
+        sig = fileSignature(statSync(filePath));
+    }
+    catch {
+        return undefined;
+    }
+    if (cached && cached.sig === sig) {
+        return cached;
+    }
+    try {
+        const entry = { sig, artifact: parseArtifactShallow(readFileSync(filePath, 'utf-8')) };
+        onChanged();
+        return entry;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Returns the searchable view of every active artifact in one research dir, using the sidecar index
+ * to skip unchanged files. Re-reads only files whose signature changed, then persists the refreshed
+ * index (best-effort; a write failure never breaks search). Corrupt/unreadable files are skipped —
+ * the real read paths in storage.ts handle archiving them.
+ */
+export function loadSearchableArtifactsForDir(researchDir) {
+    if (!existsSync(researchDir))
+        return [];
+    const indexPath = join(researchDir, INDEX_FILE);
+    const previous = loadIndex(indexPath);
+    const entries = {};
+    const results = [];
+    let changed = false;
+    for (const file of readdirSync(researchDir)) {
+        if (!isResearchFile(file))
+            continue;
+        const filePath = join(researchDir, file);
+        const entry = getOrUpdateIndexEntry(filePath, previous.entries[file], () => {
+            changed = true;
+        });
+        if (entry) {
+            entries[file] = entry;
+            results.push({ artifact: entry.artifact, filePath });
+        }
+    }
+    // Files removed since the last run leave dangling entries; drop them so the index can't grow
+    // without bound.
+    if (!changed) {
+        changed = Object.keys(previous.entries).some((key) => !(key in entries));
+    }
+    if (changed) {
+        try {
+            atomicWriteFile(indexPath, JSON.stringify({ version: INDEX_VERSION, entries }));
+        }
+        catch {
+            // Best-effort cache write; search results are already computed without it.
+        }
+    }
+    return results;
+}
+/**
+ * Multi-root variant: scans the `research/` dir of each data dir in order and deduplicates by cache
+ * key so a project entry shadows the same key in the global cache. Mirrors the dedup semantics of
+ * {@link scanCacheDirs} but reads through the per-root sidecar index.
+ */
+export function loadSearchableArtifacts(dataDirs) {
+    const seen = new Set();
+    const all = [];
+    for (const dataDir of dataDirs) {
+        for (const located of loadSearchableArtifactsForDir(join(dataDir, 'research'))) {
+            const key = located.artifact.metadata.cache_key;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            all.push(located);
+        }
+    }
+    return all;
+}

package/dist/lib/research/storage.d.ts

@@ -1,4 +1,5 @@
 import type { ResearchArtifact } from './schema.js';
+export declare function isResearchFile(file: string): boolean;
 /**
  * Scans a cache directory and maps each valid artifact file through a callback.
  * Return null from the callback to skip an entry; all non-null results are collected.

package/dist/lib/research/storage.js

@@ -2,7 +2,7 @@ import { join } from 'node:path';
 import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, renameSync, } from 'node:fs';
 import { atomicWriteFile } from '../atomic-write.js';
 import { parseArtifact, serializeArtifact } from './artifact.js';
-function isResearchFile(file) {
+export function isResearchFile(file) {
     return (file.endsWith('.md') &&
         !file.includes('.tmp') &&
         !file.includes('.corrupt') &&

package/dist/lib/research/summarize.d.ts

@@ -0,0 +1,17 @@
+import type { SummaryLevel } from '../config/schema.js';
+export type { SummaryLevel };
+export interface SummarizeOptions {
+    /** Aggressiveness of prose reduction. Default 'conservative'. */
+    level?: SummaryLevel;
+    /** Below this token count the whole document is returned unchanged. Default 200. */
+    minTokens?: number;
+    /** Prose paragraphs with fewer sentences than this are kept whole. Default 2. */
+    minSentences?: number;
+}
+export declare function splitSentences(paragraph: string): string[];
+/**
+ * Extractively shortens the prose paragraphs of a Markdown document while preserving headings,
+ * fenced code blocks, tables, and lists verbatim. Returns the input unchanged when it is below
+ * `minTokens` or contains no prose to condense. Deterministic for identical input + options.
+ */
+export declare function summarizeMarkdown(markdown: string, options?: SummarizeOptions): string;