brainbank 0.1.3 → 0.1.4

package/src/providers/vector/hnsw-loader.ts

@@ -0,0 +1,129 @@
+/**
+ * BrainBank — HNSW Loader
+ *
+ * Utilities for persisting and loading HNSW indexes to/from disk.
+ * Used by BrainBank._runInitialize() and PluginContext.loadVectors().
+ *
+ * Includes cross-process write locking and hot-reload support
+ * for multi-process coordination.
+ */
+
+import type { DatabaseAdapter, CountRow } from '@/db/adapter.ts';
+import type { HNSWIndex } from './hnsw-index.ts';
+
+import { dirname, join } from 'node:path';
+import { withLock } from '@/lib/write-lock.ts';
+
+/** Derive the HNSW index file path from the DB path. */
+export function hnswPath(dbPath: string, name: string): string {
+    return join(dirname(dbPath), `hnsw-${name}.index`);
+}
+
+/** Derive the lock directory from the DB path. */
+export function lockDir(dbPath: string): string {
+    return dirname(dbPath);
+}
+
+/** Count rows in a vector table (fast, no data transfer). */
+export function countRows(db: DatabaseAdapter, table: string): number {
+    const row = db.prepare(`SELECT COUNT(*) as c FROM ${table}`).get() as CountRow;
+    return row?.c ?? 0;
+}
+
+/**
+ * Save all HNSW indexes to disk with cross-process file locking.
+ * Prevents concurrent writes from corrupting `.index` files.
+ */
+export async function saveAllHnsw(
+    dbPath: string,
+    kvHnsw: HNSWIndex,
+    sharedHnsw: Map<string, { hnsw: HNSWIndex; vecCache: Map<number, Float32Array> }>,
+    privateHnsw: Map<string, HNSWIndex>,
+): Promise<boolean> {
+    try {
+        await withLock(lockDir(dbPath), 'hnsw', () => {
+            kvHnsw.save(hnswPath(dbPath, 'kv'));
+            for (const [name, { hnsw }] of sharedHnsw) {
+                hnsw.save(hnswPath(dbPath, name));
+            }
+            for (const [name, hnsw] of privateHnsw) {
+                hnsw.save(hnswPath(dbPath, name));
+            }
+        });
+        return true;
+    } catch {
+        // Non-fatal: next startup rebuilds from SQLite (slower).
+        return false;
+    }
+}
+
+/** Load vectors from SQLite into HNSW + cache. */
+export function loadVectors(
+    db: DatabaseAdapter,
+    table: string,
+    idCol: string,
+    hnsw: HNSWIndex,
+    cache: Map<number, Float32Array>,
+): void {
+    const iter = db.prepare(`SELECT ${idCol}, embedding FROM ${table}`).iterate() as IterableIterator<{ embedding: Uint8Array; [key: string]: unknown }>;
+    for (const row of iter) {
+        const vec = new Float32Array(
+            row.embedding.buffer.slice(
+                row.embedding.byteOffset,
+                row.embedding.byteOffset + row.embedding.byteLength,
+            ),
+        );
+        hnsw.add(vec, row[idCol] as number);
+        cache.set(row[idCol] as number, vec);
+    }
+}
+
+/** Populate only the vecCache from SQLite (HNSW already loaded from file). */
+export function loadVecCache(
+    db: DatabaseAdapter,
+    table: string,
+    idCol: string,
+    cache: Map<number, Float32Array>,
+): void {
+    const iter = db.prepare(`SELECT ${idCol}, embedding FROM ${table}`).iterate() as IterableIterator<{ embedding: Uint8Array; [key: string]: unknown }>;
+    for (const row of iter) {
+        const vec = new Float32Array(
+            row.embedding.buffer.slice(
+                row.embedding.byteOffset,
+                row.embedding.byteOffset + row.embedding.byteLength,
+            ),
+        );
+        cache.set(row[idCol] as number, vec);
+    }
+}
+
+/** Deps for reloading a single HNSW index from disk. */
+interface ReloadDeps {
+    dbPath: string;
+    db: DatabaseAdapter;
+    name: string;
+    hnsw: HNSWIndex;
+    vecCache: Map<number, Float32Array>;
+    vectorTable: string;
+    idCol: string;
+}
+
+/**
+ * Reload a single HNSW index from disk after detecting a stale version.
+ * Reinitializes the in-memory HNSW, loads the saved index file, and
+ * refreshes the vector cache from SQLite.
+ */
+export function reloadHnsw(deps: ReloadDeps): void {
+    const { dbPath, db, name, hnsw, vecCache, vectorTable, idCol } = deps;
+    const indexPath = hnswPath(dbPath, name);
+    const rowCount = countRows(db, vectorTable);
+
+    hnsw.reinit();
+    vecCache.clear();
+
+    if (hnsw.tryLoad(indexPath, rowCount)) {
+        loadVecCache(db, vectorTable, idCol, vecCache);
+    } else {
+        loadVectors(db, vectorTable, idCol, hnsw, vecCache);
+    }
+}

package/src/search/bm25-boost.ts

@@ -0,0 +1,76 @@
+/**
+ * BrainBank — BM25 Intersection Boost
+ *
+ * Pure functions for post-processing vector search results with BM25 keyword overlap.
+ * Extracted from ContextBuilder for single-responsibility and testability.
+ */
+
+import type { SearchResult } from '@/types.ts';
+import type { SearchStrategy } from './types.ts';
+
+/** BM25 boost factor applied to vector results that also match keywords. */
+export const BM25_BOOST = 0.15;
+
+/**
+ * Boost vector results that also appear in BM25 keyword results.
+ * Does NOT add new results — only re-scores and re-sorts existing vector hits.
+ * This promotes keyword-relevant files without introducing BM25-only noise.
+ */
+export async function boostWithBM25(
+    vectorResults: SearchResult[],
+    bm25: SearchStrategy,
+    query: string,
+    sources: Record<string, number>,
+): Promise<SearchResult[]> {
+    if (vectorResults.length === 0) return vectorResults;
+
+    const bm25Results = await bm25.search(query, { sources });
+    if (bm25Results.length === 0) return vectorResults;
+
+    // Build a set of BM25 hit keys for fast lookup
+    const bm25Keys = new Set<string>();
+    for (const r of bm25Results) {
+        bm25Keys.add(resultKey(r));
+    }
+
+    // Boost scores of vector results that also appear in BM25
+    const boosted = vectorResults.map(r => {
+        const k = resultKey(r);
+        if (bm25Keys.has(k)) {
+            return { ...r, score: r.score + BM25_BOOST };
+        }
+        return r;
+    });
+
+    // Re-sort by boosted score
+    boosted.sort((a, b) => b.score - a.score);
+    return boosted;
+}
+
+/** Filter results whose filePath starts with any of the given prefixes. */
+export function filterByPath(results: SearchResult[], prefix: string | string[] | undefined): SearchResult[] {
+    if (!prefix) return results;
+    const prefixes = Array.isArray(prefix) ? prefix : [prefix];
+    if (prefixes.length === 0) return results;
+    // Normalize prefixes to ensure directory-boundary matching.
+    // 'sdk' → 'sdk/' so it won't match 'sdk-server/'.
+    // Prefixes already ending with '/' or containing '.' (file match) are kept as-is.
+    const normalized = prefixes.map(p => {
+        if (p.endsWith('/') || p.includes('.')) return p;
+        return p + '/';
+    });
+    return results.filter(r => normalized.some(p => r.filePath?.startsWith(p)));
+}
+
+/** Exclude results whose filePath starts with any of the given prefixes. */
+export function filterByIgnore(results: SearchResult[], ignorePaths: string[] | undefined): SearchResult[] {
+    if (!ignorePaths || ignorePaths.length === 0) return results;
+    return results.filter(r => !r.filePath || !ignorePaths.some(p => r.filePath!.startsWith(p)));
+}
+
+/** Generate a dedup key for a search result (file:startLine:endLine). */
+export function resultKey(r: SearchResult): string {
+    const sl = 'startLine' in r.metadata ? r.metadata.startLine : '';
+    const el = 'endLine' in r.metadata ? r.metadata.endLine : '';
+    return `${r.filePath ?? ''}:${sl}:${el}`;
+}

package/src/search/context-builder.ts

@@ -0,0 +1,209 @@
+/**
+ * BrainBank — Context Builder
+ *
+ * Orchestrates the context-building pipeline:
+ *   1. Vector search (primary)
+ *   2. Path scoping (filter)
+ *   3. LLM noise pruning (optional)
+ *   4. Session dedup (filter)
+ *   5. Plugin formatters (output)
+ *
+ * All search post-processing lives in `bm25-boost.ts`.
+ * Plugin-agnostic — discovers formatters from ContextFormatterPlugin.
+ */
+
+import type { ContextOptions, EmbeddingProvider, Pruner, SearchResult } from '@/types.ts';
+import type { SearchStrategy } from './types.ts';
+import type { PluginRegistry } from '@/services/plugin-registry.ts';
+
+import { isContextFormatterPlugin, isContextFieldPlugin, isSearchable } from '@/plugin.ts';
+import { filterByPath, filterByIgnore } from './bm25-boost.ts';
+import { pruneResults } from '@/lib/prune.ts';
+import { logQuery } from '@/lib/logger.ts';
+import type { QueryLogResult } from '@/lib/logger.ts';
+import { providerKey } from '@/lib/provider-key.ts';
+
+const _debug = !!process.env.BRAINBANK_DEBUG;
+function dbg(msg: string): void { if (_debug) console.error(msg); }
+
+export class ContextBuilder {
+    constructor(
+        private _search: SearchStrategy | undefined,
+        private _registry: PluginRegistry,
+        private _pruner?: Pruner,
+        private _embedding?: EmbeddingProvider,
+        private _configFields: Record<string, unknown> = {},
+    ) {}
+
+    /** Set config-level context field defaults (from config.json "context" section). */
+    set configFields(fields: Record<string, unknown>) {
+        this._configFields = fields;
+    }
+
+    /** Build a full context block for a task. Returns markdown for system prompt. */
+    async build(task: string, options: ContextOptions = {}): Promise<string> {
+        const t0 = Date.now();
+        const src = options.sources ?? {};
+        const { minScore = 0.25, useMMR = true, mmrLambda = 0.7 } = options;
+
+        // 1. Primary: vector search (includes per-repo BM25 fusion internally)
+        let results: SearchResult[] = this._search
+            ? await this._search.search(task, {
+                sources: src,
+                minScore, useMMR, mmrLambda,
+            })
+            : [];
+
+        // 2. Path scoping + ignore filtering
+        results = filterByPath(results, options.pathPrefix);
+        results = filterByIgnore(results, options.ignorePaths);
+
+        // 3. LLM noise pruning (optional — per-request override or construction-time)
+        const pruner = options.pruner ?? this._pruner;
+        const beforePrune = results;
+
+        // 3a. Pre-score filtering: when many results, drop the weakest before LLM pruning
+        //     This keeps the pruner's input manageable and improves accuracy.
+        if (pruner && results.length > 40) {
+            const topScore = Math.max(...results.map(r => r.score));
+            const threshold = topScore * 0.35; // Keep results scoring >= 35% of top
+            const preFiltered = results.filter(r => r.score >= threshold);
+            if (preFiltered.length < results.length && preFiltered.length >= 3) {
+                dbg(`[pre-filter] Dropped ${results.length - preFiltered.length} low-score results (threshold: ${threshold.toFixed(3)}, top: ${topScore.toFixed(3)})`);
+                results = preFiltered;
+            }
+        }
+
+        if (pruner && results.length > 1) {
+            dbg(`[pruner] Running ${_prunerName(pruner)} on ${results.length} results...`);
+            const pruneT0 = Date.now();
+            // Merge context + prunerContext into a single pruner description
+            const prunerDesc = [options.context, options.prunerContext].filter(Boolean).join('\n\n') || undefined;
+            results = await pruneResults(task, results, pruner, prunerDesc);
+            const pruneMs = Date.now() - pruneT0;
+            const dropped = beforePrune.filter(r => !results.includes(r));
+            dbg(`[pruner] ${beforePrune.length} → ${results.length} in ${pruneMs}ms (${dropped.length} dropped)`);
+            if (results.length > 0) {
+                dbg(`[pruner] Kept: ${results.map(r => r.filePath ?? '?').join(', ')}`);
+            }
+            if (dropped.length > 0) {
+                dbg(`[pruner] Dropped: ${dropped.map(r => r.filePath ?? '?').join(', ')}`);
+            }
+        } else if (!pruner) {
+            dbg(`[pruner] No pruner configured — skipping`);
+        } else {
+            dbg(`[pruner] Only ${results.length} result(s) — skipping pruner (need >1)`);
+        }
+
+        // 4. Exclude already-returned files (session dedup)
+        if (options.excludeFiles && options.excludeFiles.size > 0) {
+            results = results.filter(r => !r.filePath || !options.excludeFiles!.has(r.filePath));
+        }
+
+        // 5. Format output
+        const resolvedFields = this._resolveFields(options);
+        const parts: string[] = [`# Context for: "${task}"\n`];
+        this._appendFormatterResults(results, parts, options, resolvedFields);
+        await this._appendSearchableResults(task, src, minScore, parts);
+
+        // ── Log ──
+        const prunedResults = pruner
+            ? beforePrune.filter(r => !results.includes(r))
+            : [];
+        logQuery({
+            source: options.source ?? 'api',
+            method: 'getContext',
+            query: task,
+            embedding: this._embedding ? providerKey(this._embedding) : 'unknown',
+            pruner: pruner ? _prunerName(pruner) : null,
+            expander: null,
+            options: {
+                sources: src,
+                pathPrefix: options.pathPrefix,
+                ignorePaths: options.ignorePaths,
+                minScore,
+                affectedFiles: options.affectedFiles,
+            },
+            results: results.map(_toLogResult),
+            pruned: prunedResults.length > 0 ? prunedResults.map(_toLogResult) : undefined,
+            durationMs: Date.now() - t0,
+        });
+
+        return parts.join('\n');
+    }
+
+    /** Invoke ContextFormatterPlugins. */
+    private _appendFormatterResults(
+        results: SearchResult[],
+        parts: string[],
+        options: ContextOptions,
+        resolvedFields?: Record<string, unknown>,
+    ): void {
+        const fields = resolvedFields ?? this._resolveFields(options);
+        const seenFormatters = new Set<string>();
+
+        for (const mod of this._registry.all) {
+            if (!isContextFormatterPlugin(mod)) continue;
+
+            if (seenFormatters.has(mod.name)) continue;
+            seenFormatters.add(mod.name);
+            mod.formatContext(results, parts, fields);
+        }
+    }
+
+    /**
+     * Resolve context fields: plugin defaults ← config.json ← per-query.
+     * Returns a flat Record with the final value for each field.
+     */
+    private _resolveFields(options: ContextOptions): Record<string, unknown> {
+        // 1. Collect plugin defaults
+        const defaults: Record<string, unknown> = {};
+        for (const mod of this._registry.all) {
+            if (isContextFieldPlugin(mod)) {
+                for (const field of mod.contextFields()) {
+                    defaults[field.name] = field.default;
+                }
+            }
+        }
+
+        // 2. Merge: defaults ← config ← per-query
+        return { ...defaults, ...this._configFields, ...(options.fields ?? {}) };
+    }
+
+    /** Collect results from SearchablePlugins that don't have their own formatter. */
+    private async _appendSearchableResults(
+        task: string,
+        sources: Record<string, number>,
+        minScore: number,
+        parts: string[],
+    ): Promise<void> {
+        for (const mod of this._registry.all) {
+            if (isContextFormatterPlugin(mod)) continue;
+            if (!isSearchable(mod)) continue;
+            const hits = await mod.search(task, { k: sources[mod.name] ?? 6, minScore });
+            if (hits.length > 0) {
+                parts.push(`## ${mod.name}\n`);
+                for (const r of hits) {
+                    parts.push(`- [${Math.round(r.score * 100)}%] ${r.content.slice(0, 200)}`);
+                }
+                parts.push('');
+            }
+        }
+    }
+}
+
+// ── Helpers ──────────────────────────────────────────
+
+function _toLogResult(r: SearchResult): QueryLogResult {
+    const meta = r.metadata as Record<string, unknown> | undefined;
+    return {
+        filePath: r.filePath ?? 'unknown',
+        score: r.score,
+        type: r.type,
+        name: (meta?.name as string | undefined) ?? undefined,
+    };
+}
+
+function _prunerName(pruner: Pruner): string {
+    return pruner.constructor?.name ?? 'custom';
+}

package/src/search/keyword/composite-bm25-search.ts

@@ -0,0 +1,47 @@
+/**
+ * BrainBank — Composite BM25 Search Strategy
+ *
+ * Generic BM25 coordinator that discovers BM25SearchPlugin instances
+ * from the registry and delegates per-source keyword search.
+ */
+
+import type { SearchResult } from '@/types.ts';
+import type { SearchStrategy, SearchOptions } from '@/search/types.ts';
+import type { PluginRegistry } from '@/services/plugin-registry.ts';
+
+import { isBM25SearchPlugin } from '@/plugin.ts';
+
+const DEFAULT_K = 8;
+
+export class CompositeBM25Search implements SearchStrategy {
+    constructor(private _registry: PluginRegistry) {}
+
+    /**
+     * Run BM25 keyword search across all plugins that implement BM25SearchPlugin.
+     * Each plugin searches its own FTS5 tables.
+     */
+    async search(query: string, options: SearchOptions = {}): Promise<SearchResult[]> {
+        const src = options.sources ?? {};
+        const results: SearchResult[] = [];
+
+        for (const plugin of this._registry.all) {
+            if (!isBM25SearchPlugin(plugin)) continue;
+
+            const k = src[plugin.name] ?? DEFAULT_K;
+            if (k <= 0) continue;
+
+            const hits = plugin.searchBM25(query, k);
+            results.push(...hits);
+        }
+
+        return results.sort((a, b) => b.score - a.score);
+    }
+
+    /** Rebuild FTS5 indices across all BM25 plugins. */
+    rebuild(): void {
+        for (const plugin of this._registry.all) {
+            if (!isBM25SearchPlugin(plugin)) continue;
+            plugin.rebuildFTS?.();
+        }
+    }
+}

package/src/search/query-decomposer.ts

@@ -0,0 +1,124 @@
+/**
+ * BrainBank — Query Decomposer
+ *
+ * Uses Haiku to break complex/ambiguous natural language queries into
+ * 2-3 focused sub-queries that each target a specific aspect of the
+ * original intent. This improves embedding coverage for multi-concept
+ * queries where a single embedding vector can't capture all facets.
+ *
+ * Example:
+ *   Input:  "user availability shiftStatus signIn signOut available unavailable"
+ *   Output: [
+ *     "user signIn signOut authentication session methods",
+ *     "shiftStatus isAvailable availability entity fields",
+ *     "responder availability state management workflow"
+ *   ]
+ *
+ * Latency: ~300-500ms (single Haiku call).
+ * Cost: ~0.0001$ per decomposition.
+ */
+
+const DEFAULT_MODEL = 'claude-haiku-4-5-20251001';
+const _debug = !!process.env.BRAINBANK_DEBUG;
+function dbg(msg: string): void { if (_debug) process.stderr.write(`[query-decomposer] ${msg}\n`); }
+
+/** Minimum word count to trigger decomposition. Short queries pass through. */
+const MIN_WORDS_FOR_DECOMPOSITION = 6;
+
+export interface QueryDecomposerOptions {
+    /** Anthropic API key. Falls back to ANTHROPIC_API_KEY env var. */
+    apiKey?: string;
+    /** Model to use. Default: claude-haiku-4-5-20251001 */
+    model?: string;
+}
+
+export class QueryDecomposer {
+    private readonly _apiKey: string;
+    private readonly _model: string;
+
+    constructor(options: QueryDecomposerOptions = {}) {
+        this._apiKey = options.apiKey ?? process.env.ANTHROPIC_API_KEY ?? '';
+        this._model = options.model ?? DEFAULT_MODEL;
+    }
+
+    /**
+     * Decompose a complex query into 2-3 focused sub-queries.
+     * Returns the original query + generated sub-queries.
+     * For simple queries (< 6 words), returns just the original.
+     */
+    async decompose(query: string): Promise<string[]> {
+        // Always include original query
+        const words = query.trim().split(/\s+/);
+        if (words.length < MIN_WORDS_FOR_DECOMPOSITION || !this._apiKey) {
+            dbg(`Skip decomposition: ${words.length} words (min: ${MIN_WORDS_FOR_DECOMPOSITION})`);
+            return [query];
+        }
+
+        try {
+            const prompt =
+                `You are a code search query optimizer. Given a complex search query, decompose it into 2-3 focused sub-queries that each target a DIFFERENT aspect of the original intent.\n\n` +
+                `Original query: "${query}"\n\n` +
+                `Rules:\n` +
+                `- Each sub-query should be 4-8 words, mixing natural language with code identifiers\n` +
+                `- Sub-queries should be COMPLEMENTARY, not overlapping\n` +
+                `- Preserve code identifiers (camelCase, PascalCase) exactly as written\n` +
+                `- Focus on: (1) the core action/method, (2) the data/entity, (3) the workflow/context\n` +
+                `- Return ONLY a JSON array of strings. No explanation.\n\n` +
+                `Example:\n` +
+                `Query: "offer lifecycle arrived started admin behalf responder availability"\n` +
+                `["offer state machine arrived started transition", "admin acting behalf responder proxy", "responder availability isAvailable update"]\n\n` +
+                `Respond with ONLY the JSON array:`;
+
+            const response = await fetch('https://api.anthropic.com/v1/messages', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'x-api-key': this._apiKey,
+                    'anthropic-version': '2023-06-01',
+                },
+                body: JSON.stringify({
+                    model: this._model,
+                    max_tokens: 256,
+                    messages: [{ role: 'user', content: prompt }],
+                }),
+            });
+
+            if (!response.ok) {
+                dbg(`API error: ${response.status} — using original query only`);
+                return [query];
+            }
+
+            const data = await response.json() as {
+                content: { type: string; text: string }[];
+            };
+
+            const text = data.content?.[0]?.text ?? '';
+            dbg(`Raw response: ${text}`);
+
+            // Extract JSON array from response (may be wrapped in ```json ... ```)
+            const match = text.match(/\[[\s\S]*?\]/);
+            if (!match) {
+                dbg(`No JSON array found — using original query only`);
+                return [query];
+            }
+
+            const subQueries = JSON.parse(match[0]) as string[];
+            if (!Array.isArray(subQueries) || subQueries.length === 0) {
+                return [query];
+            }
+
+            // Cap at 3 sub-queries + always include original
+            const result = [query, ...subQueries.slice(0, 3)];
+            dbg(`Decomposed into ${result.length} queries: ${JSON.stringify(result)}`);
+            return result;
+        } catch (err) {
+            dbg(`Error: ${err instanceof Error ? err.message : String(err)} — using original query only`);
+            return [query];
+        }
+    }
+
+    /** Check if the decomposer is available (has API key). */
+    get available(): boolean {
+        return !!this._apiKey;
+    }
+}

package/src/search/types.ts

@@ -0,0 +1,37 @@
+/**
+ * BrainBank — Search Types
+ * 
+ * Shared interface for all search strategies.
+ * Implement SearchStrategy to add a new search backend.
+ */
+
+import type { SearchResult } from '@/types.ts';
+
+/** Any search implementation follows this shape. */
+export interface SearchStrategy {
+    search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
+    /** Rebuild internal indices (e.g. FTS5). Optional. */
+    rebuild?(): void;
+}
+
+/** Pre-embedded vector search for a single domain (code, git, etc.). */
+export interface DomainVectorSearch {
+    /** Search using a pre-computed query vector. Optional queryText enables BM25 fusion. */
+    search(queryVec: Float32Array, k: number, minScore: number, useMMR?: boolean, mmrLambda?: number, queryText?: string): SearchResult[];
+}
+
+export interface SearchOptions {
+    /** Per-source result limits. Built-in: 'code', 'git', 'memory'. Any other key = custom plugin or KV collection. */
+    sources?: Record<string, number>;
+    /** Minimum similarity score. Default: 0.25 */
+    minScore?: number;
+    /** Use MMR for diversity. Default: true */
+    useMMR?: boolean;
+    /** MMR lambda. Default: 0.7 */
+    mmrLambda?: number;
+    /** Caller origin for debug logging. */
+    source?: 'cli' | 'mcp' | 'daemon' | 'api';
+    /** Filter results to files under these path prefixes. */
+    pathPrefix?: string | string[];
+}
+