brainbank 0.1.0-beta.1

package/src/lib/logger.ts

@@ -0,0 +1,126 @@
+/**
+ * BrainBank — Query Debug Logger
+ *
+ * Appends structured, human-readable entries to /tmp/brainbank.log.
+ * Covers all search operations: getContext, search, hybridSearch, searchBM25.
+ * Truncates at 10 MB (keeps the newest half).
+ *
+ * Layer 0 — pure functions, no state.
+ */
+
+import * as fs from 'node:fs';
+
+const LOG_PATH = '/tmp/brainbank.log';
+const MAX_BYTES = 10 * 1024 * 1024; // 10 MB
+
+// ── Public Types ─────────────────────────────────────
+
+export type QuerySource = 'cli' | 'mcp' | 'daemon' | 'api';
+
+export interface QueryLogEntry {
+    source: QuerySource;
+    method: 'getContext' | 'search' | 'hybridSearch' | 'searchBM25';
+    query: string;
+    embedding: string;
+    pruner: string | null;
+    expander?: string | null;
+    expandedCount?: number;
+    options: Record<string, unknown>;
+    results: QueryLogResult[];
+    pruned?: QueryLogResult[];
+    durationMs: number;
+}
+
+export interface QueryLogResult {
+    filePath: string;
+    score: number;
+    type: string;
+    name?: string;
+}
+
+// ── Public API ───────────────────────────────────────
+
+/** Append a query log entry to /tmp/brainbank.log. Never throws. */
+export function logQuery(entry: QueryLogEntry): void {
+    try {
+        _truncateIfNeeded();
+        fs.appendFileSync(LOG_PATH, _formatEntry(entry));
+    } catch {
+        // Logging must never break the app
+    }
+}
+
+// ── Formatting ───────────────────────────────────────
+
+function _formatEntry(e: QueryLogEntry): string {
+    const divider = '═'.repeat(70);
+    const lines: string[] = [
+        '',
+        divider,
+        `[${new Date().toISOString()}] ${e.source.toUpperCase()} · ${e.method}`,
+        `Query: "${e.query}"`,
+        `Embedding: ${e.embedding} | Pruner: ${e.pruner ?? 'none'} | Expander: ${e.expander ?? 'off'}${e.expandedCount ? ` (+${e.expandedCount})` : ''}`,
+    ];
+
+    // Options (sources, path, etc.)
+    const opts = Object.entries(e.options).filter(([, v]) => v !== undefined);
+    if (opts.length > 0) {
+        const parts = opts.map(([k, v]) =>
+            `${k}: ${typeof v === 'object' ? JSON.stringify(v) : String(v)}`
+        );
+        lines.push(parts.join(' | '));
+    }
+
+    lines.push(`Duration: ${e.durationMs}ms`);
+    lines.push('');
+
+    // Results
+    const resultCount = e.results.length;
+    const prunedCount = e.pruned?.length ?? 0;
+    const header = prunedCount > 0
+        ? `Results (${resultCount + prunedCount} → ${resultCount} after pruning):`
+        : `Results (${resultCount}):`;
+    lines.push(header);
+
+    for (let i = 0; i < e.results.length; i++) {
+        const r = e.results[i];
+        const pct = Math.round(r.score * 100);
+        const name = r.name ? `[${r.name}]` : '';
+        lines.push(`  #${String(i + 1).padStart(2)}  ${String(pct).padStart(3)}% ${r.filePath.padEnd(45)} ${name}`);
+    }
+
+    // Pruned items
+    if (e.pruned && e.pruned.length > 0) {
+        lines.push('');
+        lines.push(`Pruned (${e.pruned.length} removed):`);
+        for (const r of e.pruned) {
+            const name = r.name ? `[${r.name}]` : '';
+            lines.push(`  ✗ ${r.filePath.padEnd(45)} ${name}`);
+        }
+    }
+
+    lines.push(divider);
+    lines.push('');
+
+    return lines.join('\n');
+}
+
+// ── Truncation ───────────────────────────────────────
+
+function _truncateIfNeeded(): void {
+    try {
+        const stat = fs.statSync(LOG_PATH);
+        if (stat.size < MAX_BYTES) return;
+
+        // Keep the newest half
+        const content = fs.readFileSync(LOG_PATH, 'utf-8');
+        const half = Math.floor(content.length / 2);
+        // Find the next entry boundary after the midpoint
+        const nextEntry = content.indexOf('\n═', half);
+        if (nextEntry > 0) {
+            fs.writeFileSync(LOG_PATH, '[truncated]\n' + content.slice(nextEntry + 1));
+        }
+    } catch {
+        // File doesn't exist yet — fine
+    }
+}

package/src/lib/math.ts

@@ -0,0 +1,87 @@
+/**
+ * BrainBank — Math Utilities
+ * 
+ * Pure vector math functions for similarity calculations.
+ * No dependencies — works on Float32Array directly.
+ */
+
+/**
+ * Cosine similarity between two vectors.
+ * Assumes vectors are already normalized (unit length).
+ * Returns value between -1.0 and 1.0.
+ */
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+    if (a.length !== b.length) {
+        throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+    }
+    if (a.length === 0) return 0;
+
+    let dot = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+    }
+    return dot;
+}
+
+/**
+ * Full cosine similarity (normalizes first).
+ * Use this when vectors may not be pre-normalized.
+ */
+export function cosineSimilarityFull(a: Float32Array, b: Float32Array): number {
+    if (a.length !== b.length) {
+        throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+    }
+    if (a.length === 0) return 0;
+
+    let dot = 0, normA = 0, normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    const denom = Math.sqrt(normA) * Math.sqrt(normB);
+    return denom === 0 ? 0 : dot / denom;
+}
+
+/**
+ * L2-normalize a vector to unit length.
+ * Returns a new Float32Array.
+ */
+export function normalize(vec: Float32Array): Float32Array {
+    let norm = 0;
+    for (let i = 0; i < vec.length; i++) {
+        norm += vec[i] * vec[i];
+    }
+    norm = Math.sqrt(norm);
+    if (norm === 0) return new Float32Array(vec.length);
+
+    const result = new Float32Array(vec.length);
+    for (let i = 0; i < vec.length; i++) {
+        result[i] = vec[i] / norm;
+    }
+    return result;
+}
+
+/**
+ * Euclidean distance between two vectors.
+ */
+export function euclideanDistance(a: Float32Array, b: Float32Array): number {
+    if (a.length !== b.length) {
+        throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+    }
+    let sum = 0;
+    for (let i = 0; i < a.length; i++) {
+        const d = a[i] - b[i];
+        sum += d * d;
+    }
+    return Math.sqrt(sum);
+}
+
+/**
+ * Convert a Float32Array to a Buffer for SQLite storage.
+ * Handles views with non-zero byteOffset (e.g. from batched embedding output).
+ * Using Buffer.from(vec.buffer) directly is WRONG for views — it copies the entire parent buffer.
+ */
+export function vecToBuffer(vec: Float32Array): Buffer {
+    return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
+}

package/src/lib/provider-key.ts

@@ -0,0 +1,20 @@
+/**
+ * BrainBank — Provider Key
+ *
+ * Infers a stable key from an existing EmbeddingProvider instance.
+ * Lives in lib/ (Layer 0) to avoid db/ → providers/ dependency.
+ */
+
+import type { EmbeddingProvider } from '@/types.ts';
+
+/** Known embedding provider keys. */
+export type EmbeddingKey = 'local' | 'openai' | 'perplexity' | 'perplexity-context';
+
+/** Infer a stable key from an existing provider instance. */
+export function providerKey(p: EmbeddingProvider): EmbeddingKey {
+    const name = p.constructor?.name ?? '';
+    if (name === 'OpenAIEmbedding') return 'openai';
+    if (name === 'PerplexityEmbedding') return 'perplexity';
+    if (name === 'PerplexityContextEmbedding') return 'perplexity-context';
+    return 'local';
+}

package/src/lib/prune.ts

@@ -0,0 +1,71 @@
+/**
+ * BrainBank — Prune Utility
+ *
+ * Bridges SearchResult[] → Pruner.prune() → filtered SearchResult[].
+ * Converts results to lightweight PrunerItems with full content previews,
+ * calls the pruner, and filters out dropped results.
+ *
+ * Content strategy: send the FULL chunk content to the pruner so it can
+ * make informed decisions. A per-item character cap prevents cost blowups
+ * on monster files — when truncated, the middle is kept (imports + core
+ * logic) rather than just the top.
+ */
+
+import type { Pruner, PrunerItem, SearchResult } from '@/types.ts';
+
+/**
+ * Max characters per item sent to the pruner.
+ * ~8K chars ≈ 200-250 lines — enough for the model to understand
+ * the file's purpose without blowing up token budgets.
+ */
+const MAX_PREVIEW_CHARS = 8_000;
+
+/** Run the pruner on search results. Returns results in the order the pruner chose. */
+export async function pruneResults(
+    query: string,
+    results: SearchResult[],
+    pruner: Pruner,
+): Promise<SearchResult[]> {
+    if (results.length <= 1) return results;
+
+    // Map results to items with full content (capped per item)
+    const items: PrunerItem[] = results.map((r, i) => ({
+        id: i,
+        filePath: r.filePath ?? 'unknown',
+        preview: _buildPreview(r.content),
+        metadata: r.metadata as Record<string, unknown>,
+    }));
+
+    const keepIds = await pruner.prune(query, items);
+    const validIds = new Set(Array.from({ length: results.length }, (_, i) => i));
+
+    // Respect the pruner's ordering — map IDs to results in returned order
+    return keepIds
+        .filter(id => validIds.has(id))
+        .map(id => results[id]);
+}
+
+/**
+ * Build a preview from full content.
+ *
+ * If the content fits within MAX_PREVIEW_CHARS, return it as-is.
+ * For oversized content, keep the first half + last quarter with a
+ * "[... N lines omitted ...]" marker — this preserves imports/types
+ * at the top AND exports/key functions that often live at the bottom.
+ */
+function _buildPreview(content: string): string {
+    if (content.length <= MAX_PREVIEW_CHARS) return content;
+
+    const lines = content.split('\n');
+    const totalLines = lines.length;
+
+    // Keep ~60% from top, ~25% from bottom
+    const topCount = Math.floor(totalLines * 0.6);
+    const bottomCount = Math.floor(totalLines * 0.25);
+    const omitted = totalLines - topCount - bottomCount;
+
+    const topPart = lines.slice(0, topCount).join('\n');
+    const bottomPart = lines.slice(totalLines - bottomCount).join('\n');
+
+    return `${topPart}\n\n// [... ${omitted} lines omitted ...]\n\n${bottomPart}`;
+}

package/src/lib/rrf.ts

@@ -0,0 +1,133 @@
+/**
+ * BrainBank — Reciprocal Rank Fusion (RRF)
+ * 
+ * Combines results from multiple search systems (vector + BM25)
+ * using the RRF algorithm: score = Σ 1/(k + rank_i)
+ * 
+ * This is the same algorithm used by Elasticsearch, QMD, and most
+ * production hybrid search systems. Simple but very effective.
+ * 
+ * Reference: Cormack et al., "Reciprocal Rank Fusion outperforms
+ * Condorcet and individual Rank Learning Methods" (2009)
+ */
+
+import type { SearchResult } from '@/types.ts';
+
+/**
+ * Fuse ranked lists from different search systems into a single ranked list.
+ * 
+ * @param resultSets - Arrays of SearchResult from different systems (e.g. vector, BM25)
+ * @param k - Smoothing constant. Default: 60 (standard value). Higher = less emphasis on top ranks.
+ * @param maxResults - Maximum results to return.
+ */
+export function reciprocalRankFusion(
+    resultSets: SearchResult[][],
+    k: number = 60,
+    maxResults: number = 15,
+): SearchResult[] {
+    // Build a map: unique key → { bestResult, rrfScore }
+    const fused = new Map<string, { result: SearchResult; rrfScore: number }>();
+
+    for (const results of resultSets) {
+        for (let rank = 0; rank < results.length; rank++) {
+            const r = results[rank];
+            const key = resultKey(r);
+            const rrfContribution = 1.0 / (k + rank + 1);
+
+            const existing = fused.get(key);
+            if (existing) {
+                existing.rrfScore += rrfContribution;
+                // Keep the result with the higher original score
+                if (r.score > existing.result.score) {
+                    existing.result = { ...r };
+                }
+            } else {
+                fused.set(key, {
+                    result: { ...r },
+                    rrfScore: rrfContribution,
+                });
+            }
+        }
+    }
+
+    // Sort by RRF score descending, normalize, and return
+    const sorted = Array.from(fused.values())
+        .sort((a, b) => b.rrfScore - a.rrfScore)
+        .slice(0, maxResults);
+
+    // Normalize RRF scores to 0..1 range.
+    // Note: A single result always normalizes to 1.0. This is correct for RRF —
+    // the score is relative to the result set, not absolute relevance.
+    // Use minScore filters sparingly with RRF.
+    const maxRRF = sorted[0]?.rrfScore ?? 1;
+    return sorted.map(entry => ({
+        ...entry.result,
+        score: entry.rrfScore / maxRRF,
+        metadata: {
+            ...entry.result.metadata,
+            rrfScore: entry.rrfScore,
+        },
+    }) as SearchResult);
+}
+
+/**
+ * Generate a unique key for a search result to detect duplicates across systems.
+ */
+function resultKey(r: SearchResult): string {
+    switch (r.type) {
+        case 'code':
+            return `code:${r.filePath}:${r.metadata.startLine}-${r.metadata.endLine}`;
+        case 'commit':
+            return `commit:${r.metadata.hash || r.metadata.shortHash}`;
+        case 'document':
+            return `document:${r.filePath ?? ''}:${r.metadata.collection ?? ''}:${r.metadata.seq ?? ''}:${r.content?.slice(0, 80)}`;
+        case 'collection':
+            return `collection:${r.metadata.id ?? r.content?.slice(0, 80)}`;
+    }
+}
+
+/**
+ * Generic RRF that works on any type — no SearchResult required.
+ *
+ * @param lists - Ranked lists from different search systems.
+ * @param keyFn - Returns a stable unique string per item.
+ * @param scoreFn - Extracts the original score from an item.
+ * @param k - Smoothing constant. Default: 60.
+ * @param maxResults - Maximum results to return.
+ */
+export function fuseRankedLists<T>(
+    lists: T[][],
+    keyFn: (item: T) => string,
+    scoreFn: (item: T) => number,
+    k: number = 60,
+    maxResults: number = 15,
+): { item: T; score: number }[] {
+    const fused = new Map<string, { item: T; rrfScore: number; bestScore: number }>();
+
+    for (const list of lists) {
+        for (let rank = 0; rank < list.length; rank++) {
+            const item = list[rank];
+            const key = keyFn(item);
+            const contribution = 1.0 / (k + rank + 1);
+            const score = scoreFn(item);
+
+            const existing = fused.get(key);
+            if (existing) {
+                existing.rrfScore += contribution;
+                if (score > existing.bestScore) {
+                    existing.item = item;
+                    existing.bestScore = score;
+                }
+            } else {
+                fused.set(key, { item, rrfScore: contribution, bestScore: score });
+            }
+        }
+    }
+
+    const sorted = [...fused.values()]
+        .sort((a, b) => b.rrfScore - a.rrfScore)
+        .slice(0, maxResults);
+
+    const maxRRF = sorted[0]?.rrfScore ?? 1;
+    return sorted.map(e => ({ item: e.item, score: e.rrfScore / maxRRF }));
+}

package/src/lib/write-lock.ts

@@ -0,0 +1,108 @@
+/**
+ * BrainBank — Write Lock
+ *
+ * Advisory file lock for cross-process HNSW write exclusion.
+ * Uses `O_CREAT | O_EXCL` for atomic lock creation — works on all OS.
+ * Stale locks (dead PID) are detected and stolen automatically.
+ */
+
+import { openSync, closeSync, unlinkSync, readFileSync, writeFileSync, existsSync, mkdirSync, constants } from 'node:fs';
+import { join } from 'node:path';
+
+/** Max wait time before giving up on acquiring a lock (ms). */
+const MAX_WAIT_MS = 30_000;
+
+/** Initial retry delay (ms), doubled on each retry. */
+const INITIAL_DELAY_MS = 50;
+
+/** Check if a process is alive by sending signal 0. */
+function isProcessAlive(pid: number): boolean {
+    if (isNaN(pid)) return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+/** Resolve the lock file path. */
+function lockPath(lockDir: string, name: string): string {
+    return join(lockDir, `${name}.lock`);
+}
+
+/** Try to create the lock file atomically. Returns true on success. */
+function tryCreateLock(filePath: string): boolean {
+    try {
+        const fd = openSync(filePath, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY);
+        writeFileSync(fd, String(process.pid));
+        closeSync(fd);
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+/**
+ * Acquire an advisory lock. Blocks with exponential backoff if another
+ * process holds the lock. Steals stale locks from dead processes.
+ *
+ * @throws After MAX_WAIT_MS if the lock cannot be acquired.
+ */
+export async function acquireLock(lockDir: string, name: string): Promise<void> {
+    if (!existsSync(lockDir)) {
+        mkdirSync(lockDir, { recursive: true });
+    }
+
+    const fp = lockPath(lockDir, name);
+    let delay = INITIAL_DELAY_MS;
+    let elapsed = 0;
+
+    while (true) {
+        if (tryCreateLock(fp)) return;
+
+        // Lock exists — check if holder is alive
+        try {
+            const content = readFileSync(fp, 'utf-8').trim();
+            const pid = parseInt(content, 10);
+            if (isNaN(pid) || !isProcessAlive(pid)) {
+                // Stale lock (dead or invalid PID) — steal it
+                try { unlinkSync(fp); } catch { /* race: another process stole it first */ }
+                if (tryCreateLock(fp)) return;
+            }
+        } catch {
+            // File gone between check and read — retry
+            if (tryCreateLock(fp)) return;
+        }
+
+        if (elapsed >= MAX_WAIT_MS) {
+            throw new Error(`BrainBank: Could not acquire write lock '${name}' after ${MAX_WAIT_MS}ms. Another process may be indexing.`);
+        }
+
+        await new Promise<void>(r => setTimeout(r, delay));
+        elapsed += delay;
+        delay = Math.min(delay * 2, 2000);
+    }
+}
+
+/** Release an advisory lock. Safe to call even if not held. */
+export function releaseLock(lockDir: string, name: string): void {
+    try {
+        unlinkSync(lockPath(lockDir, name));
+    } catch {
+        // Already released or never acquired — safe to ignore
+    }
+}
+
+/**
+ * Execute a function while holding an advisory lock.
+ * Lock is always released, even on error.
+ */
+export async function withLock<T>(lockDir: string, name: string, fn: () => T | Promise<T>): Promise<T> {
+    await acquireLock(lockDir, name);
+    try {
+        return await fn();
+    } finally {
+        releaseLock(lockDir, name);
+    }
+}