brainbank 0.7.0 → 0.8.0

package/src/search/vector/mmr.ts

@@ -0,0 +1,64 @@
+/**
+ * BrainBank — Maximum Marginal Relevance (MMR)
+ * 
+ * Diversifies vector search results to avoid returning redundant items.
+ * λ=1.0 → pure relevance, λ=0.0 → pure diversity.
+ * Default λ=0.7 balances both.
+ */
+
+import type { VectorIndex, SearchHit } from '@/types.ts';
+import { cosineSimilarity } from '@/lib/math.ts';
+
+/**
+ * Search with Maximum Marginal Relevance for diversified results.
+ * 
+ * Algorithm:
+ *   1. Get 3x candidates from HNSW
+ *   2. Greedily select items that maximize: λ * relevance - (1-λ) * max_sim_to_selected
+ */
+export function searchMMR(
+    index: VectorIndex,
+    query: Float32Array,
+    vectorCache: Map<number, Float32Array>,
+    k: number,
+    lambda: number = 0.7,
+): SearchHit[] {
+    // Get more candidates than needed
+    const candidates = index.search(query, k * 3);
+    if (candidates.length <= k) return candidates;
+
+    const selected: SearchHit[] = [];
+    const remaining = [...candidates];
+
+    while (selected.length < k && remaining.length > 0) {
+        let bestScore = -Infinity;
+        let bestIdx = 0;
+
+        for (let i = 0; i < remaining.length; i++) {
+            const relevance = remaining[i].score;
+
+            // Max similarity to any already-selected item
+            let maxSim = 0;
+            for (const sel of selected) {
+                const candidateVec = vectorCache.get(remaining[i].id);
+                const selectedVec = vectorCache.get(sel.id);
+                if (candidateVec && selectedVec) {
+                    maxSim = Math.max(maxSim, cosineSimilarity(candidateVec, selectedVec));
+                }
+            }
+
+            // MMR score: balance relevance vs diversity
+            const mmrScore = lambda * relevance - (1 - lambda) * maxSim;
+
+            if (mmrScore > bestScore) {
+                bestScore = mmrScore;
+                bestIdx = i;
+            }
+        }
+
+        selected.push(remaining[bestIdx]);
+        remaining.splice(bestIdx, 1);
+    }
+
+    return selected;
+}

package/src/services/collection.ts

@@ -0,0 +1,405 @@
+/**
+ * BrainBank — Collection
+ * 
+ * Universal key-value store with vector + BM25 hybrid search.
+ * The foundation primitive — store anything, search semantically.
+ * 
+ *   const errors = brain.collection('debug_errors');
+ *   await errors.add('Fixed null check in api handler', { file: 'api.ts' });
+ *   const hits = await errors.search('null pointer');
+ */
+
+import type { DatabaseAdapter, KvDataRow, CountRow } from '@/db/adapter.ts';
+import type { HNSWIndex } from '@/providers/vector/hnsw-index.ts';
+import type { EmbeddingProvider, Reranker, SearchResult } from '@/types.ts';
+
+import { sanitizeFTS, normalizeBM25 } from '@/lib/fts.ts';
+import { vecToBuffer } from '@/lib/math.ts';
+import { rerank } from '@/lib/rerank.ts';
+import { fuseRankedLists } from '@/lib/rrf.ts';
+
+export interface CollectionItem {
+    id: number;
+    collection: string;
+    content: string;
+    metadata: Record<string, unknown>;
+    tags: string[];
+    createdAt: number;
+    expiresAt?: number;
+    score?: number;
+}
+
+export interface CollectionSearchOptions {
+    /** Max results. Default: 5 */
+    k?: number;
+    /** Search mode. Default: 'hybrid' */
+    mode?: 'hybrid' | 'vector' | 'keyword';
+    /** Minimum score threshold. Default: 0.15 */
+    minScore?: number;
+    /** Filter by tags (item must have ALL specified tags). */
+    tags?: string[];
+}
+
+export interface CollectionAddOptions {
+    /** Metadata key-value pairs. */
+    metadata?: Record<string, unknown>;
+    /** Tags for filtering. */
+    tags?: string[];
+    /** Time-to-live duration string (e.g. '7d', '24h', '30m'). */
+    ttl?: string;
+}
+
+export class Collection {
+    constructor(
+        private _name: string,
+        private _db: DatabaseAdapter,
+        private _embedding: EmbeddingProvider,
+        private _hnsw: HNSWIndex,
+        private _vecs: Map<number, Float32Array>,
+        private _reranker?: Reranker,
+    ) {}
+
+    /** Collection name. */
+    get name(): string { return this._name; }
+
+    /** Add an item. Returns its ID. */
+    async add(content: string, options: CollectionAddOptions | Record<string, unknown> = {}): Promise<number> {
+        // Support both signatures: add(content, { metadata, tags, ttl }) and add(content, metadata)
+        const opts = 'tags' in options || 'ttl' in options || 'metadata' in options
+            ? options as CollectionAddOptions
+            : { metadata: options as Record<string, unknown> };
+
+        const metadata = opts.metadata ?? {};
+        const tags = opts.tags ?? [];
+        const expiresAt = opts.ttl ? Math.floor(Date.now() / 1000) + parseDuration(opts.ttl) : null;
+
+        // Embed FIRST — if this throws, no orphaned rows are left in kv_data
+        const vec = await this._embedding.embed(content);
+
+        const result = this._db.prepare(
+            'INSERT INTO kv_data (collection, content, meta_json, tags_json, expires_at) VALUES (?, ?, ?, ?, ?)'
+        ).run(this._name, content, JSON.stringify(metadata), JSON.stringify(tags), expiresAt);
+
+        const id = Number(result.lastInsertRowid);
+        this._db.prepare(
+            'INSERT INTO kv_vectors (data_id, embedding) VALUES (?, ?)'
+        ).run(id, vecToBuffer(vec));
+
+        this._hnsw.add(vec, id);
+        this._vecs.set(id, vec);
+
+        return id;
+    }
+
+    /** Update an item's content (re-embeds). Returns the new ID. */
+    async update(id: number, content: string, options?: CollectionAddOptions): Promise<number> {
+        const row = this._db.prepare(
+            'SELECT * FROM kv_data WHERE id = ? AND collection = ?'
+        ).get(id, this._name) as KvDataRow | undefined;
+
+        if (!row) throw new Error(`BrainBank: Item ${id} not found in collection '${this._name}'.`);
+
+        // Merge: keep original metadata/tags unless overridden
+        const metadata = options?.metadata ?? JSON.parse(row.meta_json || '{}');
+        const tags = options?.tags ?? JSON.parse(row.tags_json || '[]');
+        const ttl = options?.ttl;
+
+        this._removeById(id);
+        return this.add(content, { metadata, tags, ...(ttl ? { ttl } : {}) });
+    }
+
+    /** Add multiple items. Returns their IDs. */
+    async addMany(items: { content: string; metadata?: Record<string, unknown>; tags?: string[]; ttl?: string }[]): Promise<number[]> {
+        if (items.length === 0) return [];
+
+        // Batch embed all texts at once
+        const texts = items.map(i => i.content);
+        const vecs = await this._embedding.embedBatch(texts);
+
+        // Commit DB rows atomically. HNSW is updated ONLY after this succeeds.
+        // If the transaction throws, execution never reaches the HNSW loop below.
+        const ids: number[] = [];
+        const insertData = this._db.prepare(
+            'INSERT INTO kv_data (collection, content, meta_json, tags_json, expires_at) VALUES (?, ?, ?, ?, ?)'
+        );
+        const insertVec = this._db.prepare(
+            'INSERT INTO kv_vectors (data_id, embedding) VALUES (?, ?)'
+        );
+
+        this._db.transaction(() => {
+            for (let i = 0; i < items.length; i++) {
+                const item = items[i];
+                const expiresAt = item.ttl ? Math.floor(Date.now() / 1000) + parseDuration(item.ttl) : null;
+
+                const result = insertData.run(
+                    this._name,
+                    item.content,
+                    JSON.stringify(item.metadata ?? {}),
+                    JSON.stringify(item.tags ?? []),
+                    expiresAt,
+                );
+
+                const id = Number(result.lastInsertRowid);
+                insertVec.run(id, vecToBuffer(vecs[i]));
+                ids.push(id);
+            }
+        });
+
+        // HNSW + cache updated after successful commit — no orphan risk on rollback.
+        for (let i = 0; i < ids.length; i++) {
+            this._hnsw.add(vecs[i], ids[i]);
+            this._vecs.set(ids[i], vecs[i]);
+        }
+
+        return ids;
+    }
+
+    /** Search this collection. */
+    async search(query: string, options: CollectionSearchOptions = {}): Promise<CollectionItem[]> {
+        const { k = 5, mode = 'hybrid', minScore = 0.15, tags } = options;
+
+        // Auto-prune expired items before search
+        this._pruneExpired();
+
+        if (mode === 'keyword') return this._filterByTags(this._searchBM25(query, k, minScore), tags);
+        if (mode === 'vector') return this._filterByTags(await this._searchVector(query, k, minScore), tags);
+
+        // Hybrid: vector + BM25 → generic RRF (no SearchResult conversion)
+        const [vectorHits, bm25Hits] = await Promise.all([
+            this._searchVector(query, k, 0),
+            Promise.resolve(this._searchBM25(query, k, 0)),
+        ]);
+
+        const fused = fuseRankedLists(
+            [vectorHits, bm25Hits],
+            h => String(h.id),
+            h => h.score ?? 0,
+        );
+
+        const results: CollectionItem[] = fused
+            .map(({ item, score }) => ({ ...item, score }))
+            .filter(r => r.score >= minScore)
+            .slice(0, k);
+
+        // Apply re-ranking if available
+        if (this._reranker && results.length > 1) {
+            const asSearchResults: SearchResult[] = results.map(r => ({
+                type: 'collection' as const,
+                score: r.score ?? 0,
+                content: r.content,
+                metadata: { id: r.id },
+            }));
+            const reranked = await rerank(query, asSearchResults, this._reranker);
+            const rerankedById = new Map(
+                reranked.map(r => [r.type === 'collection' ? r.metadata.id : undefined, r.score]),
+            );
+            const blended = results.map(r => ({ ...r, score: rerankedById.get(r.id) ?? r.score ?? 0 }));
+            return this._filterByTags(
+                blended.sort((a, b) => (b.score ?? 0) - (a.score ?? 0)),
+                tags,
+            );
+        }
+
+        return this._filterByTags(results, tags);
+    }
+
+    /** Search and return results as SearchResult[] for use in hybrid search pipelines. */
+    async searchAsResults(query: string, k: number): Promise<SearchResult[]> {
+        const hits = await this.search(query, { k });
+        return hits.map(h => ({
+            type: 'collection' as const,
+            score: h.score ?? 0,
+            content: h.content,
+            metadata: { ...h.metadata, id: h.id, collection: this._name },
+        }));
+    }
+
+    /** List items (newest first). */
+    list(options: { limit?: number; offset?: number; tags?: string[] } = {}): CollectionItem[] {
+        const { limit = 20, offset = 0, tags } = options;
+
+        // Auto-prune expired items
+        this._pruneExpired();
+
+        const rows = this._db.prepare(
+            'SELECT * FROM kv_data WHERE collection = ? AND (expires_at IS NULL OR expires_at > ?) ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?'
+        ).all(this._name, Math.floor(Date.now() / 1000), limit, offset) as KvDataRow[];
+        return this._filterByTags(rows.map(r => this._rowToItem(r)), tags);
+    }
+
+    /** Count items in this collection. */
+    count(): number {
+        return (this._db.prepare(
+            'SELECT COUNT(*) as c FROM kv_data WHERE collection = ? AND (expires_at IS NULL OR expires_at > ?)'
+        ).get(this._name, Math.floor(Date.now() / 1000)) as CountRow).c;
+    }
+
+    /** Keep only the N most recent items, remove the rest. */
+    async trim(options: { keep: number }): Promise<{ removed: number }> {
+        const before = this.count();
+        if (before <= options.keep) return { removed: 0 };
+
+        // Get IDs to remove (oldest first, beyond the keep window)
+        const toRemove = this._db.prepare(`
+            SELECT id FROM kv_data 
+            WHERE collection = ? 
+            ORDER BY created_at DESC, id DESC 
+            LIMIT -1 OFFSET ?
+        `).all(this._name, options.keep) as Pick<KvDataRow, 'id'>[];
+
+        for (const row of toRemove) {
+            this._removeById(row.id);
+        }
+
+        return { removed: toRemove.length };
+    }
+
+    /** Remove items older than a duration string (e.g. '30d', '12h'). */
+    async prune(options: { olderThan: string }): Promise<{ removed: number }> {
+        const seconds = parseDuration(options.olderThan);
+        const cutoff = Math.floor(Date.now() / 1000) - seconds;
+
+        const toRemove = this._db.prepare(
+            'SELECT id FROM kv_data WHERE collection = ? AND created_at < ?'
+        ).all(this._name, cutoff) as Pick<KvDataRow, 'id'>[];
+
+        for (const row of toRemove) {
+            this._removeById(row.id);
+        }
+
+        return { removed: toRemove.length };
+    }
+
+    /** Remove a specific item by ID. */
+    remove(id: number): void {
+        this._removeById(id);
+    }
+
+    /** Clear all items in this collection. */
+    clear(): void {
+        const rows = this._db.prepare(
+            'SELECT id FROM kv_data WHERE collection = ?'
+        ).all(this._name) as Pick<KvDataRow, 'id'>[];
+
+        for (const row of rows) {
+            this._removeById(row.id);
+        }
+    }
+
+
+    private _removeById(id: number): void {
+        // DB first — can fail (disk full, lock). If it throws, HNSW+cache stay consistent.
+        this._db.prepare('DELETE FROM kv_data WHERE id = ?').run(id);
+        // HNSW + cache after — these always succeed
+        this._hnsw.remove(id);
+        this._vecs.delete(id);
+    }
+
+    private async _searchVector(query: string, k: number, minScore: number): Promise<CollectionItem[]> {
+        if (this._hnsw.size === 0) return [];
+
+        const queryVec = await this._embedding.embed(query);
+        // Adaptive over-fetch: proportional to total/collection density, clamped [3, 50]
+        const searchK = this._adaptiveSearchK(k);
+        const hits = this._hnsw.search(queryVec, searchK);
+
+        const ids = hits.map(h => h.id);
+        if (ids.length === 0) return [];
+
+        const scoreMap = new Map(hits.map(h => [h.id, h.score]));
+        const placeholders = ids.map(() => '?').join(',');
+
+        const rows = this._db.prepare(
+            `SELECT * FROM kv_data WHERE id IN (${placeholders}) AND collection = ?`
+        ).all(...ids, this._name) as KvDataRow[];
+
+        return rows
+            .map(r => ({ ...this._rowToItem(r), score: scoreMap.get(r.id) ?? 0 }))
+            .filter(r => r.score >= minScore)
+            .sort((a, b) => (b.score ?? 0) - (a.score ?? 0))
+            .slice(0, k);
+    }
+
+    /** Compute adaptive over-fetch multiplier based on collection density in shared HNSW. */
+    private _adaptiveSearchK(k: number): number {
+        const totalSize = this._hnsw.size;
+        if (totalSize === 0) return 0;
+        const collectionCount = this.count();
+        if (collectionCount === 0) return Math.min(k * 3, totalSize);
+        const ratio = Math.ceil(totalSize / collectionCount);
+        const multiplier = Math.max(3, Math.min(ratio, 50));
+        return Math.min(k * multiplier, totalSize);
+    }
+
+    private _searchBM25(query: string, k: number, minScore: number): CollectionItem[] {
+        const ftsQuery = sanitizeFTS(query);
+        if (!ftsQuery) return [];
+
+        try {
+            const rows = this._db.prepare(`
+                SELECT d.*, bm25(fts_kv, 5.0, 1.0) AS score
+                FROM fts_kv f
+                JOIN kv_data d ON d.id = f.rowid
+                WHERE fts_kv MATCH ? AND d.collection = ?
+                ORDER BY score ASC
+                LIMIT ?
+            `).all(ftsQuery, this._name, k) as (KvDataRow & { score: number })[];
+
+            return rows
+                .map(r => ({
+                    ...this._rowToItem(r),
+                    score: normalizeBM25(r.score),
+                }))
+                .filter(r => (r.score ?? 0) >= minScore);
+        } catch {
+            return [];
+        }
+    }
+
+    private _rowToItem(r: KvDataRow): CollectionItem {
+        return {
+            id: r.id,
+            collection: r.collection,
+            content: r.content,
+            metadata: JSON.parse(r.meta_json || '{}') as Record<string, unknown>,
+            tags: JSON.parse(r.tags_json || '[]') as string[],
+            createdAt: r.created_at,
+            expiresAt: r.expires_at ?? undefined,
+        };
+    }
+
+    /** Filter results by tags (item must have ALL specified tags). */
+    private _filterByTags(items: CollectionItem[], tags?: string[]): CollectionItem[] {
+        if (!tags || tags.length === 0) return items;
+        return items.filter(item =>
+            tags.every(t => item.tags.includes(t))
+        );
+    }
+
+    /** Remove expired items (TTL). Called automatically on search/list. */
+    private _pruneExpired(): void {
+        const now = Math.floor(Date.now() / 1000);
+        const expired = this._db.prepare(
+            'SELECT id FROM kv_data WHERE collection = ? AND expires_at IS NOT NULL AND expires_at <= ?'
+        ).all(this._name, now) as Pick<KvDataRow, 'id'>[];
+
+        for (const row of expired) {
+            this._removeById(row.id);
+        }
+    }
+}
+
+/** Parse a duration string like '30d', '12h', '5m' to seconds. */
+function parseDuration(s: string): number {
+    const match = s.match(/^(\d+)([dhms])$/);
+    if (!match) throw new Error(`Invalid duration: "${s}". Use format like '30d', '12h', '5m'.`);
+
+    const n = parseInt(match[1], 10);
+    switch (match[2]) {
+        case 'd': return n * 86400;
+        case 'h': return n * 3600;
+        case 'm': return n * 60;
+        case 's': return n;
+        default: return n;
+    }
+}

package/src/services/daemon.ts

@@ -0,0 +1,87 @@
+/**
+ * Daemon — PID file management for the BrainBank HTTP server.
+ *
+ * PID file: ~/.cache/brainbank/server.pid
+ * Format: JSON { pid: number, port: number }
+ *
+ * Used by:
+ *   - `brainbank daemon` to write PID on start
+ *   - `brainbank daemon stop` to find and kill the daemon
+ *   - `brainbank status` to report server state
+ *   - CLI commands to detect running server for delegation
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+
+export const DEFAULT_PORT = 8181;
+
+interface PidInfo {
+    pid: number;
+    port: number;
+}
+
+/** Directory for PID file and other cache data. */
+function cacheDir(): string {
+    return path.join(os.homedir(), '.cache', 'brainbank');
+}
+
+/** Full path to the PID file. */
+function pidPath(): string {
+    return path.join(cacheDir(), 'server.pid');
+}
+
+/** Write the PID file after server starts. */
+export function writePid(pid: number, port: number): void {
+    const dir = cacheDir();
+    fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(pidPath(), JSON.stringify({ pid, port } as PidInfo));
+}
+
+/** Read PID info from file. Returns null if missing or malformed. */
+export function readPid(): PidInfo | null {
+    try {
+        const raw = fs.readFileSync(pidPath(), 'utf8');
+        const info = JSON.parse(raw) as PidInfo;
+        if (typeof info.pid !== 'number' || typeof info.port !== 'number') return null;
+        return info;
+    } catch {
+        return null;
+    }
+}
+
+/** Remove the PID file. */
+export function removePid(): void {
+    try {
+        fs.unlinkSync(pidPath());
+    } catch {
+        // File doesn't exist — safe to ignore
+    }
+}
+
+/**
+ * Check if the server process is still alive.
+ * Uses `kill(pid, 0)` which doesn't actually send a signal —
+ * it just checks whether the process exists.
+ */
+export function isServerRunning(): PidInfo | null {
+    const info = readPid();
+    if (!info) return null;
+
+    try {
+        process.kill(info.pid, 0);
+        return info;
+    } catch {
+        // Process not running — stale PID file
+        removePid();
+        return null;
+    }
+}
+
+/** Get the server URL if running, or null. */
+export function getServerUrl(): string | null {
+    const info = isServerRunning();
+    if (!info) return null;
+    return `http://localhost:${info.port}`;
+}