brainbank 0.1.0-beta.1

package/src/db/sqlite-adapter.ts

@@ -0,0 +1,218 @@
+/**
+ * BrainBank — SQLite Adapter
+ *
+ * Implements `DatabaseAdapter` using Node.js built-in `node:sqlite`.
+ * Zero native addons — no ABI issues across Node versions.
+ * Handles WAL mode, directory creation, schema init, and transactions.
+ */
+
+import type { DatabaseAdapter, AdapterCapabilities, PreparedStatement, ExecuteResult } from './adapter.ts';
+import type { DatabaseSync as DatabaseSyncType, StatementSync as StatementSyncType } from 'node:sqlite';
+
+import { DatabaseSync } from 'node:sqlite';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+
+// ── Schema ──────────────────────────────────────────────────────────
+
+export const SCHEMA_VERSION = 9;
+
+/**
+ * Create core tables and indices.
+ * Safe to call multiple times — uses IF NOT EXISTS.
+ * Domain tables are created by plugins via runPluginMigrations().
+ */
+function createSchema(adapter: DatabaseAdapter): void {
+    adapter.exec(`
+        -- ── Schema versioning ──────────────────────────
+        CREATE TABLE IF NOT EXISTS schema_version (
+            version     INTEGER PRIMARY KEY,
+            applied_at  INTEGER NOT NULL DEFAULT (unixepoch())
+        );
+        INSERT OR IGNORE INTO schema_version (version) VALUES (${SCHEMA_VERSION});
+
+        -- ── Plugin Versions (migration tracking) ──────
+        CREATE TABLE IF NOT EXISTS plugin_versions (
+            plugin_name TEXT    PRIMARY KEY,
+            version     INTEGER NOT NULL,
+            applied_at  INTEGER NOT NULL DEFAULT (unixepoch())
+        );
+
+        -- ── Dynamic Collections (KV Store) ───────────
+        CREATE TABLE IF NOT EXISTS kv_data (
+            id          INTEGER PRIMARY KEY AUTOINCREMENT,
+            collection  TEXT    NOT NULL,
+            content     TEXT    NOT NULL,
+            meta_json   TEXT    NOT NULL DEFAULT '{}',
+            tags_json   TEXT    NOT NULL DEFAULT '[]',
+            expires_at  INTEGER,
+            created_at  INTEGER NOT NULL DEFAULT (unixepoch())
+        );
+
+        CREATE TABLE IF NOT EXISTS kv_vectors (
+            data_id   INTEGER PRIMARY KEY REFERENCES kv_data(id) ON DELETE CASCADE,
+            embedding BLOB    NOT NULL
+        );
+
+        CREATE VIRTUAL TABLE IF NOT EXISTS fts_kv USING fts5(
+            content,
+            collection,
+            content='kv_data',
+            content_rowid='id',
+            tokenize='porter unicode61'
+        );
+
+        CREATE TRIGGER IF NOT EXISTS trg_fts_kv_insert AFTER INSERT ON kv_data BEGIN
+            INSERT INTO fts_kv(rowid, content, collection)
+            VALUES (new.id, new.content, new.collection);
+        END;
+        CREATE TRIGGER IF NOT EXISTS trg_fts_kv_delete AFTER DELETE ON kv_data BEGIN
+            INSERT INTO fts_kv(fts_kv, rowid, content, collection)
+            VALUES ('delete', old.id, old.content, old.collection);
+        END;
+
+        CREATE INDEX IF NOT EXISTS idx_kv_collection ON kv_data(collection);
+        CREATE INDEX IF NOT EXISTS idx_kv_created    ON kv_data(created_at DESC);
+
+        -- ── Embedding Metadata ───────────────────────
+        CREATE TABLE IF NOT EXISTS embedding_meta (
+            key     TEXT PRIMARY KEY,
+            value   TEXT NOT NULL
+        );
+
+        -- ── Index State (cross-process coordination) ─
+        CREATE TABLE IF NOT EXISTS index_state (
+            name       TEXT    PRIMARY KEY,
+            version    INTEGER NOT NULL DEFAULT 0,
+            writer_pid INTEGER NOT NULL DEFAULT 0,
+            updated_at INTEGER NOT NULL DEFAULT (unixepoch())
+        );
+
+        -- ── Plugin Tracking (incremental indexing) ────
+        CREATE TABLE IF NOT EXISTS plugin_tracking (
+            plugin       TEXT NOT NULL,
+            key          TEXT NOT NULL,
+            content_hash TEXT NOT NULL,
+            indexed_at   INTEGER NOT NULL DEFAULT (unixepoch()),
+            PRIMARY KEY (plugin, key)
+        );
+    `);
+}
+
+/** Get the current schema version from the database. */
+export function getSchemaVersion(adapter: DatabaseAdapter): number {
+    try {
+        const row = adapter.prepare('SELECT MAX(version) as v FROM schema_version').get() as { v: number } | undefined;
+        return row?.v ?? 0;
+    } catch {
+        return 0;
+    }
+}
+
+
+// ── Statement Wrapper ───────────────────────────────────────────────
+
+/** SQLite parameter type accepted by node:sqlite. */
+type SqlParam = string | number | bigint | null | Uint8Array;
+
+/** Wraps a `node:sqlite` StatementSync into a `PreparedStatement<T>`. */
+function wrapStatement<T>(stmt: StatementSyncType): PreparedStatement<T> {
+    return {
+        get(...params: unknown[]): T | undefined {
+            return stmt.get(...(params as SqlParam[])) as T | undefined;
+        },
+        all(...params: unknown[]): T[] {
+            return stmt.all(...(params as SqlParam[])) as T[];
+        },
+        run(...params: unknown[]): ExecuteResult {
+            const info = stmt.run(...(params as SqlParam[]));
+            return {
+                lastInsertRowid: info.lastInsertRowid,
+                changes: Number(info.changes),
+            };
+        },
+        iterate(...params: unknown[]): IterableIterator<T> {
+            return stmt.iterate(...(params as SqlParam[])) as IterableIterator<T>;
+        },
+    };
+}
+
+
+// ── SQLiteAdapter ───────────────────────────────────────────────────
+
+export class SQLiteAdapter implements DatabaseAdapter {
+    private _db: DatabaseSyncType;
+
+    readonly capabilities: AdapterCapabilities = {
+        fts: 'fts5',
+        upsert: 'or-replace',
+        json: true,
+        vectors: false,
+    };
+
+    constructor(dbPath: string) {
+        // Ensure parent directory exists
+        const dir = path.dirname(dbPath);
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+
+        this._db = new DatabaseSync(dbPath);
+        this._db.exec('PRAGMA journal_mode = WAL');
+        this._db.exec('PRAGMA busy_timeout = 5000');
+        this._db.exec('PRAGMA synchronous = NORMAL');
+        this._db.exec('PRAGMA foreign_keys = ON');
+
+        // Initialize schema
+        createSchema(this);
+    }
+
+    /** Prepare a reusable statement. */
+    prepare<T = unknown>(sql: string): PreparedStatement<T> {
+        return wrapStatement<T>(this._db.prepare(sql));
+    }
+
+    /** Execute raw SQL (no results). */
+    exec(sql: string): void {
+        this._db.exec(sql);
+    }
+
+    /** Run a function inside a transaction. Auto-commits on success, auto-rollbacks on error. */
+    transaction<T>(fn: () => T): T {
+        this._db.exec('BEGIN');
+        try {
+            const result = fn();
+            this._db.exec('COMMIT');
+            return result;
+        } catch (err) {
+            this._db.exec('ROLLBACK');
+            throw err;
+        }
+    }
+
+    /** Run a prepared statement on multiple rows. Wraps in a single transaction. */
+    batch<T extends unknown[]>(sql: string, rows: T[]): void {
+        const stmt = this._db.prepare(sql);
+        this.transaction(() => {
+            for (const row of rows) {
+                stmt.run(...(row as SqlParam[]));
+            }
+        });
+    }
+
+    /** Close the database. */
+    close(): void {
+        this._db.close();
+    }
+
+    /**
+     * Access the underlying `node:sqlite` DatabaseSync instance.
+     *
+     * @deprecated Use `DatabaseAdapter` methods instead. This exists
+     * only for gradual migration of plugins that depend on driver internals.
+     */
+    raw<T = unknown>(): T {
+        return this._db as unknown as T;
+    }
+}

package/src/db/tracker.ts

@@ -0,0 +1,91 @@
+/**
+ * BrainBank — Incremental Tracker
+ *
+ * Standardized helper for plugins to detect add/update/delete during indexing.
+ * Uses a shared `plugin_tracking` table with per-plugin namespacing.
+ *
+ * Usage in a plugin:
+ *   const tracker = ctx.createTracker();  // uses plugin name
+ *   for (const file of files) {
+ *       const hash = sha256(content);
+ *       if (tracker.isUnchanged(file, hash)) { skipped++; continue; }
+ *       indexFile(file, content);
+ *       tracker.markIndexed(file, hash);
+ *   }
+ *   const orphans = tracker.findOrphans(new Set(files));
+ *   for (const key of orphans) { removeData(key); tracker.remove(key); }
+ */
+
+import type { DatabaseAdapter } from './adapter.ts';
+
+/** Incremental index tracker — detects add/update/delete for plugin files. */
+export interface IncrementalTracker {
+    /** Check if a key's content is unchanged. Returns true if the hash matches (skip indexing). */
+    isUnchanged(key: string, contentHash: string): boolean;
+
+    /** Mark a key as successfully indexed with the given hash. Call after indexing completes. */
+    markIndexed(key: string, contentHash: string): void;
+
+    /** Find tracked keys that are NOT in the current set. Returns keys to delete. */
+    findOrphans(currentKeys: Set<string>): string[];
+
+    /** Remove tracking for a key. Call after cleaning up the key's data. */
+    remove(key: string): void;
+
+    /** Remove all tracking entries for this plugin. */
+    clear(): void;
+}
+
+/** Create tracking table. Called during core schema init. */
+export function createTrackingTable(db: DatabaseAdapter): void {
+    db.exec(`
+        CREATE TABLE IF NOT EXISTS plugin_tracking (
+            plugin       TEXT NOT NULL,
+            key          TEXT NOT NULL,
+            content_hash TEXT NOT NULL,
+            indexed_at   INTEGER NOT NULL DEFAULT (unixepoch()),
+            PRIMARY KEY (plugin, key)
+        );
+    `);
+}
+
+/** Create an IncrementalTracker scoped to a plugin name. */
+export function createTracker(db: DatabaseAdapter, pluginName: string): IncrementalTracker {
+    return {
+        isUnchanged(key: string, contentHash: string): boolean {
+            const row = db.prepare(
+                'SELECT content_hash FROM plugin_tracking WHERE plugin = ? AND key = ?'
+            ).get(pluginName, key) as { content_hash: string } | undefined;
+            return row?.content_hash === contentHash;
+        },
+
+        markIndexed(key: string, contentHash: string): void {
+            db.prepare(`
+                INSERT INTO plugin_tracking (plugin, key, content_hash)
+                VALUES (?, ?, ?)
+                ON CONFLICT(plugin, key) DO UPDATE SET
+                    content_hash = excluded.content_hash,
+                    indexed_at = unixepoch()
+            `).run(pluginName, key, contentHash);
+        },
+
+        findOrphans(currentKeys: Set<string>): string[] {
+            const rows = db.prepare(
+                'SELECT key FROM plugin_tracking WHERE plugin = ?'
+            ).all(pluginName) as { key: string }[];
+            return rows.filter(r => !currentKeys.has(r.key)).map(r => r.key);
+        },
+
+        remove(key: string): void {
+            db.prepare(
+                'DELETE FROM plugin_tracking WHERE plugin = ? AND key = ?'
+            ).run(pluginName, key);
+        },
+
+        clear(): void {
+            db.prepare(
+                'DELETE FROM plugin_tracking WHERE plugin = ?'
+            ).run(pluginName);
+        },
+    };
+}

package/src/engine/index-api.ts

@@ -0,0 +1,81 @@
+/**
+ * BrainBank — Index API
+ *
+ * Orchestrates indexing across all registered plugins.
+ * Plugin-agnostic — uses capability interfaces to discover what can be indexed.
+ *
+ * After each plugin finishes indexing, bumps the version in `index_state`
+ * and saves HNSW indices to disk (with cross-process file locking).
+ */
+
+import type { DatabaseAdapter } from '@/db/adapter.ts';
+import type { HNSWIndex } from '@/providers/vector/hnsw-index.ts';
+import type { PluginRegistry } from '@/services/plugin-registry.ts';
+import type { IndexResult, StageProgressCallback } from '@/types.ts';
+
+import { bumpVersion } from '@/db/metadata.ts';
+import { isIndexable } from '@/plugin.ts';
+import { saveAllHnsw } from '@/providers/vector/hnsw-loader.ts';
+
+/** Deps injected by BrainBank at init time. */
+export interface IndexDeps {
+    db: DatabaseAdapter;
+    dbPath: string;
+    sharedHnsw: Map<string, { hnsw: HNSWIndex; vecCache: Map<number, Float32Array> }>;
+    kvHnsw: HNSWIndex;
+    registry: PluginRegistry;
+    emit: (event: string, data: unknown) => void;
+}
+
+/** Merge two `IndexResult` values, accumulating counts. */
+function mergeResult(acc: IndexResult | undefined, r: IndexResult): IndexResult {
+    if (!acc) return { ...r };
+    return {
+        indexed: acc.indexed + r.indexed,
+        skipped: acc.skipped + r.skipped,
+        chunks: (acc.chunks ?? 0) + (r.chunks ?? 0),
+    };
+}
+
+/** Run indexing across all indexable plugins. Filter with `modules` (base types). */
+export async function runIndex(deps: IndexDeps, options: {
+    modules?: string[];
+    forceReindex?: boolean;
+    onProgress?: StageProgressCallback;
+    /** Plugin-specific options forwarded to `IndexablePlugin.index()`. */
+    pluginOptions?: Record<string, unknown>;
+} = {}): Promise<Record<string, unknown>> {
+    const want = options.modules ? new Set(options.modules) : null;
+    const results: Record<string, unknown> = {};
+
+    for (const mod of deps.registry.all) {
+        if (want && !want.has(mod.name)) continue;
+        if (!isIndexable(mod)) continue;
+
+        const label = mod.name;
+        options.onProgress?.(label, 'Starting...');
+
+        const r = await mod.index({
+            forceReindex: options.forceReindex,
+            onProgress: (msg: string, cur: number, total: number) =>
+                options.onProgress?.(label, `[${cur}/${total}] ${msg}`),
+            ...options.pluginOptions,
+        });
+
+        results[mod.name] = mergeResult(results[mod.name] as IndexResult | undefined, r);
+
+        // Bump version per plugin name (= HNSW key) so hot-reload resolves correctly.
+        bumpVersion(deps.db, mod.name);
+    }
+
+    // Save HNSW indices with file locking after all plugins complete
+    await saveAllHnsw(
+        deps.dbPath,
+        deps.kvHnsw,
+        deps.sharedHnsw,
+        new Map(),
+    );
+
+    deps.emit('indexed', results);
+    return results;
+}

package/src/engine/reembed.ts

@@ -0,0 +1,206 @@
+/**
+ * BrainBank — Re-embedding Engine
+ * 
+ * Regenerates all vectors without re-indexing.
+ * Reads existing text from SQLite, embeds with the current provider,
+ * and replaces vector BLOBs. No file I/O, no git parsing, no re-chunking.
+ * 
+ * Usage:
+ *   const result = await brain.reembed({ onProgress });
+ *   // → { code: 1200, git: 500, docs: 80, kv: 45, total: 1837 }
+ */
+
+import type { DatabaseAdapter, CountRow, VectorRow } from '@/db/adapter.ts';
+import type { Plugin, ReembedTable } from '@/plugin.ts';
+import type { HNSWIndex } from '@/providers/vector/hnsw-index.ts';
+import type { EmbeddingProvider, ProgressCallback } from '@/types.ts';
+
+import { setEmbeddingMeta } from '@/db/metadata.ts';
+import { vecToBuffer } from '@/lib/math.ts';
+import { isReembeddable } from '@/plugin.ts';
+import { saveAllHnsw } from '@/providers/vector/hnsw-loader.ts';
+
+
+const CORE_TABLES: ReembedTable[] = [
+    {
+        name: 'kv',
+        textTable: 'kv_data',
+        vectorTable: 'kv_vectors',
+        idColumn: 'id',
+        fkColumn: 'data_id',
+        textBuilder: (r) => String(r.content),
+    },
+];
+
+/** Collect reembed tables from plugins + core. Deduplicates by vectorTable. */
+function collectTables(plugins: Plugin[]): ReembedTable[] {
+    const byVectorTable = new Map<string, ReembedTable>();
+    for (const p of plugins) {
+        if (isReembeddable(p)) {
+            const config = p.reembedConfig();
+            byVectorTable.set(config.vectorTable, config);
+        }
+    }
+    for (const t of CORE_TABLES) {
+        byVectorTable.set(t.vectorTable, t);
+    }
+    return [...byVectorTable.values()];
+}
+
+
+export interface ReembedResult {
+    /** Per-table vector counts. Keys are table names (e.g. 'code', 'git', 'docs', 'kv'). */
+    counts: Record<string, number>;
+    total: number;
+}
+
+export interface ReembedOptions {
+    /** Progress callback: (tableName, current, total) */
+    onProgress?: ProgressCallback;
+    /** Batch size for embedBatch. Default: 50 */
+    batchSize?: number;
+}
+
+
+/**
+ * Re-embed all existing text with the current embedding provider.
+ * Does NOT re-parse files, git, or documents — only replaces vectors.
+ */
+export async function reembedAll(
+    db: DatabaseAdapter,
+    embedding: EmbeddingProvider,
+    hnswMap: Map<string, { hnsw: HNSWIndex; vecs: Map<number, Float32Array> }>,
+    plugins: Plugin[],
+    options: ReembedOptions = {},
+    persist?: {
+        dbPath: string;
+        kvHnsw: HNSWIndex;
+        sharedHnsw: Map<string, { hnsw: HNSWIndex; vecCache: Map<number, Float32Array> }>;
+    },
+): Promise<ReembedResult> {
+    const { batchSize = 50, onProgress } = options;
+    const tables = collectTables(plugins);
+    const counts: Record<string, number> = {};
+    let total = 0;
+
+    for (const table of tables) {
+        // Skip tables that don't exist (plugin not installed)
+        try {
+            const textExists = (db.prepare(
+                `SELECT COUNT(*) as c FROM sqlite_master WHERE type='table' AND name=?`
+            ).get(table.textTable) as CountRow).c;
+            const vecExists = (db.prepare(
+                `SELECT COUNT(*) as c FROM sqlite_master WHERE type='table' AND name=?`
+            ).get(table.vectorTable) as CountRow).c;
+            if (!textExists || !vecExists) continue;
+        } catch (e: unknown) {
+            if (e instanceof Error && e.message.includes('no such table')) continue;
+            throw e;
+        }
+
+        const count = await reembedTable(db, embedding, table, batchSize, onProgress);
+        counts[table.name] = count;
+        total += count;
+
+        // Rebuild HNSW if available
+        const entry = hnswMap.get(table.name);
+        if (entry && count > 0) {
+            await rebuildHnsw(db, table, entry.hnsw, entry.vecs);
+        }
+    }
+
+    // Persist provider metadata + HNSW indexes to disk
+    setEmbeddingMeta(db, embedding);
+    if (persist) {
+        saveAllHnsw(persist.dbPath, persist.kvHnsw, persist.sharedHnsw, new Map());
+    }
+
+    return {
+        counts,
+        total,
+    };
+}
+
+/**
+ * Re-embed a single table. Returns count of vectors regenerated.
+ * 
+ * Streams per-batch to avoid OOM on large tables — memory stays O(batchSize).
+ * Tradeoff: if embedBatch fails mid-way, partial vectors exist. Reembed is
+ * a destructive operation by design — re-run to completion if interrupted.
+ */
+async function reembedTable(
+    db: DatabaseAdapter,
+    embedding: EmbeddingProvider,
+    table: ReembedTable,
+    batchSize: number,
+    onProgress?: ProgressCallback,
+): Promise<number> {
+    const totalCount = (db.prepare(
+        `SELECT COUNT(*) as c FROM ${table.textTable}`
+    ).get() as CountRow).c;
+
+    if (totalCount === 0) return 0;
+
+    // Phase 1: Build new vectors in a temp table (safe — old data untouched)
+    const tempTable = `_reembed_${table.vectorTable}`;
+    db.exec(`DROP TABLE IF EXISTS ${tempTable}`);
+    db.exec(`CREATE TABLE ${tempTable} AS SELECT * FROM ${table.vectorTable} WHERE 0`);
+
+    const insertTemp = db.prepare(
+        `INSERT INTO ${tempTable} (${table.fkColumn}, embedding) VALUES (?, ?)`
+    );
+
+    let processed = 0;
+    try {
+        for (let offset = 0; offset < totalCount; offset += batchSize) {
+            const batch = db.prepare(
+                `SELECT * FROM ${table.textTable} LIMIT ? OFFSET ?`
+            ).all(batchSize, offset) as Record<string, unknown>[];
+            const texts = batch.map(r => table.textBuilder(r));
+            const vectors = await embedding.embedBatch(texts);
+
+            db.transaction(() => {
+                for (let j = 0; j < batch.length; j++) {
+                    insertTemp.run(batch[j][table.idColumn], vecToBuffer(vectors[j]));
+                }
+            });
+
+            processed += batch.length;
+            onProgress?.(table.name, processed, totalCount);
+        }
+
+        // Phase 2: Atomic swap — all or nothing
+        db.transaction(() => {
+            db.exec(`DELETE FROM ${table.vectorTable}`);
+            db.exec(`INSERT INTO ${table.vectorTable} SELECT * FROM ${tempTable}`);
+        });
+    } finally {
+        // Always clean up temp table — even if embedBatch fails mid-batch
+        db.exec(`DROP TABLE IF EXISTS ${tempTable}`);
+    }
+
+    return processed;
+}
+
+/** Rebuild HNSW index from vector table. */
+async function rebuildHnsw(
+    db: DatabaseAdapter,
+    table: ReembedTable,
+    hnsw: HNSWIndex,
+    vecs: Map<number, Float32Array>,
+): Promise<void> {
+    // Wipe stale vectors before repopulating
+    vecs.clear();
+    hnsw.reinit();
+
+    const rows = db.prepare(
+        `SELECT ${table.fkColumn} as id, embedding FROM ${table.vectorTable}`
+    ).all() as VectorRow[];
+
+    for (const row of rows) {
+        const emb = row.embedding;
+        const vec = new Float32Array(emb.buffer.slice(emb.byteOffset, emb.byteOffset + emb.byteLength));
+        hnsw.add(vec, row.id);
+        vecs.set(row.id, vec);
+    }
+}