brainbank 0.1.0-beta.1

package/src/plugin.ts

@@ -0,0 +1,381 @@
+/**
+ * BrainBank — Plugin System
+ * 
+ * Plugins are pluggable strategies that scan external data sources
+ * and push content into BrainBank. Built-in plugins handle code,
+ * git, and docs. Third-party frameworks (LangChain, etc.)
+ * can implement custom plugins.
+ * 
+ *   import { BrainBank } from 'brainbank';
+ *   import { code } from 'brainbank/indexers/code';
+ *   
+ *   const brain = new BrainBank()
+ *     .use(code({ repoPath: '.' }));
+ */
+
+import type { DatabaseAdapter } from './db/adapter.ts';
+import type { Migration } from './db/migrations.ts';
+import type { IncrementalTracker } from './db/tracker.ts';
+import type { HNSWIndex } from './providers/vector/hnsw-index.ts';
+import type { DomainVectorSearch } from './search/types.ts';
+import type { WebhookServer } from './services/webhook-server.ts';
+import type {
+    EmbeddingProvider, SearchResult, IndexResult, ProgressCallback,
+    ResolvedConfig, DocumentCollection, ICollection,
+    WatchEventHandler, WatchHandle, WatchConfig,
+    ExpanderManifestItem,
+} from './types.ts';
+
+// Provided to each plugin during initialization.
+
+export interface PluginContext {
+    /** Database adapter (shared across all plugins). */
+    db: DatabaseAdapter;
+    /** Embedding provider (shared). */
+    embedding: EmbeddingProvider;
+    /** Resolved BrainBank config. */
+    config: ResolvedConfig;
+    /**
+     * Create and initialize an HNSW index.
+     * Pass `name` to enable disk persistence (recommended).
+     *
+     * **Private vs shared:** Use `getOrCreateSharedHnsw()` for indexes that should be
+     * part of the composite search (code, git) and persisted across restarts.
+     * Use `createHnsw()` for plugin-local indexes that don't participate in the
+     * main search pipeline (e.g. internal similarity lookups).
+     */
+    createHnsw(maxElements?: number, dims?: number, name?: string): Promise<HNSWIndex>;
+    /** Load existing vectors from a SQLite vectors table into an HNSW index + cache. */
+    loadVectors(table: string, idCol: string, hnsw: HNSWIndex, cache: Map<number, Float32Array>): void;
+    /**
+     * Get or create a shared HNSW index by key.
+     *
+     * **HNSW sharing strategies:**
+     * The `type` key determines sharing behavior. Two plugins that pass the
+     * same key share one HNSW index; different keys get separate indexes.
+     *
+     * | Plugin type | Key passed        | Sharing behavior                          |
+     * |------------|------------------|------------------------------------------|
+     * | git        | `'git'`           | All `git:*` repos share one HNSW          |
+     * | docs       | `'docs'`          | All docs share one HNSW                   |
+     * | code       | `this.name`       | Each `code:*` repo gets its own HNSW      |
+     *
+     * **Rule of thumb:**
+     * - Same key = shared index (saves memory, single search covers all)
+     * - Plugin name as key = per-repo index (avoids cross-repo noise)
+     *
+     * The key is also used for hot-reload (`ensureFresh`) and disk persistence
+     * (`hnsw-<key>.index`), so it must match the key used in `bumpVersion()`.
+     */
+    getOrCreateSharedHnsw(type: string, maxElements?: number, dims?: number): Promise<{ hnsw: HNSWIndex; vecCache: Map<number, Float32Array>; isNew: boolean }>;
+    /** Get or create a dynamic collection. */
+    collection(name: string): ICollection;
+    /**
+     * Create an incremental tracker scoped to this plugin.
+     * Provides `isUnchanged`, `markIndexed`, `findOrphans`, `remove`, `clear`
+     * for standardized add/update/delete detection during indexing.
+     */
+    createTracker(): IncrementalTracker;
+    /** Optional webhook server for push-based watch plugins. undefined if not configured. */
+    webhookServer?: WebhookServer;
+}
+
+// Minimal contract: name + initialize. All capabilities are expressed
+// via composed interfaces below.
+
+export interface Plugin {
+    /** Unique plugin name (e.g. 'code', 'git', 'docs'). */
+    readonly name: string;
+    /** Initialize the plugin (create HNSW, load vectors, etc.). */
+    initialize(ctx: PluginContext): Promise<void>;
+    /** Return stats for this plugin. */
+    stats?(): Record<string, number | string>;
+    /** Clean up resources. */
+    close?(): void;
+}
+
+// Implemented by plugins that support specific capabilities.
+// Use type guards below to check at runtime.
+
+/** Options accepted by IndexablePlugin.index(). */
+export interface IndexOptions {
+    forceReindex?: boolean;
+    depth?: number;
+    onProgress?: ProgressCallback;
+}
+
+/** Plugins that can scan and index content (code, git). */
+export interface IndexablePlugin extends Plugin {
+    index(options?: IndexOptions): Promise<IndexResult>;
+    /** Incremental: re-index only specific items by ID. Falls back to index() if not implemented. */
+    indexItems?(ids: string[]): Promise<IndexResult>;
+}
+
+/** Plugins that can search indexed content (docs). */
+export interface SearchablePlugin extends Plugin {
+    search(query: string, options?: Record<string, unknown>): Promise<SearchResult[]>;
+}
+
+/** Plugins that can watch their own data source for changes. */
+export interface WatchablePlugin extends Plugin {
+    /** Start watching. Plugin controls how (fs.watch, polling, webhook, etc.). */
+    watch(onEvent: WatchEventHandler): WatchHandle;
+    /** Optional hints for the core (debounce, batching, priority). */
+    watchConfig?(): WatchConfig;
+}
+
+
+/** Check if a plugin can scan/index content. */
+export function isIndexable(i: Plugin): i is IndexablePlugin {
+    return typeof (i as IndexablePlugin).index === 'function';
+}
+
+/** Check if a plugin can search content. */
+export function isSearchable(i: Plugin): i is SearchablePlugin {
+    return typeof (i as SearchablePlugin).search === 'function';
+}
+
+/** Check if a plugin can watch its own data source. */
+export function isWatchable(i: Plugin): i is WatchablePlugin {
+    return typeof (i as WatchablePlugin).watch === 'function';
+}
+
+/** Path-specific context metadata for document collections. */
+export interface PathContext {
+    collection: string;
+    path: string;
+    context: string;
+}
+
+/** Plugins that manage document collections (docs). */
+export interface DocsPlugin extends SearchablePlugin {
+    addCollection(collection: DocumentCollection): void;
+    removeCollection(name: string): void;
+    listCollections(): DocumentCollection[];
+    indexDocs(options?: { onProgress?: (collection: string, file: string, current: number, total: number) => void }): Promise<Record<string, { indexed: number; skipped: number; removed: number; chunks: number }>>;
+    addContext(collection: string, path: string, context: string): void;
+    listContexts(): PathContext[];
+}
+
+/** Check if a plugin manages document collections. */
+export function isDocsPlugin(i: Plugin): i is DocsPlugin {
+    return typeof (i as DocsPlugin).addCollection === 'function'
+        && typeof (i as DocsPlugin).listCollections === 'function';
+}
+
+
+/** Plugin that provides co-edit suggestions (e.g. git). */
+export interface CoEditPlugin extends Plugin {
+    coEdits: {
+        suggest(filePath: string, limit: number): { file: string; count: number }[];
+    };
+}
+
+/** Check if a plugin provides co-edit suggestions. */
+export function isCoEditPlugin(p: Plugin): p is CoEditPlugin {
+    return 'coEdits' in p && typeof (p as CoEditPlugin).coEdits?.suggest === 'function';
+}
+
+
+/** Table descriptor for re-embedding — maps text rows to vector BLOBs. */
+export interface ReembedTable {
+    /** Human-readable name (for progress). */
+    name: string;
+    /** Table with text content. */
+    textTable: string;
+    /** Table with vector BLOBs. */
+    vectorTable: string;
+    /** PK column in text table. */
+    idColumn: string;
+    /** FK column in vector table. */
+    fkColumn: string;
+    /** Build the embedding text from a DB row. */
+    textBuilder: (row: Record<string, unknown>) => string;
+}
+
+/** Plugins that own vector tables and can rebuild embedding text from DB rows. */
+export interface ReembeddablePlugin extends Plugin {
+    /** Table descriptor for re-embedding. */
+    reembedConfig(): ReembedTable;
+}
+
+/** Check if a plugin supports re-embedding. */
+export function isReembeddable(p: Plugin): p is ReembeddablePlugin {
+    return typeof (p as ReembeddablePlugin).reembedConfig === 'function';
+}
+
+
+/** Plugin that provides a domain-specific vector search strategy. */
+export interface VectorSearchPlugin extends Plugin {
+    /** Create the domain vector search (called during SearchAPI wiring). */
+    createVectorSearch(): DomainVectorSearch | undefined;
+}
+
+/** Check if a plugin provides a domain vector search. */
+export function isVectorSearchPlugin(p: Plugin): p is VectorSearchPlugin {
+    return typeof (p as VectorSearchPlugin).createVectorSearch === 'function';
+}
+
+/** Describes a configurable context field that a plugin supports. */
+export interface ContextFieldDef {
+    /** Field name (e.g. 'lines', 'callTree', 'symbols'). Must be unique per plugin. */
+    name: string;
+    /** Accepted value type. 'object' allows nested config like `{ depth: 3 }`. */
+    type: 'boolean' | 'number' | 'object';
+    /** Default value (used when not specified in config or query). */
+    default: unknown;
+    /** Human-readable description for CLI --help and MCP tool descriptions. */
+    description: string;
+}
+
+/** Plugin that declares configurable context fields. */
+export interface ContextFieldPlugin extends Plugin {
+    /** Declare available context fields. Called once during setup. */
+    contextFields(): ContextFieldDef[];
+}
+
+/** Check if a plugin declares context fields. */
+export function isContextFieldPlugin(p: Plugin): p is ContextFieldPlugin {
+    return typeof (p as ContextFieldPlugin).contextFields === 'function';
+}
+
+/** Plugin that contributes sections to the context builder output. */
+export interface ContextFormatterPlugin extends Plugin {
+    /**
+     * Append formatted markdown sections to `parts`.
+     * `fields` contains resolved context fields (plugin defaults ← config ← per-query).
+     */
+    formatContext(results: SearchResult[], parts: string[], fields: Record<string, unknown>): void;
+}
+
+/** Check if a plugin provides context formatting. */
+export function isContextFormatterPlugin(p: Plugin): p is ContextFormatterPlugin {
+    return typeof (p as ContextFormatterPlugin).formatContext === 'function';
+}
+
+
+/** Plugin that owns database tables and supports versioned migrations. */
+export interface MigratablePlugin extends Plugin {
+    /** Current schema version for this plugin. */
+    readonly schemaVersion: number;
+    /** Ordered list of migrations (version 1, 2, 3, …). */
+    readonly migrations: Migration[];
+}
+
+/** Check if a plugin supports schema migrations. */
+export function isMigratable(p: Plugin): p is MigratablePlugin {
+    return typeof (p as MigratablePlugin).schemaVersion === 'number'
+        && Array.isArray((p as MigratablePlugin).migrations);
+}
+
+
+/** Plugin that can do FTS5 keyword search on its own tables. */
+export interface BM25SearchPlugin extends Plugin {
+    /** Run BM25 keyword search. Returns scored results. */
+    searchBM25(query: string, k: number, minScore?: number): SearchResult[];
+    /** Rebuild the FTS5 index from the content table. */
+    rebuildFTS?(): void;
+}
+
+/** Check if a plugin provides BM25 keyword search. */
+export function isBM25SearchPlugin(p: Plugin): p is BM25SearchPlugin {
+    return typeof (p as BM25SearchPlugin).searchBM25 === 'function';
+}
+
+/** Plugin that supports context expansion (provides manifest + resolves chunk IDs). */
+export interface ExpandablePlugin extends Plugin {
+    /**
+     * Build a manifest of candidate chunks for LLM expansion.
+     * Returns chunks from files NOT already in search results.
+     * Priority chunks (from import graph neighbors) are marked with `priority: true`.
+     *
+     * @param excludeFilePaths File paths already present in search results — excluded from manifest.
+     * @param excludeIds       Chunk IDs already in search results — excluded from manifest.
+     * @param resultFilePaths  File paths in search results — used to query import graph for priority chunks.
+     */
+    buildManifest(excludeFilePaths: string[], excludeIds: number[], resultFilePaths?: string[]): ExpanderManifestItem[];
+    /**
+     * Resolve chunk IDs back into SearchResults.
+     * Called after the expander selects additional IDs.
+     */
+    resolveChunks(ids: number[]): SearchResult[];
+}
+
+/** Check if a plugin supports context expansion. */
+export function isExpandablePlugin(p: Plugin): p is ExpandablePlugin {
+    return typeof (p as ExpandablePlugin).buildManifest === 'function'
+        && typeof (p as ExpandablePlugin).resolveChunks === 'function';
+}
+
+
+/** Plugin that can resolve file paths/patterns directly to SearchResults (no search). */
+export interface FileResolvablePlugin extends Plugin {
+    /**
+     * Resolve file paths, directories, and glob patterns to SearchResults.
+     * Each entry is resolved: exact → directory → glob → fuzzy basename fallback.
+     *
+     * @param patterns - File paths, directory prefixes (trailing `/`), or glob patterns (`*`).
+     */
+    resolveFiles(patterns: string[]): SearchResult[];
+}
+
+/** Check if a plugin can resolve files directly. */
+export function isFileResolvable(p: Plugin): p is FileResolvablePlugin {
+    return typeof (p as FileResolvablePlugin).resolveFiles === 'function';
+}
+
+
+// ── Third-party Plugin Discovery (TUI integration) ────────────────
+
+/**
+ * Scan info for the TUI sidebar. Describes what content a plugin can index.
+ * Exported standalone by plugin packages — called BEFORE plugin initialization.
+ *
+ * @example
+ * ```typescript
+ * // brainbank-csv/index.ts
+ * export function scan(repoPath: string): PluginScanInfo { ... }
+ * ```
+ */
+export interface PluginScanInfo {
+    /** Plugin name (e.g. 'csv', 'openapi'). */
+    name: string;
+    /** Whether there's content available to index. */
+    available: boolean;
+    /** Human-readable summary (e.g. '12 CSV files'). */
+    summary: string;
+    /** Emoji icon for TUI display. */
+    icon: string;
+    /** Whether checked by default in the module selector. */
+    checked: boolean;
+    /** Reason this plugin is disabled (shown when unavailable). */
+    disabled?: string;
+    /** Detail lines for the scan tree (e.g. per-file breakdown). */
+    details?: string[];
+}
+
+/**
+ * A single line in the TUI explorer preview panel.
+ * Returned by `preview()` — rendered as-is in the right panel.
+ *
+ * @example
+ * ```typescript
+ * // brainbank-csv/index.ts
+ * export function preview(repoPath: string): PluginPreviewLine[] {
+ *   return [
+ *     { text: '📊 3 CSV files', bold: true },
+ *     { text: '  sales.csv  2.3 MB', color: '#9ECE6A' },
+ *   ];
+ * }
+ * ```
+ */
+export interface PluginPreviewLine {
+    /** Text content for this line. */
+    text: string;
+    /** Optional hex color (e.g. '#9ECE6A'). */
+    color?: string;
+    /** Render bold. */
+    bold?: boolean;
+    /** Render dimmed. */
+    dim?: boolean;
+}

package/src/providers/embeddings/embedding-worker-thread.ts

@@ -0,0 +1,95 @@
+/**
+ * BrainBank — Embedding Worker Thread
+ *
+ * Worker script that runs the embedding provider in a dedicated thread.
+ * Receives text from the main thread, returns Float32Array vectors.
+ * Keeps the main thread's event loop free for serving search requests.
+ */
+
+import { parentPort, workerData } from 'node:worker_threads';
+
+/** Message types for main ↔ worker communication. */
+interface EmbedRequest {
+    id: number;
+    type: 'embed' | 'embedBatch' | 'close';
+    text?: string;
+    texts?: string[];
+}
+
+interface EmbedResponse {
+    id: number;
+    type: 'result' | 'error';
+    vector?: ArrayBuffer;
+    vectors?: ArrayBuffer[];
+    dims?: number;
+    error?: string;
+}
+
+/** Minimal interface for the provider loaded in the worker. */
+interface WorkerProvider {
+    dims: number;
+    embed(text: string): Promise<Float32Array>;
+    embedBatch(texts: string[]): Promise<Float32Array[]>;
+    close(): Promise<void>;
+}
+
+async function main(): Promise<void> {
+    if (!parentPort) throw new Error('Must run as worker thread');
+
+    const config = workerData as { providerType: string; providerOptions: Record<string, unknown> };
+    let provider: WorkerProvider;
+
+    // Dynamically instantiate the provider based on type
+    if (config.providerType === 'local') {
+        const { LocalEmbedding } = await import('./local-embedding.ts') as { LocalEmbedding: new (opts?: Record<string, unknown>) => WorkerProvider };
+        provider = new LocalEmbedding(config.providerOptions);
+    } else if (config.providerType === 'openai') {
+        const { OpenAIEmbedding } = await import('./openai-embedding.ts') as { OpenAIEmbedding: new (opts?: Record<string, unknown>) => WorkerProvider };
+        provider = new OpenAIEmbedding(config.providerOptions);
+    } else {
+        throw new Error(`BrainBank: Unknown embedding provider type '${config.providerType}' in worker.`);
+    }
+
+    parentPort.on('message', async (msg: EmbedRequest) => {
+        if (msg.type === 'close') {
+            await provider.close();
+            parentPort!.postMessage({ id: msg.id, type: 'result' } satisfies EmbedResponse);
+            process.exit(0);
+        }
+
+        try {
+            if (msg.type === 'embed' && msg.text) {
+                const vec = await provider.embed(msg.text);
+                const buffer = vec.buffer as ArrayBuffer;
+                parentPort!.postMessage(
+                    { id: msg.id, type: 'result', vector: buffer, dims: provider.dims } satisfies EmbedResponse,
+                    [buffer],
+                );
+            } else if (msg.type === 'embedBatch' && msg.texts) {
+                const vecs = await provider.embedBatch(msg.texts);
+                const buffers = vecs.map(v => v.buffer as ArrayBuffer);
+                parentPort!.postMessage(
+                    {
+                        id: msg.id,
+                        type: 'result',
+                        vectors: buffers,
+                        dims: provider.dims,
+                    } satisfies EmbedResponse,
+                    buffers,
+                );
+            }
+        } catch (err: unknown) {
+            const message = err instanceof Error ? err.message : String(err);
+            parentPort!.postMessage({ id: msg.id, type: 'error', error: message } satisfies EmbedResponse);
+        }
+    });
+
+    // Signal ready
+    parentPort.postMessage({ id: 0, type: 'result', dims: provider.dims } satisfies EmbedResponse);
+}
+
+main().catch(err => {
+    const message = err instanceof Error ? err.message : String(err);
+    parentPort?.postMessage({ id: -1, type: 'error', error: message });
+    process.exit(1);
+});

package/src/providers/embeddings/embedding-worker.ts

@@ -0,0 +1,141 @@
+/**
+ * BrainBank — Embedding Worker Proxy
+ *
+ * Drop-in replacement for any EmbeddingProvider that offloads
+ * embedding computation to a dedicated worker thread.
+ * The main thread's event loop stays free for serving searches.
+ *
+ * Usage:
+ *   const proxy = new EmbeddingWorkerProxy('local', { model: '...' });
+ *   await proxy.ready();
+ *   const vec = await proxy.embed('hello');
+ */
+
+import type { EmbeddingProvider } from '@/types.ts';
+
+import { Worker } from 'node:worker_threads';
+import { fileURLToPath } from 'node:url';
+import { join, dirname } from 'node:path';
+
+/** Message sent to the worker. */
+interface WorkerRequest {
+    id: number;
+    type: 'embed' | 'embedBatch' | 'close';
+    text?: string;
+    texts?: string[];
+}
+
+/** Message received from the worker. */
+interface WorkerResponse {
+    id: number;
+    type: 'result' | 'error';
+    vector?: ArrayBuffer;
+    vectors?: ArrayBuffer[];
+    dims?: number;
+    error?: string;
+}
+
+export class EmbeddingWorkerProxy implements EmbeddingProvider {
+    private _worker: Worker;
+    private _nextId = 1;
+    private _pending = new Map<number, { resolve: (v: WorkerResponse) => void; reject: (e: Error) => void }>();
+    private _ready: Promise<void>;
+    private _dims = 0;
+
+    /** Embedding dimensions (available after `ready()` resolves). */
+    get dims(): number { return this._dims; }
+
+    constructor(
+        providerType: string,
+        providerOptions: Record<string, unknown> = {},
+    ) {
+        const workerPath = join(
+            dirname(fileURLToPath(import.meta.url)),
+            'embedding-worker-thread.ts',
+        );
+
+        this._worker = new Worker(workerPath, {
+            workerData: { providerType, providerOptions },
+            execArgv: ['--import', 'tsx'],
+        });
+
+        this._worker.on('message', (msg: WorkerResponse) => {
+            // Initial ready signal (id === 0)
+            if (msg.id === 0 && msg.dims) {
+                this._dims = msg.dims;
+                return;
+            }
+
+            const pending = this._pending.get(msg.id);
+            if (!pending) return;
+            this._pending.delete(msg.id);
+
+            if (msg.type === 'error') {
+                pending.reject(new Error(`BrainBank: Embedding worker error: ${msg.error}`));
+            } else {
+                pending.resolve(msg);
+            }
+        });
+
+        this._worker.on('error', (err: Error) => {
+            for (const pending of this._pending.values()) {
+                pending.reject(err);
+            }
+            this._pending.clear();
+        });
+
+        // Wait for the worker to signal ready
+        this._ready = new Promise<void>((resolve, reject) => {
+            const onMsg = (msg: WorkerResponse) => {
+                if (msg.id === 0 && msg.dims) {
+                    this._dims = msg.dims;
+                    this._worker.removeListener('message', onMsg);
+                    resolve();
+                }
+            };
+            this._worker.on('message', onMsg);
+            this._worker.on('error', reject);
+        });
+    }
+
+    /** Wait for the worker to be ready (provider loaded). */
+    async ready(): Promise<void> {
+        return this._ready;
+    }
+
+    /** Send a request to the worker and wait for the response. */
+    private _send(req: Omit<WorkerRequest, 'id'>): Promise<WorkerResponse> {
+        const id = this._nextId++;
+        return new Promise<WorkerResponse>((resolve, reject) => {
+            this._pending.set(id, { resolve, reject });
+            this._worker.postMessage({ ...req, id } satisfies WorkerRequest);
+        });
+    }
+
+    /** Embed a single text string. */
+    async embed(text: string): Promise<Float32Array> {
+        await this._ready;
+        const resp = await this._send({ type: 'embed', text });
+        if (!resp.vector) throw new Error('BrainBank: Worker returned no vector.');
+        return new Float32Array(resp.vector);
+    }
+
+    /** Embed multiple texts in a batch. */
+    async embedBatch(texts: string[]): Promise<Float32Array[]> {
+        if (texts.length === 0) return [];
+        await this._ready;
+        const resp = await this._send({ type: 'embedBatch', texts });
+        if (!resp.vectors) throw new Error('BrainBank: Worker returned no vectors.');
+        return resp.vectors.map(buf => new Float32Array(buf));
+    }
+
+    /** Terminate the worker. */
+    async close(): Promise<void> {
+        try {
+            await this._send({ type: 'close' });
+        } catch {
+            // Worker may already be terminated
+        }
+        await this._worker.terminate();
+    }
+}