brainbank 0.1.0-beta.1

package/src/search/context-builder.ts

@@ -0,0 +1,251 @@
+/**
+ * BrainBank — Context Builder
+ *
+ * Orchestrates the context-building pipeline:
+ *   1. Vector search (primary)
+ *   2. Path scoping (filter)
+ *   3. LLM noise pruning (optional)
+ *   4. Session dedup (filter)
+ *   5. LLM context expansion (optional — expander field)
+ *   6. Plugin formatters (output)
+ *
+ * All search post-processing lives in `bm25-boost.ts`.
+ * Plugin-agnostic — discovers formatters from ContextFormatterPlugin.
+ */
+
+import type { ContextOptions, EmbeddingProvider, Expander, ExpanderManifestItem, Pruner, SearchResult } from '@/types.ts';
+import type { SearchStrategy } from './types.ts';
+import type { PluginRegistry } from '@/services/plugin-registry.ts';
+
+import { isContextFormatterPlugin, isContextFieldPlugin, isExpandablePlugin, isSearchable } from '@/plugin.ts';
+import { filterByPath, filterByIgnore } from './bm25-boost.ts';
+import { pruneResults } from '@/lib/prune.ts';
+import { logQuery } from '@/lib/logger.ts';
+import type { QueryLogResult } from '@/lib/logger.ts';
+import { providerKey } from '@/lib/provider-key.ts';
+
+export class ContextBuilder {
+    constructor(
+        private _search: SearchStrategy | undefined,
+        private _registry: PluginRegistry,
+        private _pruner?: Pruner,
+        private _embedding?: EmbeddingProvider,
+        private _configFields: Record<string, unknown> = {},
+        private _expander?: Expander,
+    ) {}
+
+    /** Set config-level context field defaults (from config.json "context" section). */
+    set configFields(fields: Record<string, unknown>) {
+        this._configFields = fields;
+    }
+
+    /** Set the expander instance. */
+    set expander(expander: Expander | undefined) {
+        this._expander = expander;
+    }
+
+    /** Build a full context block for a task. Returns markdown for system prompt. */
+    async build(task: string, options: ContextOptions = {}): Promise<string> {
+        const t0 = Date.now();
+        const src = options.sources ?? {};
+        const { minScore = 0.25, useMMR = true, mmrLambda = 0.7 } = options;
+
+        // 1. Primary: vector search (includes per-repo BM25 fusion internally)
+        let results: SearchResult[] = this._search
+            ? await this._search.search(task, {
+                sources: src,
+                minScore, useMMR, mmrLambda,
+            })
+            : [];
+
+        // 2. Path scoping + ignore filtering
+        results = filterByPath(results, options.pathPrefix);
+        results = filterByIgnore(results, options.ignorePaths);
+
+        // 3. LLM noise pruning (optional — per-request override or construction-time)
+        const pruner = options.pruner ?? this._pruner;
+        const beforePrune = results;
+        if (pruner && results.length > 1) {
+            results = await pruneResults(task, results, pruner);
+        }
+
+        // 4. Exclude already-returned files (session dedup)
+        if (options.excludeFiles && options.excludeFiles.size > 0) {
+            results = results.filter(r => !r.filePath || !options.excludeFiles!.has(r.filePath));
+        }
+
+        // 5. LLM context expansion (optional — only when expander field is enabled)
+        const resolvedFields = this._resolveFields(options);
+        let expanderNote: string | undefined;
+        if (resolvedFields.expander === true && this._expander && results.length > 0) {
+            const expansion = await this._expand(task, results);
+            if (expansion.results.length > 0) {
+                results = [...results, ...expansion.results];
+            }
+            expanderNote = expansion.note;
+        }
+
+        // 6. Format output
+        const parts: string[] = [`# Context for: "${task}"\n`];
+        this._appendFormatterResults(results, parts, options, resolvedFields);
+        await this._appendSearchableResults(task, src, minScore, parts);
+
+        // 7. Append expander note (last section)
+        if (expanderNote) {
+            parts.push(`\n## Expansion Notes\n\n${expanderNote}\n`);
+        }
+
+        // ── Log ──
+        const prunedResults = pruner
+            ? beforePrune.filter(r => !results.includes(r))
+            : [];
+        const expanderEnabled = resolvedFields.expander === true;
+        const expandedResults = expanderNote !== undefined
+            ? results.filter(r => !beforePrune.includes(r) && !prunedResults.includes(r))
+            : [];
+        logQuery({
+            source: options.source ?? 'api',
+            method: 'getContext',
+            query: task,
+            embedding: this._embedding ? providerKey(this._embedding) : 'unknown',
+            pruner: pruner ? _prunerName(pruner) : null,
+            expander: expanderEnabled ? (this._expander ? _expanderName(this._expander) : 'configured-no-instance') : null,
+            expandedCount: expandedResults.length > 0 ? expandedResults.length : undefined,
+            options: {
+                sources: src,
+                pathPrefix: options.pathPrefix,
+                ignorePaths: options.ignorePaths,
+                minScore,
+                affectedFiles: options.affectedFiles,
+            },
+            results: results.map(_toLogResult),
+            pruned: prunedResults.length > 0 ? prunedResults.map(_toLogResult) : undefined,
+            durationMs: Date.now() - t0,
+        });
+
+        return parts.join('\n');
+    }
+
+    /** Invoke ContextFormatterPlugins. */
+    private _appendFormatterResults(
+        results: SearchResult[],
+        parts: string[],
+        options: ContextOptions,
+        resolvedFields?: Record<string, unknown>,
+    ): void {
+        const fields = resolvedFields ?? this._resolveFields(options);
+        const seenFormatters = new Set<string>();
+
+        for (const mod of this._registry.all) {
+            if (!isContextFormatterPlugin(mod)) continue;
+
+            if (seenFormatters.has(mod.name)) continue;
+            seenFormatters.add(mod.name);
+            mod.formatContext(results, parts, fields);
+        }
+    }
+
+    /**
+     * Resolve context fields: plugin defaults ← config.json ← per-query.
+     * Returns a flat Record with the final value for each field.
+     */
+    private _resolveFields(options: ContextOptions): Record<string, unknown> {
+        // 1. Collect plugin defaults
+        const defaults: Record<string, unknown> = {};
+        for (const mod of this._registry.all) {
+            if (isContextFieldPlugin(mod)) {
+                for (const field of mod.contextFields()) {
+                    defaults[field.name] = field.default;
+                }
+            }
+        }
+
+        // 2. Merge: defaults ← config ← per-query
+        return { ...defaults, ...this._configFields, ...(options.fields ?? {}) };
+    }
+
+    /**
+     * Run LLM expansion: build manifest of candidate chunks from files
+     * NOT already in search results, call expander, resolve selected IDs.
+     */
+    private async _expand(task: string, results: SearchResult[]): Promise<{ results: SearchResult[]; note?: string }> {
+        if (!this._expander) return { results: [] };
+
+        // Collect unique file paths already in results
+        const excludeFilePaths = [...new Set(
+            results.filter(r => r.filePath).map(r => r.filePath as string),
+        )];
+
+        // Collect current chunk IDs (to exclude from manifest)
+        const excludeIds: number[] = [];
+        for (const r of results) {
+            const meta = r.metadata as Record<string, unknown> | undefined;
+            const id = meta?.id as number | undefined;
+            if (id !== undefined) excludeIds.push(id);
+        }
+
+        // Build manifest + resolve from ExpandablePlugins
+        const manifest: ExpanderManifestItem[] = [];
+        let resolver: ((ids: number[]) => SearchResult[]) | undefined;
+        for (const mod of this._registry.all) {
+            if (!isExpandablePlugin(mod)) continue;
+
+            manifest.push(...mod.buildManifest(excludeFilePaths, excludeIds));
+            if (!resolver) {
+                resolver = (ids: number[]) => mod.resolveChunks(ids);
+            }
+        }
+        if (manifest.length === 0 || !resolver) return { results: [] };
+
+        // Call expander
+        try {
+            const expandResult = await this._expander.expand(task, excludeIds, manifest);
+            if (expandResult.ids.length === 0) return { results: [], note: expandResult.note };
+            return { results: resolver(expandResult.ids), note: expandResult.note };
+        } catch {
+            // Fail-open: expansion errors are non-fatal
+            return { results: [] };
+        }
+    }
+
+    /** Collect results from SearchablePlugins that don't have their own formatter. */
+    private async _appendSearchableResults(
+        task: string,
+        sources: Record<string, number>,
+        minScore: number,
+        parts: string[],
+    ): Promise<void> {
+        for (const mod of this._registry.all) {
+            if (isContextFormatterPlugin(mod)) continue;
+            if (!isSearchable(mod)) continue;
+            const hits = await mod.search(task, { k: sources[mod.name] ?? 6, minScore });
+            if (hits.length > 0) {
+                parts.push(`## ${mod.name}\n`);
+                for (const r of hits) {
+                    parts.push(`- [${Math.round(r.score * 100)}%] ${r.content.slice(0, 200)}`);
+                }
+                parts.push('');
+            }
+        }
+    }
+}
+
+// ── Helpers ──────────────────────────────────────────
+
+function _toLogResult(r: SearchResult): QueryLogResult {
+    const meta = r.metadata as Record<string, unknown> | undefined;
+    return {
+        filePath: r.filePath ?? 'unknown',
+        score: r.score,
+        type: r.type,
+        name: (meta?.name as string | undefined) ?? undefined,
+    };
+}
+
+function _prunerName(pruner: Pruner): string {
+    return pruner.constructor?.name ?? 'custom';
+}
+
+function _expanderName(expander: Expander): string {
+    return expander.constructor?.name ?? 'custom';
+}

package/src/search/keyword/composite-bm25-search.ts

@@ -0,0 +1,47 @@
+/**
+ * BrainBank — Composite BM25 Search Strategy
+ *
+ * Generic BM25 coordinator that discovers BM25SearchPlugin instances
+ * from the registry and delegates per-source keyword search.
+ */
+
+import type { SearchResult } from '@/types.ts';
+import type { SearchStrategy, SearchOptions } from '@/search/types.ts';
+import type { PluginRegistry } from '@/services/plugin-registry.ts';
+
+import { isBM25SearchPlugin } from '@/plugin.ts';
+
+const DEFAULT_K = 8;
+
+export class CompositeBM25Search implements SearchStrategy {
+    constructor(private _registry: PluginRegistry) {}
+
+    /**
+     * Run BM25 keyword search across all plugins that implement BM25SearchPlugin.
+     * Each plugin searches its own FTS5 tables.
+     */
+    async search(query: string, options: SearchOptions = {}): Promise<SearchResult[]> {
+        const src = options.sources ?? {};
+        const results: SearchResult[] = [];
+
+        for (const plugin of this._registry.all) {
+            if (!isBM25SearchPlugin(plugin)) continue;
+
+            const k = src[plugin.name] ?? DEFAULT_K;
+            if (k <= 0) continue;
+
+            const hits = plugin.searchBM25(query, k);
+            results.push(...hits);
+        }
+
+        return results.sort((a, b) => b.score - a.score);
+    }
+
+    /** Rebuild FTS5 indices across all BM25 plugins. */
+    rebuild(): void {
+        for (const plugin of this._registry.all) {
+            if (!isBM25SearchPlugin(plugin)) continue;
+            plugin.rebuildFTS?.();
+        }
+    }
+}

package/src/search/types.ts

@@ -0,0 +1,37 @@
+/**
+ * BrainBank — Search Types
+ * 
+ * Shared interface for all search strategies.
+ * Implement SearchStrategy to add a new search backend.
+ */
+
+import type { SearchResult } from '@/types.ts';
+
+/** Any search implementation follows this shape. */
+export interface SearchStrategy {
+    search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
+    /** Rebuild internal indices (e.g. FTS5). Optional. */
+    rebuild?(): void;
+}
+
+/** Pre-embedded vector search for a single domain (code, git, etc.). */
+export interface DomainVectorSearch {
+    /** Search using a pre-computed query vector. Optional queryText enables BM25 fusion. */
+    search(queryVec: Float32Array, k: number, minScore: number, useMMR?: boolean, mmrLambda?: number, queryText?: string): SearchResult[];
+}
+
+export interface SearchOptions {
+    /** Per-source result limits. Built-in: 'code', 'git', 'memory'. Any other key = custom plugin or KV collection. */
+    sources?: Record<string, number>;
+    /** Minimum similarity score. Default: 0.25 */
+    minScore?: number;
+    /** Use MMR for diversity. Default: true */
+    useMMR?: boolean;
+    /** MMR lambda. Default: 0.7 */
+    mmrLambda?: number;
+    /** Caller origin for debug logging. */
+    source?: 'cli' | 'mcp' | 'daemon' | 'api';
+    /** Filter results to files under these path prefixes. */
+    pathPrefix?: string | string[];
+}
+

package/src/search/vector/composite-vector-search.ts

@@ -0,0 +1,61 @@
+/**
+ * BrainBank — Composite Vector Search
+ *
+ * Generic orchestrator for domain-specific vector searches.
+ * Embeds the query once, delegates to registered DomainVectorSearch strategies.
+ * Uses round-robin interleaving when multiple strategies exist to ensure
+ * balanced representation across domains.
+ * Plugin-agnostic — strategies are discovered at wiring time.
+ */
+
+import type { EmbeddingProvider, SearchResult } from '@/types.ts';
+import type { SearchStrategy, SearchOptions, DomainVectorSearch } from '@/search/types.ts';
+
+export interface CompositeVectorConfig {
+    strategies: Map<string, DomainVectorSearch>;
+    embedding: EmbeddingProvider;
+    /** Default K values per strategy name. Strategies not listed default to 0. */
+    defaults?: Record<string, number>;
+}
+
+export class CompositeVectorSearch implements SearchStrategy {
+    /** Default K when no source override is provided. */
+    private static readonly DEFAULT_K = 6;
+
+    constructor(private _c: CompositeVectorConfig) {}
+
+    /** Search across all registered domain strategies with score-based merge. */
+    async search(query: string, options: SearchOptions = {}): Promise<SearchResult[]> {
+        const src = options.sources ?? {};
+        const { minScore = 0.25, useMMR = true, mmrLambda = 0.7 } = options;
+
+        const queryVec = await this._c.embedding.embed(query);
+
+        // Each strategy gets full K for ranking quality, cap total after merge
+        const allResults: SearchResult[] = [];
+        let requestedK = 0;
+
+        for (const [name, strategy] of this._c.strategies) {
+            const k = src[name] ?? this._c.defaults?.[name] ?? CompositeVectorSearch.DEFAULT_K;
+            if (k <= 0) continue;
+            requestedK = Math.max(requestedK, k);
+            const hits = strategy.search(queryVec, k, minScore, useMMR, mmrLambda, query);
+
+            allResults.push(...hits);
+        }
+
+        if (allResults.length === 0) return [];
+
+        // Sort by raw rrfScore, cap to requested K
+        allResults.sort((a, b) => b.score - a.score);
+        const capped = allResults.slice(0, requestedK);
+
+        // Normalize scores 0-1 globally
+        const maxScore = capped[0].score;
+        if (maxScore > 0) {
+            for (const r of capped) r.score = r.score / maxScore;
+        }
+
+        return capped;
+    }
+}

package/src/search/vector/mmr.ts

@@ -0,0 +1,64 @@
+/**
+ * BrainBank — Maximum Marginal Relevance (MMR)
+ * 
+ * Diversifies vector search results to avoid returning redundant items.
+ * λ=1.0 → pure relevance, λ=0.0 → pure diversity.
+ * Default λ=0.7 balances both.
+ */
+
+import type { VectorIndex, SearchHit } from '@/types.ts';
+import { cosineSimilarity } from '@/lib/math.ts';
+
+/**
+ * Search with Maximum Marginal Relevance for diversified results.
+ * 
+ * Algorithm:
+ *   1. Get 3x candidates from HNSW
+ *   2. Greedily select items that maximize: λ * relevance - (1-λ) * max_sim_to_selected
+ */
+export function searchMMR(
+    index: VectorIndex,
+    query: Float32Array,
+    vectorCache: Map<number, Float32Array>,
+    k: number,
+    lambda: number = 0.7,
+): SearchHit[] {
+    // Get more candidates than needed
+    const candidates = index.search(query, k * 3);
+    if (candidates.length <= k) return candidates;
+
+    const selected: SearchHit[] = [];
+    const remaining = [...candidates];
+
+    while (selected.length < k && remaining.length > 0) {
+        let bestScore = -Infinity;
+        let bestIdx = 0;
+
+        for (let i = 0; i < remaining.length; i++) {
+            const relevance = remaining[i].score;
+
+            // Max similarity to any already-selected item
+            let maxSim = 0;
+            for (const sel of selected) {
+                const candidateVec = vectorCache.get(remaining[i].id);
+                const selectedVec = vectorCache.get(sel.id);
+                if (candidateVec && selectedVec) {
+                    maxSim = Math.max(maxSim, cosineSimilarity(candidateVec, selectedVec));
+                }
+            }
+
+            // MMR score: balance relevance vs diversity
+            const mmrScore = lambda * relevance - (1 - lambda) * maxSim;
+
+            if (mmrScore > bestScore) {
+                bestScore = mmrScore;
+                bestIdx = i;
+            }
+        }
+
+        selected.push(remaining[bestIdx]);
+        remaining.splice(bestIdx, 1);
+    }
+
+    return selected;
+}