brainbank 0.1.0-beta.1

package/src/engine/search-api.ts

@@ -0,0 +1,218 @@
+/**
+ * BrainBank — Search API
+ *
+ * Thin orchestrator for all search operations.
+ * Pipeline: collect → fuse (RRF).
+ *
+ * Plugin-agnostic — discovers vector strategies and searchable plugins
+ * via capability interfaces. No hardcoded plugin names.
+ */
+
+import type { DatabaseAdapter } from '@/db/adapter.ts';
+import type { HNSWIndex } from '@/providers/vector/hnsw-index.ts';
+import type { SearchStrategy, SearchOptions, DomainVectorSearch } from '@/search/types.ts';
+import type { KVService } from '@/services/kv-service.ts';
+import type { PluginRegistry } from '@/services/plugin-registry.ts';
+import type { ResolvedConfig, EmbeddingProvider, SearchResult, ContextOptions } from '@/types.ts';
+
+import { isVectorSearchPlugin, isSearchable, isCoEditPlugin, isContextFormatterPlugin } from '@/plugin.ts';
+import { reciprocalRankFusion } from '@/lib/rrf.ts';
+import { filterByPath } from '@/search/bm25-boost.ts';
+import { ContextBuilder } from '@/search/context-builder.ts';
+import { CompositeBM25Search } from '@/search/keyword/composite-bm25-search.ts';
+import { CompositeVectorSearch } from '@/search/vector/composite-vector-search.ts';
+import { logQuery } from '@/lib/logger.ts';
+import type { QueryLogResult } from '@/lib/logger.ts';
+import { providerKey } from '@/lib/provider-key.ts';
+
+/** Dependencies injected at construction time. */
+export interface SearchAPIDeps {
+    search?:          SearchStrategy;
+    bm25?:            SearchStrategy;
+    registry:         PluginRegistry;
+    config:           ResolvedConfig;
+    kvService:        KVService;
+    contextBuilder?:  ContextBuilder;
+    embedding:        EmbeddingProvider;
+}
+
+/**
+ * Build a fully-wired SearchAPI from registry state.
+ * Discovers vector strategies from VectorSearchPlugin capability.
+ * Always returns an instance — handles search-less setups internally.
+ */
+export function createSearchAPI(
+    _db: DatabaseAdapter,
+    embedding: EmbeddingProvider,
+    config: ResolvedConfig,
+    registry: PluginRegistry,
+    kvService: KVService,
+    sharedHnsw: Map<string, { hnsw: HNSWIndex; vecCache: Map<number, Float32Array> }>,
+): SearchAPI {
+    const strategies = new Map<string, DomainVectorSearch>();
+
+    for (const mod of registry.all) {
+        if (isVectorSearchPlugin(mod)) {
+            const vs = mod.createVectorSearch();
+            if (vs) {
+                // Use full plugin name (e.g. code:servicehub-backend)
+                // so each repo gets its own vector search strategy
+                strategies.set(mod.name, vs);
+            }
+        }
+    }
+
+    const search = strategies.size > 0
+        ? new CompositeVectorSearch({
+            strategies,
+            embedding,
+        })
+        : undefined;
+
+    const bm25 = new CompositeBM25Search(registry);
+
+    const contextBuilder = new ContextBuilder(search, registry, config.pruner, embedding, config.contextFields ?? {}, config.expander);
+
+    return new SearchAPI({
+        search, bm25, registry, config,
+        kvService, contextBuilder, embedding,
+    });
+}
+
+
+export class SearchAPI {
+    constructor(private _d: SearchAPIDeps) {}
+
+    /** Build formatted context block for LLM injection. */
+    async getContext(task: string, options: ContextOptions = {}): Promise<string> {
+        if (!this._d.contextBuilder) return '';
+        return this._d.contextBuilder.build(task, options);
+    }
+
+    /** Semantic search across all loaded modules. */
+    async search(query: string, options?: SearchOptions): Promise<SearchResult[]> {
+        const t0 = Date.now();
+        const lists: SearchResult[][] = [];
+
+        if (this._d.search) {
+            lists.push(await this._d.search.search(query, options));
+        }
+
+        lists.push(...await this._collectSearchablePlugins(query, options));
+
+        let results: SearchResult[];
+        if (lists.length === 0) results = [];
+        else if (lists.length === 1) results = lists[0];
+        else results = reciprocalRankFusion(lists);
+
+        results = filterByPath(results, options?.pathPrefix);
+        this._logSearch('search', query, options, results, Date.now() - t0);
+        return results;
+    }
+
+    /** Hybrid search: vector + BM25 → RRF. */
+    async hybridSearch(query: string, options?: SearchOptions): Promise<SearchResult[]> {
+        const t0 = Date.now();
+        const src = options?.sources ?? {};
+        const lists: SearchResult[][] = [];
+
+        if (this._d.search) {
+            const [vec, kw] = await Promise.all([
+                this._d.search.search(query, options),
+                Promise.resolve(this._d.bm25?.search(query, options) ?? []),
+            ]);
+            lists.push(vec, kw);
+        }
+
+        lists.push(...await this._collectSearchablePlugins(query, options));
+        lists.push(...await this._collectKvCollections(query, src));
+
+        let results: SearchResult[];
+        if (lists.length === 0) results = [];
+        else {
+            results = reciprocalRankFusion(lists);
+        }
+
+        results = filterByPath(results, options?.pathPrefix);
+        this._logSearch('hybridSearch', query, options, results, Date.now() - t0);
+        return results;
+    }
+
+    /** BM25 keyword search only. */
+    async searchBM25(query: string, options?: SearchOptions): Promise<SearchResult[]> {
+        const t0 = Date.now();
+        let results = await this._d.bm25?.search(query, options) ?? [];
+        results = filterByPath(results, options?.pathPrefix);
+        this._logSearch('searchBM25', query, options, results, Date.now() - t0);
+        return results;
+    }
+
+    /** Rebuild FTS5 indices. */
+    rebuildFTS(): void {
+        this._d.bm25?.rebuild?.();
+    }
+
+    /** Collect results from all SearchablePlugins (docs, custom). */
+    private async _collectSearchablePlugins(
+        query: string, options?: SearchOptions,
+    ): Promise<SearchResult[][]> {
+        const lists: SearchResult[][] = [];
+        for (const mod of this._d.registry.all) {
+            if (!isSearchable(mod)) continue;
+            // Skip plugins that already participate via VectorSearchPlugin
+            if (isVectorSearchPlugin(mod)) continue;
+            const hits = await mod.search(query, options ? { ...options } : undefined);
+            if (hits.length > 0) lists.push(hits);
+        }
+        return lists;
+    }
+
+    /** Collect results from KV collections named in sources. */
+    private async _collectKvCollections(
+        query: string, sources: Record<string, number>,
+    ): Promise<SearchResult[][]> {
+        const pluginNames = new Set(this._d.registry.names.map(n => n.split(':')[0]));
+        const lists: SearchResult[][] = [];
+        for (const [name, k] of Object.entries(sources)) {
+            if (pluginNames.has(name)) continue;
+            const hits = await this._d.kvService.collection(name).searchAsResults(query, k);
+            if (hits.length > 0) lists.push(hits);
+        }
+        return lists;
+    }
+
+    /** Log a search/hybridSearch/searchBM25 call. */
+    private _logSearch(
+        method: 'search' | 'hybridSearch' | 'searchBM25',
+        query: string,
+        options: SearchOptions | undefined,
+        results: SearchResult[],
+        durationMs: number,
+    ): void {
+        logQuery({
+            source: options?.source ?? 'api',
+            method,
+            query,
+            embedding: providerKey(this._d.embedding),
+            pruner: null,
+            options: {
+                sources: options?.sources,
+                minScore: options?.minScore,
+            },
+            results: results.map(_toLogResult),
+            durationMs,
+        });
+    }
+}
+
+// ── Helpers ──────────────────────────────────────────
+
+function _toLogResult(r: SearchResult): QueryLogResult {
+    const meta = r.metadata as Record<string, unknown> | undefined;
+    return {
+        filePath: r.filePath ?? 'unknown',
+        score: r.score,
+        type: r.type,
+        name: (meta?.name as string | undefined) ?? undefined,
+    };
+}

package/src/index.ts

@@ -0,0 +1,154 @@
+/**
+ * BrainBank — Public API
+ * 
+ * Semantic knowledge bank for AI agents.
+ * 
+ *   import { BrainBank } from 'brainbank';
+ *   import { code } from '@brainbank/code';
+ *   import { docs } from '@brainbank/docs';
+ *   
+ *   const brain = new BrainBank()
+ *     .use(code({ repoPath: '.' }))
+ *     .use(docs());
+ *   
+ *   // Dynamic collections — universal data primitive
+ *   const errors = brain.collection('debug_errors');
+ *   await errors.add('Fixed null check', { file: 'api.ts' });
+ */
+
+
+export { BrainBank } from './brainbank.ts';
+
+// Plugin types + capability interfaces
+export type {
+    Plugin, PluginContext,
+    IndexablePlugin, SearchablePlugin, WatchablePlugin,
+    CoEditPlugin,
+    ReembeddablePlugin, ReembedTable,
+    DocsPlugin,
+    VectorSearchPlugin, ContextFormatterPlugin, ContextFieldPlugin,
+    ContextFieldDef,
+    MigratablePlugin, BM25SearchPlugin, ExpandablePlugin,
+    FileResolvablePlugin,
+    PluginScanInfo, PluginPreviewLine,
+} from './plugin.ts';
+export {
+    isIndexable, isSearchable, isWatchable,
+    isDocsPlugin, isCoEditPlugin, isReembeddable,
+    isVectorSearchPlugin, isContextFormatterPlugin, isContextFieldPlugin,
+    isMigratable, isBM25SearchPlugin, isExpandablePlugin,
+    isFileResolvable,
+} from './plugin.ts';
+
+// Constants (core-only)
+export { HNSW } from './constants.ts';
+export type { HnswKey } from './constants.ts';
+
+// Collections
+export { Collection } from './services/collection.ts';
+export type { CollectionItem, CollectionSearchOptions, CollectionAddOptions } from './services/collection.ts';
+
+// Types
+export type {
+    BrainBankConfig, ResolvedConfig,
+    EmbeddingProvider,
+    Pruner, PrunerItem,
+    Expander, ExpanderManifestItem, ExpanderResult,
+    VectorIndex, SearchHit,
+    CodeChunk,
+    GitCommitRecord,
+    SearchResult, SearchResultType,
+    CodeResult, CommitResult, DocumentResult, CollectionResult,
+    CodeResultMetadata, CommitResultMetadata, DocumentResultMetadata,
+    ContextOptions,
+    IndexResult,
+    ProgressCallback, StageProgressCallback,
+    CoEditSuggestion,
+    DocumentCollection, DocChunk,
+    WatchEvent, WatchEventHandler, WatchHandle, WatchConfig,
+} from './types.ts';
+export {
+    isCodeResult, isCommitResult, isDocumentResult,
+    isCollectionResult,
+    matchResult,
+} from './types.ts';
+
+// Embeddings
+export { LocalEmbedding } from './providers/embeddings/local-embedding.ts';
+export { OpenAIEmbedding } from './providers/embeddings/openai-embedding.ts';
+export type { OpenAIEmbeddingOptions } from './providers/embeddings/openai-embedding.ts';
+export { PerplexityEmbedding } from './providers/embeddings/perplexity-embedding.ts';
+export type { PerplexityEmbeddingOptions } from './providers/embeddings/perplexity-embedding.ts';
+export { PerplexityContextEmbedding } from './providers/embeddings/perplexity-context-embedding.ts';
+export type { PerplexityContextEmbeddingOptions } from './providers/embeddings/perplexity-context-embedding.ts';
+export { EmbeddingWorkerProxy } from './providers/embeddings/embedding-worker.ts';
+export type { ReembedResult, ReembedOptions } from './engine/reembed.ts';
+export type { WatchOptions } from './services/watch.ts';
+export { Watcher } from './services/watch.ts';
+export { WebhookServer } from './services/webhook-server.ts';
+export type { WebhookHandler } from './services/webhook-server.ts';
+export { HttpServer } from './services/http-server.ts';
+export type { HttpServerOptions } from './services/http-server.ts';
+export { isServerRunning, getServerUrl, readPid, writePid, removePid, DEFAULT_PORT } from './services/daemon.ts';
+
+// Pruner
+export { HaikuPruner } from './providers/pruners/haiku-pruner.ts';
+export type { HaikuPrunerOptions } from './providers/pruners/haiku-pruner.ts';
+export { pruneResults } from './lib/prune.ts';
+
+// Expander
+export { HaikuExpander } from './providers/pruners/haiku-expander.ts';
+export type { HaikuExpanderOptions } from './providers/pruners/haiku-expander.ts';
+
+// Embedding resolver
+export { resolveEmbedding, providerKey } from './providers/embeddings/resolve.ts';
+export type { EmbeddingKey } from './providers/embeddings/resolve.ts';
+
+// Config
+export { resolveConfig, DEFAULTS } from './config.ts';
+
+// Migrations
+export type { Migration } from './db/migrations.ts';
+export { runPluginMigrations } from './db/migrations.ts';
+
+// Incremental indexing tracker
+export type { IncrementalTracker } from './db/tracker.ts';
+export { createTracker } from './db/tracker.ts';
+
+// Vector indices
+export { HNSWIndex } from './providers/vector/hnsw-index.ts';
+export { searchMMR } from './search/vector/mmr.ts';
+
+// Language support (used by @brainbank/code)
+export { SUPPORTED_EXTENSIONS, IGNORE_DIRS, isSupported, getLanguage, isIgnoredDir, isIgnoredFile } from './lib/languages.ts';
+
+// Math utilities (needed by plugins)
+export { vecToBuffer, cosineSimilarity, normalize } from './lib/math.ts';
+
+// KV service (collection infrastructure)
+export { KVService } from './services/kv-service.ts';
+
+// Search internals (plugins may use these)
+export { ContextBuilder } from './search/context-builder.ts';
+export { CompositeVectorSearch } from './search/vector/composite-vector-search.ts';
+export { CompositeBM25Search } from './search/keyword/composite-bm25-search.ts';
+export { reciprocalRankFusion } from './lib/rrf.ts';
+export { sanitizeFTS, normalizeBM25, escapeLike } from './lib/fts.ts';
+
+// Search types
+export type { SearchStrategy, SearchOptions, DomainVectorSearch } from './search/types.ts';
+
+// Multi-process coordination
+export { bumpVersion, getVersions, getVersion } from './db/metadata.ts';
+export { acquireLock, releaseLock, withLock } from './lib/write-lock.ts';
+
+// Database adapter (for plugin access and custom adapters)
+export type { DatabaseAdapter, PreparedStatement, ExecuteResult, AdapterCapabilities } from './db/adapter.ts';
+export type { KvDataRow, KvVectorRow, EmbeddingMetaRow, VectorRow, CountRow } from './db/adapter.ts';
+export { SQLiteAdapter } from './db/sqlite-adapter.ts';
+
+// Factory (for programmatic BrainBank creation)
+export { createBrain, resetFactoryCache, contextFromCLI } from './cli/factory/index.ts';
+export type { BrainContext } from './cli/factory/brain-context.ts';
+export type { ProjectConfig } from './cli/factory/config-loader.ts';
+

package/src/lib/fts.ts

@@ -0,0 +1,57 @@
+/**
+ * BrainBank — FTS Utilities
+ * 
+ * Shared helpers for SQLite FTS5 query sanitization.
+ */
+
+/**
+ * Split camelCase, PascalCase, and snake_case into individual words.
+ *   "MagicLinkCallback" → "Magic Link Callback"
+ *   "tenant_worker"     → "tenant worker"
+ */
+function splitCompound(word: string): string {
+    return word
+        .replace(/([a-z])([A-Z])/g, '$1 $2')   // camelCase → camel Case
+        .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') // HTMLParser → HTML Parser
+        .replace(/[_\-./\\]/g, ' ')              // snake_case, kebab-case, paths
+        .trim();
+}
+
+/**
+ * Sanitize a user query for FTS5 syntax.
+ * Strips operators that would cause parse errors, splits compound words,
+ * and converts words to implicit AND with exact-match quoting.
+ */
+export function sanitizeFTS(query: string): string {
+    const clean = query
+        .replace(/[{}[\]()^~*:]/g, ' ')
+        .replace(/\bAND\b|\bOR\b|\bNOT\b|\bNEAR\b/gi, '')
+        .trim();
+
+    // Split compound words (camelCase, PascalCase, snake_case)
+    const expanded = clean.split(/\s+/)
+        .map(w => splitCompound(w))
+        .join(' ');
+
+    const words = expanded.split(/\s+/).filter(w => w.length > 1);
+    if (words.length === 0) return '';
+
+    return words.map(w => `"${w}"`).join(' ');
+}
+
+/**
+ * Normalize BM25 score from SQLite (negative, lower = better)
+ * to 0.0–1.0 (higher = better) for consistency with vector search.
+ */
+export function normalizeBM25(rawScore: number): number {
+    const abs = Math.abs(rawScore);
+    return 1.0 / (1.0 + Math.exp(-0.3 * (abs - 5)));
+}
+
+/**
+ * Escape SQL LIKE wildcard characters (`%`, `_`, `\`).
+ * Use with `LIKE ? ESCAPE '\'` in queries.
+ */
+export function escapeLike(s: string): string {
+    return s.replace(/[%_\\]/g, '\\$&');
+}

package/src/lib/languages.ts

@@ -0,0 +1,180 @@
+/**
+ * BrainBank — Language Registry
+ * 
+ * Supported file extensions, language mappings, and ignore lists.
+ * Controls which files get indexed and how they're chunked.
+ */
+
+
+export const SUPPORTED_EXTENSIONS: Record<string, string> = {
+    // TypeScript / JavaScript
+    '.ts': 'typescript',
+    '.tsx': 'typescript',
+    '.js': 'javascript',
+    '.jsx': 'javascript',
+    '.mjs': 'javascript',
+    '.cjs': 'javascript',
+
+    // Systems
+    '.go': 'go',
+    '.rs': 'rust',
+    '.cpp': 'cpp',
+    '.cc': 'cpp',
+    '.c': 'c',
+    '.h': 'c',
+    '.hpp': 'cpp',
+
+    // JVM
+    '.java': 'java',
+    '.kt': 'kotlin',
+    '.scala': 'scala',
+
+    // Scripting
+    '.py': 'python',
+    '.rb': 'ruby',
+    '.php': 'php',
+    '.lua': 'lua',
+    '.sh': 'bash',
+    '.bash': 'bash',
+    '.zsh': 'bash',
+
+    // Web
+    '.html': 'html',
+    '.css': 'css',
+    '.scss': 'scss',
+    '.less': 'less',
+    '.svelte': 'svelte',
+    '.vue': 'vue',
+
+    // Data / Config (JSON + YAML excluded — config/CI files cause search noise)
+    '.toml': 'toml',
+    '.xml': 'xml',
+    '.graphql': 'graphql',
+    '.gql': 'graphql',
+
+    // Database
+    '.sql': 'sql',
+    '.prisma': 'prisma',
+
+    // Other
+    '.swift': 'swift',
+    '.dart': 'dart',
+    '.r': 'r',
+    '.ex': 'elixir',
+    '.exs': 'elixir',
+    '.erl': 'erlang',
+    '.zig': 'zig',
+};
+
+
+export const IGNORE_DIRS = new Set([
+    // Package managers
+    'node_modules',
+    'bower_components',
+    '.pnpm',
+
+    // Build output
+    'dist',
+    'build',
+    'out',
+    '.next',
+    '.nuxt',
+    '.output',
+    '.svelte-kit',
+
+    // Auto-generated code
+    'generated',
+    'sdk',
+    'openapi',
+
+    // Version control
+    '.git',
+    '.hg',
+    '.svn',
+
+    // IDE / Editor
+    '.idea',
+    '.vscode',
+
+    // Runtime / Cache
+    '__pycache__',
+    '.pytest_cache',
+    'venv',
+    '.venv',
+    '.env',
+    '.tox',
+
+    // Coverage / Test artifacts
+    'coverage',
+    '.nyc_output',
+    'htmlcov',
+
+    // Compiled
+    'target',     // Rust, Java
+    '.cargo',
+    'vendor',     // Go, PHP
+
+    // Database (auto-generated migrations, dumps, seeds)
+    'migrations',
+    'db_dumps',
+    'seeds',
+
+    // AI / Model cache
+    '.model-cache',
+    '.brainbank',
+
+    // OS
+    '.DS_Store',
+]);
+
+
+export const IGNORE_FILES = new Set([
+    'package-lock.json',
+    'yarn.lock',
+    'pnpm-lock.yaml',
+    'bun.lockb',
+    'Cargo.lock',
+    'Gemfile.lock',
+    'poetry.lock',
+    'composer.lock',
+    'go.sum',
+]);
+
+
+import path from 'node:path';
+
+/** Check if a file extension is supported for indexing. */
+export function isSupported(filePath: string): boolean {
+    const ext = path.extname(filePath).toLowerCase();
+    return ext in SUPPORTED_EXTENSIONS;
+}
+
+/** Get the language name for a file. Returns undefined if not supported. */
+export function getLanguage(filePath: string): string | undefined {
+    const ext = path.extname(filePath).toLowerCase();
+    return SUPPORTED_EXTENSIONS[ext];
+}
+
+/** Check if a directory name should be ignored. */
+export function isIgnoredDir(dirName: string): boolean {
+    return IGNORE_DIRS.has(dirName);
+}
+
+/** Check if a filename should be ignored. */
+export function isIgnoredFile(fileName: string): boolean {
+    return IGNORE_FILES.has(fileName);
+}
+
+/** Check if a relative path matches any of the given glob patterns. */
+export function matchesGlob(relPath: string, patterns: string[]): boolean {
+    for (const pattern of patterns) {
+        const regex = pattern
+            .replace(/[.+^${}()|[\]\\]/g, '\\$&')  // escape regex specials (not * ?)
+            .replace(/\*\*/g, '\x00')               // placeholder for **
+            .replace(/\*/g, '[^/]*')                 // * = anything except /
+            .replace(/\?/g, '.')                     // ? = any single char
+            .replace(/\x00/g, '.*');                 // ** = anything including /
+        if (new RegExp(`^${regex}$`).test(relPath)) return true;
+    }
+    return false;
+}