brainbank 0.1.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +155 -0
- package/assets/architecture.png +0 -0
- package/bin/brainbank +18 -0
- package/bin/brainbank-mcp +19 -0
- package/dist/chunk-3YBCD6DI.js +117 -0
- package/dist/chunk-3YBCD6DI.js.map +1 -0
- package/dist/chunk-63GBCDS5.js +3249 -0
- package/dist/chunk-63GBCDS5.js.map +1 -0
- package/dist/chunk-DMFMTOHF.js +123 -0
- package/dist/chunk-DMFMTOHF.js.map +1 -0
- package/dist/chunk-FQYKWB2Q.js +136 -0
- package/dist/chunk-FQYKWB2Q.js.map +1 -0
- package/dist/chunk-IMJJ2VEM.js +74 -0
- package/dist/chunk-IMJJ2VEM.js.map +1 -0
- package/dist/chunk-M744PCJQ.js +43 -0
- package/dist/chunk-M744PCJQ.js.map +1 -0
- package/dist/chunk-O3J6ZIXK.js +82 -0
- package/dist/chunk-O3J6ZIXK.js.map +1 -0
- package/dist/chunk-OPH7GZ7U.js +124 -0
- package/dist/chunk-OPH7GZ7U.js.map +1 -0
- package/dist/chunk-PXEWQMN7.js +89 -0
- package/dist/chunk-PXEWQMN7.js.map +1 -0
- package/dist/chunk-RDQYDLYZ.js +69 -0
- package/dist/chunk-RDQYDLYZ.js.map +1 -0
- package/dist/chunk-VIIHPCC4.js +254 -0
- package/dist/chunk-VIIHPCC4.js.map +1 -0
- package/dist/chunk-WCQVDF3K.js +14 -0
- package/dist/chunk-WCQVDF3K.js.map +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +3076 -0
- package/dist/cli.js.map +1 -0
- package/dist/haiku-expander-YRSIPGKP.js +8 -0
- package/dist/haiku-expander-YRSIPGKP.js.map +1 -0
- package/dist/haiku-pruner-SHAXUPY6.js +8 -0
- package/dist/haiku-pruner-SHAXUPY6.js.map +1 -0
- package/dist/http-server-QUXHLWUM.js +9 -0
- package/dist/http-server-QUXHLWUM.js.map +1 -0
- package/dist/index.d.ts +2161 -0
- package/dist/index.js +357 -0
- package/dist/index.js.map +1 -0
- package/dist/local-embedding-NZQTILGV.js +8 -0
- package/dist/local-embedding-NZQTILGV.js.map +1 -0
- package/dist/mcp.d.ts +2 -0
- package/dist/mcp.js +334 -0
- package/dist/mcp.js.map +1 -0
- package/dist/openai-embedding-ZP5TSUJG.js +8 -0
- package/dist/openai-embedding-ZP5TSUJG.js.map +1 -0
- package/dist/perplexity-context-embedding-GI5PHE6X.js +9 -0
- package/dist/perplexity-context-embedding-GI5PHE6X.js.map +1 -0
- package/dist/perplexity-embedding-KZRYGJRC.js +10 -0
- package/dist/perplexity-embedding-KZRYGJRC.js.map +1 -0
- package/dist/plugin-IKQ6IRSJ.js +32 -0
- package/dist/plugin-IKQ6IRSJ.js.map +1 -0
- package/dist/resolve-ASGLBNUC.js +10 -0
- package/dist/resolve-ASGLBNUC.js.map +1 -0
- package/dist/stats-tui-ZY2NQSEA.js +1904 -0
- package/dist/stats-tui-ZY2NQSEA.js.map +1 -0
- package/package.json +96 -0
- package/src/brainbank.ts +617 -0
- package/src/cli/commands/collection.ts +77 -0
- package/src/cli/commands/context.ts +179 -0
- package/src/cli/commands/daemon.ts +100 -0
- package/src/cli/commands/docs.ts +71 -0
- package/src/cli/commands/files.ts +69 -0
- package/src/cli/commands/help.ts +77 -0
- package/src/cli/commands/index.ts +482 -0
- package/src/cli/commands/kv.ts +140 -0
- package/src/cli/commands/mcp-export.ts +273 -0
- package/src/cli/commands/mcp.ts +6 -0
- package/src/cli/commands/reembed.ts +30 -0
- package/src/cli/commands/scan.ts +336 -0
- package/src/cli/commands/search.ts +203 -0
- package/src/cli/commands/stats.ts +68 -0
- package/src/cli/commands/status.ts +47 -0
- package/src/cli/commands/watch.ts +47 -0
- package/src/cli/factory/brain-context.ts +43 -0
- package/src/cli/factory/builtin-registration.ts +87 -0
- package/src/cli/factory/config-loader.ts +77 -0
- package/src/cli/factory/index.ts +69 -0
- package/src/cli/factory/plugin-loader.ts +325 -0
- package/src/cli/index.ts +71 -0
- package/src/cli/server-client.ts +178 -0
- package/src/cli/tui/index-tui.tsx +667 -0
- package/src/cli/tui/stats-data.ts +523 -0
- package/src/cli/tui/stats-search.ts +262 -0
- package/src/cli/tui/stats-tui.tsx +1465 -0
- package/src/cli/tui/tree-scanner.ts +650 -0
- package/src/cli/utils.ts +137 -0
- package/src/config.ts +49 -0
- package/src/constants.ts +21 -0
- package/src/db/adapter.ts +112 -0
- package/src/db/metadata.ts +130 -0
- package/src/db/migrations.ts +66 -0
- package/src/db/sqlite-adapter.ts +218 -0
- package/src/db/tracker.ts +91 -0
- package/src/engine/index-api.ts +81 -0
- package/src/engine/reembed.ts +206 -0
- package/src/engine/search-api.ts +218 -0
- package/src/index.ts +154 -0
- package/src/lib/fts.ts +57 -0
- package/src/lib/languages.ts +180 -0
- package/src/lib/logger.ts +126 -0
- package/src/lib/math.ts +87 -0
- package/src/lib/provider-key.ts +20 -0
- package/src/lib/prune.ts +71 -0
- package/src/lib/rrf.ts +133 -0
- package/src/lib/write-lock.ts +108 -0
- package/src/mcp/mcp-server.ts +195 -0
- package/src/mcp/workspace-factory.ts +68 -0
- package/src/mcp/workspace-pool.ts +224 -0
- package/src/plugin.ts +381 -0
- package/src/providers/embeddings/embedding-worker-thread.ts +95 -0
- package/src/providers/embeddings/embedding-worker.ts +141 -0
- package/src/providers/embeddings/local-embedding.ts +115 -0
- package/src/providers/embeddings/openai-embedding.ts +167 -0
- package/src/providers/embeddings/perplexity-context-embedding.ts +195 -0
- package/src/providers/embeddings/perplexity-embedding.ts +165 -0
- package/src/providers/embeddings/resolve.ts +34 -0
- package/src/providers/pruners/haiku-expander.ts +166 -0
- package/src/providers/pruners/haiku-pruner.ts +112 -0
- package/src/providers/vector/hnsw-index.ts +174 -0
- package/src/providers/vector/hnsw-loader.ts +129 -0
- package/src/search/bm25-boost.ts +69 -0
- package/src/search/context-builder.ts +251 -0
- package/src/search/keyword/composite-bm25-search.ts +47 -0
- package/src/search/types.ts +37 -0
- package/src/search/vector/composite-vector-search.ts +61 -0
- package/src/search/vector/mmr.ts +64 -0
- package/src/services/collection.ts +384 -0
- package/src/services/daemon.ts +87 -0
- package/src/services/http-server.ts +336 -0
- package/src/services/kv-service.ts +64 -0
- package/src/services/plugin-registry.ts +77 -0
- package/src/services/watch.ts +340 -0
- package/src/services/webhook-server.ts +100 -0
- package/src/types.ts +493 -0
package/src/brainbank.ts
ADDED
|
@@ -0,0 +1,617 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BrainBank — Main Orchestrator
|
|
3
|
+
*
|
|
4
|
+
* Thin facade that composes services:
|
|
5
|
+
* - **PluginRegistry** — registration + lookup
|
|
6
|
+
* - **SearchAPI** — all search + context logic
|
|
7
|
+
* - **runIndex** — code / git / docs indexing orchestration
|
|
8
|
+
*
|
|
9
|
+
* Initialization is inline — no indirection layers.
|
|
10
|
+
* All heavy logic lives in those modules; BrainBank owns state,
|
|
11
|
+
* guards (`_requireInit` / `initialize`), and public API shape.
|
|
12
|
+
*
|
|
13
|
+
* Multi-process coordination:
|
|
14
|
+
* - `ensureFresh()` detects stale HNSW indices via `index_state` table
|
|
15
|
+
* - Hot-reloads from disk when another process updated the index
|
|
16
|
+
* - Called implicitly before every search operation
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type { ReembedResult, ReembedOptions } from './engine/reembed.ts';
|
|
20
|
+
import type { IndexDeps } from './engine/index-api.ts';
|
|
21
|
+
import type { Plugin, PluginContext } from './plugin.ts';
|
|
22
|
+
import type { SearchOptions } from './search/types.ts';
|
|
23
|
+
import type { WatchOptions } from './services/watch.ts';
|
|
24
|
+
import type {
|
|
25
|
+
BrainBankConfig, ResolvedConfig, EmbeddingProvider,
|
|
26
|
+
SearchResult, ICollection,
|
|
27
|
+
ContextOptions, StageProgressCallback,
|
|
28
|
+
} from './types.ts';
|
|
29
|
+
|
|
30
|
+
import { EventEmitter } from 'node:events';
|
|
31
|
+
|
|
32
|
+
import * as path from 'node:path';
|
|
33
|
+
import { resolveConfig } from './config.ts';
|
|
34
|
+
import { HNSW } from './constants.ts';
|
|
35
|
+
import type { DatabaseAdapter } from './db/adapter.ts';
|
|
36
|
+
import { SQLiteAdapter } from './db/sqlite-adapter.ts';
|
|
37
|
+
import { createTracker } from './db/tracker.ts';
|
|
38
|
+
import { setEmbeddingMeta, getEmbeddingMeta, detectProviderMismatch, getVersions } from './db/metadata.ts';
|
|
39
|
+
import { runIndex } from './engine/index-api.ts';
|
|
40
|
+
import { reembedAll } from './engine/reembed.ts';
|
|
41
|
+
import { SearchAPI, createSearchAPI } from './engine/search-api.ts';
|
|
42
|
+
import { isReembeddable, isFileResolvable } from './plugin.ts';
|
|
43
|
+
|
|
44
|
+
import { resolveEmbedding } from './providers/embeddings/resolve.ts';
|
|
45
|
+
import { HNSWIndex } from './providers/vector/hnsw-index.ts';
|
|
46
|
+
import { hnswPath, countRows, saveAllHnsw, loadVectors, loadVecCache, reloadHnsw } from './providers/vector/hnsw-loader.ts';
|
|
47
|
+
import { KVService } from './services/kv-service.ts';
|
|
48
|
+
import { PluginRegistry } from './services/plugin-registry.ts';
|
|
49
|
+
import { Watcher } from './services/watch.ts';
|
|
50
|
+
import { WebhookServer } from './services/webhook-server.ts';
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
export class BrainBank extends EventEmitter {
|
|
54
|
+
private _config: ResolvedConfig;
|
|
55
|
+
private _db!: DatabaseAdapter;
|
|
56
|
+
private _embedding!: EmbeddingProvider;
|
|
57
|
+
private _registry = new PluginRegistry();
|
|
58
|
+
private _searchAPI?: SearchAPI;
|
|
59
|
+
private _indexDeps?: IndexDeps;
|
|
60
|
+
private _kvService?: KVService;
|
|
61
|
+
private _initialized = false;
|
|
62
|
+
private _initPromise: Promise<void> | null = null;
|
|
63
|
+
private _watcher?: Watcher;
|
|
64
|
+
private _webhookServer?: WebhookServer;
|
|
65
|
+
private _sharedHnsw = new Map<string, { hnsw: HNSWIndex; vecCache: Map<number, Float32Array> }>();
|
|
66
|
+
private _repoDBs = new Map<string, DatabaseAdapter>();
|
|
67
|
+
private _loadedVersions = new Map<string, number>();
|
|
68
|
+
|
|
69
|
+
constructor(config: BrainBankConfig = {}) {
|
|
70
|
+
super();
|
|
71
|
+
this._config = resolveConfig(config);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Whether the brainbank has been initialized. */
|
|
75
|
+
get isInitialized(): boolean {
|
|
76
|
+
return this._initialized;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** The resolved configuration. */
|
|
80
|
+
get config(): Readonly<ResolvedConfig> {
|
|
81
|
+
return this._config;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** All registered plugin names (insertion order). */
|
|
85
|
+
get plugins(): string[] {
|
|
86
|
+
return this._registry.names;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Register a plugin. Chainable.
|
|
91
|
+
*
|
|
92
|
+
* @example
|
|
93
|
+
* brain.use(code({ repoPath: '.' })).use(docs());
|
|
94
|
+
*
|
|
95
|
+
* @throws If called after `initialize()`.
|
|
96
|
+
*/
|
|
97
|
+
use(plugin: Plugin): this {
|
|
98
|
+
if (this._initialized) {
|
|
99
|
+
throw new Error(
|
|
100
|
+
`BrainBank: Cannot add plugin '${plugin.name}' after initialization. ` +
|
|
101
|
+
`Call .use() before any operations.`,
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
this._registry.register(plugin);
|
|
105
|
+
return this;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Check if a plugin is loaded.
|
|
110
|
+
* Also matches type prefix (e.g. `'code'` matches `'code:frontend'`).
|
|
111
|
+
*/
|
|
112
|
+
has(name: string): boolean {
|
|
113
|
+
return this._registry.has(name);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Get a plugin instance by name. Returns `undefined` if not loaded. */
|
|
117
|
+
plugin<T extends Plugin = Plugin>(name: string): T | undefined {
|
|
118
|
+
return this._registry.has(name) ? this._registry.get<T>(name) : undefined;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Initialize database, HNSW indices, and load existing vectors.
|
|
123
|
+
* Automatically called by `index` / `search` methods if not yet initialized.
|
|
124
|
+
* Concurrent calls are deduped via `_initPromise`.
|
|
125
|
+
*
|
|
126
|
+
* @param options.force - If `true`, skip vector load on dimension mismatch.
|
|
127
|
+
*/
|
|
128
|
+
async initialize(options: { force?: boolean } = {}): Promise<void> {
|
|
129
|
+
if (this._initialized) return;
|
|
130
|
+
if (this._initPromise) return this._initPromise;
|
|
131
|
+
|
|
132
|
+
this._initPromise = this._runInitialize(options)
|
|
133
|
+
.then(() => { this._initPromise = null; })
|
|
134
|
+
.catch(err => {
|
|
135
|
+
this._cleanupAfterFailedInit();
|
|
136
|
+
throw err;
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
return this._initPromise;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Estimated memory footprint of loaded HNSW indices (bytes).
|
|
144
|
+
* Counts only vector data: `vectorCount × dims × 4`.
|
|
145
|
+
* Returns 0 if not initialized.
|
|
146
|
+
*/
|
|
147
|
+
memoryHint(): number {
|
|
148
|
+
if (!this._initialized) return 0;
|
|
149
|
+
const dims = this._config.embeddingDims;
|
|
150
|
+
const bytesPerVector = dims * 4;
|
|
151
|
+
|
|
152
|
+
let total = 0;
|
|
153
|
+
if (this._kvService) total += this._kvService.hnsw.size * bytesPerVector;
|
|
154
|
+
for (const { hnsw } of this._sharedHnsw.values()) {
|
|
155
|
+
total += hnsw.size * bytesPerVector;
|
|
156
|
+
}
|
|
157
|
+
return total;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Close database and release all resources. Synchronous. */
|
|
161
|
+
close(): void {
|
|
162
|
+
void this._watcher?.close();
|
|
163
|
+
this._webhookServer?.close();
|
|
164
|
+
for (const plugin of this._registry.all) plugin.close?.();
|
|
165
|
+
|
|
166
|
+
const pruner = this._config.pruner as { close?: () => void } | undefined;
|
|
167
|
+
pruner?.close?.();
|
|
168
|
+
|
|
169
|
+
this._embedding?.close().catch(() => { });
|
|
170
|
+
for (const db of this._repoDBs.values()) db.close();
|
|
171
|
+
this._repoDBs.clear();
|
|
172
|
+
this._db?.close();
|
|
173
|
+
this._initialized = false;
|
|
174
|
+
this._kvService?.clear();
|
|
175
|
+
this._sharedHnsw.clear();
|
|
176
|
+
this._loadedVersions.clear();
|
|
177
|
+
this._kvService = undefined;
|
|
178
|
+
this._searchAPI = undefined;
|
|
179
|
+
this._indexDeps = undefined;
|
|
180
|
+
this._webhookServer = undefined;
|
|
181
|
+
this._registry.clear();
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Get or create a dynamic collection (universal KV primitive).
|
|
186
|
+
*
|
|
187
|
+
* @example
|
|
188
|
+
* const errors = brain.collection('debug_errors');
|
|
189
|
+
* await errors.add('Fixed null check', { file: 'api.ts' });
|
|
190
|
+
* const hits = await errors.search('null pointer');
|
|
191
|
+
*
|
|
192
|
+
* @throws If not initialized.
|
|
193
|
+
*/
|
|
194
|
+
collection(name: string): ICollection {
|
|
195
|
+
if (!this._kvService) {
|
|
196
|
+
throw new Error('BrainBank: Collections not ready. Call await brain.initialize() first.');
|
|
197
|
+
}
|
|
198
|
+
return this._kvService.collection(name);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** List all collection names that have data. */
|
|
202
|
+
listCollectionNames(): string[] {
|
|
203
|
+
this._requireInit('listCollectionNames');
|
|
204
|
+
return this._kvService!.listNames();
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Delete a collection's data and evict from cache. */
|
|
208
|
+
deleteCollection(name: string): void {
|
|
209
|
+
this._requireInit('deleteCollection');
|
|
210
|
+
this._kvService!.delete(name);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Run indexing across selected modules. Auto-initializes. */
|
|
214
|
+
async index(options: {
|
|
215
|
+
modules?: string[];
|
|
216
|
+
forceReindex?: boolean;
|
|
217
|
+
onProgress?: StageProgressCallback;
|
|
218
|
+
pluginOptions?: Record<string, unknown>;
|
|
219
|
+
} = {}): Promise<Record<string, unknown>> {
|
|
220
|
+
await this.initialize();
|
|
221
|
+
return runIndex(this._indexDeps!, options);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Detect stale HNSW indices and hot-reload from disk.
|
|
226
|
+
* Called implicitly before every search operation.
|
|
227
|
+
* Cost: one SQLite SELECT (~5μs on WAL mode).
|
|
228
|
+
*/
|
|
229
|
+
async ensureFresh(): Promise<void> {
|
|
230
|
+
if (!this._initialized) return;
|
|
231
|
+
|
|
232
|
+
const dbVersions = getVersions(this._db);
|
|
233
|
+
for (const [name, dbVersion] of dbVersions) {
|
|
234
|
+
const loaded = this._loadedVersions.get(name) ?? 0;
|
|
235
|
+
if (dbVersion <= loaded) continue;
|
|
236
|
+
|
|
237
|
+
this.emit('progress', `Hot-reload: ${name} version ${loaded} → ${dbVersion}`);
|
|
238
|
+
this._reloadIndex(name);
|
|
239
|
+
this._loadedVersions.set(name, dbVersion);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Semantic search across all loaded modules.
|
|
245
|
+
* Scope via `sources: { code: 10, git: 0 }`.
|
|
246
|
+
*/
|
|
247
|
+
async search(query: string, options?: SearchOptions): Promise<SearchResult[]> {
|
|
248
|
+
await this.initialize();
|
|
249
|
+
await this.ensureFresh();
|
|
250
|
+
return this._searchAPI?.search(query, options) ?? [];
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Hybrid search: vector + BM25 fused with Reciprocal Rank Fusion.
|
|
255
|
+
* Scope via `sources: { code: 10, git: 5, docs: 3, myNotes: 5 }`.
|
|
256
|
+
*/
|
|
257
|
+
async hybridSearch(query: string, options?: SearchOptions): Promise<SearchResult[]> {
|
|
258
|
+
await this.initialize();
|
|
259
|
+
await this.ensureFresh();
|
|
260
|
+
return this._searchAPI?.hybridSearch(query, options) ?? [];
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** BM25 keyword search only (no embeddings needed). */
|
|
264
|
+
async searchBM25(query: string, options?: SearchOptions): Promise<SearchResult[]> {
|
|
265
|
+
await this.initialize();
|
|
266
|
+
await this.ensureFresh();
|
|
267
|
+
return this._searchAPI?.searchBM25(query, options) ?? [];
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** Build formatted context block for LLM system prompt injection. Auto-initializes. */
|
|
271
|
+
async getContext(task: string, options: ContextOptions = {}): Promise<string> {
|
|
272
|
+
await this.initialize();
|
|
273
|
+
await this.ensureFresh();
|
|
274
|
+
return this._searchAPI?.getContext(task, options) ?? '';
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Resolve file paths, directories, and glob patterns to full SearchResults.
|
|
279
|
+
* Bypasses search entirely — reads directly from plugin indexes.
|
|
280
|
+
*
|
|
281
|
+
* @example
|
|
282
|
+
* const files = brain.resolveFiles(['src/auth/login.ts', 'src/graph/']);
|
|
283
|
+
*/
|
|
284
|
+
resolveFiles(patterns: string[]): SearchResult[] {
|
|
285
|
+
this._requireInit('resolveFiles');
|
|
286
|
+
const results: SearchResult[] = [];
|
|
287
|
+
for (const mod of this._registry.all) {
|
|
288
|
+
if (!isFileResolvable(mod)) continue;
|
|
289
|
+
results.push(...mod.resolveFiles(patterns));
|
|
290
|
+
}
|
|
291
|
+
return results;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** Rebuild FTS5 indices. */
|
|
295
|
+
rebuildFTS(): void {
|
|
296
|
+
this._requireInit('rebuildFTS');
|
|
297
|
+
this._searchAPI?.rebuildFTS();
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** Get statistics for all loaded plugins. */
|
|
301
|
+
stats(): Record<string, Record<string, number | string> | undefined> {
|
|
302
|
+
this._requireInit('stats');
|
|
303
|
+
const result: Record<string, Record<string, number | string> | undefined> = {};
|
|
304
|
+
|
|
305
|
+
for (const mod of this._registry.all) {
|
|
306
|
+
if (mod.stats) {
|
|
307
|
+
const baseType = mod.name.split(':')[0];
|
|
308
|
+
result[baseType] = mod.stats();
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
return result;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
/** Start watching for changes and auto-re-index. */
|
|
317
|
+
watch(options: WatchOptions = {}): Watcher {
|
|
318
|
+
this._requireInit('watch');
|
|
319
|
+
void this._watcher?.close();
|
|
320
|
+
this._watcher = new Watcher(
|
|
321
|
+
async () => { await this.index(); },
|
|
322
|
+
this._registry.all,
|
|
323
|
+
options,
|
|
324
|
+
this._config.repoPath,
|
|
325
|
+
);
|
|
326
|
+
return this._watcher;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* Re-embed all existing text with the current embedding provider.
|
|
331
|
+
* Use after switching providers (e.g. Local → OpenAI).
|
|
332
|
+
*/
|
|
333
|
+
async reembed(options: ReembedOptions = {}): Promise<ReembedResult> {
|
|
334
|
+
await this.initialize();
|
|
335
|
+
|
|
336
|
+
const hnswMap = new Map<string, { hnsw: HNSWIndex; vecs: Map<number, Float32Array> }>();
|
|
337
|
+
|
|
338
|
+
if (this._kvService) {
|
|
339
|
+
hnswMap.set(HNSW.KV, { hnsw: this._kvService.hnsw, vecs: this._kvService.vecs });
|
|
340
|
+
}
|
|
341
|
+
for (const [type, shared] of this._sharedHnsw) {
|
|
342
|
+
hnswMap.set(type, { hnsw: shared.hnsw, vecs: shared.vecCache });
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
const result = await reembedAll(this._db, this._embedding, hnswMap, this._registry.all, options, {
|
|
347
|
+
dbPath: this._config.dbPath,
|
|
348
|
+
kvHnsw: this._kvService!.hnsw,
|
|
349
|
+
sharedHnsw: this._sharedHnsw,
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
this.emit('reembedded', result);
|
|
353
|
+
return result;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* Linear 8-step initialization:
|
|
358
|
+
* 1. Open database
|
|
359
|
+
* 2. Resolve embedding provider
|
|
360
|
+
* 3. Check dimension mismatch
|
|
361
|
+
* 4. Create KV HNSW + KVService
|
|
362
|
+
* 5. Load KV vectors
|
|
363
|
+
* 6. Initialize plugins
|
|
364
|
+
* 7. Persist HNSW indices
|
|
365
|
+
* 8. Build SearchAPI + index deps
|
|
366
|
+
*/
|
|
367
|
+
private async _runInitialize(options: { force?: boolean } = {}): Promise<void> {
|
|
368
|
+
if (this._initialized) return;
|
|
369
|
+
|
|
370
|
+
this._db = new SQLiteAdapter(this._config.dbPath);
|
|
371
|
+
this._embedding = await this._resolveEmbedding();
|
|
372
|
+
|
|
373
|
+
const mismatch = detectProviderMismatch(this._db, this._embedding);
|
|
374
|
+
if (mismatch?.mismatch && !options.force) {
|
|
375
|
+
this._db.close();
|
|
376
|
+
throw new Error(
|
|
377
|
+
`BrainBank: Embedding dimension mismatch (stored: ${mismatch.stored}, current: ${mismatch.current}). ` +
|
|
378
|
+
`Run brain.reembed() to re-index with the new provider, or switch back to the original provider.`,
|
|
379
|
+
);
|
|
380
|
+
}
|
|
381
|
+
setEmbeddingMeta(this._db, this._embedding);
|
|
382
|
+
|
|
383
|
+
const skipVectorLoad = !!(options.force && mismatch?.mismatch);
|
|
384
|
+
const dims = this._embedding.dims ?? this._config.embeddingDims;
|
|
385
|
+
|
|
386
|
+
const kvHnsw = new HNSWIndex(
|
|
387
|
+
dims,
|
|
388
|
+
this._config.maxElements ?? 500_000,
|
|
389
|
+
this._config.hnswM,
|
|
390
|
+
this._config.hnswEfConstruction,
|
|
391
|
+
this._config.hnswEfSearch,
|
|
392
|
+
);
|
|
393
|
+
await kvHnsw.init();
|
|
394
|
+
|
|
395
|
+
this._kvService = new KVService(this._db, this._embedding, kvHnsw, new Map());
|
|
396
|
+
|
|
397
|
+
if (!skipVectorLoad) {
|
|
398
|
+
const kvIndexPath = hnswPath(this._config.dbPath, 'kv');
|
|
399
|
+
const kvCount = countRows(this._db, 'kv_vectors');
|
|
400
|
+
if (kvHnsw.tryLoad(kvIndexPath, kvCount)) {
|
|
401
|
+
loadVecCache(this._db, 'kv_vectors', 'data_id', this._kvService.vecs);
|
|
402
|
+
} else {
|
|
403
|
+
loadVectors(this._db, 'kv_vectors', 'data_id', kvHnsw, this._kvService.vecs);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
const privateHnsw = new Map<string, HNSWIndex>();
|
|
408
|
+
for (const mod of this._registry.all) {
|
|
409
|
+
const pluginDb = this._getOrCreatePluginDb(mod.name);
|
|
410
|
+
// Propagate embedding meta to per-repo DBs
|
|
411
|
+
if (pluginDb !== this._db) {
|
|
412
|
+
setEmbeddingMeta(pluginDb, this._embedding);
|
|
413
|
+
}
|
|
414
|
+
const ctx = this._buildPluginContext(skipVectorLoad, privateHnsw, pluginDb, mod.name);
|
|
415
|
+
await mod.initialize(ctx);
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// Start webhook server if configured (after plugins so they can register routes)
|
|
419
|
+
if (this._config.webhookPort) {
|
|
420
|
+
this._webhookServer = new WebhookServer();
|
|
421
|
+
this._webhookServer.listen(this._config.webhookPort);
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
await saveAllHnsw(this._config.dbPath, kvHnsw, this._sharedHnsw, privateHnsw);
|
|
425
|
+
|
|
426
|
+
this._searchAPI = createSearchAPI(
|
|
427
|
+
this._db, this._embedding, this._config,
|
|
428
|
+
this._registry, this._kvService, this._sharedHnsw,
|
|
429
|
+
);
|
|
430
|
+
this._indexDeps = {
|
|
431
|
+
db: this._db,
|
|
432
|
+
dbPath: this._config.dbPath,
|
|
433
|
+
sharedHnsw: this._sharedHnsw,
|
|
434
|
+
kvHnsw,
|
|
435
|
+
registry: this._registry,
|
|
436
|
+
emit: (e, d) => this.emit(e, d),
|
|
437
|
+
};
|
|
438
|
+
|
|
439
|
+
// Snapshot current versions for staleness detection
|
|
440
|
+
this._loadedVersions = getVersions(this._db);
|
|
441
|
+
|
|
442
|
+
this._initialized = true;
|
|
443
|
+
this.emit('initialized', { plugins: this.plugins });
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/** Reset shared state after a failed `_runInitialize`. */
|
|
447
|
+
private _cleanupAfterFailedInit(): void {
|
|
448
|
+
for (const { hnsw } of this._sharedHnsw.values()) {
|
|
449
|
+
try { hnsw.reinit(); } catch (e) {
|
|
450
|
+
this.emit('warn', `HNSW reinit failed during cleanup: ${e}`);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
this._kvService?.clear();
|
|
454
|
+
if (this._kvService) {
|
|
455
|
+
try { this._kvService.hnsw.reinit(); } catch (e) {
|
|
456
|
+
this.emit('warn', `KV HNSW reinit failed during cleanup: ${e}`);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
try { this._db?.close(); } catch { /* DB already closed — safe to ignore */ }
|
|
460
|
+
|
|
461
|
+
this._db = undefined!;
|
|
462
|
+
this._kvService = undefined;
|
|
463
|
+
this._searchAPI = undefined;
|
|
464
|
+
this._indexDeps = undefined;
|
|
465
|
+
this._initPromise = null;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/** Resolve embedding: explicit config > stored DB key > local default. */
|
|
469
|
+
private async _resolveEmbedding(): Promise<EmbeddingProvider> {
|
|
470
|
+
if (this._config.embeddingProvider) return this._config.embeddingProvider;
|
|
471
|
+
|
|
472
|
+
const meta = getEmbeddingMeta(this._db);
|
|
473
|
+
if (meta?.providerKey && meta.providerKey !== 'local') {
|
|
474
|
+
this.emit('progress', `Embedding: auto-resolved '${meta.providerKey}' from DB`);
|
|
475
|
+
return resolveEmbedding(meta.providerKey);
|
|
476
|
+
}
|
|
477
|
+
return resolveEmbedding('local');
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/**
|
|
481
|
+
* Get or create a per-repo SQLiteAdapter for namespaced plugins.
|
|
482
|
+
* Non-namespaced plugins use the root DB.
|
|
483
|
+
* DB path: `.brainbank/<repoName>.db` (e.g., `servicehub-backend.db`).
|
|
484
|
+
*/
|
|
485
|
+
private _getOrCreatePluginDb(pluginName: string): DatabaseAdapter {
|
|
486
|
+
if (!pluginName.includes(':')) return this._db;
|
|
487
|
+
|
|
488
|
+
const repoName = pluginName.split(':').slice(1).join(':');
|
|
489
|
+
const existing = this._repoDBs.get(repoName);
|
|
490
|
+
if (existing) return existing;
|
|
491
|
+
|
|
492
|
+
const dir = path.dirname(this._config.dbPath);
|
|
493
|
+
const repoDbPath = path.join(dir, `${repoName}.db`);
|
|
494
|
+
const db = new SQLiteAdapter(repoDbPath);
|
|
495
|
+
this._repoDBs.set(repoName, db);
|
|
496
|
+
return db;
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
/** Build a per-plugin `PluginContext` with appropriate DB and HNSW scoping. */
|
|
500
|
+
private _buildPluginContext(
|
|
501
|
+
skipVectorLoad: boolean,
|
|
502
|
+
privateHnsw: Map<string, HNSWIndex>,
|
|
503
|
+
pluginDb: DatabaseAdapter,
|
|
504
|
+
pluginName: string,
|
|
505
|
+
): PluginContext {
|
|
506
|
+
let autoId = 0;
|
|
507
|
+
const dbPath = this._config.dbPath;
|
|
508
|
+
|
|
509
|
+
return {
|
|
510
|
+
db: pluginDb,
|
|
511
|
+
embedding: this._embedding,
|
|
512
|
+
config: this._config,
|
|
513
|
+
|
|
514
|
+
createHnsw: async (maxElements?: number, dims?: number, name?: string) => {
|
|
515
|
+
const hnsw = await new HNSWIndex(
|
|
516
|
+
dims ?? this._config.embeddingDims,
|
|
517
|
+
maxElements ?? this._config.maxElements,
|
|
518
|
+
this._config.hnswM,
|
|
519
|
+
this._config.hnswEfConstruction,
|
|
520
|
+
this._config.hnswEfSearch,
|
|
521
|
+
).init();
|
|
522
|
+
privateHnsw.set(name ?? `private-${autoId++}`, hnsw);
|
|
523
|
+
return hnsw;
|
|
524
|
+
},
|
|
525
|
+
|
|
526
|
+
loadVectors: (table, idCol, hnsw, cache) => {
|
|
527
|
+
if (skipVectorLoad) return;
|
|
528
|
+
const indexName = table.replace('_vectors', '').replace('_chunks', '');
|
|
529
|
+
const indexPath = hnswPath(dbPath, `${indexName}-${pluginDb === this._db ? 'root' : 'repo'}`);
|
|
530
|
+
const rowCount = countRows(pluginDb, table);
|
|
531
|
+
if (hnsw.tryLoad(indexPath, rowCount)) {
|
|
532
|
+
loadVecCache(pluginDb, table, idCol, cache);
|
|
533
|
+
} else {
|
|
534
|
+
loadVectors(pluginDb, table, idCol, hnsw, cache);
|
|
535
|
+
}
|
|
536
|
+
},
|
|
537
|
+
|
|
538
|
+
getOrCreateSharedHnsw: async (type, maxElements, dims) => {
|
|
539
|
+
const existing = this._sharedHnsw.get(type);
|
|
540
|
+
if (existing) return { ...existing, isNew: false };
|
|
541
|
+
|
|
542
|
+
const hnsw = await new HNSWIndex(
|
|
543
|
+
dims ?? this._config.embeddingDims,
|
|
544
|
+
maxElements ?? this._config.maxElements,
|
|
545
|
+
this._config.hnswM,
|
|
546
|
+
this._config.hnswEfConstruction,
|
|
547
|
+
this._config.hnswEfSearch,
|
|
548
|
+
).init();
|
|
549
|
+
|
|
550
|
+
const vecCache = new Map<number, Float32Array>();
|
|
551
|
+
this._sharedHnsw.set(type, { hnsw, vecCache });
|
|
552
|
+
return { hnsw, vecCache, isNew: true };
|
|
553
|
+
},
|
|
554
|
+
|
|
555
|
+
collection: (name) => this._kvService!.collection(name),
|
|
556
|
+
|
|
557
|
+
createTracker: () => createTracker(pluginDb, pluginName),
|
|
558
|
+
|
|
559
|
+
webhookServer: this._webhookServer,
|
|
560
|
+
};
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
/**
|
|
564
|
+
* Reload a single HNSW index by name.
|
|
565
|
+
* Discovers the vector table via ReembeddablePlugin capability.
|
|
566
|
+
* KV is handled directly since it's core-owned.
|
|
567
|
+
*
|
|
568
|
+
* The `name` comes from `index_state` and equals the plugin's `mod.name`
|
|
569
|
+
* (e.g. `code:backend`, `git`, `docs`). This matches the key used in
|
|
570
|
+
* `getOrCreateSharedHnsw()` during initialization.
|
|
571
|
+
*/
|
|
572
|
+
private _reloadIndex(name: string): void {
|
|
573
|
+
// KV HNSW — core-owned, known table
|
|
574
|
+
if (name === HNSW.KV && this._kvService) {
|
|
575
|
+
reloadHnsw({
|
|
576
|
+
dbPath: this._config.dbPath,
|
|
577
|
+
db: this._db,
|
|
578
|
+
name,
|
|
579
|
+
hnsw: this._kvService.hnsw,
|
|
580
|
+
vecCache: this._kvService.vecs,
|
|
581
|
+
vectorTable: 'kv_vectors',
|
|
582
|
+
idCol: 'data_id',
|
|
583
|
+
});
|
|
584
|
+
return;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// Shared HNSW — exact key match against the map
|
|
588
|
+
const shared = this._sharedHnsw.get(name);
|
|
589
|
+
if (!shared) return;
|
|
590
|
+
|
|
591
|
+
// Discover vector table from the plugin that owns this HNSW key.
|
|
592
|
+
// Match by plugin name (e.g. 'code:backend') or reembed name (e.g. 'code').
|
|
593
|
+
for (const mod of this._registry.all) {
|
|
594
|
+
if (!isReembeddable(mod)) continue;
|
|
595
|
+
if (mod.name !== name) continue;
|
|
596
|
+
|
|
597
|
+
const cfg = mod.reembedConfig();
|
|
598
|
+
reloadHnsw({
|
|
599
|
+
dbPath: this._config.dbPath,
|
|
600
|
+
db: this._db,
|
|
601
|
+
name,
|
|
602
|
+
hnsw: shared.hnsw,
|
|
603
|
+
vecCache: shared.vecCache,
|
|
604
|
+
vectorTable: cfg.vectorTable,
|
|
605
|
+
idCol: cfg.fkColumn,
|
|
606
|
+
});
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
/** Guard: throw descriptive error if not initialized. */
|
|
612
|
+
private _requireInit(method: string): void {
|
|
613
|
+
if (!this._initialized) {
|
|
614
|
+
throw new Error(`BrainBank: Not initialized. Call await brain.initialize() before ${method}().`);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* brainbank collection add|list|remove — Document collection management
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { c, args, getFlag, stripFlags, findDocsPlugin } from '@/cli/utils.ts';
|
|
6
|
+
import { createBrain } from '@/cli/factory/index.ts';
|
|
7
|
+
|
|
8
|
+
export async function cmdCollection(): Promise<void> {
|
|
9
|
+
const pos = stripFlags(args);
|
|
10
|
+
const sub = pos[1];
|
|
11
|
+
|
|
12
|
+
if (sub === 'add') {
|
|
13
|
+
const path = pos[2];
|
|
14
|
+
const name = getFlag('name');
|
|
15
|
+
const pattern = getFlag('pattern') ?? '**/*.md';
|
|
16
|
+
const context = getFlag('context');
|
|
17
|
+
const ignoreRaw = getFlag('ignore');
|
|
18
|
+
|
|
19
|
+
if (!path || !name) {
|
|
20
|
+
console.log(c.red('Usage: brainbank collection add <path> --name <name> [--pattern "**/*.md"] [--ignore "glob"] [--context "description"]'));
|
|
21
|
+
process.exit(1);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const brain = await createBrain();
|
|
25
|
+
const docsPlugin = findDocsPlugin(brain);
|
|
26
|
+
if (!docsPlugin) { console.log(c.red('Docs plugin not loaded. Install @brainbank/docs.')); process.exit(1); }
|
|
27
|
+
await docsPlugin.addCollection({
|
|
28
|
+
name,
|
|
29
|
+
path,
|
|
30
|
+
pattern,
|
|
31
|
+
ignore: ignoreRaw ? ignoreRaw.split(',') : [],
|
|
32
|
+
context: context ?? undefined,
|
|
33
|
+
});
|
|
34
|
+
console.log(c.green(`✓ Collection '${name}' added: ${path} (${pattern})`));
|
|
35
|
+
if (context) console.log(c.dim(` Context: ${context}`));
|
|
36
|
+
brain.close();
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
if (sub === 'list') {
|
|
41
|
+
const brain = await createBrain();
|
|
42
|
+
await brain.initialize();
|
|
43
|
+
const docsPlugin = findDocsPlugin(brain);
|
|
44
|
+
if (!docsPlugin) { console.log(c.yellow(' Docs plugin not loaded.')); brain.close(); return; }
|
|
45
|
+
const collections = docsPlugin.listCollections();
|
|
46
|
+
if (collections.length === 0) {
|
|
47
|
+
console.log(c.yellow(' No collections registered.'));
|
|
48
|
+
} else {
|
|
49
|
+
console.log(c.bold('\n━━━ Collections ━━━\n'));
|
|
50
|
+
for (const col of collections) {
|
|
51
|
+
console.log(` ${c.cyan(col.name)} ${c.dim('→')} ${col.path}`);
|
|
52
|
+
console.log(` Pattern: ${col.pattern ?? '**/*.md'}`);
|
|
53
|
+
if (col.context) console.log(` Context: ${c.dim(col.context)}`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
brain.close();
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if (sub === 'remove') {
|
|
61
|
+
const name = pos[2];
|
|
62
|
+
if (!name) {
|
|
63
|
+
console.log(c.red('Usage: brainbank collection remove <name>'));
|
|
64
|
+
process.exit(1);
|
|
65
|
+
}
|
|
66
|
+
const brain = await createBrain();
|
|
67
|
+
const docsPlugin = findDocsPlugin(brain);
|
|
68
|
+
if (!docsPlugin) { console.log(c.red('Docs plugin not loaded.')); process.exit(1); }
|
|
69
|
+
await docsPlugin.removeCollection(name);
|
|
70
|
+
console.log(c.green(`✓ Collection '${name}' removed.`));
|
|
71
|
+
brain.close();
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
console.log(c.red('Usage: brainbank collection <add|list|remove>'));
|
|
76
|
+
process.exit(1);
|
|
77
|
+
}
|