opencode-rag-plugin 2.10.1 → 2.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/ReadMe.md +40 -28
  2. package/dist/api.js +1 -0
  3. package/dist/chunker/grammar.d.ts +4 -1
  4. package/dist/chunker/grammar.js +9 -3
  5. package/dist/chunker/skeleton.js +61 -4
  6. package/dist/cli/commands/clickhouse.d.ts +61 -0
  7. package/dist/cli/commands/clickhouse.js +335 -0
  8. package/dist/cli/commands/eval-tuner.js +1 -1
  9. package/dist/cli/commands/eval.js +1 -1
  10. package/dist/cli/commands/index-command.js +2 -1
  11. package/dist/cli/commands/index.d.ts +1 -0
  12. package/dist/cli/commands/index.js +1 -0
  13. package/dist/cli/commands/init-helpers.js +2 -2
  14. package/dist/cli/commands/init.js +49 -11
  15. package/dist/cli/commands/quirk.js +6 -6
  16. package/dist/cli/commands/status.js +70 -6
  17. package/dist/cli/commands/ui.js +2 -2
  18. package/dist/cli/index.js +2 -1
  19. package/dist/cli/types.d.ts +14 -0
  20. package/dist/core/bootstrap.d.ts +7 -0
  21. package/dist/core/bootstrap.js +40 -9
  22. package/dist/core/config.d.ts +128 -5
  23. package/dist/core/config.js +205 -8
  24. package/dist/core/interfaces.d.ts +54 -0
  25. package/dist/core/runtime-overrides.d.ts +9 -0
  26. package/dist/core/runtime-overrides.js +44 -0
  27. package/dist/core/state-sync.d.ts +91 -0
  28. package/dist/core/state-sync.js +273 -0
  29. package/dist/eval/fast-index.js +2 -2
  30. package/dist/eval/rerank-eval.js +2 -2
  31. package/dist/eval/run-branch-compare.js +2 -2
  32. package/dist/eval/run-token-test.js +2 -2
  33. package/dist/index.d.ts +5 -2
  34. package/dist/index.js +2 -1
  35. package/dist/indexer/pipeline.d.ts +6 -0
  36. package/dist/indexer/pipeline.js +173 -56
  37. package/dist/indexer/progress.d.ts +75 -0
  38. package/dist/indexer/progress.js +114 -0
  39. package/dist/indexer/stats.d.ts +14 -0
  40. package/dist/plugin.d.ts +6 -3
  41. package/dist/plugin.js +39 -20
  42. package/dist/quirks/quirk-store.d.ts +10 -0
  43. package/dist/quirks/quirk-store.js +110 -15
  44. package/dist/quirks/types.d.ts +7 -0
  45. package/dist/tui.js +128 -111
  46. package/dist/vectorstore/clickhouse.d.ts +150 -0
  47. package/dist/vectorstore/clickhouse.js +931 -0
  48. package/dist/vectorstore/dimension.d.ts +19 -0
  49. package/dist/vectorstore/dimension.js +36 -0
  50. package/dist/vectorstore/factory.d.ts +9 -3
  51. package/dist/vectorstore/factory.js +11 -2
  52. package/dist/vectorstore/lancedb.d.ts +2 -2
  53. package/dist/watcher.d.ts +29 -0
  54. package/dist/watcher.js +76 -2
  55. package/dist/web/api.d.ts +3 -4
  56. package/dist/web/api.js +18 -3
  57. package/dist/web/server.js +28 -2
  58. package/dist/web/ui/assets/{ScatterPlot3D-CKwyh1Ec.js → ScatterPlot3D-XjXIwavM.js} +5 -5
  59. package/dist/web/ui/assets/index-B-RH6fko.css +2 -0
  60. package/dist/web/ui/assets/index-FFPRbgLx.js +4 -0
  61. package/dist/web/ui/index.html +2 -2
  62. package/package.json +7 -2
  63. package/dist/web/ui/assets/index-CWcp0jNf.css +0 -2
  64. package/dist/web/ui/assets/index-DbXFrn53.js +0 -4
@@ -0,0 +1,931 @@
1
+ /**
2
+ * @fileoverview ClickHouse-backed `VectorStore` — an opt-in, central alternative
3
+ * to the default LanceDB store.
4
+ *
5
+ * Storage model (one table pair per project namespace, database `OpenCodeRAG`):
6
+ *
7
+ * - `chunks_<ns>`: `ReplacingMergeTree(ver, is_deleted)` ordered by
8
+ * `(epoch, file_path, chunk_id)`. Vector search is an exact
9
+ * `cosineDistance(...) ORDER BY ... LIMIT k` scan — ClickHouse only gained
10
+ * approximate `vector_similarity` indexes in 25.8, and the server this was
11
+ * built for runs 25.1 (see `ann` in the config).
12
+ * - `state_<ns>`: a one-row-per-name KV table used for the durable workspace
13
+ * state (manifest, keyword index, quirks, overrides, `_meta`, `_lease`) and
14
+ * by {@link StateSync}. The local `vectorStore.path` is only a disposable
15
+ * cache when this backend is active.
16
+ *
17
+ * Correctness notes:
18
+ * - Writes are insert-first: new rows and tombstones for superseded chunk ids
19
+ * go in one `insert`, so an aborted pass never loses data.
20
+ * - A full rebuild writes into `epoch = active + 1` and becomes visible when
21
+ * `_meta.active_epoch` flips — no directory swap, no downtime.
22
+ * - `FINAL` is required on reads because `ReplacingMergeTree` deduplicates
23
+ * only during background merges.
24
+ */
25
+ import { createHash } from "node:crypto";
26
+ import path from "node:path";
27
+ import { normalizeFilePath } from "../core/manifest.js";
28
+ import { normalizeFileExtensions } from "../core/filters.js";
29
+ /** Synthetic filePath prefix for quirk chunks — mirrors lancedb.ts and quirk-store.ts. */
30
+ const QUIRK_FILE_PREFIX = "quirk:";
31
+ const META_NAME = "_meta";
32
+ const LEASE_NAME = "_lease";
33
+ /** Sanitize a namespace so it is safe as a table suffix. */
34
+ export function sanitizeNamespace(raw) {
35
+ const cleaned = raw
36
+ .normalize("NFKD")
37
+ .replace(/[^A-Za-z0-9_]+/g, "_")
38
+ .replace(/^_+|_+$/g, "")
39
+ .slice(0, 40);
40
+ return cleaned.length > 0 ? cleaned : "default";
41
+ }
42
+ /** Derive a stable namespace from a workspace path (name + path hash). */
43
+ export function deriveNamespace(workspaceDir) {
44
+ const absolute = path.resolve(workspaceDir);
45
+ const base = path.basename(absolute) || "workspace";
46
+ const hash = createHash("sha1").update(absolute).digest("hex").slice(0, 8);
47
+ return sanitizeNamespace(`${base}_${hash}`);
48
+ }
49
+ /** Resolve a full ClickHouse config block (with defaults) for a workspace. */
50
+ export function resolveClickHouseOptions(config, workspaceDir) {
51
+ const ch = config.vectorStore.clickhouse;
52
+ if (!ch) {
53
+ throw new Error('vectorStore.provider is "clickhouse" but vectorStore.clickhouse is missing — add the connection block to opencode-rag.json');
54
+ }
55
+ const envPassword = ch.passwordEnv ? process.env[ch.passwordEnv] : undefined;
56
+ const namespace = sanitizeNamespace(ch.namespace?.trim() || deriveNamespace(workspaceDir));
57
+ return {
58
+ url: ch.url,
59
+ database: ch.database,
60
+ username: ch.username,
61
+ password: envPassword ?? ch.password ?? "",
62
+ namespace,
63
+ chunksTable: `chunks_${namespace}`,
64
+ stateTable: `state_${namespace}`,
65
+ requestTimeoutMs: ch.requestTimeoutMs ?? 30_000,
66
+ maxOpenConnections: ch.maxOpenConnections ?? 10,
67
+ compression: ch.compression ?? true,
68
+ ann: ch.ann ?? "off",
69
+ annFetchMultiplier: Math.min(1000, Math.max(1, ch.annFetchMultiplier ?? 4)),
70
+ annWithRescoring: ch.annWithRescoring ?? true,
71
+ leaseEnabled: ch.lease?.enabled ?? true,
72
+ leaseTtlMs: Math.max(10, ch.lease?.ttlSeconds ?? 90) * 1000,
73
+ diskFreeFloorBytes: ch.diskFreeFloorBytes ?? 5 * 1024 * 1024 * 1024,
74
+ syncDescCache: ch.state?.syncDescCache ?? false,
75
+ embeddingModel: config.embedding.model,
76
+ };
77
+ }
78
+ /** Quote a ClickHouse identifier. */
79
+ function quoteIdent(name) {
80
+ return "`" + name.replace(/`/g, "``") + "`";
81
+ }
82
+ /** Vectors are stored L2-normalized, matching the LanceDB store. */
83
+ function l2Normalize(vector) {
84
+ let norm = 0;
85
+ for (const value of vector)
86
+ norm += value * value;
87
+ const length = Math.sqrt(norm);
88
+ if (length === 0)
89
+ return vector;
90
+ return vector.map((value) => value / length);
91
+ }
92
+ /** Mirror LanceDB's glob→LIKE translation (no anchoring, same wildcards). */
93
+ function globToLike(pattern) {
94
+ return pattern.replace(/\*\*/g, "%").replace(/\*/g, "%").replace(/\?/g, "_");
95
+ }
96
+ /**
97
+ * ClickHouse-backed vector store. Instantiate through `createVectorStore()`;
98
+ * the constructor only records options and opens the client lazily.
99
+ */
100
+ export class ClickHouseStore {
101
+ opts;
102
+ dimensions;
103
+ /** Set on a rebuild target: all writes/reads are bound to this epoch. */
104
+ boundEpoch;
105
+ client = null;
106
+ schemaReady = null;
107
+ activeEpoch = null;
108
+ writeChain = Promise.resolve();
109
+ static modulePromise = null;
110
+ constructor(options, vectorDimension = 384, boundEpoch) {
111
+ this.opts = options;
112
+ this.dimensions = vectorDimension;
113
+ this.boundEpoch = boundEpoch;
114
+ }
115
+ // ── Connection & schema ────────────────────────────────────────────────
116
+ async loadClientModule() {
117
+ if (!ClickHouseStore.modulePromise) {
118
+ ClickHouseStore.modulePromise = import("@clickhouse/client").catch((err) => {
119
+ ClickHouseStore.modulePromise = null;
120
+ throw new Error('The "clickhouse" vector store provider requires the "@clickhouse/client" package. ' +
121
+ "Install it with: npm install @clickhouse/client --legacy-peer-deps " +
122
+ `(original error: ${err.message})`);
123
+ });
124
+ }
125
+ return ClickHouseStore.modulePromise;
126
+ }
127
+ async getClient() {
128
+ if (this.client)
129
+ return this.client;
130
+ const { createClient } = await this.loadClientModule();
131
+ this.client = createClient({
132
+ url: this.opts.url,
133
+ username: this.opts.username,
134
+ password: this.opts.password,
135
+ // Always talk to `default`; every statement fully qualifies the target
136
+ // database so the client survives `CREATE DATABASE` without reconnecting.
137
+ database: "default",
138
+ request_timeout: this.opts.requestTimeoutMs,
139
+ max_open_connections: this.opts.maxOpenConnections,
140
+ compression: this.opts.compression ? { request: true, response: true } : undefined,
141
+ });
142
+ return this.client;
143
+ }
144
+ async command(query, params) {
145
+ const client = await this.getClient();
146
+ await client.command({
147
+ query,
148
+ query_params: params,
149
+ clickhouse_settings: { wait_end_of_query: 1 },
150
+ });
151
+ }
152
+ async select(query, params, settings) {
153
+ const client = await this.getClient();
154
+ const resultSet = await client.query({
155
+ query,
156
+ query_params: params,
157
+ format: "JSONEachRow",
158
+ clickhouse_settings: settings,
159
+ });
160
+ return resultSet.json();
161
+ }
162
+ table(name) {
163
+ return `${quoteIdent(this.opts.database)}.${quoteIdent(name)}`;
164
+ }
165
+ insertTable(name) {
166
+ // The client interpolates this into `INSERT INTO <table> FORMAT ...`, so
167
+ // it must be a plain (unquoted) identifier path. Both parts are validated
168
+ // by sanitizeNamespace / the constant database name.
169
+ return `${this.opts.database}.${name}`;
170
+ }
171
+ async ensureSchema() {
172
+ if (!this.schemaReady) {
173
+ this.schemaReady = this.initSchema().catch((err) => {
174
+ this.schemaReady = null;
175
+ throw err;
176
+ });
177
+ }
178
+ await this.schemaReady;
179
+ // Cheap (2 s TTL) refresh of the active epoch so a long-lived handle sees
180
+ // another machine's promote/clear instead of reading a pruned epoch.
181
+ await this.maybeRefreshEpoch();
182
+ }
183
+ async initSchema() {
184
+ await this.command(`CREATE DATABASE IF NOT EXISTS ${quoteIdent(this.opts.database)}`);
185
+ const indexClause = this.opts.ann === "hnsw"
186
+ ? `, INDEX vec_idx embedding TYPE vector_similarity('hnsw', 'cosineDistance', ${this.dimensions})`
187
+ : "";
188
+ await this.command(`CREATE TABLE IF NOT EXISTS ${this.table(this.opts.chunksTable)} (` +
189
+ "chunk_id String, file_path String, start_line Int32, end_line Int32, " +
190
+ "language LowCardinality(String), kind LowCardinality(String), role LowCardinality(String), " +
191
+ "quirk_type LowCardinality(String), tags String, content String, description String, " +
192
+ "embedding Array(Float32) CODEC(NONE), epoch UInt64, " +
193
+ "ver DateTime64(6, 'UTC') DEFAULT now64(6), is_deleted UInt8 DEFAULT 0" +
194
+ indexClause +
195
+ ") ENGINE = ReplacingMergeTree(ver, is_deleted) " +
196
+ "ORDER BY (epoch, file_path, chunk_id)");
197
+ await this.command(`CREATE TABLE IF NOT EXISTS ${this.table(this.opts.stateTable)} (` +
198
+ "name String, ver DateTime64(6, 'UTC') DEFAULT now64(6), " +
199
+ "sha256 String DEFAULT '', bytes UInt64 DEFAULT 0, data String CODEC(ZSTD(3))" +
200
+ ") ENGINE = ReplacingMergeTree(ver) ORDER BY name");
201
+ // A store created before ANN was enabled has the table but not the index.
202
+ if (this.opts.ann === "hnsw") {
203
+ await this.ensureVectorIndex();
204
+ }
205
+ const meta = await this.readMeta();
206
+ if (!meta) {
207
+ await this.writeMeta({
208
+ activeEpoch: 1,
209
+ pendingEpoch: 0,
210
+ dimension: this.dimensions,
211
+ embeddingModel: this.opts.embeddingModel,
212
+ });
213
+ this.activeEpoch = 1;
214
+ return;
215
+ }
216
+ this.activeEpoch = meta.activeEpoch;
217
+ if (meta.dimension !== this.dimensions) {
218
+ if (this.opts.ann === "hnsw") {
219
+ // The ANN index is bound to the dimension in its DDL and rejects rows
220
+ // of any other length, so a model change needs a fresh table (the
221
+ // in-place epoch rebuild cannot change the index dimension).
222
+ throw new Error(`ClickHouse store ${this.opts.namespace} was built with vector dimension ${meta.dimension}, ` +
223
+ `but the ANN index is bound to ${this.dimensions}. Drop ` +
224
+ `\`${this.opts.database}\`.\`${this.opts.chunksTable}\` and ` +
225
+ `\`${this.opts.database}\`.\`${this.opts.stateTable}\`, then re-index; ` +
226
+ `or set vectorStore.clickhouse.ann = "off" to rebuild in place.`);
227
+ }
228
+ // The store was built by a different embedding model. Writes are
229
+ // refused; the pipeline detects this through getVectorDimension() and
230
+ // rebuilds into a fresh epoch.
231
+ console.warn(`[clickhouse] Store ${this.opts.namespace} was built with vector dimension ${meta.dimension} ` +
232
+ `but the current embedder produces ${this.dimensions}. ` +
233
+ "Run 'opencode-rag index' to rebuild the index with the current model.");
234
+ }
235
+ }
236
+ /** Add + materialize the HNSW index on a table created before ANN was enabled. */
237
+ async ensureVectorIndex() {
238
+ const findIndex = () => this.select("SELECT name FROM system.data_skipping_indices " +
239
+ "WHERE database = {db:String} AND table = {table:String} AND name = {name:String} LIMIT 1", { db: this.opts.database, table: this.opts.chunksTable, name: "vec_idx" });
240
+ if ((await findIndex()).length > 0)
241
+ return;
242
+ try {
243
+ await this.command(`ALTER TABLE ${this.table(this.opts.chunksTable)} ADD INDEX vec_idx embedding ` +
244
+ `TYPE vector_similarity('hnsw', 'cosineDistance', ${this.dimensions})`);
245
+ await this.command(`ALTER TABLE ${this.table(this.opts.chunksTable)} MATERIALIZE INDEX vec_idx ` +
246
+ "SETTINGS mutations_sync = 2");
247
+ }
248
+ catch (err) {
249
+ // A racing client may have created it between the check and the ALTER.
250
+ if ((await findIndex().catch(() => [])).length > 0)
251
+ return;
252
+ throw new Error(`Could not create the ANN index with dimension ${this.dimensions} on ` +
253
+ `${this.opts.chunksTable}: ${err.message}. If the store was built with another ` +
254
+ 'embedding model, drop the project tables and re-index, or set vectorStore.clickhouse.ann = "off".');
255
+ }
256
+ }
257
+ currentEpoch() {
258
+ return this.boundEpoch ?? this.activeEpoch ?? 1;
259
+ }
260
+ /**
261
+ * Refresh the cached active epoch from `_meta` (at most every 2 s).
262
+ *
263
+ * Another machine may promote a rebuild or clear the namespace at any time;
264
+ * a handle that kept a stale epoch would read (or write) a superseded epoch
265
+ * forever — and after `optimize()` prunes it, silently see an empty store.
266
+ */
267
+ epochCheckedAt = 0;
268
+ async maybeRefreshEpoch() {
269
+ if (this.boundEpoch !== undefined)
270
+ return;
271
+ const now = Date.now();
272
+ if (now - this.epochCheckedAt < 2000)
273
+ return;
274
+ this.epochCheckedAt = now;
275
+ try {
276
+ const meta = await this.readMeta();
277
+ if (meta)
278
+ this.activeEpoch = meta.activeEpoch;
279
+ }
280
+ catch {
281
+ // Keep the cached epoch; a transient meta read must not break reads.
282
+ }
283
+ }
284
+ async readMeta() {
285
+ const rows = await this.select(`SELECT data FROM ${this.table(this.opts.stateTable)} FINAL WHERE name = {name:String} LIMIT 1`, { name: META_NAME });
286
+ const raw = rows[0]?.data;
287
+ if (!raw)
288
+ return undefined;
289
+ try {
290
+ const parsed = JSON.parse(raw);
291
+ return {
292
+ activeEpoch: Number(parsed.activeEpoch ?? 1),
293
+ pendingEpoch: Number(parsed.pendingEpoch ?? 0),
294
+ dimension: Number(parsed.dimension ?? this.dimensions),
295
+ embeddingModel: String(parsed.embeddingModel ?? ""),
296
+ };
297
+ }
298
+ catch {
299
+ return undefined;
300
+ }
301
+ }
302
+ async writeMeta(meta) {
303
+ const client = await this.getClient();
304
+ const data = JSON.stringify(meta);
305
+ await client.insert({
306
+ table: this.insertTable(this.opts.stateTable),
307
+ values: [{ name: META_NAME, data, sha256: "", bytes: Buffer.byteLength(data) }],
308
+ format: "JSONEachRow",
309
+ });
310
+ this.activeEpoch = meta.activeEpoch;
311
+ }
312
+ /** Generic state-bundle read used by `StateSync`. */
313
+ async readState(name) {
314
+ await this.ensureSchema();
315
+ const rows = await this.select(`SELECT data, toString(ver) AS ver FROM ${this.table(this.opts.stateTable)} FINAL WHERE name = {name:String} LIMIT 1`, { name });
316
+ return rows[0];
317
+ }
318
+ /** Generic state-bundle write used by `StateSync`. */
319
+ async writeState(name, data, sha256 = "") {
320
+ await this.ensureSchema();
321
+ const client = await this.getClient();
322
+ await client.insert({
323
+ table: this.insertTable(this.opts.stateTable),
324
+ values: [{ name, data, sha256, bytes: Buffer.byteLength(data) }],
325
+ format: "JSONEachRow",
326
+ });
327
+ }
328
+ /** Names currently present in the state bundle (newest row per name). */
329
+ async listStateNames() {
330
+ await this.ensureSchema();
331
+ const rows = await this.select(`SELECT name FROM ${this.table(this.opts.stateTable)} FINAL ORDER BY name`);
332
+ return rows.map((row) => row.name);
333
+ }
334
+ // ── Writes ─────────────────────────────────────────────────────────────
335
+ async withWriteLock(fn) {
336
+ const run = this.writeChain.then(fn, fn);
337
+ this.writeChain = run.then(() => undefined, () => undefined);
338
+ return run;
339
+ }
340
+ async addChunks(chunks, options) {
341
+ if (chunks.length === 0)
342
+ return;
343
+ await this.withWriteLock(() => this.addChunksInternal(chunks, options?.dedup ?? true));
344
+ }
345
+ async addChunksBulk(items) {
346
+ const active = items.filter((item) => item.chunks.length > 0);
347
+ if (active.length === 0)
348
+ return;
349
+ await this.withWriteLock(async () => {
350
+ for (const item of active) {
351
+ await this.addChunksInternal(item.chunks, item.dedup);
352
+ }
353
+ });
354
+ }
355
+ async addChunksInternal(chunks, dedup) {
356
+ await this.ensureSchema();
357
+ if (this.boundEpoch === undefined) {
358
+ // Refuse writes whose vector length disagrees with the active epoch's
359
+ // schema. A rebuild target is exempt: it is building the new dimension
360
+ // and only promotes (updating `_meta.dimension`) at the end.
361
+ const meta = await this.readMeta();
362
+ if (meta && meta.dimension !== this.dimensions) {
363
+ throw new Error(`ClickHouse store ${this.opts.namespace} was built with vector dimension ${meta.dimension} ` +
364
+ `but this handle produces ${this.dimensions} — run 'opencode-rag index' to rebuild it.`);
365
+ }
366
+ }
367
+ const epoch = this.currentEpoch();
368
+ const rows = chunks
369
+ .map((chunk) => this.chunkToRow(chunk, epoch))
370
+ .filter((row) => row !== null);
371
+ if (rows.length === 0)
372
+ return;
373
+ const tombstones = [];
374
+ if (dedup) {
375
+ const paths = Array.from(new Set(rows.map((row) => row.file_path)));
376
+ const oldRows = await this.select(`SELECT file_path, chunk_id, embedding FROM ${this.table(this.opts.chunksTable)} FINAL ` +
377
+ "WHERE epoch = {epoch:UInt64} AND file_path IN ({paths:Array(String)})", { epoch, paths });
378
+ const newIdsByPath = new Map();
379
+ for (const row of rows) {
380
+ const set = newIdsByPath.get(row.file_path) ?? new Set();
381
+ set.add(row.chunk_id);
382
+ newIdsByPath.set(row.file_path, set);
383
+ }
384
+ for (const old of oldRows) {
385
+ if (newIdsByPath.get(old.file_path)?.has(old.chunk_id))
386
+ continue;
387
+ tombstones.push(this.tombstoneRow(old.file_path, old.chunk_id, epoch, old.embedding ?? []));
388
+ }
389
+ }
390
+ const client = await this.getClient();
391
+ await client.insert({
392
+ table: this.insertTable(this.opts.chunksTable),
393
+ values: [...rows, ...tombstones],
394
+ format: "JSONEachRow",
395
+ });
396
+ }
397
+ chunkToRow(chunk, epoch) {
398
+ if (!chunk.embedding || chunk.embedding.length === 0)
399
+ return null;
400
+ const filePath = chunk.metadata.filePath.startsWith(QUIRK_FILE_PREFIX)
401
+ // Synthetic quirk ids stay RAW — normalizing them keys rows by the cwd
402
+ // of whichever process wrote them (same trap as LanceDB).
403
+ ? chunk.metadata.filePath
404
+ : normalizeFilePath(chunk.metadata.filePath);
405
+ return {
406
+ chunk_id: chunk.id,
407
+ file_path: filePath,
408
+ start_line: chunk.metadata.startLine,
409
+ end_line: chunk.metadata.endLine,
410
+ language: chunk.metadata.language ?? "",
411
+ kind: chunk.metadata.kind ?? "",
412
+ role: chunk.metadata.role ?? "",
413
+ quirk_type: chunk.metadata.quirkType ?? "",
414
+ tags: chunk.metadata.tags ? JSON.stringify(chunk.metadata.tags) : "",
415
+ content: chunk.content,
416
+ description: chunk.description ?? "",
417
+ embedding: l2Normalize(chunk.embedding),
418
+ epoch,
419
+ is_deleted: 0,
420
+ };
421
+ }
422
+ tombstoneRow(filePath, chunkId, epoch, embedding) {
423
+ // The ANN index rejects rows whose vector length differs from its DDL
424
+ // dimension and rejects empty arrays. A tombstone reuses the superseded
425
+ // row's vector (FINAL removes it from results); the fallback unit vector
426
+ // only covers legacy/corrupt rows without one.
427
+ let vector = [];
428
+ if (this.opts.ann === "hnsw") {
429
+ if (embedding.length === this.dimensions) {
430
+ vector = embedding;
431
+ }
432
+ else {
433
+ vector = new Array(this.dimensions).fill(0);
434
+ vector[0] = 1;
435
+ }
436
+ }
437
+ return {
438
+ chunk_id: chunkId,
439
+ file_path: filePath,
440
+ start_line: 0,
441
+ end_line: 0,
442
+ language: "",
443
+ kind: "",
444
+ role: "",
445
+ quirk_type: "",
446
+ tags: "",
447
+ content: "",
448
+ description: "",
449
+ embedding: vector,
450
+ epoch,
451
+ is_deleted: 1,
452
+ };
453
+ }
454
+ /** Remove all chunks for a file (tombstones; no table mutation). */
455
+ async deleteByFilePath(filePath) {
456
+ await this.withWriteLock(async () => {
457
+ await this.ensureSchema();
458
+ const epoch = this.currentEpoch();
459
+ const normalized = filePath.startsWith(QUIRK_FILE_PREFIX)
460
+ ? filePath
461
+ : normalizeFilePath(filePath);
462
+ const oldRows = await this.select(`SELECT chunk_id, embedding FROM ${this.table(this.opts.chunksTable)} FINAL ` +
463
+ "WHERE epoch = {epoch:UInt64} AND file_path = {path:String}", { epoch, path: normalized });
464
+ if (oldRows.length === 0)
465
+ return;
466
+ const client = await this.getClient();
467
+ await client.insert({
468
+ table: this.insertTable(this.opts.chunksTable),
469
+ values: oldRows.map((row) => this.tombstoneRow(normalized, row.chunk_id, epoch, row.embedding ?? [])),
470
+ format: "JSONEachRow",
471
+ });
472
+ });
473
+ }
474
+ /** Make every stored chunk invisible (bumps the active epoch). */
475
+ async clear() {
476
+ await this.withWriteLock(async () => {
477
+ await this.ensureSchema();
478
+ const meta = (await this.readMeta()) ?? {
479
+ activeEpoch: 1,
480
+ pendingEpoch: 0,
481
+ dimension: this.dimensions,
482
+ embeddingModel: "",
483
+ };
484
+ await this.writeMeta({
485
+ ...meta,
486
+ activeEpoch: Math.max(meta.activeEpoch, meta.pendingEpoch) + 1,
487
+ pendingEpoch: 0,
488
+ });
489
+ });
490
+ }
491
+ // ── Rebuild (epoch promotion) ──────────────────────────────────────────
492
+ /** Start a full rebuild into a private epoch and return the target handle. */
493
+ async beginRebuild() {
494
+ await this.ensureSchema();
495
+ const free = await this.select("SELECT min(free_space) AS free_space FROM system.disks");
496
+ const freeBytes = Number(free?.[0]?.free_space ?? Number.MAX_SAFE_INTEGER);
497
+ if (Number.isFinite(freeBytes) && freeBytes < this.opts.diskFreeFloorBytes) {
498
+ throw new Error(`ClickHouse server has only ${(freeBytes / 1024 ** 3).toFixed(1)} GiB free — ` +
499
+ `refusing a full rebuild (floor: ${(this.opts.diskFreeFloorBytes / 1024 ** 3).toFixed(1)} GiB). ` +
500
+ "Prune superseded epochs first: run an index pass (which optimizes the store) or " +
501
+ "OPTIMIZE TABLE … FINAL CLEANUP on the server.");
502
+ }
503
+ const meta = (await this.readMeta()) ?? {
504
+ activeEpoch: 1,
505
+ pendingEpoch: 0,
506
+ dimension: this.dimensions,
507
+ embeddingModel: "",
508
+ };
509
+ const pending = Math.max(meta.activeEpoch, meta.pendingEpoch) + 1;
510
+ await this.writeMeta({ ...meta, pendingEpoch: pending });
511
+ return new ClickHouseStore(this.opts, this.dimensions, pending);
512
+ }
513
+ /** Make the rebuild target visible in one atomic `_meta` write. */
514
+ async promoteRebuild() {
515
+ await this.ensureSchema();
516
+ const meta = await this.readMeta();
517
+ if (!meta || meta.pendingEpoch <= 0)
518
+ return;
519
+ await this.writeMeta({
520
+ ...meta,
521
+ activeEpoch: meta.pendingEpoch,
522
+ pendingEpoch: 0,
523
+ // The rebuild may have been triggered by an embedding-model change, so
524
+ // the promoted epoch defines the store's dimension from now on.
525
+ dimension: this.dimensions,
526
+ embeddingModel: this.opts.embeddingModel,
527
+ });
528
+ }
529
+ /** Abandon the rebuild target; its rows become unreachable orphans. */
530
+ async discardRebuild() {
531
+ await this.ensureSchema();
532
+ const meta = await this.readMeta();
533
+ if (!meta || meta.pendingEpoch <= 0)
534
+ return;
535
+ await this.writeMeta({ ...meta, pendingEpoch: 0 });
536
+ }
537
+ // ── Reads ──────────────────────────────────────────────────────────────
538
+ async search(embedding, topK) {
539
+ return this.searchWithFilter(embedding, topK);
540
+ }
541
+ async searchWithFilter(embedding, topK, filter) {
542
+ try {
543
+ await this.ensureSchema();
544
+ const storeDimension = await this.getVectorDimension();
545
+ if (storeDimension !== undefined && embedding.length !== storeDimension) {
546
+ console.warn(`[clickhouse] searchWithFilter: query embedding dimension ${embedding.length} != ` +
547
+ `store dimension ${storeDimension} — returning empty. Rebuild with 'opencode-rag index --force'.`);
548
+ return [];
549
+ }
550
+ const epoch = this.currentEpoch();
551
+ const requestedK = Math.max(1, Math.floor(topK));
552
+ const maxAnnLimit = 1000; // max_limit_for_vector_search_queries default (26.x)
553
+ if (this.opts.ann === "hnsw" && requestedK > maxAnnLimit) {
554
+ console.warn(`[clickhouse] topK ${requestedK} exceeds the ANN query limit ${maxAnnLimit} — using ${maxAnnLimit}.`);
555
+ }
556
+ const limit = this.opts.ann === "hnsw" ? Math.min(requestedK, maxAnnLimit) : requestedK;
557
+ const params = { epoch, k: limit };
558
+ const where = [`epoch = {epoch:UInt64}`];
559
+ if (filter?.languages?.length) {
560
+ where.push("language IN ({languages:Array(String)})");
561
+ params.languages = filter.languages;
562
+ }
563
+ if (filter?.kinds?.length) {
564
+ where.push("kind IN ({kinds:Array(String)})");
565
+ params.kinds = filter.kinds;
566
+ }
567
+ if (filter?.pathPatterns?.length) {
568
+ const likes = filter.pathPatterns.map((pattern, index) => {
569
+ params[`path${index}`] = globToLike(pattern);
570
+ return `file_path LIKE {path${index}:String}`;
571
+ });
572
+ where.push(`(${likes.join(" OR ")})`);
573
+ }
574
+ if (filter?.fileExtensions?.length) {
575
+ const exts = normalizeFileExtensions(filter.fileExtensions);
576
+ const likes = exts.map((ext, index) => {
577
+ params[`ext${index}`] = `%${ext}`;
578
+ return `file_path LIKE {ext${index}:String}`;
579
+ });
580
+ where.push(`(${likes.join(" OR ")})`);
581
+ }
582
+ const hasFilter = where.length > 1;
583
+ // Filtered ANN searches post-filter the candidate list; over-fetch so a
584
+ // selective path/language filter cannot starve the result set.
585
+ const settings = this.opts.ann === "hnsw"
586
+ ? {
587
+ vector_search_with_rescoring: this.opts.annWithRescoring ? 1 : 0,
588
+ vector_search_index_fetch_multiplier: hasFilter ? this.opts.annFetchMultiplier : 1,
589
+ }
590
+ : undefined;
591
+ const rows = await this.select(`SELECT chunk_id, file_path, start_line, end_line, language, kind, role, quirk_type, ` +
592
+ "tags, content, description, epoch, is_deleted, " +
593
+ "cosineDistance(embedding, {q:Array(Float32)}) AS dist " +
594
+ `FROM ${this.table(this.opts.chunksTable)} FINAL ` +
595
+ `WHERE ${where.join(" AND ")} ORDER BY dist ASC LIMIT {k:UInt32}`, { ...params, q: embedding }, settings);
596
+ return rows.map((row) => this.rowToSearchResult(row));
597
+ }
598
+ catch (err) {
599
+ // Mirror LanceDB: a failed search logs and returns no results rather
600
+ // than crashing the plugin's retrieval path.
601
+ console.warn(`[clickhouse] searchWithFilter failed (returning []): ${err.message}`);
602
+ return [];
603
+ }
604
+ }
605
+ rowToSearchResult(row) {
606
+ let tags;
607
+ try {
608
+ tags = row.tags ? JSON.parse(row.tags) : undefined;
609
+ }
610
+ catch {
611
+ tags = undefined;
612
+ }
613
+ const rawDistance = typeof row.dist === "number" && Number.isFinite(row.dist) ? row.dist : 2;
614
+ return {
615
+ // ClickHouse cosineDistance is 1 - cosine similarity (range 0..2), the
616
+ // same domain as LanceDB's cosine distance; normalize identically so
617
+ // fusion/minScore stay comparable across backends.
618
+ score: Math.min(1, Math.max(0, 1 - rawDistance / 2)),
619
+ chunk: {
620
+ id: row.chunk_id,
621
+ content: row.content,
622
+ description: row.description ?? "",
623
+ metadata: {
624
+ filePath: row.file_path,
625
+ startLine: row.start_line,
626
+ endLine: row.end_line,
627
+ language: row.language,
628
+ kind: row.kind || undefined,
629
+ role: row.role === "source" || row.role === "test" || row.role === "doc"
630
+ ? row.role
631
+ : undefined,
632
+ quirkType: row.quirk_type || undefined,
633
+ tags,
634
+ },
635
+ },
636
+ };
637
+ }
638
+ async count() {
639
+ await this.ensureSchema();
640
+ const rows = await this.select(`SELECT count() AS c FROM ${this.table(this.opts.chunksTable)} FINAL WHERE epoch = {epoch:UInt64}`, { epoch: this.currentEpoch() });
641
+ return Number(rows[0]?.c ?? 0);
642
+ }
643
+ async getFilePaths() {
644
+ await this.ensureSchema();
645
+ const rows = await this.select(`SELECT DISTINCT file_path FROM ${this.table(this.opts.chunksTable)} FINAL ` +
646
+ "WHERE epoch = {epoch:UInt64} ORDER BY file_path", { epoch: this.currentEpoch() });
647
+ return rows.map((row) => row.file_path);
648
+ }
649
+ async listFiles() {
650
+ await this.ensureSchema();
651
+ const rows = await this.select(`SELECT file_path, any(language) AS language, count() AS chunk_count ` +
652
+ `FROM ${this.table(this.opts.chunksTable)} FINAL ` +
653
+ "WHERE epoch = {epoch:UInt64} GROUP BY file_path ORDER BY file_path", { epoch: this.currentEpoch() });
654
+ return rows.map((row) => ({
655
+ filePath: row.file_path,
656
+ language: row.language,
657
+ chunkCount: Number(row.chunk_count),
658
+ }));
659
+ }
660
+ async getChunks(offset, limit) {
661
+ await this.ensureSchema();
662
+ const rows = await this.select(`SELECT ${this.summaryColumns()} FROM ${this.table(this.opts.chunksTable)} FINAL ` +
663
+ "WHERE epoch = {epoch:UInt64} ORDER BY file_path, start_line " +
664
+ "LIMIT {limit:UInt32} OFFSET {offset:UInt32}", {
665
+ epoch: this.currentEpoch(),
666
+ limit: Math.max(1, Math.floor(limit)),
667
+ offset: Math.max(0, Math.floor(offset)),
668
+ });
669
+ return rows.map((row) => this.rowToSummary(row));
670
+ }
671
+ async getChunksFiltered(offset, limit, lang, filePrefix) {
672
+ await this.ensureSchema();
673
+ const params = { epoch: this.currentEpoch() };
674
+ const where = ["epoch = {epoch:UInt64}"];
675
+ if (lang) {
676
+ where.push("language = {lang:String}");
677
+ params.lang = lang;
678
+ }
679
+ if (filePrefix) {
680
+ where.push("file_path LIKE {prefix:String}");
681
+ params.prefix = `${filePrefix}%`;
682
+ }
683
+ const whereSql = where.join(" AND ");
684
+ const [rows, totals] = await Promise.all([
685
+ this.select(`SELECT ${this.summaryColumns()} FROM ${this.table(this.opts.chunksTable)} FINAL ` +
686
+ `WHERE ${whereSql} ORDER BY file_path, start_line ` +
687
+ "LIMIT {limit:UInt32} OFFSET {offset:UInt32}", {
688
+ ...params,
689
+ limit: Math.max(1, Math.min(Math.floor(limit), 1000)),
690
+ offset: Math.max(0, Math.floor(offset)),
691
+ }),
692
+ this.select(`SELECT count() AS c FROM ${this.table(this.opts.chunksTable)} FINAL WHERE ${whereSql}`, params),
693
+ ]);
694
+ return {
695
+ chunks: rows.map((row) => this.rowToSummary(row)),
696
+ total: Number(totals[0]?.c ?? 0),
697
+ };
698
+ }
699
+ async getChunkById(id) {
700
+ await this.ensureSchema();
701
+ const rows = await this.select(`SELECT ${this.summaryColumns()} FROM ${this.table(this.opts.chunksTable)} FINAL ` +
702
+ "WHERE epoch = {epoch:UInt64} AND chunk_id = {id:String} LIMIT 1", { epoch: this.currentEpoch(), id });
703
+ return rows[0] ? this.rowToSummary(rows[0]) : undefined;
704
+ }
705
+ async getChunksByIds(ids) {
706
+ const bounded = ids.slice(0, 100);
707
+ if (bounded.length === 0)
708
+ return [];
709
+ await this.ensureSchema();
710
+ const rows = await this.select(`SELECT ${this.summaryColumns()} FROM ${this.table(this.opts.chunksTable)} FINAL ` +
711
+ "WHERE epoch = {epoch:UInt64} AND chunk_id IN ({ids:Array(String)})", { epoch: this.currentEpoch(), ids: bounded });
712
+ return rows.map((row) => this.rowToSummary(row));
713
+ }
714
+ async getChunksWithEmbeddings(limit) {
715
+ await this.ensureSchema();
716
+ const rows = await this.select("SELECT chunk_id, file_path, language, start_line, end_line, description, embedding " +
717
+ `FROM ${this.table(this.opts.chunksTable)} FINAL ` +
718
+ "WHERE epoch = {epoch:UInt64} ORDER BY file_path, start_line LIMIT {limit:UInt32}", { epoch: this.currentEpoch(), limit: Math.max(1, Math.floor(limit)) });
719
+ return rows.map((row) => ({
720
+ id: row.chunk_id,
721
+ filePath: row.file_path,
722
+ language: row.language,
723
+ startLine: row.start_line,
724
+ endLine: row.end_line,
725
+ description: row.description ?? "",
726
+ embedding: Array.from(row.embedding ?? []),
727
+ }));
728
+ }
729
+ async getChunksByFilePath(filePath) {
730
+ await this.ensureSchema();
731
+ const normalized = filePath.startsWith(QUIRK_FILE_PREFIX)
732
+ ? filePath
733
+ : normalizeFilePath(filePath);
734
+ const rows = await this.select(`SELECT ${this.summaryColumns()}, embedding FROM ${this.table(this.opts.chunksTable)} FINAL ` +
735
+ "WHERE epoch = {epoch:UInt64} AND file_path = {path:String} ORDER BY start_line", { epoch: this.currentEpoch(), path: normalized });
736
+ return rows.map((row) => ({
737
+ id: row.chunk_id,
738
+ content: row.content,
739
+ description: row.description ?? "",
740
+ embedding: Array.from(row.embedding ?? []),
741
+ metadata: {
742
+ filePath: row.file_path,
743
+ startLine: row.start_line,
744
+ endLine: row.end_line,
745
+ language: row.language,
746
+ kind: row.kind || undefined,
747
+ role: row.role === "source" || row.role === "test" || row.role === "doc"
748
+ ? row.role
749
+ : undefined,
750
+ quirkType: row.quirk_type || undefined,
751
+ tags: this.parseTags(row.tags),
752
+ },
753
+ }));
754
+ }
755
+ summaryColumns() {
756
+ return "chunk_id, file_path, start_line, end_line, language, kind, role, quirk_type, tags, content, description";
757
+ }
758
+ parseTags(raw) {
759
+ if (!raw)
760
+ return undefined;
761
+ try {
762
+ return JSON.parse(raw);
763
+ }
764
+ catch {
765
+ return undefined;
766
+ }
767
+ }
768
+ rowToSummary(row) {
769
+ return {
770
+ id: row.chunk_id,
771
+ filePath: row.file_path,
772
+ language: row.language,
773
+ startLine: row.start_line,
774
+ endLine: row.end_line,
775
+ content: row.content,
776
+ description: row.description ?? "",
777
+ kind: row.kind ?? "",
778
+ role: row.role ?? "",
779
+ quirkType: row.quirk_type ?? "",
780
+ tags: row.tags ?? "",
781
+ };
782
+ }
783
+ async getVectorDimension() {
784
+ await this.ensureSchema();
785
+ const meta = await this.readMeta();
786
+ return meta?.dimension;
787
+ }
788
+ /**
789
+ * Read the stored dimension without creating anything. Used by the
790
+ * bootstrap dimension probe, which must stay read-only for `status`.
791
+ */
792
+ async peekDimension() {
793
+ const rows = await this.select("SELECT name FROM system.tables WHERE database = {db:String} AND name = {table:String} LIMIT 1", { db: this.opts.database, table: this.opts.stateTable });
794
+ if (rows.length === 0)
795
+ return undefined;
796
+ const meta = await this.readMeta();
797
+ return meta?.dimension;
798
+ }
799
+ async checkIntegrity() {
800
+ try {
801
+ await this.select("SELECT 1 AS ok");
802
+ return true;
803
+ }
804
+ catch {
805
+ return false;
806
+ }
807
+ }
808
+ /** Whether the HNSW vector index is present on the chunks table (diagnostics). */
809
+ async hasVectorIndex() {
810
+ try {
811
+ await this.ensureSchema();
812
+ const rows = await this.select("SELECT name FROM system.data_skipping_indices " +
813
+ "WHERE database = {db:String} AND table = {table:String} AND name = {name:String} LIMIT 1", { db: this.opts.database, table: this.opts.chunksTable, name: "vec_idx" });
814
+ return rows.length > 0;
815
+ }
816
+ catch {
817
+ return false;
818
+ }
819
+ }
820
+ async optimize(options) {
821
+ try {
822
+ await this.ensureSchema();
823
+ const table = this.table(this.opts.chunksTable);
824
+ await this.command(`OPTIMIZE TABLE ${table} FINAL`);
825
+ // Old epochs are already invisible (all reads filter on the active
826
+ // epoch); pruning them only reclaims disk. Never let a cleanup failure
827
+ // break an index pass.
828
+ const meta = await this.readMeta();
829
+ if (meta && (options?.aggressive || meta.activeEpoch > 1)) {
830
+ await this.command(
831
+ // Only superseded epochs (`< active`). A pending rebuild epoch is
832
+ // `> active` and must survive an optimize run.
833
+ `ALTER TABLE ${table} DELETE WHERE epoch < {epoch:UInt64}`, { epoch: meta.activeEpoch });
834
+ options?.logger?.(`[clickhouse] Pruned chunks from superseded epochs (active ${meta.activeEpoch}).`);
835
+ }
836
+ await this.command(`OPTIMIZE TABLE ${this.table(this.opts.stateTable)} FINAL`);
837
+ }
838
+ catch (err) {
839
+ options?.logger?.(`[clickhouse] optimize failed: ${err.message}`);
840
+ }
841
+ }
842
+ // ── Lease (advisory single-writer coordination) ────────────────────────
843
+ async readLeaseRow(kind) {
844
+ const row = await this.readState(`${LEASE_NAME}:${kind}`);
845
+ if (!row?.data)
846
+ return undefined;
847
+ try {
848
+ const parsed = JSON.parse(row.data);
849
+ if (!parsed.owner)
850
+ return undefined;
851
+ return { owner: parsed.owner, expiresAt: Number(parsed.expiresAt ?? 0) };
852
+ }
853
+ catch {
854
+ return undefined;
855
+ }
856
+ }
857
+ async acquireLease(owner, kind, ttlMs) {
858
+ await this.ensureSchema();
859
+ const current = await this.readLeaseRow(kind);
860
+ const now = Date.now();
861
+ if (current && current.owner !== owner && current.expiresAt > now)
862
+ return false;
863
+ await this.writeState(`${LEASE_NAME}:${kind}`, JSON.stringify({ owner, expiresAt: now + ttlMs }));
864
+ // Read-back confirmation: ClickHouse has no compare-and-swap, so two
865
+ // racing writers can both insert. The loser sees the winner here.
866
+ await new Promise((resolve) => setTimeout(resolve, 120));
867
+ const confirmed = await this.readLeaseRow(kind);
868
+ return confirmed?.owner === owner;
869
+ }
870
+ async renewLease(owner, kind, ttlMs) {
871
+ await this.ensureSchema();
872
+ const current = await this.readLeaseRow(kind);
873
+ if (current && current.owner !== owner && current.expiresAt > Date.now())
874
+ return false;
875
+ await this.writeState(`${LEASE_NAME}:${kind}`, JSON.stringify({ owner, expiresAt: Date.now() + ttlMs }));
876
+ return true;
877
+ }
878
+ async releaseLease(owner, kind) {
879
+ try {
880
+ await this.ensureSchema();
881
+ const current = await this.readLeaseRow(kind);
882
+ if (current?.owner !== owner)
883
+ return;
884
+ await this.writeState(`${LEASE_NAME}:${kind}`, JSON.stringify({ owner: "", expiresAt: 0 }));
885
+ }
886
+ catch {
887
+ // Best effort — an expired lease is reclaimed by the next writer.
888
+ }
889
+ }
890
+ async getLease(kind) {
891
+ try {
892
+ await this.ensureSchema();
893
+ const lease = await this.readLeaseRow(kind);
894
+ if (!lease || lease.expiresAt <= Date.now())
895
+ return undefined;
896
+ return lease;
897
+ }
898
+ catch {
899
+ return undefined;
900
+ }
901
+ }
902
+ async close() {
903
+ const client = this.client;
904
+ this.client = null;
905
+ this.schemaReady = null;
906
+ this.activeEpoch = null;
907
+ if (client) {
908
+ try {
909
+ await client.close();
910
+ }
911
+ catch {
912
+ // Closing a pooled HTTP client is best effort.
913
+ }
914
+ }
915
+ }
916
+ async reopen() {
917
+ await this.close();
918
+ }
919
+ /**
920
+ * Drop this project's tables. Used by `opencode-rag clear --hard` tooling and
921
+ * by tests; irreversible. Other namespaces are untouched.
922
+ */
923
+ async dropProjectTables() {
924
+ await this.ensureSchema();
925
+ await this.command(`DROP TABLE IF EXISTS ${this.table(this.opts.chunksTable)}`);
926
+ await this.command(`DROP TABLE IF EXISTS ${this.table(this.opts.stateTable)}`);
927
+ this.schemaReady = null;
928
+ this.activeEpoch = null;
929
+ }
930
+ }
931
+ //# sourceMappingURL=clickhouse.js.map