opencode-rag-plugin 2.10.0 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/ReadMe.md +40 -28
  2. package/dist/api.js +1 -0
  3. package/dist/cli/commands/clickhouse.d.ts +61 -0
  4. package/dist/cli/commands/clickhouse.js +335 -0
  5. package/dist/cli/commands/eval-tuner.js +1 -1
  6. package/dist/cli/commands/eval.js +1 -1
  7. package/dist/cli/commands/index-command.js +2 -1
  8. package/dist/cli/commands/index.d.ts +1 -0
  9. package/dist/cli/commands/index.js +1 -0
  10. package/dist/cli/commands/init-helpers.js +2 -2
  11. package/dist/cli/commands/init.js +49 -11
  12. package/dist/cli/commands/quirk.js +6 -6
  13. package/dist/cli/commands/status.js +70 -6
  14. package/dist/cli/commands/ui.js +2 -2
  15. package/dist/cli/index.js +2 -1
  16. package/dist/cli/types.d.ts +14 -0
  17. package/dist/core/bootstrap.d.ts +7 -0
  18. package/dist/core/bootstrap.js +40 -9
  19. package/dist/core/config.d.ts +128 -5
  20. package/dist/core/config.js +205 -8
  21. package/dist/core/interfaces.d.ts +54 -0
  22. package/dist/core/runtime-overrides.d.ts +9 -0
  23. package/dist/core/runtime-overrides.js +44 -0
  24. package/dist/core/state-sync.d.ts +91 -0
  25. package/dist/core/state-sync.js +273 -0
  26. package/dist/eval/fast-index.js +2 -2
  27. package/dist/eval/rerank-eval.js +2 -2
  28. package/dist/eval/run-branch-compare.js +2 -2
  29. package/dist/eval/run-token-test.js +2 -2
  30. package/dist/index.d.ts +5 -2
  31. package/dist/index.js +2 -1
  32. package/dist/indexer/pipeline.d.ts +32 -0
  33. package/dist/indexer/pipeline.js +230 -65
  34. package/dist/indexer/progress.d.ts +75 -0
  35. package/dist/indexer/progress.js +114 -0
  36. package/dist/indexer/stats.d.ts +14 -0
  37. package/dist/plugin.d.ts +6 -3
  38. package/dist/plugin.js +39 -20
  39. package/dist/quirks/quirk-store.d.ts +10 -0
  40. package/dist/quirks/quirk-store.js +110 -15
  41. package/dist/quirks/types.d.ts +7 -0
  42. package/dist/tui.js +134 -113
  43. package/dist/vectorstore/clickhouse.d.ts +150 -0
  44. package/dist/vectorstore/clickhouse.js +931 -0
  45. package/dist/vectorstore/dimension.d.ts +19 -0
  46. package/dist/vectorstore/dimension.js +36 -0
  47. package/dist/vectorstore/factory.d.ts +9 -3
  48. package/dist/vectorstore/factory.js +11 -2
  49. package/dist/vectorstore/lancedb.d.ts +27 -4
  50. package/dist/vectorstore/lancedb.js +180 -15
  51. package/dist/watcher.d.ts +29 -0
  52. package/dist/watcher.js +76 -2
  53. package/dist/web/api.d.ts +3 -4
  54. package/dist/web/api.js +18 -3
  55. package/dist/web/server.js +28 -2
  56. package/dist/web/ui/assets/index-B-RH6fko.css +2 -0
  57. package/dist/web/ui/index.html +2 -2
  58. package/package.json +2 -1
  59. package/dist/web/ui/assets/index-CWcp0jNf.css +0 -2
  60. /package/dist/web/ui/assets/{index-DbXFrn53.js → index-ChGi2bSp.js} +0 -0
@@ -0,0 +1,19 @@
1
+ /**
2
+ * @fileoverview Backend-neutral dimension probe.
3
+ *
4
+ * The pipeline needs the dimension of an existing store before it can decide
5
+ * whether to rebuild. LanceDB keeps it in the table schema; ClickHouse keeps
6
+ * it in the project's `_meta` state row. Both probes are read-only.
7
+ */
8
+ import type { RagConfig } from "../core/config.js";
9
+ /**
10
+ * Read the embedding dimension of the configured store.
11
+ *
12
+ * @param config - Resolved RAG configuration.
13
+ * @param storePath - Local store directory (LanceDB only).
14
+ * @param workspaceDir - Workspace root (used to derive the ClickHouse namespace).
15
+ * @returns The stored dimension, or undefined when the store does not exist yet.
16
+ */
17
+ export declare function readConfiguredStoreDimension(config: RagConfig, storePath: string, workspaceDir: string): Promise<number | undefined>;
18
+ /** Read the dimension from a ClickHouse store without creating any schema. */
19
+ export declare function readClickHouseDimension(config: RagConfig, workspaceDir: string): Promise<number | undefined>;
@@ -0,0 +1,36 @@
1
+ import { readStoreDimension } from "./lancedb.js";
2
+ import { ClickHouseStore, resolveClickHouseOptions } from "./clickhouse.js";
3
+ /**
4
+ * Read the embedding dimension of the configured store.
5
+ *
6
+ * @param config - Resolved RAG configuration.
7
+ * @param storePath - Local store directory (LanceDB only).
8
+ * @param workspaceDir - Workspace root (used to derive the ClickHouse namespace).
9
+ * @returns The stored dimension, or undefined when the store does not exist yet.
10
+ */
11
+ export async function readConfiguredStoreDimension(config, storePath, workspaceDir) {
12
+ const provider = config.vectorStore.provider ?? "lancedb";
13
+ if (provider === "lancedb") {
14
+ return readStoreDimension(storePath);
15
+ }
16
+ if (provider === "clickhouse") {
17
+ return readClickHouseDimension(config, workspaceDir);
18
+ }
19
+ return undefined;
20
+ }
21
+ /** Read the dimension from a ClickHouse store without creating any schema. */
22
+ export async function readClickHouseDimension(config, workspaceDir) {
23
+ const options = resolveClickHouseOptions(config, workspaceDir);
24
+ const store = new ClickHouseStore(options, config.embedding.vectorDimension ?? 384);
25
+ try {
26
+ return await store.peekDimension();
27
+ }
28
+ catch (err) {
29
+ console.warn(`[bootstrap] Could not read ClickHouse store dimension: ${err.message}`);
30
+ return undefined;
31
+ }
32
+ finally {
33
+ await store.close().catch(() => { });
34
+ }
35
+ }
36
+ //# sourceMappingURL=dimension.js.map
@@ -1,5 +1,6 @@
1
1
  /**
2
- * @fileoverview Factory function to create the configured VectorStore instance (LanceDB or in-memory).
2
+ * @fileoverview Factory function to create the configured VectorStore instance
3
+ * (LanceDB, in-memory, or ClickHouse).
3
4
  */
4
5
  import type { VectorStore } from "../core/interfaces.js";
5
6
  import type { RagConfig } from "../core/config.js";
@@ -9,11 +10,16 @@ import type { RagConfig } from "../core/config.js";
9
10
  * Supported providers:
10
11
  * - `"lancedb"`: Persistent LanceDB-backed store (default).
11
12
  * - `"memory"`: Ephemeral in-memory store.
13
+ * - `"clickhouse"`: Central ClickHouse-backed store; `storePath` is only a
14
+ * local cache directory for workspace state.
12
15
  *
13
16
  * @param config - The RAG configuration containing the vector store provider setting.
14
- * @param storePath - The filesystem path for persistent stores (LanceDB).
17
+ * @param storePath - The filesystem path for persistent local stores (LanceDB)
18
+ * and for the disposable state cache when ClickHouse is active.
15
19
  * @param dimension - The embedding vector dimension.
20
+ * @param workspaceDir - Workspace root, used to derive the default ClickHouse
21
+ * namespace. Defaults to `process.cwd()`.
16
22
  * @returns A VectorStore instance matching the configured provider.
17
23
  * @throws If the provider name is unknown.
18
24
  */
19
- export declare function createVectorStore(config: RagConfig, storePath: string, dimension: number): VectorStore;
25
+ export declare function createVectorStore(config: RagConfig, storePath: string, dimension: number, workspaceDir?: string): VectorStore;
@@ -1,19 +1,25 @@
1
1
  import { LanceDbStore } from "./lancedb.js";
2
2
  import { InMemoryVectorStore } from "./memory.js";
3
+ import { ClickHouseStore, resolveClickHouseOptions } from "./clickhouse.js";
3
4
  /**
4
5
  * Create a VectorStore instance based on the configured provider.
5
6
  *
6
7
  * Supported providers:
7
8
  * - `"lancedb"`: Persistent LanceDB-backed store (default).
8
9
  * - `"memory"`: Ephemeral in-memory store.
10
+ * - `"clickhouse"`: Central ClickHouse-backed store; `storePath` is only a
11
+ * local cache directory for workspace state.
9
12
  *
10
13
  * @param config - The RAG configuration containing the vector store provider setting.
11
- * @param storePath - The filesystem path for persistent stores (LanceDB).
14
+ * @param storePath - The filesystem path for persistent local stores (LanceDB)
15
+ * and for the disposable state cache when ClickHouse is active.
12
16
  * @param dimension - The embedding vector dimension.
17
+ * @param workspaceDir - Workspace root, used to derive the default ClickHouse
18
+ * namespace. Defaults to `process.cwd()`.
13
19
  * @returns A VectorStore instance matching the configured provider.
14
20
  * @throws If the provider name is unknown.
15
21
  */
16
- export function createVectorStore(config, storePath, dimension) {
22
+ export function createVectorStore(config, storePath, dimension, workspaceDir = process.cwd()) {
17
23
  const provider = config.vectorStore.provider ?? "lancedb";
18
24
  if (provider === "lancedb") {
19
25
  return new LanceDbStore(storePath, dimension);
@@ -21,6 +27,9 @@ export function createVectorStore(config, storePath, dimension) {
21
27
  if (provider === "memory") {
22
28
  return new InMemoryVectorStore();
23
29
  }
30
+ if (provider === "clickhouse") {
31
+ return new ClickHouseStore(resolveClickHouseOptions(config, workspaceDir), dimension);
32
+ }
24
33
  throw new Error(`Unknown vector store provider: ${provider}`);
25
34
  }
26
35
  //# sourceMappingURL=factory.js.map
@@ -1,4 +1,4 @@
1
- import type { VectorStore, Chunk, ChunkSummary, SearchResult, MetadataFilter } from "../core/interfaces.js";
1
+ import type { Chunk, ChunkSummary, SearchResult, MetadataFilter, ChunkBrowseStore } from "../core/interfaces.js";
2
2
  /** Minimal warning sink for store diagnostics (defaults to console.warn). */
3
3
  export type StoreWarn = (message: string) => void;
4
4
  /**
@@ -112,7 +112,7 @@ export interface BulkChunkWrite {
112
112
  * and chunk metadata queries. Supports automatic corruption recovery by falling
113
113
  * back to prior table versions or dropping and rebuilding the table.
114
114
  */
115
- export declare class LanceDbStore implements VectorStore {
115
+ export declare class LanceDbStore implements ChunkBrowseStore {
116
116
  private dbPath;
117
117
  private readonly vectorDimension;
118
118
  private db;
@@ -333,14 +333,37 @@ export declare class LanceDbStore implements VectorStore {
333
333
  * @param options - `aggressive: true` prunes every version but the current
334
334
  * one with `deleteUnverified`. Only safe for a private store that no other
335
335
  * process reads (e.g. a temporary rebuild store). For the shared store,
336
- * versions newer than 1 hour are retained so in-flight queries (Web UI,
337
- * background auto-index) can finish before their data files are reclaimed.
336
+ * versions newer than {@link REGULAR_CLEANUP_AGE_MS} are retained so
337
+ * in-flight queries (Web UI, background auto-index) can finish before their
338
+ * data files are reclaimed. The window matches the pipeline's optimize
339
+ * throttle, so every run can reclaim the build the previous one left behind.
338
340
  */
339
341
  optimize(options?: {
340
342
  aggressive?: boolean;
341
343
  skipIndex?: boolean;
342
344
  logger?: StoreWarn;
343
345
  }): Promise<void>;
346
+ /** Names of the index build directories that currently carry files. */
347
+ private indexBuildDirs;
348
+ /**
349
+ * Delete superseded index build directories and rebuild the ANN index once.
350
+ *
351
+ * LanceDB writes a new `_indices/<uuid>/` on every `optimize()` and every
352
+ * `createIndex()`, each holding an uncompressed copy of the vector column
353
+ * (4096 x float32 = 16 KB per row, ~190 MB at 12k chunks), and it never
354
+ * removes the old ones. A store that optimized a few times a day reached
355
+ * 2.5 GB of dead builds against 380 MB of live data.
356
+ *
357
+ * Above {@link MAX_INDEX_BUILDS} builds this deletes every build directory and
358
+ * creates one fresh index (a `replace: true` build also replaces the dangling
359
+ * index versions the deleted directories leave behind). Skipped while any
360
+ * build is younger than {@link INDEX_BUILD_MIN_AGE_MS} so a build running in
361
+ * another process is never hit; if the recreate fails the index is simply
362
+ * rebuilt by the next repair pass.
363
+ *
364
+ * @param logger - Warning sink for the reclaimed-space message.
365
+ */
366
+ private reclaimIndexBuilds;
344
367
  /**
345
368
  * Return all unique file paths currently stored in the index.
346
369
  * @returns An array of normalized file paths.
@@ -33,6 +33,44 @@ const MAX_STALE_INDEX_VERSIONS = 40;
33
33
  * meaningless 9 MB index file.
34
34
  */
35
35
  const MIN_ROWS_FOR_IVF_INDEX = 4096;
36
+ /**
37
+ * When more than this many index build directories carry files, `optimize()`
38
+ * deletes them all and builds the ANN index once.
39
+ *
40
+ * LanceDB never reclaims superseded index builds: every `Table.optimize()` and
41
+ * every `createIndex()` writes a new `_indices/<uuid>/` holding an uncompressed
42
+ * copy of the vector column (12k chunks x 4096 float32 = ~190 MB), and the old
43
+ * directories stay on disk indefinitely. Three builds is the working ceiling;
44
+ * beyond that a single rebuild is cheaper than the dead copies.
45
+ */
46
+ const MAX_INDEX_BUILDS = 3;
47
+ /** Builds younger than this are treated as in-flight (another process may be writing). */
48
+ const INDEX_BUILD_MIN_AGE_MS = 60 * 1000;
49
+ /**
50
+ * Version retention for a regular (non-aggressive) optimize run.
51
+ *
52
+ * Matches the pipeline's optimize throttle (`MIN_OPTIMIZE_INTERVAL_MS`): long
53
+ * enough for an in-flight query to finish, short enough that the next due run
54
+ * reclaims the index build the previous one superseded.
55
+ */
56
+ const REGULAR_CLEANUP_AGE_MS = 15 * 60 * 1000;
57
+ /** Recursively sum the file sizes of a directory (0 when unreadable). */
58
+ function dirSizeBytes(dir) {
59
+ let total = 0;
60
+ for (const entry of fsSync.readdirSync(dir, { withFileTypes: true })) {
61
+ const full = path.join(dir, entry.name);
62
+ total += entry.isDirectory() ? dirSizeBytes(full) : fsSync.statSync(full).size;
63
+ }
64
+ return total;
65
+ }
66
+ /** Newest modification time inside a directory (its own when empty). */
67
+ function newestMtimeMs(dir) {
68
+ let newest = fsSync.statSync(dir).mtimeMs;
69
+ for (const entry of fsSync.readdirSync(dir, { withFileTypes: true })) {
70
+ newest = Math.max(newest, fsSync.statSync(path.join(dir, entry.name)).mtimeMs);
71
+ }
72
+ return newest;
73
+ }
36
74
  /**
37
75
  * Count index-version directories in a LanceDB table directory. Each
38
76
  * `createIndex` writes a new `<uuid>/` directory under `_indices`; a healthy
@@ -338,7 +376,11 @@ function buildFilePathMatch(filePath) {
338
376
  // uuids — no LIKE metacharacters; paths never contain quotes unescaped)
339
377
  return `(filePath = '${escBase}' OR filePath LIKE '%/${escBase}')`;
340
378
  }
341
- return `filePath = '${normalizeFilePath(filePath).replace(/'/g, "''")}'`;
379
+ // Case-insensitive on purpose: Windows paths reach the store in more than one
380
+ // spelling (`c:/Daten/...` from a watcher event vs `C:/Daten/...` from
381
+ // path.resolve), and a case-sensitive delete leaves the other spelling behind
382
+ // forever - that is how 262 files ended up stored twice.
383
+ return `lower(filePath) = lower('${normalizeFilePath(filePath).replace(/'/g, "''")}')`;
342
384
  }
343
385
  /**
344
386
  * A LanceDB-backed vector store with persistent on-disk storage, vector search,
@@ -738,12 +780,15 @@ export class LanceDbStore {
738
780
  // keeps an abort between insert and delete from losing data.
739
781
  // Quirk paths are skipped — they were fully cleaned above.
740
782
  const byFile = this.rowsByFilePath(rows);
741
- for (const [filePath, ids] of byFile) {
783
+ // NOT IN every id written by this call, not just the group's: two spellings
784
+ // of the same file can land in one batch, and a per-group list would let the
785
+ // first delete wipe the second group's freshly inserted rows.
786
+ const allIds = rows.map((row) => `'${row.id.replace(/'/g, "''")}'`).join(", ");
787
+ for (const filePath of byFile.keys()) {
742
788
  if (filePath.startsWith(QUIRK_FILE_PREFIX))
743
789
  continue;
744
790
  const escapedPath = filePath.replace(/'/g, "''");
745
- const idList = ids.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
746
- await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
791
+ await table.delete(`lower(filePath) = lower('${escapedPath}') AND id NOT IN (${allIds})`);
747
792
  }
748
793
  }
749
794
  async addChunksBulkInternal(items) {
@@ -778,12 +823,12 @@ export class LanceDbStore {
778
823
  }
779
824
  // INSERT FIRST (single add for the whole batch), then per-file dedup
780
825
  await table.add(allRows);
781
- for (const [filePath, ids] of dedupByFile) {
826
+ const allBulkIds = allRows.map((row) => `'${row.id.replace(/'/g, "''")}'`).join(", ");
827
+ for (const filePath of dedupByFile.keys()) {
782
828
  if (filePath.startsWith(QUIRK_FILE_PREFIX))
783
829
  continue;
784
830
  const escapedPath = filePath.replace(/'/g, "''");
785
- const idList = ids.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
786
- await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
831
+ await table.delete(`lower(filePath) = lower('${escapedPath}') AND id NOT IN (${allBulkIds})`);
787
832
  }
788
833
  }
789
834
  /**
@@ -1215,10 +1260,22 @@ export class LanceDbStore {
1215
1260
  const indices = await table.listIndices();
1216
1261
  const vecIndex = indices.find((i) => i.columns.includes("embedding"));
1217
1262
  const idxName = vecIndex?.name ?? "embedding_idx";
1218
- const stats = await table.indexStats(idxName);
1219
- if (stats && stats.distanceType === "cosine")
1263
+ // indexStats throws when the registered index points at files that no
1264
+ // longer exist (a dangling `_indices/<uuid>` after a prune, or a
1265
+ // half-written build). Such an index is unusable and must be rebuilt -
1266
+ // aborting here would leave every search on the brute-force path for the
1267
+ // whole process lifetime. `undefined` without a throw just means "no
1268
+ // index yet" and stays on the old path.
1269
+ let unreadable = false;
1270
+ const stats = await table.indexStats(idxName).catch(() => {
1271
+ unreadable = true;
1272
+ return undefined;
1273
+ });
1274
+ if (unreadable)
1275
+ report(`[lancedb] index '${idxName}' is unreadable - rebuilding it`);
1276
+ if (stats?.distanceType === "cosine")
1220
1277
  return; // already healthy
1221
- if (!stats && count < MIN_ROWS_FOR_IVF_INDEX)
1278
+ if (!unreadable && count < MIN_ROWS_FOR_IVF_INDEX)
1222
1279
  return; // small store — brute-force is optimal, and IVF would only trip Lance's "too small for a meaningful index" KMeans warning
1223
1280
  const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
1224
1281
  for (let attempt = 1; attempt <= 3; attempt++) {
@@ -1278,18 +1335,26 @@ export class LanceDbStore {
1278
1335
  * @param options - `aggressive: true` prunes every version but the current
1279
1336
  * one with `deleteUnverified`. Only safe for a private store that no other
1280
1337
  * process reads (e.g. a temporary rebuild store). For the shared store,
1281
- * versions newer than 1 hour are retained so in-flight queries (Web UI,
1282
- * background auto-index) can finish before their data files are reclaimed.
1338
+ * versions newer than {@link REGULAR_CLEANUP_AGE_MS} are retained so
1339
+ * in-flight queries (Web UI, background auto-index) can finish before their
1340
+ * data files are reclaimed. The window matches the pipeline's optimize
1341
+ * throttle, so every run can reclaim the build the previous one left behind.
1283
1342
  */
1284
1343
  async optimize(options) {
1285
1344
  await this.withWriteLock(async () => {
1286
1345
  try {
1287
1346
  const table = await this.getTable();
1347
+ // Snapshot before the optimize: table.optimize() and the index repair
1348
+ // both write their own builds, so only the pre-existing set says which
1349
+ // directories are dead copies worth reclaiming.
1350
+ const preExistingDirs = this.dbPath.startsWith("memory://")
1351
+ ? new Set()
1352
+ : this.indexBuildDirs();
1288
1353
  if (options?.aggressive) {
1289
1354
  await table.optimize({ cleanupOlderThan: new Date(), deleteUnverified: true });
1290
1355
  }
1291
1356
  else {
1292
- const threshold = new Date(Date.now() - 60 * 60 * 1000);
1357
+ const threshold = new Date(Date.now() - REGULAR_CLEANUP_AGE_MS);
1293
1358
  await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
1294
1359
  }
1295
1360
  // Remove stale empty index-version directories (husks Lance leaves
@@ -1297,19 +1362,21 @@ export class LanceDbStore {
1297
1362
  // from growing unboundedly and prevents false positives in the
1298
1363
  // index-repair stale-version guard.
1299
1364
  if (!this.dbPath.startsWith("memory://")) {
1300
- const swept = sweepEmptyIndexVersionDirs(path.join(this.dbPath, TABLE_NAME + ".lance"));
1365
+ const tablePath = path.join(this.dbPath, TABLE_NAME + ".lance");
1366
+ const swept = sweepEmptyIndexVersionDirs(tablePath);
1301
1367
  if (swept > 0) {
1302
1368
  const message = `[lancedb] swept ${swept} stale index-version directories`;
1303
1369
  (options?.logger ?? console.warn)(message);
1304
1370
  }
1305
1371
  }
1306
- // Build the ANN vector index — without it every vectorSearch() is a
1372
+ // Build the ANN vector index - without it every vectorSearch() is a
1307
1373
  // brute-force O(N) flat scan (slow at 50k+ chunks). Skips early when
1308
1374
  // an index with the correct cosine metric already exists, and can be
1309
1375
  // skipped entirely for a private temporary store that is never
1310
1376
  // searched (the rebuild pipeline builds the index once at the end).
1311
1377
  if (!options?.skipIndex) {
1312
1378
  await this.ensureCosineIndex(options?.logger);
1379
+ await this.reclaimIndexBuilds(preExistingDirs, options?.logger);
1313
1380
  }
1314
1381
  }
1315
1382
  catch (err) {
@@ -1320,6 +1387,104 @@ export class LanceDbStore {
1320
1387
  }
1321
1388
  });
1322
1389
  }
1390
+ /** Names of the index build directories that currently carry files. */
1391
+ indexBuildDirs() {
1392
+ const indicesDir = path.join(this.dbPath, TABLE_NAME + ".lance", "_indices");
1393
+ const builds = new Set();
1394
+ let names;
1395
+ try {
1396
+ names = fsSync.readdirSync(indicesDir);
1397
+ }
1398
+ catch {
1399
+ return builds;
1400
+ }
1401
+ for (const name of names) {
1402
+ try {
1403
+ if (fsSync.readdirSync(path.join(indicesDir, name)).length > 0)
1404
+ builds.add(name);
1405
+ }
1406
+ catch {
1407
+ // unreadable/racing entry - ignore
1408
+ }
1409
+ }
1410
+ return builds;
1411
+ }
1412
+ /**
1413
+ * Delete superseded index build directories and rebuild the ANN index once.
1414
+ *
1415
+ * LanceDB writes a new `_indices/<uuid>/` on every `optimize()` and every
1416
+ * `createIndex()`, each holding an uncompressed copy of the vector column
1417
+ * (4096 x float32 = 16 KB per row, ~190 MB at 12k chunks), and it never
1418
+ * removes the old ones. A store that optimized a few times a day reached
1419
+ * 2.5 GB of dead builds against 380 MB of live data.
1420
+ *
1421
+ * Above {@link MAX_INDEX_BUILDS} builds this deletes every build directory and
1422
+ * creates one fresh index (a `replace: true` build also replaces the dangling
1423
+ * index versions the deleted directories leave behind). Skipped while any
1424
+ * build is younger than {@link INDEX_BUILD_MIN_AGE_MS} so a build running in
1425
+ * another process is never hit; if the recreate fails the index is simply
1426
+ * rebuilt by the next repair pass.
1427
+ *
1428
+ * @param logger - Warning sink for the reclaimed-space message.
1429
+ */
1430
+ async reclaimIndexBuilds(preExisting, logger) {
1431
+ if (this.dbPath.startsWith("memory://"))
1432
+ return;
1433
+ if (preExisting.size <= MAX_INDEX_BUILDS)
1434
+ return;
1435
+ const indicesDir = path.join(this.dbPath, TABLE_NAME + ".lance", "_indices");
1436
+ let names;
1437
+ try {
1438
+ names = fsSync.readdirSync(indicesDir);
1439
+ }
1440
+ catch {
1441
+ return; // no index directories: nothing to reclaim
1442
+ }
1443
+ // Only builds that existed before this optimize and are older than the
1444
+ // in-flight window are candidates; builds written by this call (or by
1445
+ // another process a moment ago) are kept.
1446
+ const deletable = new Set();
1447
+ for (const name of preExisting) {
1448
+ try {
1449
+ if (Date.now() - newestMtimeMs(path.join(indicesDir, name)) >= INDEX_BUILD_MIN_AGE_MS) {
1450
+ deletable.add(name);
1451
+ }
1452
+ }
1453
+ catch {
1454
+ // unreadable/racing entry - leave it alone
1455
+ }
1456
+ }
1457
+ if (deletable.size === 0)
1458
+ return;
1459
+ let reclaimedBytes = 0;
1460
+ for (const name of names) {
1461
+ if (!deletable.has(name))
1462
+ continue;
1463
+ const full = path.join(indicesDir, name);
1464
+ try {
1465
+ reclaimedBytes += dirSizeBytes(full);
1466
+ fsSync.rmSync(full, { recursive: true, force: true });
1467
+ }
1468
+ catch {
1469
+ // best-effort - keep going, a rebuild fixes the metadata either way
1470
+ }
1471
+ }
1472
+ // The deleted builds leave their index versions dangling in the manifest;
1473
+ // a replace build re-registers the index and drops them.
1474
+ const table = await this.getTable();
1475
+ const count = await table.countRows().catch(() => 0);
1476
+ const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
1477
+ await table.createIndex("embedding", {
1478
+ config: lancedb.Index.ivfFlat({ numPartitions, distanceType: "cosine" }),
1479
+ replace: true,
1480
+ name: "embedding_idx",
1481
+ waitTimeoutSeconds: 120,
1482
+ });
1483
+ if (reclaimedBytes > 0) {
1484
+ const message = `[lancedb] reclaimed ${(reclaimedBytes / (1024 * 1024)).toFixed(1)} MB of superseded index builds`;
1485
+ (logger ?? console.warn)(message);
1486
+ }
1487
+ }
1323
1488
  /**
1324
1489
  * Return all unique file paths currently stored in the index.
1325
1490
  * @returns An array of normalized file paths.
package/dist/watcher.d.ts CHANGED
@@ -5,6 +5,7 @@
5
5
  */
6
6
  import type { RagConfig } from "./core/config.js";
7
7
  import type { DescriptionProvider, EmbeddingProvider, KeywordIndex, VectorStore } from "./core/interfaces.js";
8
+ import type { StateSync } from "./core/state-sync.js";
8
9
  /** A background indexer that can be shut down gracefully. */
9
10
  export interface BackgroundIndexer {
10
11
  /** Stop the watcher, cancel pending index passes, and clean up status files. */
@@ -32,6 +33,8 @@ export interface CreateBackgroundIndexerOptions {
32
33
  descriptionProvider?: DescriptionProvider;
33
34
  /** Embedding vector dimension — enables atomic rebuilds instead of destructive clear. */
34
35
  dimension?: number;
36
+ /** Durable-state synchronizer (ClickHouse); flushed at the end of each pass. */
37
+ stateSync?: StateSync;
35
38
  }
36
39
  /** The current operational status of the background indexer watcher. */
37
40
  export type WatcherStatus = {
@@ -73,6 +76,32 @@ export declare function deriveWatcherHealth(status: WatcherStatus): {
73
76
  export declare function tryAcquireWatcherLock(storePath: string): boolean;
74
77
  /** Release the watcher lock — only if this process owns it. */
75
78
  export declare function releaseWatcherLock(storePath: string): void;
79
+ /** A watcher observed on disk: its lock, its persisted status, and liveness. */
80
+ export interface ObservedWatcher {
81
+ /** Store directory holding `watcher.lock` / `watcher-status.json`. */
82
+ storePath: string;
83
+ /** Contents of `watcher.lock`, when present and parseable. */
84
+ lock?: {
85
+ pid: number;
86
+ startedAt: number;
87
+ };
88
+ /** PID in the lock is alive — the watcher process is currently running. */
89
+ alive: boolean;
90
+ /** Parsed `watcher-status.json`, when present and parseable. */
91
+ status?: WatcherStatus;
92
+ }
93
+ /** Read one store directory's watcher lock + status. Returns undefined when the dir holds neither. */
94
+ export declare function readWatcherState(storePath: string): ObservedWatcher | undefined;
95
+ /**
96
+ * Find every watcher observable for a workspace — including one running in a
97
+ * DIFFERENT store dir than the configured `vectorStore.path` (the classic
98
+ * symptom of a config change the already-running plugin never picked up).
99
+ *
100
+ * Scans the configured store dir, its sibling directories, and every child of
101
+ * `<workspaceRoot>/.opencode/` for `watcher.lock` / `watcher-status.json`.
102
+ * Results are sorted live-first, then by most recent activity.
103
+ */
104
+ export declare function discoverWatcherStates(configuredStorePath: string, workspaceRoot: string): ObservedWatcher[];
76
105
  /**
77
106
  * Create a background file watcher that automatically re-indexes the
78
107
  * workspace when files change. Uses chokidar for file system events and
package/dist/watcher.js CHANGED
@@ -5,7 +5,7 @@
5
5
  */
6
6
  import chokidar from "chokidar";
7
7
  import path from "node:path";
8
- import { writeFileSync, unlinkSync, existsSync, readFileSync, openSync, closeSync } from "node:fs";
8
+ import { writeFileSync, unlinkSync, existsSync, readFileSync, openSync, closeSync, readdirSync } from "node:fs";
9
9
  import { appendDebugLog } from "./core/fileLogger.js";
10
10
  import { isCorruptionError } from "./vectorstore/lancedb.js";
11
11
  import { createWatchPassScheduler, createWatchIgnore, runIndexPass, } from "./indexer.js";
@@ -113,6 +113,79 @@ export function releaseWatcherLock(storePath) {
113
113
  catch { /* ignore */ }
114
114
  }
115
115
  }
116
+ /** Read one store directory's watcher lock + status. Returns undefined when the dir holds neither. */
117
+ export function readWatcherState(storePath) {
118
+ let lock;
119
+ try {
120
+ lock = readWatcherLock(storePath);
121
+ }
122
+ catch {
123
+ /* unreadable dir */
124
+ }
125
+ let status;
126
+ try {
127
+ const statusPath = path.join(storePath, "watcher-status.json");
128
+ if (existsSync(statusPath)) {
129
+ status = JSON.parse(readFileSync(statusPath, "utf-8"));
130
+ }
131
+ }
132
+ catch {
133
+ /* corrupt status file — ignore */
134
+ }
135
+ if (!lock && !status)
136
+ return undefined;
137
+ return {
138
+ storePath,
139
+ lock,
140
+ alive: lock ? isPidAlive(lock.pid) : false,
141
+ status,
142
+ };
143
+ }
144
+ /**
145
+ * Find every watcher observable for a workspace — including one running in a
146
+ * DIFFERENT store dir than the configured `vectorStore.path` (the classic
147
+ * symptom of a config change the already-running plugin never picked up).
148
+ *
149
+ * Scans the configured store dir, its sibling directories, and every child of
150
+ * `<workspaceRoot>/.opencode/` for `watcher.lock` / `watcher-status.json`.
151
+ * Results are sorted live-first, then by most recent activity.
152
+ */
153
+ export function discoverWatcherStates(configuredStorePath, workspaceRoot) {
154
+ const candidates = new Set();
155
+ const add = (dir) => {
156
+ try {
157
+ if (existsSync(dir))
158
+ candidates.add(path.resolve(dir));
159
+ }
160
+ catch {
161
+ /* skip unreadable candidate */
162
+ }
163
+ };
164
+ add(configuredStorePath);
165
+ add(path.dirname(configuredStorePath));
166
+ const dotDir = path.join(workspaceRoot, ".opencode");
167
+ add(dotDir);
168
+ try {
169
+ if (existsSync(dotDir)) {
170
+ for (const entry of readdirSync(dotDir, { withFileTypes: true })) {
171
+ if (entry.isDirectory())
172
+ add(path.join(dotDir, entry.name));
173
+ }
174
+ }
175
+ }
176
+ catch {
177
+ /* unreadable .opencode dir */
178
+ }
179
+ const found = [];
180
+ for (const dir of candidates) {
181
+ const state = readWatcherState(dir);
182
+ if (state)
183
+ found.push(state);
184
+ }
185
+ const lastActivity = (w) => Math.max(w.status?.lastSuccessAt ?? 0, w.status?.lastRunAt ?? 0, w.status?.lastChangeAt ?? 0, w.lock?.startedAt ?? 0);
186
+ found.sort((a, b) => (b.alive ? 1 : 0) - (a.alive ? 1 : 0) || lastActivity(b) - lastActivity(a));
187
+ return found;
188
+ }
116
189
  /**
117
190
  * Create a background file watcher that automatically re-indexes the
118
191
  * workspace when files change. Uses chokidar for file system events and
@@ -123,7 +196,7 @@ export function releaseWatcherLock(storePath) {
123
196
  * @returns A BackgroundIndexer handle with a close() method for shutdown.
124
197
  */
125
198
  export function createBackgroundIndexer(options) {
126
- const { cwd, storePath, config, store, embedder, logFilePath, logLevel, keywordIndex, descriptionProvider, dimension } = options;
199
+ const { cwd, storePath, config, store, embedder, logFilePath, logLevel, keywordIndex, descriptionProvider, dimension, stateSync } = options;
127
200
  const autoIndexCfg = config.openCode.autoIndex ?? { enabled: false, debounceMs: 5000, intervalMs: 300000 };
128
201
  let active = false;
129
202
  let recheckTimer;
@@ -159,6 +232,7 @@ export function createBackgroundIndexer(options) {
159
232
  embedder,
160
233
  keywordIndex,
161
234
  descriptionProvider,
235
+ stateSync,
162
236
  dimension,
163
237
  abortSignal: ac.signal,
164
238
  logger: {
package/dist/web/api.d.ts CHANGED
@@ -2,10 +2,9 @@
2
2
  * @fileoverview REST API handler for the OpenCodeRAG Web UI with search, file, chunk, eval, and token analysis endpoints.
3
3
  */
4
4
  import type { IncomingMessage, ServerResponse } from "node:http";
5
- import { LanceDbStore } from "../vectorstore/lancedb.js";
6
5
  import { KeywordIndex } from "../retriever/keyword-index.js";
7
6
  import { type RagConfig } from "../core/config.js";
8
- import { type EmbeddingProvider } from "../core/interfaces.js";
7
+ import { type ChunkBrowseStore, type EmbeddingProvider } from "../core/interfaces.js";
9
8
  /** Internal shape for a JSON API response: an HTTP status code and a serialisable body. */
10
9
  interface ApiResponse {
11
10
  status: number;
@@ -21,7 +20,7 @@ interface ApiResponse {
21
20
  * - `/api/eval/sessions`, `/api/eval/sessions/:id`, `/api/eval/compare`
22
21
  * - `/api/eval/sessions/:id/analysis`, `/api/eval/token-compare`, `/api/eval/project-savings`
23
22
  *
24
- * @param store - The LanceDB vector store instance.
23
+ * @param store - The vector store instance (LanceDB or ClickHouse).
25
24
  * @param keywordIndex - The keyword-index instance for text search.
26
25
  * @param storePath - Filesystem path to the store directory (used by eval endpoints).
27
26
  * @param cwd - Optional workspace root for resolving file paths.
@@ -31,7 +30,7 @@ interface ApiResponse {
31
30
  * @param configPath - Path to the opencode-rag.json config file, used by PUT /api/config.
32
31
  * @returns An async handler that returns `true` when a route matched or `false` otherwise.
33
32
  */
34
- export declare function createApiHandler(store: LanceDbStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig, getEmbedder?: () => Promise<EmbeddingProvider>, token?: string, configPath?: string): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
33
+ export declare function createApiHandler(store: ChunkBrowseStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig, getEmbedder?: () => Promise<EmbeddingProvider>, token?: string, configPath?: string): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
35
34
  /**
36
35
  * Apply a validated config patch (`{ indexing: { includeDirs: [...] } }`)
37
36
  * to the on-disk config file and refresh the in-memory config used by