opencode-rag-plugin 2.10.0 → 2.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -50,6 +50,32 @@ export interface Logger {
50
50
  * @param options - Configuration for the index pass.
51
51
  * @returns Aggregate statistics for the pass.
52
52
  */
53
+ /** Minimum gap between two regular (non-aggressive) store optimize runs. */
54
+ export declare const MIN_OPTIMIZE_INTERVAL_MS: number;
55
+ /**
56
+ * Decide whether a regular store optimize may run now, and stamp the attempt.
57
+ *
58
+ * LanceDB's `Table.optimize()` always rewrites the ANN index (it decides the
59
+ * index is stale and rebuilds it), and one IVF_FLAT build is a full,
60
+ * uncompressed copy of the vector column - 12k chunks x 4096 float32 is about
61
+ * 190 MB. The watch loop optimized at the end of EVERY pass, so a busy session
62
+ * left one dead build per pass behind: on 2026-10-26 the store carried 15 builds
63
+ * (2.5 GB) against 380 MB of live data. Optimizing at most every
64
+ * {@link MIN_OPTIMIZE_INTERVAL_MS} bounds that to the builds of the last
65
+ * interval, and the version prune inside the same call reclaims the rest.
66
+ *
67
+ * Aggressive runs (a private temp store during a full rebuild) always return
68
+ * true.
69
+ *
70
+ * @param options - `force` bypasses the throttle; `now` overrides the clock (tests).
71
+ * @returns True when the caller should run the optimize.
72
+ */
73
+ export declare function optimizeDue(options?: {
74
+ force?: boolean;
75
+ now?: number;
76
+ }): boolean;
77
+ /** Reset the optimize throttle (tests only). */
78
+ export declare function resetOptimizeThrottle(): void;
53
79
  export declare function runIndexPass(options: RunIndexPassOptions): Promise<IndexRunStats>;
54
80
  /**
55
81
  * Warning text for description settings that leave chain-of-thought enabled,
@@ -59,6 +59,43 @@ function isPidAlive(pid) {
59
59
  * @param options - Configuration for the index pass.
60
60
  * @returns Aggregate statistics for the pass.
61
61
  */
62
+ /** Minimum gap between two regular (non-aggressive) store optimize runs. */
63
+ export const MIN_OPTIMIZE_INTERVAL_MS = 15 * 60 * 1000;
64
+ /** Last time a regular optimize actually ran in this process. */
65
+ let lastOptimizeAt = 0;
66
+ /**
67
+ * Decide whether a regular store optimize may run now, and stamp the attempt.
68
+ *
69
+ * LanceDB's `Table.optimize()` always rewrites the ANN index (it decides the
70
+ * index is stale and rebuilds it), and one IVF_FLAT build is a full,
71
+ * uncompressed copy of the vector column - 12k chunks x 4096 float32 is about
72
+ * 190 MB. The watch loop optimized at the end of EVERY pass, so a busy session
73
+ * left one dead build per pass behind: on 2026-10-26 the store carried 15 builds
74
+ * (2.5 GB) against 380 MB of live data. Optimizing at most every
75
+ * {@link MIN_OPTIMIZE_INTERVAL_MS} bounds that to the builds of the last
76
+ * interval, and the version prune inside the same call reclaims the rest.
77
+ *
78
+ * Aggressive runs (a private temp store during a full rebuild) always return
79
+ * true.
80
+ *
81
+ * @param options - `force` bypasses the throttle; `now` overrides the clock (tests).
82
+ * @returns True when the caller should run the optimize.
83
+ */
84
+ export function optimizeDue(options) {
85
+ const now = options?.now ?? Date.now();
86
+ if (options?.force) {
87
+ lastOptimizeAt = now;
88
+ return true;
89
+ }
90
+ if (now - lastOptimizeAt < MIN_OPTIMIZE_INTERVAL_MS)
91
+ return false;
92
+ lastOptimizeAt = now;
93
+ return true;
94
+ }
95
+ /** Reset the optimize throttle (tests only). */
96
+ export function resetOptimizeThrottle() {
97
+ lastOptimizeAt = 0;
98
+ }
62
99
  export async function runIndexPass(options) {
63
100
  const logger = createLogger(options.logger);
64
101
  const lockPath = path.join(options.storePath, LOCK_FILE);
@@ -170,11 +207,14 @@ async function runIndexPassInner(options, logger) {
170
207
  if (!options.force && manifestStatus === "ok" && Object.keys(manifest.files).length > 0) {
171
208
  try {
172
209
  const storedPaths = await options.store.getFilePaths();
173
- const storedSet = new Set(storedPaths);
210
+ // Case-insensitive on Windows: the store may hold `c:/...` while the
211
+ // manifest keys `C:/...` (path.resolve keeps the drive-letter case).
212
+ // A case-sensitive compare re-indexes every such file on every pass.
213
+ const storedSet = new Set(storedPaths.map((p) => p.toLowerCase()));
174
214
  let removedCount = 0;
175
215
  for (const [p, entry] of Object.entries(manifest.files)) {
176
216
  // Path missing from store entirely (aborted before store finished)
177
- if (entry.chunkCount > 0 && !storedSet.has(p)) {
217
+ if (entry.chunkCount > 0 && !storedSet.has(p.toLowerCase())) {
178
218
  delete manifest.files[p];
179
219
  removedCount++;
180
220
  }
@@ -1016,7 +1056,7 @@ async function runIndexPassInner(options, logger) {
1016
1056
  if (optimizeInterval > 0 && windowCount % optimizeInterval === 0) {
1017
1057
  const storedInWindow = await prevStore;
1018
1058
  prevStore = null;
1019
- if (storedInWindow > 0) {
1059
+ if (storedInWindow > 0 && optimizeDue({ force: Boolean(tempStorePath) })) {
1020
1060
  logger.info("Optimizing vector store (mid-run compaction, pruning old versions)...");
1021
1061
  await effectiveStore.optimize?.(tempStorePath
1022
1062
  ? { aggressive: true, skipIndex: true, logger: logger.warn }
@@ -1191,15 +1231,23 @@ async function runIndexPassInner(options, logger) {
1191
1231
  // swapped-in store is private to this process. Skipped when the rebuild did
1192
1232
  // not promote (the old store is untouched and its handle may be closed).
1193
1233
  if (!aborted() && !keepPreviousIndexState) {
1194
- logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
1195
- const optimizeStart = Date.now();
1196
- try {
1197
- const optimizeTarget = tempStorePath ? options.store : effectiveStore;
1198
- await optimizeTarget.optimize?.(tempStorePath ? { aggressive: true, logger: logger.warn } : { logger: logger.warn });
1199
- logger.info(`Vector store optimized in ${((Date.now() - optimizeStart) / 1000).toFixed(1)}s`);
1234
+ // Every optimize rebuilds the ANN index (a ~190 MB vector copy at 4096
1235
+ // dims), so passes within the throttle window skip it; the next due pass
1236
+ // compacts and prunes, which also reclaims the skipped builds.
1237
+ if (!optimizeDue({ force: Boolean(tempStorePath) })) {
1238
+ logger.debug(`Skipping vector store optimize (throttled: < ${MIN_OPTIMIZE_INTERVAL_MS / 60000} min since the last run)`);
1200
1239
  }
1201
- catch (err) {
1202
- logger.warn(`Vector store optimization failed: ${err.message}`);
1240
+ else {
1241
+ logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
1242
+ const optimizeStart = Date.now();
1243
+ try {
1244
+ const optimizeTarget = tempStorePath ? options.store : effectiveStore;
1245
+ await optimizeTarget.optimize?.(tempStorePath ? { aggressive: true, logger: logger.warn } : { logger: logger.warn });
1246
+ logger.info(`Vector store optimized in ${((Date.now() - optimizeStart) / 1000).toFixed(1)}s`);
1247
+ }
1248
+ catch (err) {
1249
+ logger.warn(`Vector store optimization failed: ${err.message}`);
1250
+ }
1203
1251
  }
1204
1252
  }
1205
1253
  // Persist description cache
package/dist/tui.js CHANGED
@@ -65,20 +65,18 @@ function loadWatcherStatus(storePath) {
65
65
  const statusPath = join(storePath, "watcher-status.json");
66
66
  if (!existsSync(statusPath))
67
67
  return { running: false, lastRunAt: undefined, disabled: true };
68
- try {
69
- const raw = JSON.parse(readFileSync(statusPath, "utf-8"));
70
- return {
71
- running: raw.running === true,
72
- lastRunAt: typeof raw.lastRunAt === "number" ? raw.lastRunAt : undefined,
73
- lastSuccessAt: typeof raw.lastSuccessAt === "number" ? raw.lastSuccessAt : undefined,
74
- lastChangeAt: typeof raw.lastChangeAt === "number" ? raw.lastChangeAt : undefined,
75
- consecutiveFailures: typeof raw.consecutiveFailures === "number" ? raw.consecutiveFailures : undefined,
76
- lastError: typeof raw.lastError === "string" ? raw.lastError : undefined,
77
- };
78
- }
79
- catch {
68
+ // BOM-tolerant read (readJsonFile) - a BOM'd status file must not report "disabled".
69
+ const raw = readJsonFile(statusPath);
70
+ if (!raw)
80
71
  return { running: false, lastRunAt: undefined, disabled: true };
81
- }
72
+ return {
73
+ running: raw.running === true,
74
+ lastRunAt: typeof raw.lastRunAt === "number" ? raw.lastRunAt : undefined,
75
+ lastSuccessAt: typeof raw.lastSuccessAt === "number" ? raw.lastSuccessAt : undefined,
76
+ lastChangeAt: typeof raw.lastChangeAt === "number" ? raw.lastChangeAt : undefined,
77
+ consecutiveFailures: typeof raw.consecutiveFailures === "number" ? raw.consecutiveFailures : undefined,
78
+ lastError: typeof raw.lastError === "string" ? raw.lastError : undefined,
79
+ };
82
80
  }
83
81
  /** True when the watched index may be stale (failed passes or pending changes). */
84
82
  function watcherIsStale(w) {
@@ -97,7 +95,11 @@ function loadRagStatus(worktree) {
97
95
  if (!existsSync(configPath))
98
96
  continue;
99
97
  try {
100
- const cfg = JSON.parse(readFileSync(configPath, "utf-8"));
98
+ // BOM-tolerant read - a leading U+FEFF must not throw and silently skip
99
+ // every config candidate (leaving the sidebar stuck on DEFAULT_STATUS).
100
+ const cfg = readJsonFile(configPath);
101
+ if (!cfg)
102
+ continue; // unparseable (e.g. malformed JSON) - try next candidate
101
103
  const embedding = cfg.embedding;
102
104
  if (embedding) {
103
105
  status.provider = embedding.provider ?? status.provider;
@@ -108,13 +110,15 @@ function loadRagStatus(worktree) {
108
110
  const storePath = resolve(worktree, storeRelPath);
109
111
  const manifestPath = join(storePath, "manifest.json");
110
112
  if (existsSync(manifestPath)) {
111
- const manifest = JSON.parse(readFileSync(manifestPath, "utf-8"));
112
- const files = manifest.files;
113
- if (files && typeof files === "object") {
114
- status.chunkCount = Object.values(files).reduce((sum, entry) => sum + (entry.chunkCount ?? 0), 0);
113
+ const manifest = readJsonFile(manifestPath);
114
+ if (manifest) {
115
+ const files = manifest.files;
116
+ if (files && typeof files === "object") {
117
+ status.chunkCount = Object.values(files).reduce((sum, entry) => sum + (entry.chunkCount ?? 0), 0);
118
+ }
119
+ status.lastIndexedAt = manifest.lastIndexedAt;
120
+ status.indexed = status.chunkCount > 0;
115
121
  }
116
- status.lastIndexedAt = manifest.lastIndexedAt;
117
- status.indexed = status.chunkCount > 0;
118
122
  }
119
123
  status.watcher = loadWatcherStatus(storePath);
120
124
  break;
@@ -333,14 +333,37 @@ export declare class LanceDbStore implements VectorStore {
333
333
  * @param options - `aggressive: true` prunes every version but the current
334
334
  * one with `deleteUnverified`. Only safe for a private store that no other
335
335
  * process reads (e.g. a temporary rebuild store). For the shared store,
336
- * versions newer than 1 hour are retained so in-flight queries (Web UI,
337
- * background auto-index) can finish before their data files are reclaimed.
336
+ * versions newer than {@link REGULAR_CLEANUP_AGE_MS} are retained so
337
+ * in-flight queries (Web UI, background auto-index) can finish before their
338
+ * data files are reclaimed. The window matches the pipeline's optimize
339
+ * throttle, so every run can reclaim the build the previous one left behind.
338
340
  */
339
341
  optimize(options?: {
340
342
  aggressive?: boolean;
341
343
  skipIndex?: boolean;
342
344
  logger?: StoreWarn;
343
345
  }): Promise<void>;
346
+ /** Names of the index build directories that currently carry files. */
347
+ private indexBuildDirs;
348
+ /**
349
+ * Delete superseded index build directories and rebuild the ANN index once.
350
+ *
351
+ * LanceDB writes a new `_indices/<uuid>/` on every `optimize()` and every
352
+ * `createIndex()`, each holding an uncompressed copy of the vector column
353
+ * (4096 x float32 = 16 KB per row, ~190 MB at 12k chunks), and it never
354
+ * removes the old ones. A store that optimized a few times a day reached
355
+ * 2.5 GB of dead builds against 380 MB of live data.
356
+ *
357
+ * Above {@link MAX_INDEX_BUILDS} builds this deletes every build directory and
358
+ * creates one fresh index (a `replace: true` build also replaces the dangling
359
+ * index versions the deleted directories leave behind). Skipped while any
360
+ * build is younger than {@link INDEX_BUILD_MIN_AGE_MS} so a build running in
361
+ * another process is never hit; if the recreate fails the index is simply
362
+ * rebuilt by the next repair pass.
363
+ *
364
+ * @param logger - Warning sink for the reclaimed-space message.
365
+ */
366
+ private reclaimIndexBuilds;
344
367
  /**
345
368
  * Return all unique file paths currently stored in the index.
346
369
  * @returns An array of normalized file paths.
@@ -33,6 +33,44 @@ const MAX_STALE_INDEX_VERSIONS = 40;
33
33
  * meaningless 9 MB index file.
34
34
  */
35
35
  const MIN_ROWS_FOR_IVF_INDEX = 4096;
36
+ /**
37
+ * When more than this many index build directories carry files, `optimize()`
38
+ * deletes them all and builds the ANN index once.
39
+ *
40
+ * LanceDB never reclaims superseded index builds: every `Table.optimize()` and
41
+ * every `createIndex()` writes a new `_indices/<uuid>/` holding an uncompressed
42
+ * copy of the vector column (12k chunks x 4096 float32 = ~190 MB), and the old
43
+ * directories stay on disk indefinitely. Three builds is the working ceiling;
44
+ * beyond that a single rebuild is cheaper than the dead copies.
45
+ */
46
+ const MAX_INDEX_BUILDS = 3;
47
+ /** Builds younger than this are treated as in-flight (another process may be writing). */
48
+ const INDEX_BUILD_MIN_AGE_MS = 60 * 1000;
49
+ /**
50
+ * Version retention for a regular (non-aggressive) optimize run.
51
+ *
52
+ * Matches the pipeline's optimize throttle (`MIN_OPTIMIZE_INTERVAL_MS`): long
53
+ * enough for an in-flight query to finish, short enough that the next due run
54
+ * reclaims the index build the previous one superseded.
55
+ */
56
+ const REGULAR_CLEANUP_AGE_MS = 15 * 60 * 1000;
57
+ /** Recursively sum the file sizes of a directory (0 when unreadable). */
58
+ function dirSizeBytes(dir) {
59
+ let total = 0;
60
+ for (const entry of fsSync.readdirSync(dir, { withFileTypes: true })) {
61
+ const full = path.join(dir, entry.name);
62
+ total += entry.isDirectory() ? dirSizeBytes(full) : fsSync.statSync(full).size;
63
+ }
64
+ return total;
65
+ }
66
+ /** Newest modification time inside a directory (its own when empty). */
67
+ function newestMtimeMs(dir) {
68
+ let newest = fsSync.statSync(dir).mtimeMs;
69
+ for (const entry of fsSync.readdirSync(dir, { withFileTypes: true })) {
70
+ newest = Math.max(newest, fsSync.statSync(path.join(dir, entry.name)).mtimeMs);
71
+ }
72
+ return newest;
73
+ }
36
74
  /**
37
75
  * Count index-version directories in a LanceDB table directory. Each
38
76
  * `createIndex` writes a new `<uuid>/` directory under `_indices`; a healthy
@@ -338,7 +376,11 @@ function buildFilePathMatch(filePath) {
338
376
  // uuids — no LIKE metacharacters; paths never contain quotes unescaped)
339
377
  return `(filePath = '${escBase}' OR filePath LIKE '%/${escBase}')`;
340
378
  }
341
- return `filePath = '${normalizeFilePath(filePath).replace(/'/g, "''")}'`;
379
+ // Case-insensitive on purpose: Windows paths reach the store in more than one
380
+ // spelling (`c:/Daten/...` from a watcher event vs `C:/Daten/...` from
381
+ // path.resolve), and a case-sensitive delete leaves the other spelling behind
382
+ // forever - that is how 262 files ended up stored twice.
383
+ return `lower(filePath) = lower('${normalizeFilePath(filePath).replace(/'/g, "''")}')`;
342
384
  }
343
385
  /**
344
386
  * A LanceDB-backed vector store with persistent on-disk storage, vector search,
@@ -738,12 +780,15 @@ export class LanceDbStore {
738
780
  // keeps an abort between insert and delete from losing data.
739
781
  // Quirk paths are skipped — they were fully cleaned above.
740
782
  const byFile = this.rowsByFilePath(rows);
741
- for (const [filePath, ids] of byFile) {
783
+ // NOT IN every id written by this call, not just the group's: two spellings
784
+ // of the same file can land in one batch, and a per-group list would let the
785
+ // first delete wipe the second group's freshly inserted rows.
786
+ const allIds = rows.map((row) => `'${row.id.replace(/'/g, "''")}'`).join(", ");
787
+ for (const filePath of byFile.keys()) {
742
788
  if (filePath.startsWith(QUIRK_FILE_PREFIX))
743
789
  continue;
744
790
  const escapedPath = filePath.replace(/'/g, "''");
745
- const idList = ids.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
746
- await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
791
+ await table.delete(`lower(filePath) = lower('${escapedPath}') AND id NOT IN (${allIds})`);
747
792
  }
748
793
  }
749
794
  async addChunksBulkInternal(items) {
@@ -778,12 +823,12 @@ export class LanceDbStore {
778
823
  }
779
824
  // INSERT FIRST (single add for the whole batch), then per-file dedup
780
825
  await table.add(allRows);
781
- for (const [filePath, ids] of dedupByFile) {
826
+ const allBulkIds = allRows.map((row) => `'${row.id.replace(/'/g, "''")}'`).join(", ");
827
+ for (const filePath of dedupByFile.keys()) {
782
828
  if (filePath.startsWith(QUIRK_FILE_PREFIX))
783
829
  continue;
784
830
  const escapedPath = filePath.replace(/'/g, "''");
785
- const idList = ids.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
786
- await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
831
+ await table.delete(`lower(filePath) = lower('${escapedPath}') AND id NOT IN (${allBulkIds})`);
787
832
  }
788
833
  }
789
834
  /**
@@ -1215,10 +1260,22 @@ export class LanceDbStore {
1215
1260
  const indices = await table.listIndices();
1216
1261
  const vecIndex = indices.find((i) => i.columns.includes("embedding"));
1217
1262
  const idxName = vecIndex?.name ?? "embedding_idx";
1218
- const stats = await table.indexStats(idxName);
1219
- if (stats && stats.distanceType === "cosine")
1263
+ // indexStats throws when the registered index points at files that no
1264
+ // longer exist (a dangling `_indices/<uuid>` after a prune, or a
1265
+ // half-written build). Such an index is unusable and must be rebuilt -
1266
+ // aborting here would leave every search on the brute-force path for the
1267
+ // whole process lifetime. `undefined` without a throw just means "no
1268
+ // index yet" and stays on the old path.
1269
+ let unreadable = false;
1270
+ const stats = await table.indexStats(idxName).catch(() => {
1271
+ unreadable = true;
1272
+ return undefined;
1273
+ });
1274
+ if (unreadable)
1275
+ report(`[lancedb] index '${idxName}' is unreadable - rebuilding it`);
1276
+ if (stats?.distanceType === "cosine")
1220
1277
  return; // already healthy
1221
- if (!stats && count < MIN_ROWS_FOR_IVF_INDEX)
1278
+ if (!unreadable && count < MIN_ROWS_FOR_IVF_INDEX)
1222
1279
  return; // small store — brute-force is optimal, and IVF would only trip Lance's "too small for a meaningful index" KMeans warning
1223
1280
  const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
1224
1281
  for (let attempt = 1; attempt <= 3; attempt++) {
@@ -1278,18 +1335,26 @@ export class LanceDbStore {
1278
1335
  * @param options - `aggressive: true` prunes every version but the current
1279
1336
  * one with `deleteUnverified`. Only safe for a private store that no other
1280
1337
  * process reads (e.g. a temporary rebuild store). For the shared store,
1281
- * versions newer than 1 hour are retained so in-flight queries (Web UI,
1282
- * background auto-index) can finish before their data files are reclaimed.
1338
+ * versions newer than {@link REGULAR_CLEANUP_AGE_MS} are retained so
1339
+ * in-flight queries (Web UI, background auto-index) can finish before their
1340
+ * data files are reclaimed. The window matches the pipeline's optimize
1341
+ * throttle, so every run can reclaim the build the previous one left behind.
1283
1342
  */
1284
1343
  async optimize(options) {
1285
1344
  await this.withWriteLock(async () => {
1286
1345
  try {
1287
1346
  const table = await this.getTable();
1347
+ // Snapshot before the optimize: table.optimize() and the index repair
1348
+ // both write their own builds, so only the pre-existing set says which
1349
+ // directories are dead copies worth reclaiming.
1350
+ const preExistingDirs = this.dbPath.startsWith("memory://")
1351
+ ? new Set()
1352
+ : this.indexBuildDirs();
1288
1353
  if (options?.aggressive) {
1289
1354
  await table.optimize({ cleanupOlderThan: new Date(), deleteUnverified: true });
1290
1355
  }
1291
1356
  else {
1292
- const threshold = new Date(Date.now() - 60 * 60 * 1000);
1357
+ const threshold = new Date(Date.now() - REGULAR_CLEANUP_AGE_MS);
1293
1358
  await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
1294
1359
  }
1295
1360
  // Remove stale empty index-version directories (husks Lance leaves
@@ -1297,19 +1362,21 @@ export class LanceDbStore {
1297
1362
  // from growing unboundedly and prevents false positives in the
1298
1363
  // index-repair stale-version guard.
1299
1364
  if (!this.dbPath.startsWith("memory://")) {
1300
- const swept = sweepEmptyIndexVersionDirs(path.join(this.dbPath, TABLE_NAME + ".lance"));
1365
+ const tablePath = path.join(this.dbPath, TABLE_NAME + ".lance");
1366
+ const swept = sweepEmptyIndexVersionDirs(tablePath);
1301
1367
  if (swept > 0) {
1302
1368
  const message = `[lancedb] swept ${swept} stale index-version directories`;
1303
1369
  (options?.logger ?? console.warn)(message);
1304
1370
  }
1305
1371
  }
1306
- // Build the ANN vector index — without it every vectorSearch() is a
1372
+ // Build the ANN vector index - without it every vectorSearch() is a
1307
1373
  // brute-force O(N) flat scan (slow at 50k+ chunks). Skips early when
1308
1374
  // an index with the correct cosine metric already exists, and can be
1309
1375
  // skipped entirely for a private temporary store that is never
1310
1376
  // searched (the rebuild pipeline builds the index once at the end).
1311
1377
  if (!options?.skipIndex) {
1312
1378
  await this.ensureCosineIndex(options?.logger);
1379
+ await this.reclaimIndexBuilds(preExistingDirs, options?.logger);
1313
1380
  }
1314
1381
  }
1315
1382
  catch (err) {
@@ -1320,6 +1387,104 @@ export class LanceDbStore {
1320
1387
  }
1321
1388
  });
1322
1389
  }
1390
+ /** Names of the index build directories that currently carry files. */
1391
+ indexBuildDirs() {
1392
+ const indicesDir = path.join(this.dbPath, TABLE_NAME + ".lance", "_indices");
1393
+ const builds = new Set();
1394
+ let names;
1395
+ try {
1396
+ names = fsSync.readdirSync(indicesDir);
1397
+ }
1398
+ catch {
1399
+ return builds;
1400
+ }
1401
+ for (const name of names) {
1402
+ try {
1403
+ if (fsSync.readdirSync(path.join(indicesDir, name)).length > 0)
1404
+ builds.add(name);
1405
+ }
1406
+ catch {
1407
+ // unreadable/racing entry - ignore
1408
+ }
1409
+ }
1410
+ return builds;
1411
+ }
1412
+ /**
1413
+ * Delete superseded index build directories and rebuild the ANN index once.
1414
+ *
1415
+ * LanceDB writes a new `_indices/<uuid>/` on every `optimize()` and every
1416
+ * `createIndex()`, each holding an uncompressed copy of the vector column
1417
+ * (4096 x float32 = 16 KB per row, ~190 MB at 12k chunks), and it never
1418
+ * removes the old ones. A store that optimized a few times a day reached
1419
+ * 2.5 GB of dead builds against 380 MB of live data.
1420
+ *
1421
+ * Above {@link MAX_INDEX_BUILDS} builds this deletes every build directory and
1422
+ * creates one fresh index (a `replace: true` build also replaces the dangling
1423
+ * index versions the deleted directories leave behind). Skipped while any
1424
+ * build is younger than {@link INDEX_BUILD_MIN_AGE_MS} so a build running in
1425
+ * another process is never hit; if the recreate fails the index is simply
1426
+ * rebuilt by the next repair pass.
1427
+ *
1428
+ * @param logger - Warning sink for the reclaimed-space message.
1429
+ */
1430
+ async reclaimIndexBuilds(preExisting, logger) {
1431
+ if (this.dbPath.startsWith("memory://"))
1432
+ return;
1433
+ if (preExisting.size <= MAX_INDEX_BUILDS)
1434
+ return;
1435
+ const indicesDir = path.join(this.dbPath, TABLE_NAME + ".lance", "_indices");
1436
+ let names;
1437
+ try {
1438
+ names = fsSync.readdirSync(indicesDir);
1439
+ }
1440
+ catch {
1441
+ return; // no index directories: nothing to reclaim
1442
+ }
1443
+ // Only builds that existed before this optimize and are older than the
1444
+ // in-flight window are candidates; builds written by this call (or by
1445
+ // another process a moment ago) are kept.
1446
+ const deletable = new Set();
1447
+ for (const name of preExisting) {
1448
+ try {
1449
+ if (Date.now() - newestMtimeMs(path.join(indicesDir, name)) >= INDEX_BUILD_MIN_AGE_MS) {
1450
+ deletable.add(name);
1451
+ }
1452
+ }
1453
+ catch {
1454
+ // unreadable/racing entry - leave it alone
1455
+ }
1456
+ }
1457
+ if (deletable.size === 0)
1458
+ return;
1459
+ let reclaimedBytes = 0;
1460
+ for (const name of names) {
1461
+ if (!deletable.has(name))
1462
+ continue;
1463
+ const full = path.join(indicesDir, name);
1464
+ try {
1465
+ reclaimedBytes += dirSizeBytes(full);
1466
+ fsSync.rmSync(full, { recursive: true, force: true });
1467
+ }
1468
+ catch {
1469
+ // best-effort - keep going, a rebuild fixes the metadata either way
1470
+ }
1471
+ }
1472
+ // The deleted builds leave their index versions dangling in the manifest;
1473
+ // a replace build re-registers the index and drops them.
1474
+ const table = await this.getTable();
1475
+ const count = await table.countRows().catch(() => 0);
1476
+ const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
1477
+ await table.createIndex("embedding", {
1478
+ config: lancedb.Index.ivfFlat({ numPartitions, distanceType: "cosine" }),
1479
+ replace: true,
1480
+ name: "embedding_idx",
1481
+ waitTimeoutSeconds: 120,
1482
+ });
1483
+ if (reclaimedBytes > 0) {
1484
+ const message = `[lancedb] reclaimed ${(reclaimedBytes / (1024 * 1024)).toFixed(1)} MB of superseded index builds`;
1485
+ (logger ?? console.warn)(message);
1486
+ }
1487
+ }
1323
1488
  /**
1324
1489
  * Return all unique file paths currently stored in the index.
1325
1490
  * @returns An array of normalized file paths.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "opencode-rag-plugin",
3
- "version": "2.10.0",
3
+ "version": "2.10.1",
4
4
  "description": "OpenCode plugin for local-first RAG-based semantic code search",
5
5
  "type": "module",
6
6
  "main": "./dist/plugin-entry.js",