opencode-rag-plugin 2.10.0 → 2.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/indexer/pipeline.d.ts +26 -0
- package/dist/indexer/pipeline.js +59 -11
- package/dist/tui.js +24 -20
- package/dist/vectorstore/lancedb.d.ts +25 -2
- package/dist/vectorstore/lancedb.js +180 -15
- package/package.json +1 -1
|
@@ -50,6 +50,32 @@ export interface Logger {
|
|
|
50
50
|
* @param options - Configuration for the index pass.
|
|
51
51
|
* @returns Aggregate statistics for the pass.
|
|
52
52
|
*/
|
|
53
|
+
/** Minimum gap between two regular (non-aggressive) store optimize runs. */
|
|
54
|
+
export declare const MIN_OPTIMIZE_INTERVAL_MS: number;
|
|
55
|
+
/**
|
|
56
|
+
* Decide whether a regular store optimize may run now, and stamp the attempt.
|
|
57
|
+
*
|
|
58
|
+
* LanceDB's `Table.optimize()` always rewrites the ANN index (it decides the
|
|
59
|
+
* index is stale and rebuilds it), and one IVF_FLAT build is a full,
|
|
60
|
+
* uncompressed copy of the vector column - 12k chunks x 4096 float32 is about
|
|
61
|
+
* 190 MB. The watch loop optimized at the end of EVERY pass, so a busy session
|
|
62
|
+
* left one dead build per pass behind: on 2026-10-26 the store carried 15 builds
|
|
63
|
+
* (2.5 GB) against 380 MB of live data. Optimizing at most every
|
|
64
|
+
* {@link MIN_OPTIMIZE_INTERVAL_MS} bounds that to the builds of the last
|
|
65
|
+
* interval, and the version prune inside the same call reclaims the rest.
|
|
66
|
+
*
|
|
67
|
+
* Aggressive runs (a private temp store during a full rebuild) always return
|
|
68
|
+
* true.
|
|
69
|
+
*
|
|
70
|
+
* @param options - `force` bypasses the throttle; `now` overrides the clock (tests).
|
|
71
|
+
* @returns True when the caller should run the optimize.
|
|
72
|
+
*/
|
|
73
|
+
export declare function optimizeDue(options?: {
|
|
74
|
+
force?: boolean;
|
|
75
|
+
now?: number;
|
|
76
|
+
}): boolean;
|
|
77
|
+
/** Reset the optimize throttle (tests only). */
|
|
78
|
+
export declare function resetOptimizeThrottle(): void;
|
|
53
79
|
export declare function runIndexPass(options: RunIndexPassOptions): Promise<IndexRunStats>;
|
|
54
80
|
/**
|
|
55
81
|
* Warning text for description settings that leave chain-of-thought enabled,
|
package/dist/indexer/pipeline.js
CHANGED
|
@@ -59,6 +59,43 @@ function isPidAlive(pid) {
|
|
|
59
59
|
* @param options - Configuration for the index pass.
|
|
60
60
|
* @returns Aggregate statistics for the pass.
|
|
61
61
|
*/
|
|
62
|
+
/** Minimum gap between two regular (non-aggressive) store optimize runs. */
|
|
63
|
+
export const MIN_OPTIMIZE_INTERVAL_MS = 15 * 60 * 1000;
|
|
64
|
+
/** Last time a regular optimize actually ran in this process. */
|
|
65
|
+
let lastOptimizeAt = 0;
|
|
66
|
+
/**
|
|
67
|
+
* Decide whether a regular store optimize may run now, and stamp the attempt.
|
|
68
|
+
*
|
|
69
|
+
* LanceDB's `Table.optimize()` always rewrites the ANN index (it decides the
|
|
70
|
+
* index is stale and rebuilds it), and one IVF_FLAT build is a full,
|
|
71
|
+
* uncompressed copy of the vector column - 12k chunks x 4096 float32 is about
|
|
72
|
+
* 190 MB. The watch loop optimized at the end of EVERY pass, so a busy session
|
|
73
|
+
* left one dead build per pass behind: on 2026-10-26 the store carried 15 builds
|
|
74
|
+
* (2.5 GB) against 380 MB of live data. Optimizing at most every
|
|
75
|
+
* {@link MIN_OPTIMIZE_INTERVAL_MS} bounds that to the builds of the last
|
|
76
|
+
* interval, and the version prune inside the same call reclaims the rest.
|
|
77
|
+
*
|
|
78
|
+
* Aggressive runs (a private temp store during a full rebuild) always return
|
|
79
|
+
* true.
|
|
80
|
+
*
|
|
81
|
+
* @param options - `force` bypasses the throttle; `now` overrides the clock (tests).
|
|
82
|
+
* @returns True when the caller should run the optimize.
|
|
83
|
+
*/
|
|
84
|
+
export function optimizeDue(options) {
|
|
85
|
+
const now = options?.now ?? Date.now();
|
|
86
|
+
if (options?.force) {
|
|
87
|
+
lastOptimizeAt = now;
|
|
88
|
+
return true;
|
|
89
|
+
}
|
|
90
|
+
if (now - lastOptimizeAt < MIN_OPTIMIZE_INTERVAL_MS)
|
|
91
|
+
return false;
|
|
92
|
+
lastOptimizeAt = now;
|
|
93
|
+
return true;
|
|
94
|
+
}
|
|
95
|
+
/** Reset the optimize throttle (tests only). */
|
|
96
|
+
export function resetOptimizeThrottle() {
|
|
97
|
+
lastOptimizeAt = 0;
|
|
98
|
+
}
|
|
62
99
|
export async function runIndexPass(options) {
|
|
63
100
|
const logger = createLogger(options.logger);
|
|
64
101
|
const lockPath = path.join(options.storePath, LOCK_FILE);
|
|
@@ -170,11 +207,14 @@ async function runIndexPassInner(options, logger) {
|
|
|
170
207
|
if (!options.force && manifestStatus === "ok" && Object.keys(manifest.files).length > 0) {
|
|
171
208
|
try {
|
|
172
209
|
const storedPaths = await options.store.getFilePaths();
|
|
173
|
-
|
|
210
|
+
// Case-insensitive on Windows: the store may hold `c:/...` while the
|
|
211
|
+
// manifest keys `C:/...` (path.resolve keeps the drive-letter case).
|
|
212
|
+
// A case-sensitive compare re-indexes every such file on every pass.
|
|
213
|
+
const storedSet = new Set(storedPaths.map((p) => p.toLowerCase()));
|
|
174
214
|
let removedCount = 0;
|
|
175
215
|
for (const [p, entry] of Object.entries(manifest.files)) {
|
|
176
216
|
// Path missing from store entirely (aborted before store finished)
|
|
177
|
-
if (entry.chunkCount > 0 && !storedSet.has(p)) {
|
|
217
|
+
if (entry.chunkCount > 0 && !storedSet.has(p.toLowerCase())) {
|
|
178
218
|
delete manifest.files[p];
|
|
179
219
|
removedCount++;
|
|
180
220
|
}
|
|
@@ -1016,7 +1056,7 @@ async function runIndexPassInner(options, logger) {
|
|
|
1016
1056
|
if (optimizeInterval > 0 && windowCount % optimizeInterval === 0) {
|
|
1017
1057
|
const storedInWindow = await prevStore;
|
|
1018
1058
|
prevStore = null;
|
|
1019
|
-
if (storedInWindow > 0) {
|
|
1059
|
+
if (storedInWindow > 0 && optimizeDue({ force: Boolean(tempStorePath) })) {
|
|
1020
1060
|
logger.info("Optimizing vector store (mid-run compaction, pruning old versions)...");
|
|
1021
1061
|
await effectiveStore.optimize?.(tempStorePath
|
|
1022
1062
|
? { aggressive: true, skipIndex: true, logger: logger.warn }
|
|
@@ -1191,15 +1231,23 @@ async function runIndexPassInner(options, logger) {
|
|
|
1191
1231
|
// swapped-in store is private to this process. Skipped when the rebuild did
|
|
1192
1232
|
// not promote (the old store is untouched and its handle may be closed).
|
|
1193
1233
|
if (!aborted() && !keepPreviousIndexState) {
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
logger.info(`Vector store optimized in ${((Date.now() - optimizeStart) / 1000).toFixed(1)}s`);
|
|
1234
|
+
// Every optimize rebuilds the ANN index (a ~190 MB vector copy at 4096
|
|
1235
|
+
// dims), so passes within the throttle window skip it; the next due pass
|
|
1236
|
+
// compacts and prunes, which also reclaims the skipped builds.
|
|
1237
|
+
if (!optimizeDue({ force: Boolean(tempStorePath) })) {
|
|
1238
|
+
logger.debug(`Skipping vector store optimize (throttled: < ${MIN_OPTIMIZE_INTERVAL_MS / 60000} min since the last run)`);
|
|
1200
1239
|
}
|
|
1201
|
-
|
|
1202
|
-
logger.
|
|
1240
|
+
else {
|
|
1241
|
+
logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
|
|
1242
|
+
const optimizeStart = Date.now();
|
|
1243
|
+
try {
|
|
1244
|
+
const optimizeTarget = tempStorePath ? options.store : effectiveStore;
|
|
1245
|
+
await optimizeTarget.optimize?.(tempStorePath ? { aggressive: true, logger: logger.warn } : { logger: logger.warn });
|
|
1246
|
+
logger.info(`Vector store optimized in ${((Date.now() - optimizeStart) / 1000).toFixed(1)}s`);
|
|
1247
|
+
}
|
|
1248
|
+
catch (err) {
|
|
1249
|
+
logger.warn(`Vector store optimization failed: ${err.message}`);
|
|
1250
|
+
}
|
|
1203
1251
|
}
|
|
1204
1252
|
}
|
|
1205
1253
|
// Persist description cache
|
package/dist/tui.js
CHANGED
|
@@ -65,20 +65,18 @@ function loadWatcherStatus(storePath) {
|
|
|
65
65
|
const statusPath = join(storePath, "watcher-status.json");
|
|
66
66
|
if (!existsSync(statusPath))
|
|
67
67
|
return { running: false, lastRunAt: undefined, disabled: true };
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
running: raw.running === true,
|
|
72
|
-
lastRunAt: typeof raw.lastRunAt === "number" ? raw.lastRunAt : undefined,
|
|
73
|
-
lastSuccessAt: typeof raw.lastSuccessAt === "number" ? raw.lastSuccessAt : undefined,
|
|
74
|
-
lastChangeAt: typeof raw.lastChangeAt === "number" ? raw.lastChangeAt : undefined,
|
|
75
|
-
consecutiveFailures: typeof raw.consecutiveFailures === "number" ? raw.consecutiveFailures : undefined,
|
|
76
|
-
lastError: typeof raw.lastError === "string" ? raw.lastError : undefined,
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
catch {
|
|
68
|
+
// BOM-tolerant read (readJsonFile) - a BOM'd status file must not report "disabled".
|
|
69
|
+
const raw = readJsonFile(statusPath);
|
|
70
|
+
if (!raw)
|
|
80
71
|
return { running: false, lastRunAt: undefined, disabled: true };
|
|
81
|
-
|
|
72
|
+
return {
|
|
73
|
+
running: raw.running === true,
|
|
74
|
+
lastRunAt: typeof raw.lastRunAt === "number" ? raw.lastRunAt : undefined,
|
|
75
|
+
lastSuccessAt: typeof raw.lastSuccessAt === "number" ? raw.lastSuccessAt : undefined,
|
|
76
|
+
lastChangeAt: typeof raw.lastChangeAt === "number" ? raw.lastChangeAt : undefined,
|
|
77
|
+
consecutiveFailures: typeof raw.consecutiveFailures === "number" ? raw.consecutiveFailures : undefined,
|
|
78
|
+
lastError: typeof raw.lastError === "string" ? raw.lastError : undefined,
|
|
79
|
+
};
|
|
82
80
|
}
|
|
83
81
|
/** True when the watched index may be stale (failed passes or pending changes). */
|
|
84
82
|
function watcherIsStale(w) {
|
|
@@ -97,7 +95,11 @@ function loadRagStatus(worktree) {
|
|
|
97
95
|
if (!existsSync(configPath))
|
|
98
96
|
continue;
|
|
99
97
|
try {
|
|
100
|
-
|
|
98
|
+
// BOM-tolerant read - a leading U+FEFF must not throw and silently skip
|
|
99
|
+
// every config candidate (leaving the sidebar stuck on DEFAULT_STATUS).
|
|
100
|
+
const cfg = readJsonFile(configPath);
|
|
101
|
+
if (!cfg)
|
|
102
|
+
continue; // unparseable (e.g. malformed JSON) - try next candidate
|
|
101
103
|
const embedding = cfg.embedding;
|
|
102
104
|
if (embedding) {
|
|
103
105
|
status.provider = embedding.provider ?? status.provider;
|
|
@@ -108,13 +110,15 @@ function loadRagStatus(worktree) {
|
|
|
108
110
|
const storePath = resolve(worktree, storeRelPath);
|
|
109
111
|
const manifestPath = join(storePath, "manifest.json");
|
|
110
112
|
if (existsSync(manifestPath)) {
|
|
111
|
-
const manifest =
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
113
|
+
const manifest = readJsonFile(manifestPath);
|
|
114
|
+
if (manifest) {
|
|
115
|
+
const files = manifest.files;
|
|
116
|
+
if (files && typeof files === "object") {
|
|
117
|
+
status.chunkCount = Object.values(files).reduce((sum, entry) => sum + (entry.chunkCount ?? 0), 0);
|
|
118
|
+
}
|
|
119
|
+
status.lastIndexedAt = manifest.lastIndexedAt;
|
|
120
|
+
status.indexed = status.chunkCount > 0;
|
|
115
121
|
}
|
|
116
|
-
status.lastIndexedAt = manifest.lastIndexedAt;
|
|
117
|
-
status.indexed = status.chunkCount > 0;
|
|
118
122
|
}
|
|
119
123
|
status.watcher = loadWatcherStatus(storePath);
|
|
120
124
|
break;
|
|
@@ -333,14 +333,37 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
333
333
|
* @param options - `aggressive: true` prunes every version but the current
|
|
334
334
|
* one with `deleteUnverified`. Only safe for a private store that no other
|
|
335
335
|
* process reads (e.g. a temporary rebuild store). For the shared store,
|
|
336
|
-
* versions newer than
|
|
337
|
-
* background auto-index) can finish before their
|
|
336
|
+
* versions newer than {@link REGULAR_CLEANUP_AGE_MS} are retained so
|
|
337
|
+
* in-flight queries (Web UI, background auto-index) can finish before their
|
|
338
|
+
* data files are reclaimed. The window matches the pipeline's optimize
|
|
339
|
+
* throttle, so every run can reclaim the build the previous one left behind.
|
|
338
340
|
*/
|
|
339
341
|
optimize(options?: {
|
|
340
342
|
aggressive?: boolean;
|
|
341
343
|
skipIndex?: boolean;
|
|
342
344
|
logger?: StoreWarn;
|
|
343
345
|
}): Promise<void>;
|
|
346
|
+
/** Names of the index build directories that currently carry files. */
|
|
347
|
+
private indexBuildDirs;
|
|
348
|
+
/**
|
|
349
|
+
* Delete superseded index build directories and rebuild the ANN index once.
|
|
350
|
+
*
|
|
351
|
+
* LanceDB writes a new `_indices/<uuid>/` on every `optimize()` and every
|
|
352
|
+
* `createIndex()`, each holding an uncompressed copy of the vector column
|
|
353
|
+
* (4096 x float32 = 16 KB per row, ~190 MB at 12k chunks), and it never
|
|
354
|
+
* removes the old ones. A store that optimized a few times a day reached
|
|
355
|
+
* 2.5 GB of dead builds against 380 MB of live data.
|
|
356
|
+
*
|
|
357
|
+
* Above {@link MAX_INDEX_BUILDS} builds this deletes every build directory and
|
|
358
|
+
* creates one fresh index (a `replace: true` build also replaces the dangling
|
|
359
|
+
* index versions the deleted directories leave behind). Skipped while any
|
|
360
|
+
* build is younger than {@link INDEX_BUILD_MIN_AGE_MS} so a build running in
|
|
361
|
+
* another process is never hit; if the recreate fails the index is simply
|
|
362
|
+
* rebuilt by the next repair pass.
|
|
363
|
+
*
|
|
364
|
+
* @param logger - Warning sink for the reclaimed-space message.
|
|
365
|
+
*/
|
|
366
|
+
private reclaimIndexBuilds;
|
|
344
367
|
/**
|
|
345
368
|
* Return all unique file paths currently stored in the index.
|
|
346
369
|
* @returns An array of normalized file paths.
|
|
@@ -33,6 +33,44 @@ const MAX_STALE_INDEX_VERSIONS = 40;
|
|
|
33
33
|
* meaningless 9 MB index file.
|
|
34
34
|
*/
|
|
35
35
|
const MIN_ROWS_FOR_IVF_INDEX = 4096;
|
|
36
|
+
/**
|
|
37
|
+
* When more than this many index build directories carry files, `optimize()`
|
|
38
|
+
* deletes them all and builds the ANN index once.
|
|
39
|
+
*
|
|
40
|
+
* LanceDB never reclaims superseded index builds: every `Table.optimize()` and
|
|
41
|
+
* every `createIndex()` writes a new `_indices/<uuid>/` holding an uncompressed
|
|
42
|
+
* copy of the vector column (12k chunks x 4096 float32 = ~190 MB), and the old
|
|
43
|
+
* directories stay on disk indefinitely. Three builds is the working ceiling;
|
|
44
|
+
* beyond that a single rebuild is cheaper than the dead copies.
|
|
45
|
+
*/
|
|
46
|
+
const MAX_INDEX_BUILDS = 3;
|
|
47
|
+
/** Builds younger than this are treated as in-flight (another process may be writing). */
|
|
48
|
+
const INDEX_BUILD_MIN_AGE_MS = 60 * 1000;
|
|
49
|
+
/**
|
|
50
|
+
* Version retention for a regular (non-aggressive) optimize run.
|
|
51
|
+
*
|
|
52
|
+
* Matches the pipeline's optimize throttle (`MIN_OPTIMIZE_INTERVAL_MS`): long
|
|
53
|
+
* enough for an in-flight query to finish, short enough that the next due run
|
|
54
|
+
* reclaims the index build the previous one superseded.
|
|
55
|
+
*/
|
|
56
|
+
const REGULAR_CLEANUP_AGE_MS = 15 * 60 * 1000;
|
|
57
|
+
/** Recursively sum the file sizes of a directory (0 when unreadable). */
|
|
58
|
+
function dirSizeBytes(dir) {
|
|
59
|
+
let total = 0;
|
|
60
|
+
for (const entry of fsSync.readdirSync(dir, { withFileTypes: true })) {
|
|
61
|
+
const full = path.join(dir, entry.name);
|
|
62
|
+
total += entry.isDirectory() ? dirSizeBytes(full) : fsSync.statSync(full).size;
|
|
63
|
+
}
|
|
64
|
+
return total;
|
|
65
|
+
}
|
|
66
|
+
/** Newest modification time inside a directory (its own when empty). */
|
|
67
|
+
function newestMtimeMs(dir) {
|
|
68
|
+
let newest = fsSync.statSync(dir).mtimeMs;
|
|
69
|
+
for (const entry of fsSync.readdirSync(dir, { withFileTypes: true })) {
|
|
70
|
+
newest = Math.max(newest, fsSync.statSync(path.join(dir, entry.name)).mtimeMs);
|
|
71
|
+
}
|
|
72
|
+
return newest;
|
|
73
|
+
}
|
|
36
74
|
/**
|
|
37
75
|
* Count index-version directories in a LanceDB table directory. Each
|
|
38
76
|
* `createIndex` writes a new `<uuid>/` directory under `_indices`; a healthy
|
|
@@ -338,7 +376,11 @@ function buildFilePathMatch(filePath) {
|
|
|
338
376
|
// uuids — no LIKE metacharacters; paths never contain quotes unescaped)
|
|
339
377
|
return `(filePath = '${escBase}' OR filePath LIKE '%/${escBase}')`;
|
|
340
378
|
}
|
|
341
|
-
|
|
379
|
+
// Case-insensitive on purpose: Windows paths reach the store in more than one
|
|
380
|
+
// spelling (`c:/Daten/...` from a watcher event vs `C:/Daten/...` from
|
|
381
|
+
// path.resolve), and a case-sensitive delete leaves the other spelling behind
|
|
382
|
+
// forever - that is how 262 files ended up stored twice.
|
|
383
|
+
return `lower(filePath) = lower('${normalizeFilePath(filePath).replace(/'/g, "''")}')`;
|
|
342
384
|
}
|
|
343
385
|
/**
|
|
344
386
|
* A LanceDB-backed vector store with persistent on-disk storage, vector search,
|
|
@@ -738,12 +780,15 @@ export class LanceDbStore {
|
|
|
738
780
|
// keeps an abort between insert and delete from losing data.
|
|
739
781
|
// Quirk paths are skipped — they were fully cleaned above.
|
|
740
782
|
const byFile = this.rowsByFilePath(rows);
|
|
741
|
-
|
|
783
|
+
// NOT IN every id written by this call, not just the group's: two spellings
|
|
784
|
+
// of the same file can land in one batch, and a per-group list would let the
|
|
785
|
+
// first delete wipe the second group's freshly inserted rows.
|
|
786
|
+
const allIds = rows.map((row) => `'${row.id.replace(/'/g, "''")}'`).join(", ");
|
|
787
|
+
for (const filePath of byFile.keys()) {
|
|
742
788
|
if (filePath.startsWith(QUIRK_FILE_PREFIX))
|
|
743
789
|
continue;
|
|
744
790
|
const escapedPath = filePath.replace(/'/g, "''");
|
|
745
|
-
|
|
746
|
-
await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
|
|
791
|
+
await table.delete(`lower(filePath) = lower('${escapedPath}') AND id NOT IN (${allIds})`);
|
|
747
792
|
}
|
|
748
793
|
}
|
|
749
794
|
async addChunksBulkInternal(items) {
|
|
@@ -778,12 +823,12 @@ export class LanceDbStore {
|
|
|
778
823
|
}
|
|
779
824
|
// INSERT FIRST (single add for the whole batch), then per-file dedup
|
|
780
825
|
await table.add(allRows);
|
|
781
|
-
|
|
826
|
+
const allBulkIds = allRows.map((row) => `'${row.id.replace(/'/g, "''")}'`).join(", ");
|
|
827
|
+
for (const filePath of dedupByFile.keys()) {
|
|
782
828
|
if (filePath.startsWith(QUIRK_FILE_PREFIX))
|
|
783
829
|
continue;
|
|
784
830
|
const escapedPath = filePath.replace(/'/g, "''");
|
|
785
|
-
|
|
786
|
-
await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
|
|
831
|
+
await table.delete(`lower(filePath) = lower('${escapedPath}') AND id NOT IN (${allBulkIds})`);
|
|
787
832
|
}
|
|
788
833
|
}
|
|
789
834
|
/**
|
|
@@ -1215,10 +1260,22 @@ export class LanceDbStore {
|
|
|
1215
1260
|
const indices = await table.listIndices();
|
|
1216
1261
|
const vecIndex = indices.find((i) => i.columns.includes("embedding"));
|
|
1217
1262
|
const idxName = vecIndex?.name ?? "embedding_idx";
|
|
1218
|
-
|
|
1219
|
-
|
|
1263
|
+
// indexStats throws when the registered index points at files that no
|
|
1264
|
+
// longer exist (a dangling `_indices/<uuid>` after a prune, or a
|
|
1265
|
+
// half-written build). Such an index is unusable and must be rebuilt -
|
|
1266
|
+
// aborting here would leave every search on the brute-force path for the
|
|
1267
|
+
// whole process lifetime. `undefined` without a throw just means "no
|
|
1268
|
+
// index yet" and stays on the old path.
|
|
1269
|
+
let unreadable = false;
|
|
1270
|
+
const stats = await table.indexStats(idxName).catch(() => {
|
|
1271
|
+
unreadable = true;
|
|
1272
|
+
return undefined;
|
|
1273
|
+
});
|
|
1274
|
+
if (unreadable)
|
|
1275
|
+
report(`[lancedb] index '${idxName}' is unreadable - rebuilding it`);
|
|
1276
|
+
if (stats?.distanceType === "cosine")
|
|
1220
1277
|
return; // already healthy
|
|
1221
|
-
if (!
|
|
1278
|
+
if (!unreadable && count < MIN_ROWS_FOR_IVF_INDEX)
|
|
1222
1279
|
return; // small store — brute-force is optimal, and IVF would only trip Lance's "too small for a meaningful index" KMeans warning
|
|
1223
1280
|
const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
|
|
1224
1281
|
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
@@ -1278,18 +1335,26 @@ export class LanceDbStore {
|
|
|
1278
1335
|
* @param options - `aggressive: true` prunes every version but the current
|
|
1279
1336
|
* one with `deleteUnverified`. Only safe for a private store that no other
|
|
1280
1337
|
* process reads (e.g. a temporary rebuild store). For the shared store,
|
|
1281
|
-
* versions newer than
|
|
1282
|
-
* background auto-index) can finish before their
|
|
1338
|
+
* versions newer than {@link REGULAR_CLEANUP_AGE_MS} are retained so
|
|
1339
|
+
* in-flight queries (Web UI, background auto-index) can finish before their
|
|
1340
|
+
* data files are reclaimed. The window matches the pipeline's optimize
|
|
1341
|
+
* throttle, so every run can reclaim the build the previous one left behind.
|
|
1283
1342
|
*/
|
|
1284
1343
|
async optimize(options) {
|
|
1285
1344
|
await this.withWriteLock(async () => {
|
|
1286
1345
|
try {
|
|
1287
1346
|
const table = await this.getTable();
|
|
1347
|
+
// Snapshot before the optimize: table.optimize() and the index repair
|
|
1348
|
+
// both write their own builds, so only the pre-existing set says which
|
|
1349
|
+
// directories are dead copies worth reclaiming.
|
|
1350
|
+
const preExistingDirs = this.dbPath.startsWith("memory://")
|
|
1351
|
+
? new Set()
|
|
1352
|
+
: this.indexBuildDirs();
|
|
1288
1353
|
if (options?.aggressive) {
|
|
1289
1354
|
await table.optimize({ cleanupOlderThan: new Date(), deleteUnverified: true });
|
|
1290
1355
|
}
|
|
1291
1356
|
else {
|
|
1292
|
-
const threshold = new Date(Date.now() -
|
|
1357
|
+
const threshold = new Date(Date.now() - REGULAR_CLEANUP_AGE_MS);
|
|
1293
1358
|
await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
|
|
1294
1359
|
}
|
|
1295
1360
|
// Remove stale empty index-version directories (husks Lance leaves
|
|
@@ -1297,19 +1362,21 @@ export class LanceDbStore {
|
|
|
1297
1362
|
// from growing unboundedly and prevents false positives in the
|
|
1298
1363
|
// index-repair stale-version guard.
|
|
1299
1364
|
if (!this.dbPath.startsWith("memory://")) {
|
|
1300
|
-
const
|
|
1365
|
+
const tablePath = path.join(this.dbPath, TABLE_NAME + ".lance");
|
|
1366
|
+
const swept = sweepEmptyIndexVersionDirs(tablePath);
|
|
1301
1367
|
if (swept > 0) {
|
|
1302
1368
|
const message = `[lancedb] swept ${swept} stale index-version directories`;
|
|
1303
1369
|
(options?.logger ?? console.warn)(message);
|
|
1304
1370
|
}
|
|
1305
1371
|
}
|
|
1306
|
-
// Build the ANN vector index
|
|
1372
|
+
// Build the ANN vector index - without it every vectorSearch() is a
|
|
1307
1373
|
// brute-force O(N) flat scan (slow at 50k+ chunks). Skips early when
|
|
1308
1374
|
// an index with the correct cosine metric already exists, and can be
|
|
1309
1375
|
// skipped entirely for a private temporary store that is never
|
|
1310
1376
|
// searched (the rebuild pipeline builds the index once at the end).
|
|
1311
1377
|
if (!options?.skipIndex) {
|
|
1312
1378
|
await this.ensureCosineIndex(options?.logger);
|
|
1379
|
+
await this.reclaimIndexBuilds(preExistingDirs, options?.logger);
|
|
1313
1380
|
}
|
|
1314
1381
|
}
|
|
1315
1382
|
catch (err) {
|
|
@@ -1320,6 +1387,104 @@ export class LanceDbStore {
|
|
|
1320
1387
|
}
|
|
1321
1388
|
});
|
|
1322
1389
|
}
|
|
1390
|
+
/** Names of the index build directories that currently carry files. */
|
|
1391
|
+
indexBuildDirs() {
|
|
1392
|
+
const indicesDir = path.join(this.dbPath, TABLE_NAME + ".lance", "_indices");
|
|
1393
|
+
const builds = new Set();
|
|
1394
|
+
let names;
|
|
1395
|
+
try {
|
|
1396
|
+
names = fsSync.readdirSync(indicesDir);
|
|
1397
|
+
}
|
|
1398
|
+
catch {
|
|
1399
|
+
return builds;
|
|
1400
|
+
}
|
|
1401
|
+
for (const name of names) {
|
|
1402
|
+
try {
|
|
1403
|
+
if (fsSync.readdirSync(path.join(indicesDir, name)).length > 0)
|
|
1404
|
+
builds.add(name);
|
|
1405
|
+
}
|
|
1406
|
+
catch {
|
|
1407
|
+
// unreadable/racing entry - ignore
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
return builds;
|
|
1411
|
+
}
|
|
1412
|
+
/**
|
|
1413
|
+
* Delete superseded index build directories and rebuild the ANN index once.
|
|
1414
|
+
*
|
|
1415
|
+
* LanceDB writes a new `_indices/<uuid>/` on every `optimize()` and every
|
|
1416
|
+
* `createIndex()`, each holding an uncompressed copy of the vector column
|
|
1417
|
+
* (4096 x float32 = 16 KB per row, ~190 MB at 12k chunks), and it never
|
|
1418
|
+
* removes the old ones. A store that optimized a few times a day reached
|
|
1419
|
+
* 2.5 GB of dead builds against 380 MB of live data.
|
|
1420
|
+
*
|
|
1421
|
+
* Above {@link MAX_INDEX_BUILDS} builds this deletes every build directory and
|
|
1422
|
+
* creates one fresh index (a `replace: true` build also replaces the dangling
|
|
1423
|
+
* index versions the deleted directories leave behind). Skipped while any
|
|
1424
|
+
* build is younger than {@link INDEX_BUILD_MIN_AGE_MS} so a build running in
|
|
1425
|
+
* another process is never hit; if the recreate fails the index is simply
|
|
1426
|
+
* rebuilt by the next repair pass.
|
|
1427
|
+
*
|
|
1428
|
+
* @param logger - Warning sink for the reclaimed-space message.
|
|
1429
|
+
*/
|
|
1430
|
+
async reclaimIndexBuilds(preExisting, logger) {
|
|
1431
|
+
if (this.dbPath.startsWith("memory://"))
|
|
1432
|
+
return;
|
|
1433
|
+
if (preExisting.size <= MAX_INDEX_BUILDS)
|
|
1434
|
+
return;
|
|
1435
|
+
const indicesDir = path.join(this.dbPath, TABLE_NAME + ".lance", "_indices");
|
|
1436
|
+
let names;
|
|
1437
|
+
try {
|
|
1438
|
+
names = fsSync.readdirSync(indicesDir);
|
|
1439
|
+
}
|
|
1440
|
+
catch {
|
|
1441
|
+
return; // no index directories: nothing to reclaim
|
|
1442
|
+
}
|
|
1443
|
+
// Only builds that existed before this optimize and are older than the
|
|
1444
|
+
// in-flight window are candidates; builds written by this call (or by
|
|
1445
|
+
// another process a moment ago) are kept.
|
|
1446
|
+
const deletable = new Set();
|
|
1447
|
+
for (const name of preExisting) {
|
|
1448
|
+
try {
|
|
1449
|
+
if (Date.now() - newestMtimeMs(path.join(indicesDir, name)) >= INDEX_BUILD_MIN_AGE_MS) {
|
|
1450
|
+
deletable.add(name);
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
catch {
|
|
1454
|
+
// unreadable/racing entry - leave it alone
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
if (deletable.size === 0)
|
|
1458
|
+
return;
|
|
1459
|
+
let reclaimedBytes = 0;
|
|
1460
|
+
for (const name of names) {
|
|
1461
|
+
if (!deletable.has(name))
|
|
1462
|
+
continue;
|
|
1463
|
+
const full = path.join(indicesDir, name);
|
|
1464
|
+
try {
|
|
1465
|
+
reclaimedBytes += dirSizeBytes(full);
|
|
1466
|
+
fsSync.rmSync(full, { recursive: true, force: true });
|
|
1467
|
+
}
|
|
1468
|
+
catch {
|
|
1469
|
+
// best-effort - keep going, a rebuild fixes the metadata either way
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
// The deleted builds leave their index versions dangling in the manifest;
|
|
1473
|
+
// a replace build re-registers the index and drops them.
|
|
1474
|
+
const table = await this.getTable();
|
|
1475
|
+
const count = await table.countRows().catch(() => 0);
|
|
1476
|
+
const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
|
|
1477
|
+
await table.createIndex("embedding", {
|
|
1478
|
+
config: lancedb.Index.ivfFlat({ numPartitions, distanceType: "cosine" }),
|
|
1479
|
+
replace: true,
|
|
1480
|
+
name: "embedding_idx",
|
|
1481
|
+
waitTimeoutSeconds: 120,
|
|
1482
|
+
});
|
|
1483
|
+
if (reclaimedBytes > 0) {
|
|
1484
|
+
const message = `[lancedb] reclaimed ${(reclaimedBytes / (1024 * 1024)).toFixed(1)} MB of superseded index builds`;
|
|
1485
|
+
(logger ?? console.warn)(message);
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1323
1488
|
/**
|
|
1324
1489
|
* Return all unique file paths currently stored in the index.
|
|
1325
1490
|
* @returns An array of normalized file paths.
|