opencode-rag-plugin 2.9.3 → 2.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +27 -2
- package/dist/cli/commands/init-helpers.js +7 -0
- package/dist/cli/commands/init.js +7 -4
- package/dist/core/bootstrap.js +13 -10
- package/dist/core/config.d.ts +99 -1
- package/dist/core/config.js +148 -5
- package/dist/embedder/health.d.ts +8 -2
- package/dist/embedder/health.js +12 -5
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/indexer/pipeline.d.ts +26 -0
- package/dist/indexer/pipeline.js +59 -11
- package/dist/plugin.d.ts +14 -9
- package/dist/plugin.js +50 -28
- package/dist/safety/shell-guard.d.ts +157 -0
- package/dist/safety/shell-guard.js +412 -0
- package/dist/tui.js +24 -20
- package/dist/v2/v2-adapter.d.ts +49 -3
- package/dist/v2/v2-adapter.js +63 -7
- package/dist/vectorstore/lancedb.d.ts +25 -2
- package/dist/vectorstore/lancedb.js +180 -15
- package/package.json +1 -1
|
@@ -333,14 +333,37 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
333
333
|
* @param options - `aggressive: true` prunes every version but the current
|
|
334
334
|
* one with `deleteUnverified`. Only safe for a private store that no other
|
|
335
335
|
* process reads (e.g. a temporary rebuild store). For the shared store,
|
|
336
|
-
* versions newer than
|
|
337
|
-
* background auto-index) can finish before their
|
|
336
|
+
* versions newer than {@link REGULAR_CLEANUP_AGE_MS} are retained so
|
|
337
|
+
* in-flight queries (Web UI, background auto-index) can finish before their
|
|
338
|
+
* data files are reclaimed. The window matches the pipeline's optimize
|
|
339
|
+
* throttle, so every run can reclaim the build the previous one left behind.
|
|
338
340
|
*/
|
|
339
341
|
optimize(options?: {
|
|
340
342
|
aggressive?: boolean;
|
|
341
343
|
skipIndex?: boolean;
|
|
342
344
|
logger?: StoreWarn;
|
|
343
345
|
}): Promise<void>;
|
|
346
|
+
/** Names of the index build directories that currently carry files. */
|
|
347
|
+
private indexBuildDirs;
|
|
348
|
+
/**
|
|
349
|
+
* Delete superseded index build directories and rebuild the ANN index once.
|
|
350
|
+
*
|
|
351
|
+
* LanceDB writes a new `_indices/<uuid>/` on every `optimize()` and every
|
|
352
|
+
* `createIndex()`, each holding an uncompressed copy of the vector column
|
|
353
|
+
* (4096 x float32 = 16 KB per row, ~190 MB at 12k chunks), and it never
|
|
354
|
+
* removes the old ones. A store that optimized a few times a day reached
|
|
355
|
+
* 2.5 GB of dead builds against 380 MB of live data.
|
|
356
|
+
*
|
|
357
|
+
* Above {@link MAX_INDEX_BUILDS} builds this deletes every build directory and
|
|
358
|
+
* creates one fresh index (a `replace: true` build also replaces the dangling
|
|
359
|
+
* index versions the deleted directories leave behind). Skipped while any
|
|
360
|
+
* build is younger than {@link INDEX_BUILD_MIN_AGE_MS} so a build running in
|
|
361
|
+
* another process is never hit; if the recreate fails the index is simply
|
|
362
|
+
* rebuilt by the next repair pass.
|
|
363
|
+
*
|
|
364
|
+
* @param logger - Warning sink for the reclaimed-space message.
|
|
365
|
+
*/
|
|
366
|
+
private reclaimIndexBuilds;
|
|
344
367
|
/**
|
|
345
368
|
* Return all unique file paths currently stored in the index.
|
|
346
369
|
* @returns An array of normalized file paths.
|
|
@@ -33,6 +33,44 @@ const MAX_STALE_INDEX_VERSIONS = 40;
|
|
|
33
33
|
* meaningless 9 MB index file.
|
|
34
34
|
*/
|
|
35
35
|
const MIN_ROWS_FOR_IVF_INDEX = 4096;
|
|
36
|
+
/**
|
|
37
|
+
* When more than this many index build directories carry files, `optimize()`
|
|
38
|
+
* deletes them all and builds the ANN index once.
|
|
39
|
+
*
|
|
40
|
+
* LanceDB never reclaims superseded index builds: every `Table.optimize()` and
|
|
41
|
+
* every `createIndex()` writes a new `_indices/<uuid>/` holding an uncompressed
|
|
42
|
+
* copy of the vector column (12k chunks x 4096 float32 = ~190 MB), and the old
|
|
43
|
+
* directories stay on disk indefinitely. Three builds is the working ceiling;
|
|
44
|
+
* beyond that a single rebuild is cheaper than the dead copies.
|
|
45
|
+
*/
|
|
46
|
+
const MAX_INDEX_BUILDS = 3;
|
|
47
|
+
/** Builds younger than this are treated as in-flight (another process may be writing). */
|
|
48
|
+
const INDEX_BUILD_MIN_AGE_MS = 60 * 1000;
|
|
49
|
+
/**
|
|
50
|
+
* Version retention for a regular (non-aggressive) optimize run.
|
|
51
|
+
*
|
|
52
|
+
* Matches the pipeline's optimize throttle (`MIN_OPTIMIZE_INTERVAL_MS`): long
|
|
53
|
+
* enough for an in-flight query to finish, short enough that the next due run
|
|
54
|
+
* reclaims the index build the previous one superseded.
|
|
55
|
+
*/
|
|
56
|
+
const REGULAR_CLEANUP_AGE_MS = 15 * 60 * 1000;
|
|
57
|
+
/** Recursively sum the file sizes of a directory (0 when unreadable). */
|
|
58
|
+
function dirSizeBytes(dir) {
|
|
59
|
+
let total = 0;
|
|
60
|
+
for (const entry of fsSync.readdirSync(dir, { withFileTypes: true })) {
|
|
61
|
+
const full = path.join(dir, entry.name);
|
|
62
|
+
total += entry.isDirectory() ? dirSizeBytes(full) : fsSync.statSync(full).size;
|
|
63
|
+
}
|
|
64
|
+
return total;
|
|
65
|
+
}
|
|
66
|
+
/** Newest modification time inside a directory (its own when empty). */
|
|
67
|
+
function newestMtimeMs(dir) {
|
|
68
|
+
let newest = fsSync.statSync(dir).mtimeMs;
|
|
69
|
+
for (const entry of fsSync.readdirSync(dir, { withFileTypes: true })) {
|
|
70
|
+
newest = Math.max(newest, fsSync.statSync(path.join(dir, entry.name)).mtimeMs);
|
|
71
|
+
}
|
|
72
|
+
return newest;
|
|
73
|
+
}
|
|
36
74
|
/**
|
|
37
75
|
* Count index-version directories in a LanceDB table directory. Each
|
|
38
76
|
* `createIndex` writes a new `<uuid>/` directory under `_indices`; a healthy
|
|
@@ -338,7 +376,11 @@ function buildFilePathMatch(filePath) {
|
|
|
338
376
|
// uuids — no LIKE metacharacters; paths never contain quotes unescaped)
|
|
339
377
|
return `(filePath = '${escBase}' OR filePath LIKE '%/${escBase}')`;
|
|
340
378
|
}
|
|
341
|
-
|
|
379
|
+
// Case-insensitive on purpose: Windows paths reach the store in more than one
|
|
380
|
+
// spelling (`c:/Daten/...` from a watcher event vs `C:/Daten/...` from
|
|
381
|
+
// path.resolve), and a case-sensitive delete leaves the other spelling behind
|
|
382
|
+
// forever - that is how 262 files ended up stored twice.
|
|
383
|
+
return `lower(filePath) = lower('${normalizeFilePath(filePath).replace(/'/g, "''")}')`;
|
|
342
384
|
}
|
|
343
385
|
/**
|
|
344
386
|
* A LanceDB-backed vector store with persistent on-disk storage, vector search,
|
|
@@ -738,12 +780,15 @@ export class LanceDbStore {
|
|
|
738
780
|
// keeps an abort between insert and delete from losing data.
|
|
739
781
|
// Quirk paths are skipped — they were fully cleaned above.
|
|
740
782
|
const byFile = this.rowsByFilePath(rows);
|
|
741
|
-
|
|
783
|
+
// NOT IN every id written by this call, not just the group's: two spellings
|
|
784
|
+
// of the same file can land in one batch, and a per-group list would let the
|
|
785
|
+
// first delete wipe the second group's freshly inserted rows.
|
|
786
|
+
const allIds = rows.map((row) => `'${row.id.replace(/'/g, "''")}'`).join(", ");
|
|
787
|
+
for (const filePath of byFile.keys()) {
|
|
742
788
|
if (filePath.startsWith(QUIRK_FILE_PREFIX))
|
|
743
789
|
continue;
|
|
744
790
|
const escapedPath = filePath.replace(/'/g, "''");
|
|
745
|
-
|
|
746
|
-
await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
|
|
791
|
+
await table.delete(`lower(filePath) = lower('${escapedPath}') AND id NOT IN (${allIds})`);
|
|
747
792
|
}
|
|
748
793
|
}
|
|
749
794
|
async addChunksBulkInternal(items) {
|
|
@@ -778,12 +823,12 @@ export class LanceDbStore {
|
|
|
778
823
|
}
|
|
779
824
|
// INSERT FIRST (single add for the whole batch), then per-file dedup
|
|
780
825
|
await table.add(allRows);
|
|
781
|
-
|
|
826
|
+
const allBulkIds = allRows.map((row) => `'${row.id.replace(/'/g, "''")}'`).join(", ");
|
|
827
|
+
for (const filePath of dedupByFile.keys()) {
|
|
782
828
|
if (filePath.startsWith(QUIRK_FILE_PREFIX))
|
|
783
829
|
continue;
|
|
784
830
|
const escapedPath = filePath.replace(/'/g, "''");
|
|
785
|
-
|
|
786
|
-
await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
|
|
831
|
+
await table.delete(`lower(filePath) = lower('${escapedPath}') AND id NOT IN (${allBulkIds})`);
|
|
787
832
|
}
|
|
788
833
|
}
|
|
789
834
|
/**
|
|
@@ -1215,10 +1260,22 @@ export class LanceDbStore {
|
|
|
1215
1260
|
const indices = await table.listIndices();
|
|
1216
1261
|
const vecIndex = indices.find((i) => i.columns.includes("embedding"));
|
|
1217
1262
|
const idxName = vecIndex?.name ?? "embedding_idx";
|
|
1218
|
-
|
|
1219
|
-
|
|
1263
|
+
// indexStats throws when the registered index points at files that no
|
|
1264
|
+
// longer exist (a dangling `_indices/<uuid>` after a prune, or a
|
|
1265
|
+
// half-written build). Such an index is unusable and must be rebuilt -
|
|
1266
|
+
// aborting here would leave every search on the brute-force path for the
|
|
1267
|
+
// whole process lifetime. `undefined` without a throw just means "no
|
|
1268
|
+
// index yet" and stays on the old path.
|
|
1269
|
+
let unreadable = false;
|
|
1270
|
+
const stats = await table.indexStats(idxName).catch(() => {
|
|
1271
|
+
unreadable = true;
|
|
1272
|
+
return undefined;
|
|
1273
|
+
});
|
|
1274
|
+
if (unreadable)
|
|
1275
|
+
report(`[lancedb] index '${idxName}' is unreadable - rebuilding it`);
|
|
1276
|
+
if (stats?.distanceType === "cosine")
|
|
1220
1277
|
return; // already healthy
|
|
1221
|
-
if (!
|
|
1278
|
+
if (!unreadable && count < MIN_ROWS_FOR_IVF_INDEX)
|
|
1222
1279
|
return; // small store — brute-force is optimal, and IVF would only trip Lance's "too small for a meaningful index" KMeans warning
|
|
1223
1280
|
const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
|
|
1224
1281
|
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
@@ -1278,18 +1335,26 @@ export class LanceDbStore {
|
|
|
1278
1335
|
* @param options - `aggressive: true` prunes every version but the current
|
|
1279
1336
|
* one with `deleteUnverified`. Only safe for a private store that no other
|
|
1280
1337
|
* process reads (e.g. a temporary rebuild store). For the shared store,
|
|
1281
|
-
* versions newer than
|
|
1282
|
-
* background auto-index) can finish before their
|
|
1338
|
+
* versions newer than {@link REGULAR_CLEANUP_AGE_MS} are retained so
|
|
1339
|
+
* in-flight queries (Web UI, background auto-index) can finish before their
|
|
1340
|
+
* data files are reclaimed. The window matches the pipeline's optimize
|
|
1341
|
+
* throttle, so every run can reclaim the build the previous one left behind.
|
|
1283
1342
|
*/
|
|
1284
1343
|
async optimize(options) {
|
|
1285
1344
|
await this.withWriteLock(async () => {
|
|
1286
1345
|
try {
|
|
1287
1346
|
const table = await this.getTable();
|
|
1347
|
+
// Snapshot before the optimize: table.optimize() and the index repair
|
|
1348
|
+
// both write their own builds, so only the pre-existing set says which
|
|
1349
|
+
// directories are dead copies worth reclaiming.
|
|
1350
|
+
const preExistingDirs = this.dbPath.startsWith("memory://")
|
|
1351
|
+
? new Set()
|
|
1352
|
+
: this.indexBuildDirs();
|
|
1288
1353
|
if (options?.aggressive) {
|
|
1289
1354
|
await table.optimize({ cleanupOlderThan: new Date(), deleteUnverified: true });
|
|
1290
1355
|
}
|
|
1291
1356
|
else {
|
|
1292
|
-
const threshold = new Date(Date.now() -
|
|
1357
|
+
const threshold = new Date(Date.now() - REGULAR_CLEANUP_AGE_MS);
|
|
1293
1358
|
await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
|
|
1294
1359
|
}
|
|
1295
1360
|
// Remove stale empty index-version directories (husks Lance leaves
|
|
@@ -1297,19 +1362,21 @@ export class LanceDbStore {
|
|
|
1297
1362
|
// from growing unboundedly and prevents false positives in the
|
|
1298
1363
|
// index-repair stale-version guard.
|
|
1299
1364
|
if (!this.dbPath.startsWith("memory://")) {
|
|
1300
|
-
const
|
|
1365
|
+
const tablePath = path.join(this.dbPath, TABLE_NAME + ".lance");
|
|
1366
|
+
const swept = sweepEmptyIndexVersionDirs(tablePath);
|
|
1301
1367
|
if (swept > 0) {
|
|
1302
1368
|
const message = `[lancedb] swept ${swept} stale index-version directories`;
|
|
1303
1369
|
(options?.logger ?? console.warn)(message);
|
|
1304
1370
|
}
|
|
1305
1371
|
}
|
|
1306
|
-
// Build the ANN vector index
|
|
1372
|
+
// Build the ANN vector index - without it every vectorSearch() is a
|
|
1307
1373
|
// brute-force O(N) flat scan (slow at 50k+ chunks). Skips early when
|
|
1308
1374
|
// an index with the correct cosine metric already exists, and can be
|
|
1309
1375
|
// skipped entirely for a private temporary store that is never
|
|
1310
1376
|
// searched (the rebuild pipeline builds the index once at the end).
|
|
1311
1377
|
if (!options?.skipIndex) {
|
|
1312
1378
|
await this.ensureCosineIndex(options?.logger);
|
|
1379
|
+
await this.reclaimIndexBuilds(preExistingDirs, options?.logger);
|
|
1313
1380
|
}
|
|
1314
1381
|
}
|
|
1315
1382
|
catch (err) {
|
|
@@ -1320,6 +1387,104 @@ export class LanceDbStore {
|
|
|
1320
1387
|
}
|
|
1321
1388
|
});
|
|
1322
1389
|
}
|
|
1390
|
+
/** Names of the index build directories that currently carry files. */
|
|
1391
|
+
indexBuildDirs() {
|
|
1392
|
+
const indicesDir = path.join(this.dbPath, TABLE_NAME + ".lance", "_indices");
|
|
1393
|
+
const builds = new Set();
|
|
1394
|
+
let names;
|
|
1395
|
+
try {
|
|
1396
|
+
names = fsSync.readdirSync(indicesDir);
|
|
1397
|
+
}
|
|
1398
|
+
catch {
|
|
1399
|
+
return builds;
|
|
1400
|
+
}
|
|
1401
|
+
for (const name of names) {
|
|
1402
|
+
try {
|
|
1403
|
+
if (fsSync.readdirSync(path.join(indicesDir, name)).length > 0)
|
|
1404
|
+
builds.add(name);
|
|
1405
|
+
}
|
|
1406
|
+
catch {
|
|
1407
|
+
// unreadable/racing entry - ignore
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
return builds;
|
|
1411
|
+
}
|
|
1412
|
+
/**
|
|
1413
|
+
* Delete superseded index build directories and rebuild the ANN index once.
|
|
1414
|
+
*
|
|
1415
|
+
* LanceDB writes a new `_indices/<uuid>/` on every `optimize()` and every
|
|
1416
|
+
* `createIndex()`, each holding an uncompressed copy of the vector column
|
|
1417
|
+
* (4096 x float32 = 16 KB per row, ~190 MB at 12k chunks), and it never
|
|
1418
|
+
* removes the old ones. A store that optimized a few times a day reached
|
|
1419
|
+
* 2.5 GB of dead builds against 380 MB of live data.
|
|
1420
|
+
*
|
|
1421
|
+
* Above {@link MAX_INDEX_BUILDS} builds this deletes every build directory and
|
|
1422
|
+
* creates one fresh index (a `replace: true` build also replaces the dangling
|
|
1423
|
+
* index versions the deleted directories leave behind). Skipped while any
|
|
1424
|
+
* build is younger than {@link INDEX_BUILD_MIN_AGE_MS} so a build running in
|
|
1425
|
+
* another process is never hit; if the recreate fails the index is simply
|
|
1426
|
+
* rebuilt by the next repair pass.
|
|
1427
|
+
*
|
|
1428
|
+
* @param logger - Warning sink for the reclaimed-space message.
|
|
1429
|
+
*/
|
|
1430
|
+
async reclaimIndexBuilds(preExisting, logger) {
|
|
1431
|
+
if (this.dbPath.startsWith("memory://"))
|
|
1432
|
+
return;
|
|
1433
|
+
if (preExisting.size <= MAX_INDEX_BUILDS)
|
|
1434
|
+
return;
|
|
1435
|
+
const indicesDir = path.join(this.dbPath, TABLE_NAME + ".lance", "_indices");
|
|
1436
|
+
let names;
|
|
1437
|
+
try {
|
|
1438
|
+
names = fsSync.readdirSync(indicesDir);
|
|
1439
|
+
}
|
|
1440
|
+
catch {
|
|
1441
|
+
return; // no index directories: nothing to reclaim
|
|
1442
|
+
}
|
|
1443
|
+
// Only builds that existed before this optimize and are older than the
|
|
1444
|
+
// in-flight window are candidates; builds written by this call (or by
|
|
1445
|
+
// another process a moment ago) are kept.
|
|
1446
|
+
const deletable = new Set();
|
|
1447
|
+
for (const name of preExisting) {
|
|
1448
|
+
try {
|
|
1449
|
+
if (Date.now() - newestMtimeMs(path.join(indicesDir, name)) >= INDEX_BUILD_MIN_AGE_MS) {
|
|
1450
|
+
deletable.add(name);
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
catch {
|
|
1454
|
+
// unreadable/racing entry - leave it alone
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
if (deletable.size === 0)
|
|
1458
|
+
return;
|
|
1459
|
+
let reclaimedBytes = 0;
|
|
1460
|
+
for (const name of names) {
|
|
1461
|
+
if (!deletable.has(name))
|
|
1462
|
+
continue;
|
|
1463
|
+
const full = path.join(indicesDir, name);
|
|
1464
|
+
try {
|
|
1465
|
+
reclaimedBytes += dirSizeBytes(full);
|
|
1466
|
+
fsSync.rmSync(full, { recursive: true, force: true });
|
|
1467
|
+
}
|
|
1468
|
+
catch {
|
|
1469
|
+
// best-effort - keep going, a rebuild fixes the metadata either way
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
// The deleted builds leave their index versions dangling in the manifest;
|
|
1473
|
+
// a replace build re-registers the index and drops them.
|
|
1474
|
+
const table = await this.getTable();
|
|
1475
|
+
const count = await table.countRows().catch(() => 0);
|
|
1476
|
+
const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
|
|
1477
|
+
await table.createIndex("embedding", {
|
|
1478
|
+
config: lancedb.Index.ivfFlat({ numPartitions, distanceType: "cosine" }),
|
|
1479
|
+
replace: true,
|
|
1480
|
+
name: "embedding_idx",
|
|
1481
|
+
waitTimeoutSeconds: 120,
|
|
1482
|
+
});
|
|
1483
|
+
if (reclaimedBytes > 0) {
|
|
1484
|
+
const message = `[lancedb] reclaimed ${(reclaimedBytes / (1024 * 1024)).toFixed(1)} MB of superseded index builds`;
|
|
1485
|
+
(logger ?? console.warn)(message);
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1323
1488
|
/**
|
|
1324
1489
|
* Return all unique file paths currently stored in the index.
|
|
1325
1490
|
* @returns An array of normalized file paths.
|