opencode-rag-plugin 2.10.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +40 -28
- package/dist/api.js +1 -0
- package/dist/cli/commands/clickhouse.d.ts +61 -0
- package/dist/cli/commands/clickhouse.js +335 -0
- package/dist/cli/commands/eval-tuner.js +1 -1
- package/dist/cli/commands/eval.js +1 -1
- package/dist/cli/commands/index-command.js +2 -1
- package/dist/cli/commands/index.d.ts +1 -0
- package/dist/cli/commands/index.js +1 -0
- package/dist/cli/commands/init-helpers.js +2 -2
- package/dist/cli/commands/init.js +49 -11
- package/dist/cli/commands/quirk.js +6 -6
- package/dist/cli/commands/status.js +70 -6
- package/dist/cli/commands/ui.js +2 -2
- package/dist/cli/index.js +2 -1
- package/dist/cli/types.d.ts +14 -0
- package/dist/core/bootstrap.d.ts +7 -0
- package/dist/core/bootstrap.js +40 -9
- package/dist/core/config.d.ts +128 -5
- package/dist/core/config.js +205 -8
- package/dist/core/interfaces.d.ts +54 -0
- package/dist/core/runtime-overrides.d.ts +9 -0
- package/dist/core/runtime-overrides.js +44 -0
- package/dist/core/state-sync.d.ts +91 -0
- package/dist/core/state-sync.js +273 -0
- package/dist/eval/fast-index.js +2 -2
- package/dist/eval/rerank-eval.js +2 -2
- package/dist/eval/run-branch-compare.js +2 -2
- package/dist/eval/run-token-test.js +2 -2
- package/dist/index.d.ts +5 -2
- package/dist/index.js +2 -1
- package/dist/indexer/pipeline.d.ts +32 -0
- package/dist/indexer/pipeline.js +230 -65
- package/dist/indexer/progress.d.ts +75 -0
- package/dist/indexer/progress.js +114 -0
- package/dist/indexer/stats.d.ts +14 -0
- package/dist/plugin.d.ts +6 -3
- package/dist/plugin.js +39 -20
- package/dist/quirks/quirk-store.d.ts +10 -0
- package/dist/quirks/quirk-store.js +110 -15
- package/dist/quirks/types.d.ts +7 -0
- package/dist/tui.js +134 -113
- package/dist/vectorstore/clickhouse.d.ts +150 -0
- package/dist/vectorstore/clickhouse.js +931 -0
- package/dist/vectorstore/dimension.d.ts +19 -0
- package/dist/vectorstore/dimension.js +36 -0
- package/dist/vectorstore/factory.d.ts +9 -3
- package/dist/vectorstore/factory.js +11 -2
- package/dist/vectorstore/lancedb.d.ts +27 -4
- package/dist/vectorstore/lancedb.js +180 -15
- package/dist/watcher.d.ts +29 -0
- package/dist/watcher.js +76 -2
- package/dist/web/api.d.ts +3 -4
- package/dist/web/api.js +18 -3
- package/dist/web/server.js +28 -2
- package/dist/web/ui/assets/index-B-RH6fko.css +2 -0
- package/dist/web/ui/index.html +2 -2
- package/package.json +2 -1
- package/dist/web/ui/assets/index-CWcp0jNf.css +0 -2
- /package/dist/web/ui/assets/{index-DbXFrn53.js → index-ChGi2bSp.js} +0 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Backend-neutral dimension probe.
|
|
3
|
+
*
|
|
4
|
+
* The pipeline needs the dimension of an existing store before it can decide
|
|
5
|
+
* whether to rebuild. LanceDB keeps it in the table schema; ClickHouse keeps
|
|
6
|
+
* it in the project's `_meta` state row. Both probes are read-only.
|
|
7
|
+
*/
|
|
8
|
+
import type { RagConfig } from "../core/config.js";
|
|
9
|
+
/**
|
|
10
|
+
* Read the embedding dimension of the configured store.
|
|
11
|
+
*
|
|
12
|
+
* @param config - Resolved RAG configuration.
|
|
13
|
+
* @param storePath - Local store directory (LanceDB only).
|
|
14
|
+
* @param workspaceDir - Workspace root (used to derive the ClickHouse namespace).
|
|
15
|
+
* @returns The stored dimension, or undefined when the store does not exist yet.
|
|
16
|
+
*/
|
|
17
|
+
export declare function readConfiguredStoreDimension(config: RagConfig, storePath: string, workspaceDir: string): Promise<number | undefined>;
|
|
18
|
+
/** Read the dimension from a ClickHouse store without creating any schema. */
|
|
19
|
+
export declare function readClickHouseDimension(config: RagConfig, workspaceDir: string): Promise<number | undefined>;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { readStoreDimension } from "./lancedb.js";
|
|
2
|
+
import { ClickHouseStore, resolveClickHouseOptions } from "./clickhouse.js";
|
|
3
|
+
/**
|
|
4
|
+
* Read the embedding dimension of the configured store.
|
|
5
|
+
*
|
|
6
|
+
* @param config - Resolved RAG configuration.
|
|
7
|
+
* @param storePath - Local store directory (LanceDB only).
|
|
8
|
+
* @param workspaceDir - Workspace root (used to derive the ClickHouse namespace).
|
|
9
|
+
* @returns The stored dimension, or undefined when the store does not exist yet.
|
|
10
|
+
*/
|
|
11
|
+
export async function readConfiguredStoreDimension(config, storePath, workspaceDir) {
|
|
12
|
+
const provider = config.vectorStore.provider ?? "lancedb";
|
|
13
|
+
if (provider === "lancedb") {
|
|
14
|
+
return readStoreDimension(storePath);
|
|
15
|
+
}
|
|
16
|
+
if (provider === "clickhouse") {
|
|
17
|
+
return readClickHouseDimension(config, workspaceDir);
|
|
18
|
+
}
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
/** Read the dimension from a ClickHouse store without creating any schema. */
|
|
22
|
+
export async function readClickHouseDimension(config, workspaceDir) {
|
|
23
|
+
const options = resolveClickHouseOptions(config, workspaceDir);
|
|
24
|
+
const store = new ClickHouseStore(options, config.embedding.vectorDimension ?? 384);
|
|
25
|
+
try {
|
|
26
|
+
return await store.peekDimension();
|
|
27
|
+
}
|
|
28
|
+
catch (err) {
|
|
29
|
+
console.warn(`[bootstrap] Could not read ClickHouse store dimension: ${err.message}`);
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
finally {
|
|
33
|
+
await store.close().catch(() => { });
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=dimension.js.map
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* @fileoverview Factory function to create the configured VectorStore instance
|
|
2
|
+
* @fileoverview Factory function to create the configured VectorStore instance
|
|
3
|
+
* (LanceDB, in-memory, or ClickHouse).
|
|
3
4
|
*/
|
|
4
5
|
import type { VectorStore } from "../core/interfaces.js";
|
|
5
6
|
import type { RagConfig } from "../core/config.js";
|
|
@@ -9,11 +10,16 @@ import type { RagConfig } from "../core/config.js";
|
|
|
9
10
|
* Supported providers:
|
|
10
11
|
* - `"lancedb"`: Persistent LanceDB-backed store (default).
|
|
11
12
|
* - `"memory"`: Ephemeral in-memory store.
|
|
13
|
+
* - `"clickhouse"`: Central ClickHouse-backed store; `storePath` is only a
|
|
14
|
+
* local cache directory for workspace state.
|
|
12
15
|
*
|
|
13
16
|
* @param config - The RAG configuration containing the vector store provider setting.
|
|
14
|
-
* @param storePath - The filesystem path for persistent stores (LanceDB)
|
|
17
|
+
* @param storePath - The filesystem path for persistent local stores (LanceDB)
|
|
18
|
+
* and for the disposable state cache when ClickHouse is active.
|
|
15
19
|
* @param dimension - The embedding vector dimension.
|
|
20
|
+
* @param workspaceDir - Workspace root, used to derive the default ClickHouse
|
|
21
|
+
* namespace. Defaults to `process.cwd()`.
|
|
16
22
|
* @returns A VectorStore instance matching the configured provider.
|
|
17
23
|
* @throws If the provider name is unknown.
|
|
18
24
|
*/
|
|
19
|
-
export declare function createVectorStore(config: RagConfig, storePath: string, dimension: number): VectorStore;
|
|
25
|
+
export declare function createVectorStore(config: RagConfig, storePath: string, dimension: number, workspaceDir?: string): VectorStore;
|
|
@@ -1,19 +1,25 @@
|
|
|
1
1
|
import { LanceDbStore } from "./lancedb.js";
|
|
2
2
|
import { InMemoryVectorStore } from "./memory.js";
|
|
3
|
+
import { ClickHouseStore, resolveClickHouseOptions } from "./clickhouse.js";
|
|
3
4
|
/**
|
|
4
5
|
* Create a VectorStore instance based on the configured provider.
|
|
5
6
|
*
|
|
6
7
|
* Supported providers:
|
|
7
8
|
* - `"lancedb"`: Persistent LanceDB-backed store (default).
|
|
8
9
|
* - `"memory"`: Ephemeral in-memory store.
|
|
10
|
+
* - `"clickhouse"`: Central ClickHouse-backed store; `storePath` is only a
|
|
11
|
+
* local cache directory for workspace state.
|
|
9
12
|
*
|
|
10
13
|
* @param config - The RAG configuration containing the vector store provider setting.
|
|
11
|
-
* @param storePath - The filesystem path for persistent stores (LanceDB)
|
|
14
|
+
* @param storePath - The filesystem path for persistent local stores (LanceDB)
|
|
15
|
+
* and for the disposable state cache when ClickHouse is active.
|
|
12
16
|
* @param dimension - The embedding vector dimension.
|
|
17
|
+
* @param workspaceDir - Workspace root, used to derive the default ClickHouse
|
|
18
|
+
* namespace. Defaults to `process.cwd()`.
|
|
13
19
|
* @returns A VectorStore instance matching the configured provider.
|
|
14
20
|
* @throws If the provider name is unknown.
|
|
15
21
|
*/
|
|
16
|
-
export function createVectorStore(config, storePath, dimension) {
|
|
22
|
+
export function createVectorStore(config, storePath, dimension, workspaceDir = process.cwd()) {
|
|
17
23
|
const provider = config.vectorStore.provider ?? "lancedb";
|
|
18
24
|
if (provider === "lancedb") {
|
|
19
25
|
return new LanceDbStore(storePath, dimension);
|
|
@@ -21,6 +27,9 @@ export function createVectorStore(config, storePath, dimension) {
|
|
|
21
27
|
if (provider === "memory") {
|
|
22
28
|
return new InMemoryVectorStore();
|
|
23
29
|
}
|
|
30
|
+
if (provider === "clickhouse") {
|
|
31
|
+
return new ClickHouseStore(resolveClickHouseOptions(config, workspaceDir), dimension);
|
|
32
|
+
}
|
|
24
33
|
throw new Error(`Unknown vector store provider: ${provider}`);
|
|
25
34
|
}
|
|
26
35
|
//# sourceMappingURL=factory.js.map
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Chunk, ChunkSummary, SearchResult, MetadataFilter, ChunkBrowseStore } from "../core/interfaces.js";
|
|
2
2
|
/** Minimal warning sink for store diagnostics (defaults to console.warn). */
|
|
3
3
|
export type StoreWarn = (message: string) => void;
|
|
4
4
|
/**
|
|
@@ -112,7 +112,7 @@ export interface BulkChunkWrite {
|
|
|
112
112
|
* and chunk metadata queries. Supports automatic corruption recovery by falling
|
|
113
113
|
* back to prior table versions or dropping and rebuilding the table.
|
|
114
114
|
*/
|
|
115
|
-
export declare class LanceDbStore implements
|
|
115
|
+
export declare class LanceDbStore implements ChunkBrowseStore {
|
|
116
116
|
private dbPath;
|
|
117
117
|
private readonly vectorDimension;
|
|
118
118
|
private db;
|
|
@@ -333,14 +333,37 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
333
333
|
* @param options - `aggressive: true` prunes every version but the current
|
|
334
334
|
* one with `deleteUnverified`. Only safe for a private store that no other
|
|
335
335
|
* process reads (e.g. a temporary rebuild store). For the shared store,
|
|
336
|
-
* versions newer than
|
|
337
|
-
* background auto-index) can finish before their
|
|
336
|
+
* versions newer than {@link REGULAR_CLEANUP_AGE_MS} are retained so
|
|
337
|
+
* in-flight queries (Web UI, background auto-index) can finish before their
|
|
338
|
+
* data files are reclaimed. The window matches the pipeline's optimize
|
|
339
|
+
* throttle, so every run can reclaim the build the previous one left behind.
|
|
338
340
|
*/
|
|
339
341
|
optimize(options?: {
|
|
340
342
|
aggressive?: boolean;
|
|
341
343
|
skipIndex?: boolean;
|
|
342
344
|
logger?: StoreWarn;
|
|
343
345
|
}): Promise<void>;
|
|
346
|
+
/** Names of the index build directories that currently carry files. */
|
|
347
|
+
private indexBuildDirs;
|
|
348
|
+
/**
|
|
349
|
+
* Delete superseded index build directories and rebuild the ANN index once.
|
|
350
|
+
*
|
|
351
|
+
* LanceDB writes a new `_indices/<uuid>/` on every `optimize()` and every
|
|
352
|
+
* `createIndex()`, each holding an uncompressed copy of the vector column
|
|
353
|
+
* (4096 x float32 = 16 KB per row, ~190 MB at 12k chunks), and it never
|
|
354
|
+
* removes the old ones. A store that optimized a few times a day reached
|
|
355
|
+
* 2.5 GB of dead builds against 380 MB of live data.
|
|
356
|
+
*
|
|
357
|
+
* Above {@link MAX_INDEX_BUILDS} builds this deletes every build directory and
|
|
358
|
+
* creates one fresh index (a `replace: true` build also replaces the dangling
|
|
359
|
+
* index versions the deleted directories leave behind). Skipped while any
|
|
360
|
+
* build is younger than {@link INDEX_BUILD_MIN_AGE_MS} so a build running in
|
|
361
|
+
* another process is never hit; if the recreate fails the index is simply
|
|
362
|
+
* rebuilt by the next repair pass.
|
|
363
|
+
*
|
|
364
|
+
* @param logger - Warning sink for the reclaimed-space message.
|
|
365
|
+
*/
|
|
366
|
+
private reclaimIndexBuilds;
|
|
344
367
|
/**
|
|
345
368
|
* Return all unique file paths currently stored in the index.
|
|
346
369
|
* @returns An array of normalized file paths.
|
|
@@ -33,6 +33,44 @@ const MAX_STALE_INDEX_VERSIONS = 40;
|
|
|
33
33
|
* meaningless 9 MB index file.
|
|
34
34
|
*/
|
|
35
35
|
const MIN_ROWS_FOR_IVF_INDEX = 4096;
|
|
36
|
+
/**
|
|
37
|
+
* When more than this many index build directories carry files, `optimize()`
|
|
38
|
+
* deletes them all and builds the ANN index once.
|
|
39
|
+
*
|
|
40
|
+
* LanceDB never reclaims superseded index builds: every `Table.optimize()` and
|
|
41
|
+
* every `createIndex()` writes a new `_indices/<uuid>/` holding an uncompressed
|
|
42
|
+
* copy of the vector column (12k chunks x 4096 float32 = ~190 MB), and the old
|
|
43
|
+
* directories stay on disk indefinitely. Three builds is the working ceiling;
|
|
44
|
+
* beyond that a single rebuild is cheaper than the dead copies.
|
|
45
|
+
*/
|
|
46
|
+
const MAX_INDEX_BUILDS = 3;
|
|
47
|
+
/** Builds younger than this are treated as in-flight (another process may be writing). */
|
|
48
|
+
const INDEX_BUILD_MIN_AGE_MS = 60 * 1000;
|
|
49
|
+
/**
|
|
50
|
+
* Version retention for a regular (non-aggressive) optimize run.
|
|
51
|
+
*
|
|
52
|
+
* Matches the pipeline's optimize throttle (`MIN_OPTIMIZE_INTERVAL_MS`): long
|
|
53
|
+
* enough for an in-flight query to finish, short enough that the next due run
|
|
54
|
+
* reclaims the index build the previous one superseded.
|
|
55
|
+
*/
|
|
56
|
+
const REGULAR_CLEANUP_AGE_MS = 15 * 60 * 1000;
|
|
57
|
+
/** Recursively sum the file sizes of a directory (0 when unreadable). */
|
|
58
|
+
function dirSizeBytes(dir) {
|
|
59
|
+
let total = 0;
|
|
60
|
+
for (const entry of fsSync.readdirSync(dir, { withFileTypes: true })) {
|
|
61
|
+
const full = path.join(dir, entry.name);
|
|
62
|
+
total += entry.isDirectory() ? dirSizeBytes(full) : fsSync.statSync(full).size;
|
|
63
|
+
}
|
|
64
|
+
return total;
|
|
65
|
+
}
|
|
66
|
+
/** Newest modification time inside a directory (its own when empty). */
|
|
67
|
+
function newestMtimeMs(dir) {
|
|
68
|
+
let newest = fsSync.statSync(dir).mtimeMs;
|
|
69
|
+
for (const entry of fsSync.readdirSync(dir, { withFileTypes: true })) {
|
|
70
|
+
newest = Math.max(newest, fsSync.statSync(path.join(dir, entry.name)).mtimeMs);
|
|
71
|
+
}
|
|
72
|
+
return newest;
|
|
73
|
+
}
|
|
36
74
|
/**
|
|
37
75
|
* Count index-version directories in a LanceDB table directory. Each
|
|
38
76
|
* `createIndex` writes a new `<uuid>/` directory under `_indices`; a healthy
|
|
@@ -338,7 +376,11 @@ function buildFilePathMatch(filePath) {
|
|
|
338
376
|
// uuids — no LIKE metacharacters; paths never contain quotes unescaped)
|
|
339
377
|
return `(filePath = '${escBase}' OR filePath LIKE '%/${escBase}')`;
|
|
340
378
|
}
|
|
341
|
-
|
|
379
|
+
// Case-insensitive on purpose: Windows paths reach the store in more than one
|
|
380
|
+
// spelling (`c:/Daten/...` from a watcher event vs `C:/Daten/...` from
|
|
381
|
+
// path.resolve), and a case-sensitive delete leaves the other spelling behind
|
|
382
|
+
// forever - that is how 262 files ended up stored twice.
|
|
383
|
+
return `lower(filePath) = lower('${normalizeFilePath(filePath).replace(/'/g, "''")}')`;
|
|
342
384
|
}
|
|
343
385
|
/**
|
|
344
386
|
* A LanceDB-backed vector store with persistent on-disk storage, vector search,
|
|
@@ -738,12 +780,15 @@ export class LanceDbStore {
|
|
|
738
780
|
// keeps an abort between insert and delete from losing data.
|
|
739
781
|
// Quirk paths are skipped — they were fully cleaned above.
|
|
740
782
|
const byFile = this.rowsByFilePath(rows);
|
|
741
|
-
|
|
783
|
+
// NOT IN every id written by this call, not just the group's: two spellings
|
|
784
|
+
// of the same file can land in one batch, and a per-group list would let the
|
|
785
|
+
// first delete wipe the second group's freshly inserted rows.
|
|
786
|
+
const allIds = rows.map((row) => `'${row.id.replace(/'/g, "''")}'`).join(", ");
|
|
787
|
+
for (const filePath of byFile.keys()) {
|
|
742
788
|
if (filePath.startsWith(QUIRK_FILE_PREFIX))
|
|
743
789
|
continue;
|
|
744
790
|
const escapedPath = filePath.replace(/'/g, "''");
|
|
745
|
-
|
|
746
|
-
await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
|
|
791
|
+
await table.delete(`lower(filePath) = lower('${escapedPath}') AND id NOT IN (${allIds})`);
|
|
747
792
|
}
|
|
748
793
|
}
|
|
749
794
|
async addChunksBulkInternal(items) {
|
|
@@ -778,12 +823,12 @@ export class LanceDbStore {
|
|
|
778
823
|
}
|
|
779
824
|
// INSERT FIRST (single add for the whole batch), then per-file dedup
|
|
780
825
|
await table.add(allRows);
|
|
781
|
-
|
|
826
|
+
const allBulkIds = allRows.map((row) => `'${row.id.replace(/'/g, "''")}'`).join(", ");
|
|
827
|
+
for (const filePath of dedupByFile.keys()) {
|
|
782
828
|
if (filePath.startsWith(QUIRK_FILE_PREFIX))
|
|
783
829
|
continue;
|
|
784
830
|
const escapedPath = filePath.replace(/'/g, "''");
|
|
785
|
-
|
|
786
|
-
await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
|
|
831
|
+
await table.delete(`lower(filePath) = lower('${escapedPath}') AND id NOT IN (${allBulkIds})`);
|
|
787
832
|
}
|
|
788
833
|
}
|
|
789
834
|
/**
|
|
@@ -1215,10 +1260,22 @@ export class LanceDbStore {
|
|
|
1215
1260
|
const indices = await table.listIndices();
|
|
1216
1261
|
const vecIndex = indices.find((i) => i.columns.includes("embedding"));
|
|
1217
1262
|
const idxName = vecIndex?.name ?? "embedding_idx";
|
|
1218
|
-
|
|
1219
|
-
|
|
1263
|
+
// indexStats throws when the registered index points at files that no
|
|
1264
|
+
// longer exist (a dangling `_indices/<uuid>` after a prune, or a
|
|
1265
|
+
// half-written build). Such an index is unusable and must be rebuilt -
|
|
1266
|
+
// aborting here would leave every search on the brute-force path for the
|
|
1267
|
+
// whole process lifetime. `undefined` without a throw just means "no
|
|
1268
|
+
// index yet" and stays on the old path.
|
|
1269
|
+
let unreadable = false;
|
|
1270
|
+
const stats = await table.indexStats(idxName).catch(() => {
|
|
1271
|
+
unreadable = true;
|
|
1272
|
+
return undefined;
|
|
1273
|
+
});
|
|
1274
|
+
if (unreadable)
|
|
1275
|
+
report(`[lancedb] index '${idxName}' is unreadable - rebuilding it`);
|
|
1276
|
+
if (stats?.distanceType === "cosine")
|
|
1220
1277
|
return; // already healthy
|
|
1221
|
-
if (!
|
|
1278
|
+
if (!unreadable && count < MIN_ROWS_FOR_IVF_INDEX)
|
|
1222
1279
|
return; // small store — brute-force is optimal, and IVF would only trip Lance's "too small for a meaningful index" KMeans warning
|
|
1223
1280
|
const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
|
|
1224
1281
|
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
@@ -1278,18 +1335,26 @@ export class LanceDbStore {
|
|
|
1278
1335
|
* @param options - `aggressive: true` prunes every version but the current
|
|
1279
1336
|
* one with `deleteUnverified`. Only safe for a private store that no other
|
|
1280
1337
|
* process reads (e.g. a temporary rebuild store). For the shared store,
|
|
1281
|
-
* versions newer than
|
|
1282
|
-
* background auto-index) can finish before their
|
|
1338
|
+
* versions newer than {@link REGULAR_CLEANUP_AGE_MS} are retained so
|
|
1339
|
+
* in-flight queries (Web UI, background auto-index) can finish before their
|
|
1340
|
+
* data files are reclaimed. The window matches the pipeline's optimize
|
|
1341
|
+
* throttle, so every run can reclaim the build the previous one left behind.
|
|
1283
1342
|
*/
|
|
1284
1343
|
async optimize(options) {
|
|
1285
1344
|
await this.withWriteLock(async () => {
|
|
1286
1345
|
try {
|
|
1287
1346
|
const table = await this.getTable();
|
|
1347
|
+
// Snapshot before the optimize: table.optimize() and the index repair
|
|
1348
|
+
// both write their own builds, so only the pre-existing set says which
|
|
1349
|
+
// directories are dead copies worth reclaiming.
|
|
1350
|
+
const preExistingDirs = this.dbPath.startsWith("memory://")
|
|
1351
|
+
? new Set()
|
|
1352
|
+
: this.indexBuildDirs();
|
|
1288
1353
|
if (options?.aggressive) {
|
|
1289
1354
|
await table.optimize({ cleanupOlderThan: new Date(), deleteUnverified: true });
|
|
1290
1355
|
}
|
|
1291
1356
|
else {
|
|
1292
|
-
const threshold = new Date(Date.now() -
|
|
1357
|
+
const threshold = new Date(Date.now() - REGULAR_CLEANUP_AGE_MS);
|
|
1293
1358
|
await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
|
|
1294
1359
|
}
|
|
1295
1360
|
// Remove stale empty index-version directories (husks Lance leaves
|
|
@@ -1297,19 +1362,21 @@ export class LanceDbStore {
|
|
|
1297
1362
|
// from growing unboundedly and prevents false positives in the
|
|
1298
1363
|
// index-repair stale-version guard.
|
|
1299
1364
|
if (!this.dbPath.startsWith("memory://")) {
|
|
1300
|
-
const
|
|
1365
|
+
const tablePath = path.join(this.dbPath, TABLE_NAME + ".lance");
|
|
1366
|
+
const swept = sweepEmptyIndexVersionDirs(tablePath);
|
|
1301
1367
|
if (swept > 0) {
|
|
1302
1368
|
const message = `[lancedb] swept ${swept} stale index-version directories`;
|
|
1303
1369
|
(options?.logger ?? console.warn)(message);
|
|
1304
1370
|
}
|
|
1305
1371
|
}
|
|
1306
|
-
// Build the ANN vector index
|
|
1372
|
+
// Build the ANN vector index - without it every vectorSearch() is a
|
|
1307
1373
|
// brute-force O(N) flat scan (slow at 50k+ chunks). Skips early when
|
|
1308
1374
|
// an index with the correct cosine metric already exists, and can be
|
|
1309
1375
|
// skipped entirely for a private temporary store that is never
|
|
1310
1376
|
// searched (the rebuild pipeline builds the index once at the end).
|
|
1311
1377
|
if (!options?.skipIndex) {
|
|
1312
1378
|
await this.ensureCosineIndex(options?.logger);
|
|
1379
|
+
await this.reclaimIndexBuilds(preExistingDirs, options?.logger);
|
|
1313
1380
|
}
|
|
1314
1381
|
}
|
|
1315
1382
|
catch (err) {
|
|
@@ -1320,6 +1387,104 @@ export class LanceDbStore {
|
|
|
1320
1387
|
}
|
|
1321
1388
|
});
|
|
1322
1389
|
}
|
|
1390
|
+
/** Names of the index build directories that currently carry files. */
|
|
1391
|
+
indexBuildDirs() {
|
|
1392
|
+
const indicesDir = path.join(this.dbPath, TABLE_NAME + ".lance", "_indices");
|
|
1393
|
+
const builds = new Set();
|
|
1394
|
+
let names;
|
|
1395
|
+
try {
|
|
1396
|
+
names = fsSync.readdirSync(indicesDir);
|
|
1397
|
+
}
|
|
1398
|
+
catch {
|
|
1399
|
+
return builds;
|
|
1400
|
+
}
|
|
1401
|
+
for (const name of names) {
|
|
1402
|
+
try {
|
|
1403
|
+
if (fsSync.readdirSync(path.join(indicesDir, name)).length > 0)
|
|
1404
|
+
builds.add(name);
|
|
1405
|
+
}
|
|
1406
|
+
catch {
|
|
1407
|
+
// unreadable/racing entry - ignore
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
return builds;
|
|
1411
|
+
}
|
|
1412
|
+
/**
|
|
1413
|
+
* Delete superseded index build directories and rebuild the ANN index once.
|
|
1414
|
+
*
|
|
1415
|
+
* LanceDB writes a new `_indices/<uuid>/` on every `optimize()` and every
|
|
1416
|
+
* `createIndex()`, each holding an uncompressed copy of the vector column
|
|
1417
|
+
* (4096 x float32 = 16 KB per row, ~190 MB at 12k chunks), and it never
|
|
1418
|
+
* removes the old ones. A store that optimized a few times a day reached
|
|
1419
|
+
* 2.5 GB of dead builds against 380 MB of live data.
|
|
1420
|
+
*
|
|
1421
|
+
* Above {@link MAX_INDEX_BUILDS} builds this deletes every build directory and
|
|
1422
|
+
* creates one fresh index (a `replace: true` build also replaces the dangling
|
|
1423
|
+
* index versions the deleted directories leave behind). Skipped while any
|
|
1424
|
+
* build is younger than {@link INDEX_BUILD_MIN_AGE_MS} so a build running in
|
|
1425
|
+
* another process is never hit; if the recreate fails the index is simply
|
|
1426
|
+
* rebuilt by the next repair pass.
|
|
1427
|
+
*
|
|
1428
|
+
* @param logger - Warning sink for the reclaimed-space message.
|
|
1429
|
+
*/
|
|
1430
|
+
async reclaimIndexBuilds(preExisting, logger) {
|
|
1431
|
+
if (this.dbPath.startsWith("memory://"))
|
|
1432
|
+
return;
|
|
1433
|
+
if (preExisting.size <= MAX_INDEX_BUILDS)
|
|
1434
|
+
return;
|
|
1435
|
+
const indicesDir = path.join(this.dbPath, TABLE_NAME + ".lance", "_indices");
|
|
1436
|
+
let names;
|
|
1437
|
+
try {
|
|
1438
|
+
names = fsSync.readdirSync(indicesDir);
|
|
1439
|
+
}
|
|
1440
|
+
catch {
|
|
1441
|
+
return; // no index directories: nothing to reclaim
|
|
1442
|
+
}
|
|
1443
|
+
// Only builds that existed before this optimize and are older than the
|
|
1444
|
+
// in-flight window are candidates; builds written by this call (or by
|
|
1445
|
+
// another process a moment ago) are kept.
|
|
1446
|
+
const deletable = new Set();
|
|
1447
|
+
for (const name of preExisting) {
|
|
1448
|
+
try {
|
|
1449
|
+
if (Date.now() - newestMtimeMs(path.join(indicesDir, name)) >= INDEX_BUILD_MIN_AGE_MS) {
|
|
1450
|
+
deletable.add(name);
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
catch {
|
|
1454
|
+
// unreadable/racing entry - leave it alone
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
if (deletable.size === 0)
|
|
1458
|
+
return;
|
|
1459
|
+
let reclaimedBytes = 0;
|
|
1460
|
+
for (const name of names) {
|
|
1461
|
+
if (!deletable.has(name))
|
|
1462
|
+
continue;
|
|
1463
|
+
const full = path.join(indicesDir, name);
|
|
1464
|
+
try {
|
|
1465
|
+
reclaimedBytes += dirSizeBytes(full);
|
|
1466
|
+
fsSync.rmSync(full, { recursive: true, force: true });
|
|
1467
|
+
}
|
|
1468
|
+
catch {
|
|
1469
|
+
// best-effort - keep going, a rebuild fixes the metadata either way
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
// The deleted builds leave their index versions dangling in the manifest;
|
|
1473
|
+
// a replace build re-registers the index and drops them.
|
|
1474
|
+
const table = await this.getTable();
|
|
1475
|
+
const count = await table.countRows().catch(() => 0);
|
|
1476
|
+
const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
|
|
1477
|
+
await table.createIndex("embedding", {
|
|
1478
|
+
config: lancedb.Index.ivfFlat({ numPartitions, distanceType: "cosine" }),
|
|
1479
|
+
replace: true,
|
|
1480
|
+
name: "embedding_idx",
|
|
1481
|
+
waitTimeoutSeconds: 120,
|
|
1482
|
+
});
|
|
1483
|
+
if (reclaimedBytes > 0) {
|
|
1484
|
+
const message = `[lancedb] reclaimed ${(reclaimedBytes / (1024 * 1024)).toFixed(1)} MB of superseded index builds`;
|
|
1485
|
+
(logger ?? console.warn)(message);
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1323
1488
|
/**
|
|
1324
1489
|
* Return all unique file paths currently stored in the index.
|
|
1325
1490
|
* @returns An array of normalized file paths.
|
package/dist/watcher.d.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import type { RagConfig } from "./core/config.js";
|
|
7
7
|
import type { DescriptionProvider, EmbeddingProvider, KeywordIndex, VectorStore } from "./core/interfaces.js";
|
|
8
|
+
import type { StateSync } from "./core/state-sync.js";
|
|
8
9
|
/** A background indexer that can be shut down gracefully. */
|
|
9
10
|
export interface BackgroundIndexer {
|
|
10
11
|
/** Stop the watcher, cancel pending index passes, and clean up status files. */
|
|
@@ -32,6 +33,8 @@ export interface CreateBackgroundIndexerOptions {
|
|
|
32
33
|
descriptionProvider?: DescriptionProvider;
|
|
33
34
|
/** Embedding vector dimension — enables atomic rebuilds instead of destructive clear. */
|
|
34
35
|
dimension?: number;
|
|
36
|
+
/** Durable-state synchronizer (ClickHouse); flushed at the end of each pass. */
|
|
37
|
+
stateSync?: StateSync;
|
|
35
38
|
}
|
|
36
39
|
/** The current operational status of the background indexer watcher. */
|
|
37
40
|
export type WatcherStatus = {
|
|
@@ -73,6 +76,32 @@ export declare function deriveWatcherHealth(status: WatcherStatus): {
|
|
|
73
76
|
export declare function tryAcquireWatcherLock(storePath: string): boolean;
|
|
74
77
|
/** Release the watcher lock — only if this process owns it. */
|
|
75
78
|
export declare function releaseWatcherLock(storePath: string): void;
|
|
79
|
+
/** A watcher observed on disk: its lock, its persisted status, and liveness. */
|
|
80
|
+
export interface ObservedWatcher {
|
|
81
|
+
/** Store directory holding `watcher.lock` / `watcher-status.json`. */
|
|
82
|
+
storePath: string;
|
|
83
|
+
/** Contents of `watcher.lock`, when present and parseable. */
|
|
84
|
+
lock?: {
|
|
85
|
+
pid: number;
|
|
86
|
+
startedAt: number;
|
|
87
|
+
};
|
|
88
|
+
/** PID in the lock is alive — the watcher process is currently running. */
|
|
89
|
+
alive: boolean;
|
|
90
|
+
/** Parsed `watcher-status.json`, when present and parseable. */
|
|
91
|
+
status?: WatcherStatus;
|
|
92
|
+
}
|
|
93
|
+
/** Read one store directory's watcher lock + status. Returns undefined when the dir holds neither. */
|
|
94
|
+
export declare function readWatcherState(storePath: string): ObservedWatcher | undefined;
|
|
95
|
+
/**
|
|
96
|
+
* Find every watcher observable for a workspace — including one running in a
|
|
97
|
+
* DIFFERENT store dir than the configured `vectorStore.path` (the classic
|
|
98
|
+
* symptom of a config change the already-running plugin never picked up).
|
|
99
|
+
*
|
|
100
|
+
* Scans the configured store dir, its sibling directories, and every child of
|
|
101
|
+
* `<workspaceRoot>/.opencode/` for `watcher.lock` / `watcher-status.json`.
|
|
102
|
+
* Results are sorted live-first, then by most recent activity.
|
|
103
|
+
*/
|
|
104
|
+
export declare function discoverWatcherStates(configuredStorePath: string, workspaceRoot: string): ObservedWatcher[];
|
|
76
105
|
/**
|
|
77
106
|
* Create a background file watcher that automatically re-indexes the
|
|
78
107
|
* workspace when files change. Uses chokidar for file system events and
|
package/dist/watcher.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import chokidar from "chokidar";
|
|
7
7
|
import path from "node:path";
|
|
8
|
-
import { writeFileSync, unlinkSync, existsSync, readFileSync, openSync, closeSync } from "node:fs";
|
|
8
|
+
import { writeFileSync, unlinkSync, existsSync, readFileSync, openSync, closeSync, readdirSync } from "node:fs";
|
|
9
9
|
import { appendDebugLog } from "./core/fileLogger.js";
|
|
10
10
|
import { isCorruptionError } from "./vectorstore/lancedb.js";
|
|
11
11
|
import { createWatchPassScheduler, createWatchIgnore, runIndexPass, } from "./indexer.js";
|
|
@@ -113,6 +113,79 @@ export function releaseWatcherLock(storePath) {
|
|
|
113
113
|
catch { /* ignore */ }
|
|
114
114
|
}
|
|
115
115
|
}
|
|
116
|
+
/** Read one store directory's watcher lock + status. Returns undefined when the dir holds neither. */
|
|
117
|
+
export function readWatcherState(storePath) {
|
|
118
|
+
let lock;
|
|
119
|
+
try {
|
|
120
|
+
lock = readWatcherLock(storePath);
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
/* unreadable dir */
|
|
124
|
+
}
|
|
125
|
+
let status;
|
|
126
|
+
try {
|
|
127
|
+
const statusPath = path.join(storePath, "watcher-status.json");
|
|
128
|
+
if (existsSync(statusPath)) {
|
|
129
|
+
status = JSON.parse(readFileSync(statusPath, "utf-8"));
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
/* corrupt status file — ignore */
|
|
134
|
+
}
|
|
135
|
+
if (!lock && !status)
|
|
136
|
+
return undefined;
|
|
137
|
+
return {
|
|
138
|
+
storePath,
|
|
139
|
+
lock,
|
|
140
|
+
alive: lock ? isPidAlive(lock.pid) : false,
|
|
141
|
+
status,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Find every watcher observable for a workspace — including one running in a
|
|
146
|
+
* DIFFERENT store dir than the configured `vectorStore.path` (the classic
|
|
147
|
+
* symptom of a config change the already-running plugin never picked up).
|
|
148
|
+
*
|
|
149
|
+
* Scans the configured store dir, its sibling directories, and every child of
|
|
150
|
+
* `<workspaceRoot>/.opencode/` for `watcher.lock` / `watcher-status.json`.
|
|
151
|
+
* Results are sorted live-first, then by most recent activity.
|
|
152
|
+
*/
|
|
153
|
+
export function discoverWatcherStates(configuredStorePath, workspaceRoot) {
|
|
154
|
+
const candidates = new Set();
|
|
155
|
+
const add = (dir) => {
|
|
156
|
+
try {
|
|
157
|
+
if (existsSync(dir))
|
|
158
|
+
candidates.add(path.resolve(dir));
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
/* skip unreadable candidate */
|
|
162
|
+
}
|
|
163
|
+
};
|
|
164
|
+
add(configuredStorePath);
|
|
165
|
+
add(path.dirname(configuredStorePath));
|
|
166
|
+
const dotDir = path.join(workspaceRoot, ".opencode");
|
|
167
|
+
add(dotDir);
|
|
168
|
+
try {
|
|
169
|
+
if (existsSync(dotDir)) {
|
|
170
|
+
for (const entry of readdirSync(dotDir, { withFileTypes: true })) {
|
|
171
|
+
if (entry.isDirectory())
|
|
172
|
+
add(path.join(dotDir, entry.name));
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
/* unreadable .opencode dir */
|
|
178
|
+
}
|
|
179
|
+
const found = [];
|
|
180
|
+
for (const dir of candidates) {
|
|
181
|
+
const state = readWatcherState(dir);
|
|
182
|
+
if (state)
|
|
183
|
+
found.push(state);
|
|
184
|
+
}
|
|
185
|
+
const lastActivity = (w) => Math.max(w.status?.lastSuccessAt ?? 0, w.status?.lastRunAt ?? 0, w.status?.lastChangeAt ?? 0, w.lock?.startedAt ?? 0);
|
|
186
|
+
found.sort((a, b) => (b.alive ? 1 : 0) - (a.alive ? 1 : 0) || lastActivity(b) - lastActivity(a));
|
|
187
|
+
return found;
|
|
188
|
+
}
|
|
116
189
|
/**
|
|
117
190
|
* Create a background file watcher that automatically re-indexes the
|
|
118
191
|
* workspace when files change. Uses chokidar for file system events and
|
|
@@ -123,7 +196,7 @@ export function releaseWatcherLock(storePath) {
|
|
|
123
196
|
* @returns A BackgroundIndexer handle with a close() method for shutdown.
|
|
124
197
|
*/
|
|
125
198
|
export function createBackgroundIndexer(options) {
|
|
126
|
-
const { cwd, storePath, config, store, embedder, logFilePath, logLevel, keywordIndex, descriptionProvider, dimension } = options;
|
|
199
|
+
const { cwd, storePath, config, store, embedder, logFilePath, logLevel, keywordIndex, descriptionProvider, dimension, stateSync } = options;
|
|
127
200
|
const autoIndexCfg = config.openCode.autoIndex ?? { enabled: false, debounceMs: 5000, intervalMs: 300000 };
|
|
128
201
|
let active = false;
|
|
129
202
|
let recheckTimer;
|
|
@@ -159,6 +232,7 @@ export function createBackgroundIndexer(options) {
|
|
|
159
232
|
embedder,
|
|
160
233
|
keywordIndex,
|
|
161
234
|
descriptionProvider,
|
|
235
|
+
stateSync,
|
|
162
236
|
dimension,
|
|
163
237
|
abortSignal: ac.signal,
|
|
164
238
|
logger: {
|
package/dist/web/api.d.ts
CHANGED
|
@@ -2,10 +2,9 @@
|
|
|
2
2
|
* @fileoverview REST API handler for the OpenCodeRAG Web UI with search, file, chunk, eval, and token analysis endpoints.
|
|
3
3
|
*/
|
|
4
4
|
import type { IncomingMessage, ServerResponse } from "node:http";
|
|
5
|
-
import { LanceDbStore } from "../vectorstore/lancedb.js";
|
|
6
5
|
import { KeywordIndex } from "../retriever/keyword-index.js";
|
|
7
6
|
import { type RagConfig } from "../core/config.js";
|
|
8
|
-
import { type EmbeddingProvider } from "../core/interfaces.js";
|
|
7
|
+
import { type ChunkBrowseStore, type EmbeddingProvider } from "../core/interfaces.js";
|
|
9
8
|
/** Internal shape for a JSON API response: an HTTP status code and a serialisable body. */
|
|
10
9
|
interface ApiResponse {
|
|
11
10
|
status: number;
|
|
@@ -21,7 +20,7 @@ interface ApiResponse {
|
|
|
21
20
|
* - `/api/eval/sessions`, `/api/eval/sessions/:id`, `/api/eval/compare`
|
|
22
21
|
* - `/api/eval/sessions/:id/analysis`, `/api/eval/token-compare`, `/api/eval/project-savings`
|
|
23
22
|
*
|
|
24
|
-
* @param store - The
|
|
23
|
+
* @param store - The vector store instance (LanceDB or ClickHouse).
|
|
25
24
|
* @param keywordIndex - The keyword-index instance for text search.
|
|
26
25
|
* @param storePath - Filesystem path to the store directory (used by eval endpoints).
|
|
27
26
|
* @param cwd - Optional workspace root for resolving file paths.
|
|
@@ -31,7 +30,7 @@ interface ApiResponse {
|
|
|
31
30
|
* @param configPath - Path to the opencode-rag.json config file, used by PUT /api/config.
|
|
32
31
|
* @returns An async handler that returns `true` when a route matched or `false` otherwise.
|
|
33
32
|
*/
|
|
34
|
-
export declare function createApiHandler(store:
|
|
33
|
+
export declare function createApiHandler(store: ChunkBrowseStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig, getEmbedder?: () => Promise<EmbeddingProvider>, token?: string, configPath?: string): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
|
|
35
34
|
/**
|
|
36
35
|
* Apply a validated config patch (`{ indexing: { includeDirs: [...] } }`)
|
|
37
36
|
* to the on-disk config file and refresh the in-memory config used by
|