opencode-rag-plugin 1.22.1 → 1.22.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/index-command.js +10 -1
- package/dist/cli/commands/status.js +14 -0
- package/dist/cli/format.js +10 -1
- package/dist/core/bootstrap.d.ts +2 -0
- package/dist/core/bootstrap.js +45 -16
- package/dist/core/config.d.ts +12 -0
- package/dist/core/config.js +1 -0
- package/dist/core/interfaces.d.ts +8 -0
- package/dist/embedder/factory.d.ts +20 -0
- package/dist/embedder/factory.js +24 -0
- package/dist/indexer/pipeline.js +175 -64
- package/dist/indexer/stats.d.ts +4 -0
- package/dist/indexer/stats.js +2 -0
- package/dist/indexer/worker.d.ts +8 -2
- package/dist/indexer/worker.js +9 -4
- package/dist/plugin.js +32 -15
- package/dist/vectorstore/lancedb.d.ts +60 -1
- package/dist/vectorstore/lancedb.js +204 -5
- package/package.json +1 -1
|
@@ -98,6 +98,7 @@ export function registerIndexCommand(program) {
|
|
|
98
98
|
}
|
|
99
99
|
}
|
|
100
100
|
logCliInfo(logFilePath, "index", `${c.label("Scanning:")} ${c.file(cwd)}`);
|
|
101
|
+
let incomplete = false;
|
|
101
102
|
const runPass = async (watchTriggered = false, abortSignal, filterPaths) => {
|
|
102
103
|
const passStarted = Date.now();
|
|
103
104
|
const stats = await runIndexPass({
|
|
@@ -116,6 +117,14 @@ export function registerIndexCommand(program) {
|
|
|
116
117
|
});
|
|
117
118
|
if (!watchTriggered) {
|
|
118
119
|
logIndexSummary(logFilePath, stats);
|
|
120
|
+
if (stats.embeddingUnavailable) {
|
|
121
|
+
incomplete = true;
|
|
122
|
+
logCliInfo(logFilePath, "index", `\n${c.error("Indexing incomplete:")} the embedding provider was unavailable — nothing was stored. Fix the provider and run the index again.`);
|
|
123
|
+
}
|
|
124
|
+
else if (stats.totalChunks === 0 && stats.newFiles + stats.modifiedFiles > 0) {
|
|
125
|
+
incomplete = true;
|
|
126
|
+
logCliInfo(logFilePath, "index", `\n${c.warn("Indexing incomplete:")} files were processed but no chunks were stored. Check the log for details.`);
|
|
127
|
+
}
|
|
119
128
|
logCliInfo(logFilePath, "index", `\n${c.success("Indexing complete.")} ${c.num(stats.finalCount)} chunks stored (${formatDuration(Date.now() - passStarted)}).`);
|
|
120
129
|
}
|
|
121
130
|
if (sigReceived && !watchTriggered) {
|
|
@@ -128,7 +137,7 @@ export function registerIndexCommand(program) {
|
|
|
128
137
|
process.removeListener("SIGTERM", handleSigint);
|
|
129
138
|
if (!options.watch) {
|
|
130
139
|
await cleanupContext(ctx);
|
|
131
|
-
process.exit(sigReceived ? 130 : 0);
|
|
140
|
+
process.exit(sigReceived ? 130 : incomplete ? 1 : 0);
|
|
132
141
|
}
|
|
133
142
|
// Only one watcher may run per workspace — a background auto-indexer
|
|
134
143
|
// in an OpenCode session (or another `index --watch`) may already own
|
|
@@ -75,6 +75,20 @@ export function registerStatusCommand(program) {
|
|
|
75
75
|
logCliInfo(logFilePath, "status", `${c.label("Store path:")} ${c.file(storePath)}`);
|
|
76
76
|
logCliInfo(logFilePath, "status", `${c.label("Embedding provider:")} ${c.value(config.embedding.provider)}`);
|
|
77
77
|
logCliInfo(logFilePath, "status", `${c.label("Embedding model:")} ${c.value(config.embedding.model)}`);
|
|
78
|
+
const storeDimension = await store.getVectorDimension?.();
|
|
79
|
+
const embedderDimension = config.embedding.vectorDimension;
|
|
80
|
+
if (embedderDimension && embedderDimension > 0) {
|
|
81
|
+
logCliInfo(logFilePath, "status", `${c.label("Embedder dim:")} ${c.num(embedderDimension)}`);
|
|
82
|
+
}
|
|
83
|
+
if (storeDimension !== undefined) {
|
|
84
|
+
logCliInfo(logFilePath, "status", `${c.label("Store dimension:")} ${c.num(storeDimension)}`);
|
|
85
|
+
}
|
|
86
|
+
if (storeDimension !== undefined &&
|
|
87
|
+
embedderDimension !== undefined &&
|
|
88
|
+
embedderDimension > 0 &&
|
|
89
|
+
storeDimension !== embedderDimension) {
|
|
90
|
+
logCliInfo(logFilePath, "status", `${c.label("Dimension mismatch:")} ${c.warn("yes")} — the store was built with a different embedding model. Run 'opencode-rag index' to rebuild.`);
|
|
91
|
+
}
|
|
78
92
|
logCliInfo(logFilePath, "status", `${c.label("File extensions:")} ${config.indexing.includeExtensions.join(", ")}`);
|
|
79
93
|
logCliInfo(logFilePath, "status", `${c.label("Excluded dirs:")} ${config.indexing.excludeDirs.join(", ")}`);
|
|
80
94
|
logCliInfo(logFilePath, "status", `${c.label("Default top-K:")} ${c.num(config.retrieval.topK)}`);
|
package/dist/cli/format.js
CHANGED
|
@@ -92,7 +92,7 @@ export async function resolveCliContext(opt, logFilePath, bootstrapOpts) {
|
|
|
92
92
|
configPath: opt.config,
|
|
93
93
|
...bootstrapOpts,
|
|
94
94
|
});
|
|
95
|
-
logCliInfo(logFilePath, "config", `${c.label("Config:")} ${c.file(ctx.
|
|
95
|
+
logCliInfo(logFilePath, "config", `${c.label("Config:")} ${c.file(ctx.configPath ?? "(built-in defaults)")}`);
|
|
96
96
|
logConfigDetails(logFilePath, ctx.config);
|
|
97
97
|
return ctx;
|
|
98
98
|
}
|
|
@@ -105,6 +105,9 @@ export async function resolveCliContext(opt, logFilePath, bootstrapOpts) {
|
|
|
105
105
|
function logConfigDetails(logFilePath, config) {
|
|
106
106
|
logCliInfo(logFilePath, "config", ` ${c.label("Embedding provider:")} ${c.value(config.embedding.provider)}`);
|
|
107
107
|
logCliInfo(logFilePath, "config", ` ${c.label("Embedding model:")} ${c.value(config.embedding.model)}`);
|
|
108
|
+
if (config.embedding.vectorDimension && config.embedding.vectorDimension > 0) {
|
|
109
|
+
logCliInfo(logFilePath, "config", ` ${c.label("Embedding dim:")} ${c.num(config.embedding.vectorDimension)}`);
|
|
110
|
+
}
|
|
108
111
|
logCliInfo(logFilePath, "config", ` ${c.label("Vector store:")} ${c.file(config.vectorStore.path)}`);
|
|
109
112
|
}
|
|
110
113
|
/**
|
|
@@ -153,7 +156,13 @@ export function logIndexSummary(logFilePath, stats) {
|
|
|
153
156
|
if (stats.descriptionFailedFiles > 0) {
|
|
154
157
|
logCliInfo(logFilePath, "index", ` ${c.label("Desc failed:")} ${c.num(stats.descriptionFailedFiles)}`);
|
|
155
158
|
}
|
|
159
|
+
if (stats.embeddingFailures > 0) {
|
|
160
|
+
logCliInfo(logFilePath, "index", ` ${c.label("Embed failed:")} ${c.num(stats.embeddingFailures)} chunk(s) without vectors — retried next pass`);
|
|
161
|
+
}
|
|
156
162
|
logCliInfo(logFilePath, "index", ` ${c.label("Chunks written:")} ${c.num(stats.totalChunks)}`);
|
|
163
|
+
if (stats.embeddingUnavailable) {
|
|
164
|
+
logCliInfo(logFilePath, "index", ` ${c.error("Embedding provider unavailable during this pass — no chunks were stored.")}`);
|
|
165
|
+
}
|
|
157
166
|
}
|
|
158
167
|
/**
|
|
159
168
|
* Format a millisecond duration into a compact human-readable string.
|
package/dist/core/bootstrap.d.ts
CHANGED
|
@@ -35,6 +35,8 @@ export interface RagContext {
|
|
|
35
35
|
dimension: number;
|
|
36
36
|
/** Resolved path to the debug log file. */
|
|
37
37
|
logFilePath: string;
|
|
38
|
+
/** Resolved path to the config file, or undefined when built-in defaults are used. */
|
|
39
|
+
configPath?: string;
|
|
38
40
|
}
|
|
39
41
|
/** Bootstrap the full RAG pipeline context: load config, resolve API keys, create embedder, vector store, keyword index, and description provider. */
|
|
40
42
|
export declare function resolveRagContext(opts?: BootstrapOptions): Promise<RagContext>;
|
package/dist/core/bootstrap.js
CHANGED
|
@@ -6,26 +6,46 @@ import path from "node:path";
|
|
|
6
6
|
import { loadConfig, findConfigFile, resolveLogConfig, DEFAULT_CONFIG } from "./config.js";
|
|
7
7
|
import { resolveApiKey } from "./resolve-api-key.js";
|
|
8
8
|
import { loadChunkersFromConfig } from "../chunker/loader.js";
|
|
9
|
-
import { createEmbedder } from "../embedder/factory.js";
|
|
9
|
+
import { createEmbedder, probeEmbeddingDimension } from "../embedder/factory.js";
|
|
10
10
|
import { createDescriptionProvider } from "../describer/factory.js";
|
|
11
11
|
import { createVectorStore } from "../vectorstore/factory.js";
|
|
12
|
+
import { readStoreDimension } from "../vectorstore/lancedb.js";
|
|
12
13
|
import { KeywordIndex } from "../retriever/keyword-index.js";
|
|
13
|
-
/**
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
14
|
+
/**
|
|
15
|
+
* Resolve the embedding dimension for the store.
|
|
16
|
+
*
|
|
17
|
+
* Precedence: an explicit `embedding.vectorDimension` in the config wins (it
|
|
18
|
+
* is persisted after the first successful probe), then a live probe of the
|
|
19
|
+
* provider, then the dimension of an existing store table, then the 384
|
|
20
|
+
* fallback. The old behavior always probed and fell back to 384 — which
|
|
21
|
+
* silently created 384-dimensional stores whenever the provider was down,
|
|
22
|
+
* even though the config declared the real dimension.
|
|
23
|
+
*
|
|
24
|
+
* @param embedder - Configured embedding provider (probed only when needed).
|
|
25
|
+
* @param storePath - Vector store path, used to read an existing schema.
|
|
26
|
+
* @param skipProbe - When true, never call the provider (read-only commands).
|
|
27
|
+
* @returns The resolved dimension.
|
|
28
|
+
*/
|
|
29
|
+
async function resolveDimension(embedder, storePath, skipProbe, configured) {
|
|
30
|
+
if (configured && configured > 0) {
|
|
31
|
+
return configured;
|
|
20
32
|
}
|
|
21
|
-
|
|
22
|
-
|
|
33
|
+
if (!skipProbe) {
|
|
34
|
+
const probe = await probeEmbeddingDimension(embedder);
|
|
35
|
+
if (probe.dimension !== undefined) {
|
|
36
|
+
return probe.dimension;
|
|
37
|
+
}
|
|
38
|
+
const storeDimension = await readStoreDimension(storePath);
|
|
39
|
+
if (storeDimension !== undefined) {
|
|
40
|
+
console.warn(`[bootstrap] Could not probe embedding dimension (${probe.error?.message ?? "unknown error"}) — ` +
|
|
41
|
+
`using the existing store's dimension ${storeDimension}.`);
|
|
42
|
+
return storeDimension;
|
|
43
|
+
}
|
|
44
|
+
console.warn(`[bootstrap] Could not probe embedding dimension (${probe.error?.message ?? "unknown error"}) — ` +
|
|
45
|
+
"falling back to 384. Set embedding.vectorDimension explicitly if the provider produces a different size.");
|
|
46
|
+
return 384;
|
|
23
47
|
}
|
|
24
|
-
|
|
25
|
-
// surface the fallback loudly so misconfigured providers are easy to spot.
|
|
26
|
-
console.warn("[bootstrap] Could not probe embedding dimension — falling back to 384. " +
|
|
27
|
-
"If indexing later fails with dimension errors, set embedding.vectorDimension explicitly.");
|
|
28
|
-
return 384;
|
|
48
|
+
return (await readStoreDimension(storePath)) ?? 384;
|
|
29
49
|
}
|
|
30
50
|
/** Load the keyword index from disk, or create a new empty one if loading fails. */
|
|
31
51
|
async function loadKeywordIndex(storePath) {
|
|
@@ -61,8 +81,16 @@ export async function resolveRagContext(opts = {}) {
|
|
|
61
81
|
}
|
|
62
82
|
const logFilePath = path.resolve(workDir, resolveLogConfig(cfg).logFilePath);
|
|
63
83
|
const embedder = createEmbedder(cfg);
|
|
64
|
-
const dimension = opts.skipProbe ? 384 : await probeDimension(embedder);
|
|
65
84
|
const storePath = path.resolve(workDir, cfg.vectorStore.path);
|
|
85
|
+
const dimension = await resolveDimension(embedder, storePath, opts.skipProbe ?? false, cfg.embedding.vectorDimension);
|
|
86
|
+
// Warn (but do not block) when the existing store schema disagrees with the
|
|
87
|
+
// resolved dimension: the next index pass rebuilds automatically, and search
|
|
88
|
+
// callers would otherwise see only cryptic LanceDB errors.
|
|
89
|
+
const storeDimension = await readStoreDimension(storePath);
|
|
90
|
+
if (storeDimension !== undefined && storeDimension !== dimension) {
|
|
91
|
+
console.warn(`[bootstrap] Store was built with vector dimension ${storeDimension} but the current embedder produces ${dimension} — ` +
|
|
92
|
+
"run 'opencode-rag index' to rebuild the index with the current model.");
|
|
93
|
+
}
|
|
66
94
|
const store = createVectorStore(cfg, storePath, dimension);
|
|
67
95
|
const keywordIndex = opts.skipKeywordIndex
|
|
68
96
|
? new KeywordIndex(storePath)
|
|
@@ -80,6 +108,7 @@ export async function resolveRagContext(opts = {}) {
|
|
|
80
108
|
descriptionProvider,
|
|
81
109
|
dimension,
|
|
82
110
|
logFilePath,
|
|
111
|
+
configPath,
|
|
83
112
|
};
|
|
84
113
|
}
|
|
85
114
|
//# sourceMappingURL=bootstrap.js.map
|
package/dist/core/config.d.ts
CHANGED
|
@@ -296,6 +296,18 @@ export interface RagConfig {
|
|
|
296
296
|
ollamaMaxBatchSize?: number;
|
|
297
297
|
/** Maximum concurrent description generation requests. */
|
|
298
298
|
descriptionConcurrency?: number;
|
|
299
|
+
/**
|
|
300
|
+
* Include LLM-generated chunk descriptions in the text that gets embedded.
|
|
301
|
+
*
|
|
302
|
+
* Descriptions help general-purpose embedding models align natural-language
|
|
303
|
+
* queries with raw code. Code-specialized models (e.g. jina-code-embeddings,
|
|
304
|
+
* trained for NL→code retrieval with instruction prefixes) do not need the
|
|
305
|
+
* crutch — and their passage prompt expects a code snippet, not prose.
|
|
306
|
+
* When false, descriptions are still generated and stored (search results,
|
|
307
|
+
* web UI), but only path/meta header/content are embedded.
|
|
308
|
+
* @default true
|
|
309
|
+
*/
|
|
310
|
+
embedDescriptions?: boolean;
|
|
299
311
|
/**
|
|
300
312
|
* Maximum file size in bytes for SVG/XML files before chunking is skipped.
|
|
301
313
|
* Large SVGs can cause tree-sitter to hang. Set to 0 for no limit.
|
package/dist/core/config.js
CHANGED
|
@@ -224,6 +224,14 @@ export interface VectorStore {
|
|
|
224
224
|
* Used by the index pipeline to detect silent corruption before scanning.
|
|
225
225
|
*/
|
|
226
226
|
checkIntegrity?(): Promise<boolean>;
|
|
227
|
+
/**
|
|
228
|
+
* Return the actual embedding dimension of the stored vector column, or
|
|
229
|
+
* `undefined` when the store has no table yet. Used to detect a store that
|
|
230
|
+
* was built with a different embedding model than the currently configured
|
|
231
|
+
* one — a mismatch makes every vector search fail and silently corrupts
|
|
232
|
+
* writes, so the pipeline rebuilds the index instead.
|
|
233
|
+
*/
|
|
234
|
+
getVectorDimension?(): Promise<number | undefined>;
|
|
227
235
|
}
|
|
228
236
|
/** Filter criteria for narrowing search results by file path, language, kind, or extension. */
|
|
229
237
|
export interface MetadataFilter {
|
|
@@ -15,6 +15,26 @@ import type { RagConfig } from "../core/config.js";
|
|
|
15
15
|
* @throws If the provider is unsupported or a required apiKey is not set
|
|
16
16
|
*/
|
|
17
17
|
export declare function createEmbedder(config: RagConfig): EmbeddingProvider;
|
|
18
|
+
/** Result of probing an embedding provider for its output dimension. */
|
|
19
|
+
export interface EmbeddingProbeResult {
|
|
20
|
+
/** Detected vector dimension, or `undefined` when the probe failed. */
|
|
21
|
+
dimension?: number;
|
|
22
|
+
/** Error raised by the provider (or an empty response), when the probe failed. */
|
|
23
|
+
error?: Error;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Probe an embedding provider with a single short text to discover the
|
|
27
|
+
* dimension of the vectors it produces.
|
|
28
|
+
*
|
|
29
|
+
* Used as a health/dimension preflight before expensive index work: a provider
|
|
30
|
+
* that is down (server not running, model not pulled) fails here in
|
|
31
|
+
* milliseconds instead of after chunking and describing every file. Errors are
|
|
32
|
+
* captured in the result rather than thrown.
|
|
33
|
+
*
|
|
34
|
+
* @param embedder - The embedding provider to probe.
|
|
35
|
+
* @returns The detected dimension, or the failure reason.
|
|
36
|
+
*/
|
|
37
|
+
export declare function probeEmbeddingDimension(embedder: EmbeddingProvider): Promise<EmbeddingProbeResult>;
|
|
18
38
|
/**
|
|
19
39
|
* Embed a list of texts in batches with optional concurrency control and per-batch retry.
|
|
20
40
|
*
|
package/dist/embedder/factory.js
CHANGED
|
@@ -40,6 +40,30 @@ export function createEmbedder(config) {
|
|
|
40
40
|
}
|
|
41
41
|
throw new Error(`Unknown embedding provider: ${provider}`);
|
|
42
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Probe an embedding provider with a single short text to discover the
|
|
45
|
+
* dimension of the vectors it produces.
|
|
46
|
+
*
|
|
47
|
+
* Used as a health/dimension preflight before expensive index work: a provider
|
|
48
|
+
* that is down (server not running, model not pulled) fails here in
|
|
49
|
+
* milliseconds instead of after chunking and describing every file. Errors are
|
|
50
|
+
* captured in the result rather than thrown.
|
|
51
|
+
*
|
|
52
|
+
* @param embedder - The embedding provider to probe.
|
|
53
|
+
* @returns The detected dimension, or the failure reason.
|
|
54
|
+
*/
|
|
55
|
+
export async function probeEmbeddingDimension(embedder) {
|
|
56
|
+
try {
|
|
57
|
+
const probe = await embedder.embed(["dimension-probe"], "query");
|
|
58
|
+
if (probe && probe[0] && probe[0].length > 0 && typeof probe[0][0] === "number") {
|
|
59
|
+
return { dimension: probe[0].length };
|
|
60
|
+
}
|
|
61
|
+
return { error: new Error("Embedding provider returned an empty probe vector") };
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
return { error: err instanceof Error ? err : new Error(String(err)) };
|
|
65
|
+
}
|
|
66
|
+
}
|
|
43
67
|
/**
|
|
44
68
|
* HTTP statuses that indicate a permanent failure — retrying cannot help
|
|
45
69
|
* (auth errors, bad requests, missing resources). Providers raise these as
|
package/dist/indexer/pipeline.js
CHANGED
|
@@ -7,7 +7,7 @@ import pLimit from "p-limit";
|
|
|
7
7
|
import { scanWorkspaceFiles } from "../content/reader.js";
|
|
8
8
|
import { loadManifest, saveManifest, normalizeFilePath, computeDescriptionConfigHash } from "../core/manifest.js";
|
|
9
9
|
import { DescriptionCache } from "../core/desc-cache.js";
|
|
10
|
-
import { embedBatch } from "../embedder/factory.js";
|
|
10
|
+
import { embedBatch, probeEmbeddingDimension } from "../embedder/factory.js";
|
|
11
11
|
import { createVectorStore } from "../vectorstore/factory.js";
|
|
12
12
|
import { swapStoreDirectories } from "../vectorstore/lancedb.js";
|
|
13
13
|
import { createIndexStats } from "./stats.js";
|
|
@@ -157,46 +157,12 @@ async function runIndexPassInner(options, logger) {
|
|
|
157
157
|
// file content IS different — these would be caught by hash comparison below but
|
|
158
158
|
// this pre-clear ensures the description cache is consulted during re-description.
|
|
159
159
|
// (Files with same hash but different descHash already fall through in prepareFile.)
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
if (diffResult) {
|
|
167
|
-
const untracked = getUntrackedFiles(options.cwd);
|
|
168
|
-
const changedSet = new Set();
|
|
169
|
-
for (const f of diffResult.changedFiles)
|
|
170
|
-
changedSet.add(f);
|
|
171
|
-
for (const f of untracked)
|
|
172
|
-
changedSet.add(f);
|
|
173
|
-
// Git paths are relative to the REPO ROOT. When the workspace is a
|
|
174
|
-
// subdirectory of the repo (monorepos), convert them to workspace-
|
|
175
|
-
// relative paths before passing them to the scan — resolving repo-root
|
|
176
|
-
// paths against `cwd` would silently miss every change.
|
|
177
|
-
const toCwdRelative = (p) => {
|
|
178
|
-
const abs = path.resolve(repoRoot, p);
|
|
179
|
-
const rel = path.relative(options.cwd, abs);
|
|
180
|
-
if (rel.startsWith("..") || path.isAbsolute(rel))
|
|
181
|
-
return null;
|
|
182
|
-
return rel;
|
|
183
|
-
};
|
|
184
|
-
const cwdRelative = [];
|
|
185
|
-
for (const p of changedSet) {
|
|
186
|
-
const rel = toCwdRelative(p);
|
|
187
|
-
if (rel !== null)
|
|
188
|
-
cwdRelative.push(rel);
|
|
189
|
-
}
|
|
190
|
-
filterPaths = cwdRelative;
|
|
191
|
-
gitDeletedPaths = diffResult.deletedFiles;
|
|
192
|
-
logger.debug(`Git incremental: ${filterPaths.length} changed/untracked, ${gitDeletedPaths.length} deleted since ${manifest.lastGitCommit.slice(0, 8)}`);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
const scanStart = Date.now();
|
|
197
|
-
const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger, options.force ? undefined : manifest, filterPaths, options.imageVisionProvider, descCache);
|
|
198
|
-
const scanSec = ((Date.now() - scanStart) / 1000).toFixed(1);
|
|
199
|
-
logger.info(`Workspace scan complete: ${workspaceFiles.length} files in ${scanSec}s`);
|
|
160
|
+
// ── Store health checks (run BEFORE the scan) ──────────────────────────
|
|
161
|
+
// The checks below can invalidate the manifest and trigger a full rebuild.
|
|
162
|
+
// They must run before scanWorkspaceFiles: the reader skips reading the
|
|
163
|
+
// contents of unchanged files when a manifest entry matches (mtime + size +
|
|
164
|
+
// descHash unchanged), so clearing the manifest after the scan would leave
|
|
165
|
+
// those files with empty content and turn them into bogus "removed" entries.
|
|
200
166
|
logger.info(`Querying vector store for existing chunk count...`);
|
|
201
167
|
const storeCountStart = Date.now();
|
|
202
168
|
const existingCount = await options.store.count();
|
|
@@ -236,15 +202,44 @@ async function runIndexPassInner(options, logger) {
|
|
|
236
202
|
manifestStatus = "missing";
|
|
237
203
|
}
|
|
238
204
|
}
|
|
205
|
+
// ── Detect a store built by a different embedding model ────────────────
|
|
206
|
+
// LanceDB silently zero-pads/truncates mismatched vectors on write, and
|
|
207
|
+
// every vector query then fails with "No vector column found to match with
|
|
208
|
+
// the query vector dimension". Treat a schema dimension mismatch like a
|
|
209
|
+
// corrupt store: clear the manifest so this pass rebuilds everything into a
|
|
210
|
+
// fresh store at the configured dimension.
|
|
211
|
+
let dimensionMismatch = false;
|
|
212
|
+
try {
|
|
213
|
+
const storeDimension = await options.store.getVectorDimension?.();
|
|
214
|
+
if (!options.force &&
|
|
215
|
+
storeDimension !== undefined &&
|
|
216
|
+
options.dimension !== undefined &&
|
|
217
|
+
storeDimension !== options.dimension) {
|
|
218
|
+
dimensionMismatch = true;
|
|
219
|
+
logger.warn(`Store was built with vector dimension ${storeDimension} but the configured embedding model produces ` +
|
|
220
|
+
`${options.dimension} — rebuilding the full index with the current model.`);
|
|
221
|
+
if (manifestStatus === "ok") {
|
|
222
|
+
for (const key of Object.keys(manifest.files))
|
|
223
|
+
delete manifest.files[key];
|
|
224
|
+
manifest.lastIndexedAt = undefined;
|
|
225
|
+
manifestStatus = "missing";
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
catch {
|
|
230
|
+
// getVectorDimension is best-effort — proceed without the drift check.
|
|
231
|
+
}
|
|
239
232
|
// Effective store used throughout the pass — may be a temp store for atomic rebuild.
|
|
240
233
|
let effectiveStore = options.store;
|
|
241
234
|
let tempStorePath;
|
|
242
|
-
|
|
235
|
+
/** Set when the temporary rebuild store actually received rows. */
|
|
236
|
+
let tempStoreWroteChunks = false;
|
|
237
|
+
if (options.force || (manifestStatus !== "ok" && (existingCount > 0 || dimensionMismatch))) {
|
|
243
238
|
for (const key of Object.keys(manifest.files)) {
|
|
244
239
|
delete manifest.files[key];
|
|
245
240
|
}
|
|
246
241
|
manifest.lastIndexedAt = undefined;
|
|
247
|
-
rebuildPerformed = existingCount > 0 || !!options.force;
|
|
242
|
+
rebuildPerformed = existingCount > 0 || !!options.force || dimensionMismatch;
|
|
248
243
|
if (manifestStatus !== "ok" && existingCount > 0) {
|
|
249
244
|
logger.warn("Manifest missing or corrupt; rebuilding full index.");
|
|
250
245
|
}
|
|
@@ -271,7 +266,7 @@ async function runIndexPassInner(options, logger) {
|
|
|
271
266
|
logger.warn("Cannot rebuild safely without embedding dimension — aborting to protect existing data. " +
|
|
272
267
|
"Run 'opencode-rag index --force' manually to rebuild.");
|
|
273
268
|
// Restore manifest entries we just deleted so the next pass can retry incrementally
|
|
274
|
-
return createIndexStats(
|
|
269
|
+
return createIndexStats(0, manifestStatus);
|
|
275
270
|
}
|
|
276
271
|
else {
|
|
277
272
|
// No existing data — safe to proceed with in-place indexing (no clear needed)
|
|
@@ -279,8 +274,76 @@ async function runIndexPassInner(options, logger) {
|
|
|
279
274
|
logger.debug("No existing data; indexing from scratch.");
|
|
280
275
|
}
|
|
281
276
|
}
|
|
277
|
+
let filterPaths;
|
|
278
|
+
let gitDeletedPaths = [];
|
|
279
|
+
if (!options.force && manifestStatus === "ok" && manifest.lastGitCommit) {
|
|
280
|
+
const repoRoot = getRepoRoot(options.cwd);
|
|
281
|
+
if (repoRoot) {
|
|
282
|
+
const diffResult = getChangedFilesSince(options.cwd, manifest.lastGitCommit);
|
|
283
|
+
if (diffResult) {
|
|
284
|
+
const untracked = getUntrackedFiles(options.cwd);
|
|
285
|
+
const changedSet = new Set();
|
|
286
|
+
for (const f of diffResult.changedFiles)
|
|
287
|
+
changedSet.add(f);
|
|
288
|
+
for (const f of untracked)
|
|
289
|
+
changedSet.add(f);
|
|
290
|
+
// Git paths are relative to the REPO ROOT. When the workspace is a
|
|
291
|
+
// subdirectory of the repo (monorepos), convert them to workspace-
|
|
292
|
+
// relative paths before passing them to the scan — resolving repo-root
|
|
293
|
+
// paths against `cwd` would silently miss every change.
|
|
294
|
+
const toCwdRelative = (p) => {
|
|
295
|
+
const abs = path.resolve(repoRoot, p);
|
|
296
|
+
const rel = path.relative(options.cwd, abs);
|
|
297
|
+
if (rel.startsWith("..") || path.isAbsolute(rel))
|
|
298
|
+
return null;
|
|
299
|
+
return rel;
|
|
300
|
+
};
|
|
301
|
+
const cwdRelative = [];
|
|
302
|
+
for (const p of changedSet) {
|
|
303
|
+
const rel = toCwdRelative(p);
|
|
304
|
+
if (rel !== null)
|
|
305
|
+
cwdRelative.push(rel);
|
|
306
|
+
}
|
|
307
|
+
filterPaths = cwdRelative;
|
|
308
|
+
gitDeletedPaths = diffResult.deletedFiles;
|
|
309
|
+
logger.debug(`Git incremental: ${filterPaths.length} changed/untracked, ${gitDeletedPaths.length} deleted since ${manifest.lastGitCommit.slice(0, 8)}`);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
const scanStart = Date.now();
|
|
314
|
+
const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger, options.force ? undefined : manifest, filterPaths, options.imageVisionProvider, descCache);
|
|
315
|
+
const scanSec = ((Date.now() - scanStart) / 1000).toFixed(1);
|
|
316
|
+
logger.info(`Workspace scan complete: ${workspaceFiles.length} files in ${scanSec}s`);
|
|
317
|
+
// ── Preflight: verify the embedding provider before expensive work ─────
|
|
318
|
+
// A provider outage used to burn the entire pass (chunking + describing
|
|
319
|
+
// every file) and then store nothing, reporting success. Probe once here
|
|
320
|
+
// so the pass aborts in milliseconds with an actionable message.
|
|
321
|
+
const pendingEmbeddingWork = options.force || workspaceFiles.some((f) => {
|
|
322
|
+
if (f.isEmpty || f.isTooSmall)
|
|
323
|
+
return false;
|
|
324
|
+
const previous = manifest.files[f.normalizedPath];
|
|
325
|
+
return !previous || previous.hash !== f.hash;
|
|
326
|
+
});
|
|
327
|
+
if (pendingEmbeddingWork && !(options.abortSignal?.aborted ?? false)) {
|
|
328
|
+
const probe = await probeEmbeddingDimension(options.embedder);
|
|
329
|
+
if (probe.dimension === undefined) {
|
|
330
|
+
logger.warn(`Embedding provider unavailable (${probe.error?.message ?? "probe failed"}) — ` +
|
|
331
|
+
"aborting index pass before chunking/description; nothing was stored.");
|
|
332
|
+
const failedStats = createIndexStats(workspaceFiles.length, manifestStatus);
|
|
333
|
+
failedStats.embeddingUnavailable = true;
|
|
334
|
+
return failedStats;
|
|
335
|
+
}
|
|
336
|
+
if (options.dimension !== undefined && probe.dimension !== options.dimension) {
|
|
337
|
+
logger.warn(`Embedding provider produces ${probe.dimension}-dimensional vectors but this index is configured for ` +
|
|
338
|
+
`${options.dimension} — aborting to avoid writing incompatible vectors. ` +
|
|
339
|
+
"Set embedding.vectorDimension to the provider's dimension and run the index again.");
|
|
340
|
+
const failedStats = createIndexStats(workspaceFiles.length, manifestStatus);
|
|
341
|
+
failedStats.embeddingUnavailable = true;
|
|
342
|
+
return failedStats;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
282
345
|
const stats = createIndexStats(workspaceFiles.length, manifestStatus);
|
|
283
|
-
stats.rebuildPerformed = rebuildPerformed;
|
|
346
|
+
stats.rebuildPerformed = rebuildPerformed || dimensionMismatch;
|
|
284
347
|
for (const file of workspaceFiles) {
|
|
285
348
|
if (file.extractionStatus === "failed" && file.extractionError) {
|
|
286
349
|
stats.extractionFailures++;
|
|
@@ -416,6 +479,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
416
479
|
if (!allEmbedded && (prep.chunks?.length ?? 0) > 0) {
|
|
417
480
|
options.logger?.warn?.(` ${prep.fileLabel}: ${validChunks.length}/${prep.chunks?.length} chunks embedded — marking for retry on next pass`);
|
|
418
481
|
}
|
|
482
|
+
if (validChunks.length > 0)
|
|
483
|
+
tempStoreWroteChunks = true;
|
|
419
484
|
storePayloads.push({ prep, validChunks, allEmbedded });
|
|
420
485
|
}
|
|
421
486
|
// One bulk write per window. Dedup (per-modified-file deletes) is skipped
|
|
@@ -738,7 +803,7 @@ async function runIndexPassInner(options, logger) {
|
|
|
738
803
|
}
|
|
739
804
|
}
|
|
740
805
|
for (const prep of deferredPreps) {
|
|
741
|
-
prep.textToEmbed = buildTextsToEmbed(prep.chunks, prep.relPath, prep.metaHeader ?? "", prep.docPrefix ?? "", prep.isImageFile ?? false);
|
|
806
|
+
prep.textToEmbed = buildTextsToEmbed(prep.chunks, prep.relPath, prep.metaHeader ?? "", prep.docPrefix ?? "", prep.isImageFile ?? false, options.config.indexing.embedDescriptions !== false);
|
|
742
807
|
}
|
|
743
808
|
const totalDescribedChunks = deferredPreps.reduce((s, p) => s + (p.chunks?.length ?? 0), 0);
|
|
744
809
|
logger.info(`Description phase complete: ${totalDescribedChunks} chunks across ${deferredPreps.length} files`);
|
|
@@ -801,6 +866,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
801
866
|
}
|
|
802
867
|
catch (err) {
|
|
803
868
|
logger.warn(` Global embedding failed: ${err.message}`);
|
|
869
|
+
stats.embeddingFailures += allTexts.length;
|
|
870
|
+
stats.embeddingUnavailable = true;
|
|
804
871
|
for (const { fileIdx } of embedQueue) {
|
|
805
872
|
options.progress?.failFile(prepared[fileIdx].fileLabel);
|
|
806
873
|
earlyWorkerResults.set(fileIdx, {
|
|
@@ -814,6 +881,21 @@ async function runIndexPassInner(options, logger) {
|
|
|
814
881
|
}
|
|
815
882
|
embedQueue.length = 0; // prevent double-processing in store phase
|
|
816
883
|
}
|
|
884
|
+
// `embedBatch` returns empty vectors for batches whose retries were
|
|
885
|
+
// exhausted — count those so the summary and exit code report a failed
|
|
886
|
+
// pass instead of a successful "0 chunks stored".
|
|
887
|
+
const failedVectors = allEmbeddings.filter((v) => !Array.isArray(v) || v.length === 0).length;
|
|
888
|
+
if (failedVectors > 0) {
|
|
889
|
+
stats.embeddingFailures += failedVectors;
|
|
890
|
+
if (failedVectors >= allTexts.length) {
|
|
891
|
+
stats.embeddingUnavailable = true;
|
|
892
|
+
logger.warn(`All ${allTexts.length} embedding requests failed — no vectors were produced, so nothing was stored. ` +
|
|
893
|
+
"Check that the embedding provider is running and the model is available, then run the index again.");
|
|
894
|
+
}
|
|
895
|
+
else {
|
|
896
|
+
logger.warn(` ${failedVectors}/${allTexts.length} embedding requests failed — those chunks are kept for retry on the next pass.`);
|
|
897
|
+
}
|
|
898
|
+
}
|
|
817
899
|
}
|
|
818
900
|
// ── Distribute embeddings back to per-file chunks ─────────────────────
|
|
819
901
|
for (let i = 0; i < embedQueue.length; i++) {
|
|
@@ -875,25 +957,43 @@ async function runIndexPassInner(options, logger) {
|
|
|
875
957
|
tryUpdateLastGitCommit(options.cwd, manifest);
|
|
876
958
|
}
|
|
877
959
|
// ── Atomically promote temp store if a full rebuild was performed ──
|
|
960
|
+
let tempStorePromoted = false;
|
|
878
961
|
if (tempStorePath) {
|
|
879
962
|
if (!aborted()) {
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
//
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
catch (err) {
|
|
890
|
-
logger.warn(`Could not promote temporary store: ${err.message}. ` +
|
|
891
|
-
`Original data preserved at ${options.storePath}`);
|
|
963
|
+
if (!tempStoreWroteChunks) {
|
|
964
|
+
// Nothing was written (e.g. every embedding request failed). Keep the
|
|
965
|
+
// existing store and manifest untouched rather than promoting an
|
|
966
|
+
// empty store over good data.
|
|
967
|
+
logger.warn("Rebuild produced no chunks — keeping the existing index and manifest unchanged.");
|
|
968
|
+
try {
|
|
969
|
+
await effectiveStore.close();
|
|
970
|
+
}
|
|
971
|
+
catch { }
|
|
892
972
|
try {
|
|
893
973
|
await fs.rm(tempStorePath, { recursive: true, force: true });
|
|
894
974
|
}
|
|
895
975
|
catch { }
|
|
896
976
|
}
|
|
977
|
+
else {
|
|
978
|
+
try {
|
|
979
|
+
await effectiveStore.close();
|
|
980
|
+
await options.store.close();
|
|
981
|
+
// Swap the newly-built temp directory into the real path
|
|
982
|
+
await swapStoreDirectories(tempStorePath, options.storePath);
|
|
983
|
+
// Re-open the original store handle so callers can search the new data
|
|
984
|
+
await options.store.reopen?.(options.storePath);
|
|
985
|
+
tempStorePromoted = true;
|
|
986
|
+
logger.debug(`Promoted temporary store ${tempStorePath} → ${options.storePath}`);
|
|
987
|
+
}
|
|
988
|
+
catch (err) {
|
|
989
|
+
logger.warn(`Could not promote temporary store: ${err.message}. ` +
|
|
990
|
+
`Original data preserved at ${options.storePath}`);
|
|
991
|
+
try {
|
|
992
|
+
await fs.rm(tempStorePath, { recursive: true, force: true });
|
|
993
|
+
}
|
|
994
|
+
catch { }
|
|
995
|
+
}
|
|
996
|
+
}
|
|
897
997
|
}
|
|
898
998
|
else {
|
|
899
999
|
// Aborted — discard temp, keep original data intact.
|
|
@@ -912,16 +1012,27 @@ async function runIndexPassInner(options, logger) {
|
|
|
912
1012
|
return stats;
|
|
913
1013
|
}
|
|
914
1014
|
}
|
|
915
|
-
//
|
|
916
|
-
//
|
|
917
|
-
|
|
918
|
-
|
|
1015
|
+
// The in-memory manifest was cleared at rebuild start and only repopulated
|
|
1016
|
+
// for the new store. When the rebuild did not promote (empty temp store or
|
|
1017
|
+
// failed swap), keep the previous manifest/keyword index on disk — saving
|
|
1018
|
+
// the cleared state would orphan the existing store's data.
|
|
1019
|
+
const keepPreviousIndexState = tempStorePath !== undefined && !tempStorePromoted;
|
|
1020
|
+
if (keepPreviousIndexState) {
|
|
1021
|
+
logger.warn("Rebuild did not complete — keeping the previous manifest and keyword index.");
|
|
1022
|
+
}
|
|
1023
|
+
else {
|
|
1024
|
+
// Save manifest and keyword index (always to the real store path — after
|
|
1025
|
+
// a successful swap this points to the new data).
|
|
1026
|
+
await saveManifest(options.storePath, manifest);
|
|
1027
|
+
await options.keywordIndex?.save(options.storePath);
|
|
1028
|
+
}
|
|
919
1029
|
// Compact fragments and prune old version manifests so countRows() can't
|
|
920
1030
|
// hang on accumulated versions from many add/delete cycles. After a temp
|
|
921
1031
|
// store rebuild, optimize the reopened real store handle (the temp handle
|
|
922
1032
|
// was closed and its directory moved); use aggressive pruning since the
|
|
923
|
-
// swapped-in store is private to this process.
|
|
924
|
-
|
|
1033
|
+
// swapped-in store is private to this process. Skipped when the rebuild did
|
|
1034
|
+
// not promote (the old store is untouched and its handle may be closed).
|
|
1035
|
+
if (!aborted() && !keepPreviousIndexState) {
|
|
925
1036
|
logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
|
|
926
1037
|
const optimizeStart = Date.now();
|
|
927
1038
|
try {
|
package/dist/indexer/stats.d.ts
CHANGED
|
@@ -38,6 +38,10 @@ export interface IndexRunStats {
|
|
|
38
38
|
}>;
|
|
39
39
|
/** Number of files where description generation failed. */
|
|
40
40
|
descriptionFailedFiles: number;
|
|
41
|
+
/** Number of chunks whose embedding request failed (kept for retry on the next pass). */
|
|
42
|
+
embeddingFailures: number;
|
|
43
|
+
/** True when the pass could not embed at all (provider unavailable or dimension mismatch) — nothing was stored. */
|
|
44
|
+
embeddingUnavailable: boolean;
|
|
41
45
|
/** True when the pass was skipped because another pass holds the lock. */
|
|
42
46
|
skipped: boolean;
|
|
43
47
|
}
|
package/dist/indexer/stats.js
CHANGED
package/dist/indexer/worker.d.ts
CHANGED
|
@@ -61,16 +61,21 @@ export interface PreparedFile {
|
|
|
61
61
|
/**
|
|
62
62
|
* Build the list of text strings that will be sent to the embedding provider.
|
|
63
63
|
* Each chunk is prefixed with the document prefix, relative path, metadata
|
|
64
|
-
* header, and (if available) a description.
|
|
64
|
+
* header, and (if available and enabled) a description.
|
|
65
|
+
*
|
|
66
|
+
* Code-specialized embedding models do not need the description crutch: with
|
|
67
|
+
* `indexing.embedDescriptions: false` the prose description is omitted from
|
|
68
|
+
* the embedded text (it is still stored on the chunk for display).
|
|
65
69
|
*
|
|
66
70
|
* @param chunks - Chunks to build embedding texts from.
|
|
67
71
|
* @param relPath - Relative file path used as context prefix.
|
|
68
72
|
* @param metaHeader - Assembled metadata header (file type, directory, etc.).
|
|
69
73
|
* @param docPrefix - Optional document-level prefix from configuration.
|
|
70
74
|
* @param isImage - Whether the source is an image (uses description only).
|
|
75
|
+
* @param includeDescription - Whether to include non-image descriptions (default true).
|
|
71
76
|
* @returns An array of formatted text strings, one per chunk.
|
|
72
77
|
*/
|
|
73
|
-
export declare function buildTextsToEmbed(chunks: Chunk[], relPath: string, metaHeader: string, docPrefix: string, isImage: boolean): string[];
|
|
78
|
+
export declare function buildTextsToEmbed(chunks: Chunk[], relPath: string, metaHeader: string, docPrefix: string, isImage: boolean, includeDescription?: boolean): string[];
|
|
74
79
|
interface Logger {
|
|
75
80
|
info(message: string): void;
|
|
76
81
|
warn(message: string): void;
|
|
@@ -115,6 +120,7 @@ export declare function prepareFile(file: WorkspaceFile, cwd: string, previous:
|
|
|
115
120
|
};
|
|
116
121
|
indexing?: {
|
|
117
122
|
maxSvgSizeBytes?: number;
|
|
123
|
+
embedDescriptions?: boolean;
|
|
118
124
|
};
|
|
119
125
|
}, keywordIndex: KeywordIndex | undefined, descriptionProvider: DescriptionProvider | undefined, logger: Logger, deferDescriptions?: boolean, descHash?: string): Promise<PreparedFile>;
|
|
120
126
|
/**
|
package/dist/indexer/worker.js
CHANGED
|
@@ -10,23 +10,28 @@ import { generateDescriptions, buildFallbackDescription } from "./description-st
|
|
|
10
10
|
/**
|
|
11
11
|
* Build the list of text strings that will be sent to the embedding provider.
|
|
12
12
|
* Each chunk is prefixed with the document prefix, relative path, metadata
|
|
13
|
-
* header, and (if available) a description.
|
|
13
|
+
* header, and (if available and enabled) a description.
|
|
14
|
+
*
|
|
15
|
+
* Code-specialized embedding models do not need the description crutch: with
|
|
16
|
+
* `indexing.embedDescriptions: false` the prose description is omitted from
|
|
17
|
+
* the embedded text (it is still stored on the chunk for display).
|
|
14
18
|
*
|
|
15
19
|
* @param chunks - Chunks to build embedding texts from.
|
|
16
20
|
* @param relPath - Relative file path used as context prefix.
|
|
17
21
|
* @param metaHeader - Assembled metadata header (file type, directory, etc.).
|
|
18
22
|
* @param docPrefix - Optional document-level prefix from configuration.
|
|
19
23
|
* @param isImage - Whether the source is an image (uses description only).
|
|
24
|
+
* @param includeDescription - Whether to include non-image descriptions (default true).
|
|
20
25
|
* @returns An array of formatted text strings, one per chunk.
|
|
21
26
|
*/
|
|
22
|
-
export function buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage) {
|
|
27
|
+
export function buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage, includeDescription = true) {
|
|
23
28
|
const textToEmbed = [];
|
|
24
29
|
for (const chunk of chunks) {
|
|
25
30
|
if (isImage) {
|
|
26
31
|
textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description);
|
|
27
32
|
}
|
|
28
33
|
else {
|
|
29
|
-
const desc = chunk.description ?? "";
|
|
34
|
+
const desc = includeDescription ? chunk.description ?? "" : "";
|
|
30
35
|
if (desc.trim().length > 0) {
|
|
31
36
|
textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + desc + "\n\n" + chunk.content);
|
|
32
37
|
}
|
|
@@ -210,7 +215,7 @@ export async function prepareFile(file, cwd, previous, config, keywordIndex, des
|
|
|
210
215
|
}
|
|
211
216
|
}
|
|
212
217
|
}
|
|
213
|
-
const textToEmbed = buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage);
|
|
218
|
+
const textToEmbed = buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage, config.indexing?.embedDescriptions !== false);
|
|
214
219
|
logger.debug(` ${fileLabel}: textToEmbed ${textToEmbed.length} entries (descProvider: ${descriptionProvider ? "yes" : "no"})`);
|
|
215
220
|
return {
|
|
216
221
|
normalizedPath: file.normalizedPath,
|
package/dist/plugin.js
CHANGED
|
@@ -7,7 +7,8 @@ import { tool } from "@opencode-ai/plugin/tool";
|
|
|
7
7
|
import { CODE_SEARCH_FILTER } from "./core/interfaces.js";
|
|
8
8
|
import { normalizeFileExtensions } from "./core/filters.js";
|
|
9
9
|
import { loadConfig, findConfigFile, DEFAULT_CONFIG, resolveLogConfig, persistProbedDimension } from "./core/config.js";
|
|
10
|
-
import { createEmbedder } from "./embedder/factory.js";
|
|
10
|
+
import { createEmbedder, probeEmbeddingDimension } from "./embedder/factory.js";
|
|
11
|
+
import { readStoreDimension } from "./vectorstore/lancedb.js";
|
|
11
12
|
import { createDescriptionProvider } from "./describer/factory.js";
|
|
12
13
|
import { createVectorStore } from "./vectorstore/factory.js";
|
|
13
14
|
import { retrieve } from "./retriever/retriever.js";
|
|
@@ -419,7 +420,8 @@ export function createRagHooks(options) {
|
|
|
419
420
|
...options.dependencies,
|
|
420
421
|
};
|
|
421
422
|
const embedder = options.embedder ?? dependencies.createEmbedder(options.cfg);
|
|
422
|
-
const
|
|
423
|
+
const configuredDimension = options.cfg.embedding.vectorDimension;
|
|
424
|
+
const store = options.store ?? dependencies.createStore(options.storePath, configuredDimension && configuredDimension > 0 ? configuredDimension : 384, options.cfg);
|
|
423
425
|
const keywordIndex = options.keywordIndex;
|
|
424
426
|
// Runtime overrides for live config editing from TUI
|
|
425
427
|
let cachedOverrides = loadRuntimeOverrides(options.storePath);
|
|
@@ -1394,6 +1396,8 @@ export const ragPlugin = async (input, _options) => {
|
|
|
1394
1396
|
}, logLevel);
|
|
1395
1397
|
// Use cached dimension from config if available (avoids blocking startup with an API call)
|
|
1396
1398
|
// If not set, probe the embedding provider once and persist the result.
|
|
1399
|
+
// When the probe fails, prefer the existing store's schema over the 384
|
|
1400
|
+
// default — a transient outage must not downgrade the store's dimension.
|
|
1397
1401
|
const embedder = createEmbedder(effectiveCfg);
|
|
1398
1402
|
let vectorDimension = effectiveCfg.embedding.vectorDimension;
|
|
1399
1403
|
if (vectorDimension && vectorDimension > 0) {
|
|
@@ -1403,34 +1407,47 @@ export const ragPlugin = async (input, _options) => {
|
|
|
1403
1407
|
}, logLevel);
|
|
1404
1408
|
}
|
|
1405
1409
|
else {
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
try {
|
|
1414
|
-
persistProbedDimension(configPath, vectorDimension);
|
|
1415
|
-
}
|
|
1416
|
-
catch { /* best-effort */ }
|
|
1410
|
+
const probe = await probeEmbeddingDimension(embedder);
|
|
1411
|
+
if (probe.dimension !== undefined) {
|
|
1412
|
+
vectorDimension = probe.dimension;
|
|
1413
|
+
const configPath = findConfigFile(input.directory);
|
|
1414
|
+
if (configPath) {
|
|
1415
|
+
try {
|
|
1416
|
+
persistProbedDimension(configPath, vectorDimension);
|
|
1417
1417
|
}
|
|
1418
|
+
catch { /* best-effort */ }
|
|
1418
1419
|
}
|
|
1419
1420
|
appendDebugLog(logFilePath, {
|
|
1420
1421
|
scope: "plugin",
|
|
1421
1422
|
message: `Vector dimension: ${vectorDimension}`,
|
|
1422
1423
|
}, logLevel);
|
|
1423
1424
|
}
|
|
1424
|
-
|
|
1425
|
+
else {
|
|
1426
|
+
vectorDimension = (await readStoreDimension(storePath)) ?? 384;
|
|
1425
1427
|
appendDebugLog(logFilePath, {
|
|
1426
1428
|
scope: "plugin",
|
|
1427
1429
|
message: `Dimension probe failed, falling back to ${vectorDimension}`,
|
|
1428
|
-
error:
|
|
1430
|
+
error: probe.error,
|
|
1429
1431
|
}, logLevel);
|
|
1430
1432
|
}
|
|
1431
1433
|
}
|
|
1432
1434
|
const store = createVectorStore(effectiveCfg, storePath, vectorDimension);
|
|
1433
1435
|
ragStores.set(input.directory, store);
|
|
1436
|
+
// Warn when the store was built by a different embedding model — vector
|
|
1437
|
+
// search fails (or silently degrades) until the index is rebuilt.
|
|
1438
|
+
try {
|
|
1439
|
+
const storeDimension = await store.getVectorDimension?.();
|
|
1440
|
+
if (storeDimension !== undefined && storeDimension !== vectorDimension) {
|
|
1441
|
+
appendDebugLog(logFilePath, {
|
|
1442
|
+
scope: "plugin",
|
|
1443
|
+
message: `Store vector dimension is ${storeDimension} but the configured embedder produces ${vectorDimension} — ` +
|
|
1444
|
+
"run 'opencode-rag index' to rebuild the index with the current model.",
|
|
1445
|
+
}, logLevel);
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
catch {
|
|
1449
|
+
// best-effort — dimension introspection must never block plugin startup
|
|
1450
|
+
}
|
|
1434
1451
|
// Load or create keyword index for hybrid search
|
|
1435
1452
|
const keywordIndex = await loadKeywordIndex(storePath, logFilePath, logLevel);
|
|
1436
1453
|
// Create description provider (enabled by default)
|
|
@@ -53,10 +53,45 @@ export declare function isCorruptionError(err: unknown): boolean;
|
|
|
53
53
|
* @returns True if the error matches a transient transaction conflict.
|
|
54
54
|
*/
|
|
55
55
|
export declare function isTransientConflictError(err: unknown): boolean;
|
|
56
|
+
/**
|
|
57
|
+
* Thrown when a write supplies embeddings whose length differs from the
|
|
58
|
+
* store's vector column dimension. LanceDB itself does NOT validate this —
|
|
59
|
+
* it silently zero-pads or truncates vectors into the fixed-size column,
|
|
60
|
+
* producing rows that can never be found by a query. Failing loudly here
|
|
61
|
+
* keeps that silent corruption out of the store.
|
|
62
|
+
*/
|
|
63
|
+
export declare class DimensionMismatchError extends Error {
|
|
64
|
+
/** Dimension of the store's vector column. */
|
|
65
|
+
readonly storeDimension: number;
|
|
66
|
+
/** Dimension of the embedding that was supplied. */
|
|
67
|
+
readonly embeddingDimension: number;
|
|
68
|
+
constructor(storeDimension: number, embeddingDimension: number, action?: string);
|
|
69
|
+
}
|
|
70
|
+
/** Type guard for {@link DimensionMismatchError}. */
|
|
71
|
+
export declare function isDimensionMismatchError(err: unknown): err is DimensionMismatchError;
|
|
72
|
+
/**
|
|
73
|
+
* Extract the fixed-size vector dimension of the `embedding` column from a
|
|
74
|
+
* LanceDB/Arrow schema. Returns `undefined` when the column is missing or is
|
|
75
|
+
* not a fixed-size list (e.g. before the table is created).
|
|
76
|
+
*/
|
|
77
|
+
export declare function extractEmbeddingDimension(fields: ReadonlyArray<{
|
|
78
|
+
name: string;
|
|
79
|
+
type: unknown;
|
|
80
|
+
}>): number | undefined;
|
|
81
|
+
/**
|
|
82
|
+
* Read the embedding dimension of an existing store without creating a table
|
|
83
|
+
* or holding a long-lived connection. Returns `undefined` when the store or
|
|
84
|
+
* table does not exist (or cannot be read).
|
|
85
|
+
*
|
|
86
|
+
* Used by the CLI bootstrap to avoid creating a brand-new store with a
|
|
87
|
+
* speculative dimension when the embedding provider cannot be probed.
|
|
88
|
+
*/
|
|
89
|
+
export declare function readStoreDimension(storePath: string): Promise<number | undefined>;
|
|
56
90
|
/**
|
|
57
91
|
* Atomically replace one LanceDB store directory with another.
|
|
58
92
|
* Swaps the real directory with a temporary one that was built during a rebuild.
|
|
59
|
-
* The old directory is moved to `${realPath}_old` and deleted asynchronously
|
|
93
|
+
* The old directory is moved to `${realPath}_old` and deleted asynchronously
|
|
94
|
+
* after non-Lance artifacts (quirk memory, caches) have been carried over.
|
|
60
95
|
*
|
|
61
96
|
* @param tempPath - Path to the newly built store (source).
|
|
62
97
|
* @param realPath - Path to the current store (destination, will be replaced).
|
|
@@ -94,6 +129,13 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
94
129
|
private indexRepairPromise;
|
|
95
130
|
/** Consecutive failed repair attempts — bounded so a broken store cannot retrain forever. */
|
|
96
131
|
private indexRepairFailures;
|
|
132
|
+
/**
|
|
133
|
+
* Actual dimension of the `embedding` column in the opened table. Read from
|
|
134
|
+
* the schema on first table access and used to validate writes/searches —
|
|
135
|
+
* the constructor's `vectorDimension` describes what the caller *expects*,
|
|
136
|
+
* which can drift from the store when the embedding model changes.
|
|
137
|
+
*/
|
|
138
|
+
private knownDimension;
|
|
97
139
|
/**
|
|
98
140
|
* Execute an async function under an exclusive write lock.
|
|
99
141
|
*
|
|
@@ -120,6 +162,23 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
120
162
|
private getTable;
|
|
121
163
|
private initTable;
|
|
122
164
|
private tableHasDescriptionColumn;
|
|
165
|
+
/**
|
|
166
|
+
* Read and cache the actual fixed-size dimension of the `embedding` column.
|
|
167
|
+
* Called whenever the table is (re)opened so write/search validation uses
|
|
168
|
+
* the store's real schema instead of the constructor's expectation.
|
|
169
|
+
*/
|
|
170
|
+
private cacheTableDimension;
|
|
171
|
+
/**
|
|
172
|
+
* Return the actual embedding dimension of the store's vector column, or
|
|
173
|
+
* `undefined` when no table exists yet. Never creates the table.
|
|
174
|
+
*/
|
|
175
|
+
getVectorDimension(): Promise<number | undefined>;
|
|
176
|
+
/**
|
|
177
|
+
* Fail loudly when rows carry embeddings whose length differs from the
|
|
178
|
+
* store's vector column. LanceDB would silently pad/truncate them instead,
|
|
179
|
+
* leaving rows that no query can match.
|
|
180
|
+
*/
|
|
181
|
+
private assertEmbeddingDimensions;
|
|
123
182
|
private hasColumn;
|
|
124
183
|
/** Add kind/quirkType/tags columns if missing from an existing table. */
|
|
125
184
|
private migrateNewColumns;
|
|
@@ -148,10 +148,129 @@ export function isTransientConflictError(err) {
|
|
|
148
148
|
}
|
|
149
149
|
return false;
|
|
150
150
|
}
|
|
151
|
+
/**
|
|
152
|
+
* Thrown when a write supplies embeddings whose length differs from the
|
|
153
|
+
* store's vector column dimension. LanceDB itself does NOT validate this —
|
|
154
|
+
* it silently zero-pads or truncates vectors into the fixed-size column,
|
|
155
|
+
* producing rows that can never be found by a query. Failing loudly here
|
|
156
|
+
* keeps that silent corruption out of the store.
|
|
157
|
+
*/
|
|
158
|
+
export class DimensionMismatchError extends Error {
|
|
159
|
+
/** Dimension of the store's vector column. */
|
|
160
|
+
storeDimension;
|
|
161
|
+
/** Dimension of the embedding that was supplied. */
|
|
162
|
+
embeddingDimension;
|
|
163
|
+
constructor(storeDimension, embeddingDimension, action = "rebuild it with 'opencode-rag index --force'") {
|
|
164
|
+
super(`Embedding dimension mismatch: the store holds ${storeDimension}-dimensional vectors ` +
|
|
165
|
+
`but the embedding model produced ${embeddingDimension}. The index was built with a ` +
|
|
166
|
+
`different embedding model — ${action}.`);
|
|
167
|
+
this.name = "DimensionMismatchError";
|
|
168
|
+
this.storeDimension = storeDimension;
|
|
169
|
+
this.embeddingDimension = embeddingDimension;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
/** Type guard for {@link DimensionMismatchError}. */
|
|
173
|
+
export function isDimensionMismatchError(err) {
|
|
174
|
+
return err instanceof DimensionMismatchError || (err instanceof Error && err.name === "DimensionMismatchError");
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Extract the fixed-size vector dimension of the `embedding` column from a
|
|
178
|
+
* LanceDB/Arrow schema. Returns `undefined` when the column is missing or is
|
|
179
|
+
* not a fixed-size list (e.g. before the table is created).
|
|
180
|
+
*/
|
|
181
|
+
export function extractEmbeddingDimension(fields) {
|
|
182
|
+
const field = fields.find((f) => f.name === "embedding");
|
|
183
|
+
const type = field?.type;
|
|
184
|
+
if (type && typeof type.listSize === "number" && type.listSize > 0) {
|
|
185
|
+
return type.listSize;
|
|
186
|
+
}
|
|
187
|
+
return undefined;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Read the embedding dimension of an existing store without creating a table
|
|
191
|
+
* or holding a long-lived connection. Returns `undefined` when the store or
|
|
192
|
+
* table does not exist (or cannot be read).
|
|
193
|
+
*
|
|
194
|
+
* Used by the CLI bootstrap to avoid creating a brand-new store with a
|
|
195
|
+
* speculative dimension when the embedding provider cannot be probed.
|
|
196
|
+
*/
|
|
197
|
+
export async function readStoreDimension(storePath) {
|
|
198
|
+
if (storePath.startsWith("memory:"))
|
|
199
|
+
return undefined;
|
|
200
|
+
try {
|
|
201
|
+
// Do not create a store directory just to read a schema.
|
|
202
|
+
await fs.access(storePath);
|
|
203
|
+
}
|
|
204
|
+
catch {
|
|
205
|
+
return undefined;
|
|
206
|
+
}
|
|
207
|
+
try {
|
|
208
|
+
const db = await lancedb.connect(storePath);
|
|
209
|
+
const tableNames = await db.tableNames();
|
|
210
|
+
if (!tableNames.includes(TABLE_NAME))
|
|
211
|
+
return undefined;
|
|
212
|
+
const table = await db.openTable(TABLE_NAME);
|
|
213
|
+
const schema = await table.schema();
|
|
214
|
+
return extractEmbeddingDimension(schema.fields);
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
return undefined;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Non-Lance artifacts that live in the store directory next to the LanceDB
|
|
222
|
+
* data and must survive a rebuild swap. `chunks.lance`/`manifest.json` are
|
|
223
|
+
* rebuilt by the pipeline; the files below are not derivable from the new
|
|
224
|
+
* index and would otherwise be destroyed with the old directory.
|
|
225
|
+
*/
|
|
226
|
+
const PRESERVED_STORE_ENTRIES = [
|
|
227
|
+
"quirks.jsonl",
|
|
228
|
+
"runtime-overrides.json",
|
|
229
|
+
"watcher-status.json",
|
|
230
|
+
".desc-cache.json",
|
|
231
|
+
"keyword-index.json",
|
|
232
|
+
"eval-sessions",
|
|
233
|
+
];
|
|
234
|
+
/**
|
|
235
|
+
* Carry non-Lance store artifacts (quirk memory, caches, overrides) from the
|
|
236
|
+
* pre-swap directory into the freshly promoted one. Best-effort: a failure to
|
|
237
|
+
* preserve one entry must not abort the swap.
|
|
238
|
+
*/
|
|
239
|
+
async function preserveStoreArtifacts(oldPath, realPath) {
|
|
240
|
+
for (const entry of PRESERVED_STORE_ENTRIES) {
|
|
241
|
+
const src = path.join(oldPath, entry);
|
|
242
|
+
const dest = path.join(realPath, entry);
|
|
243
|
+
try {
|
|
244
|
+
await fs.access(src);
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
continue; // artifact not present in the old store
|
|
248
|
+
}
|
|
249
|
+
try {
|
|
250
|
+
try {
|
|
251
|
+
await fs.access(dest);
|
|
252
|
+
continue; // destination already has this artifact (e.g. re-saved desc cache)
|
|
253
|
+
}
|
|
254
|
+
catch {
|
|
255
|
+
// destination missing — move or copy it over
|
|
256
|
+
}
|
|
257
|
+
await fs.rename(src, dest);
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
try {
|
|
261
|
+
await fs.cp(src, dest, { recursive: true, force: true });
|
|
262
|
+
}
|
|
263
|
+
catch {
|
|
264
|
+
// best-effort — keep the swap result even if an artifact cannot be carried over
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
151
269
|
/**
|
|
152
270
|
* Atomically replace one LanceDB store directory with another.
|
|
153
271
|
* Swaps the real directory with a temporary one that was built during a rebuild.
|
|
154
|
-
* The old directory is moved to `${realPath}_old` and deleted asynchronously
|
|
272
|
+
* The old directory is moved to `${realPath}_old` and deleted asynchronously
|
|
273
|
+
* after non-Lance artifacts (quirk memory, caches) have been carried over.
|
|
155
274
|
*
|
|
156
275
|
* @param tempPath - Path to the newly built store (source).
|
|
157
276
|
* @param realPath - Path to the current store (destination, will be replaced).
|
|
@@ -176,7 +295,9 @@ export async function swapStoreDirectories(tempPath, realPath) {
|
|
|
176
295
|
catch { }
|
|
177
296
|
throw err;
|
|
178
297
|
}
|
|
179
|
-
//
|
|
298
|
+
// Preserve quirks.jsonl & friends (the temp store contains only Lance data),
|
|
299
|
+
// then clean up the old directory best-effort.
|
|
300
|
+
await preserveStoreArtifacts(oldPath, realPath);
|
|
180
301
|
fs.rm(oldPath, { recursive: true, force: true }).catch(() => { });
|
|
181
302
|
}
|
|
182
303
|
/**
|
|
@@ -201,6 +322,13 @@ export class LanceDbStore {
|
|
|
201
322
|
indexRepairPromise = null;
|
|
202
323
|
/** Consecutive failed repair attempts — bounded so a broken store cannot retrain forever. */
|
|
203
324
|
indexRepairFailures = 0;
|
|
325
|
+
/**
|
|
326
|
+
* Actual dimension of the `embedding` column in the opened table. Read from
|
|
327
|
+
* the schema on first table access and used to validate writes/searches —
|
|
328
|
+
* the constructor's `vectorDimension` describes what the caller *expects*,
|
|
329
|
+
* which can drift from the store when the embedding model changes.
|
|
330
|
+
*/
|
|
331
|
+
knownDimension = null;
|
|
204
332
|
/**
|
|
205
333
|
* Execute an async function under an exclusive write lock.
|
|
206
334
|
*
|
|
@@ -270,6 +398,7 @@ export class LanceDbStore {
|
|
|
270
398
|
const tableNames = await db.tableNames();
|
|
271
399
|
if (tableNames.includes(TABLE_NAME)) {
|
|
272
400
|
this.table = await db.openTable(TABLE_NAME);
|
|
401
|
+
await this.cacheTableDimension(this.table);
|
|
273
402
|
if (await this.tableHasDescriptionColumn()) {
|
|
274
403
|
await this.migrateNewColumns();
|
|
275
404
|
return this.table;
|
|
@@ -323,6 +452,7 @@ export class LanceDbStore {
|
|
|
323
452
|
data: [seedRow],
|
|
324
453
|
mode: "overwrite",
|
|
325
454
|
});
|
|
455
|
+
this.knownDimension = this.vectorDimension;
|
|
326
456
|
const deleted = await this.table.delete('id = "__seed__"');
|
|
327
457
|
if (deleted === undefined) {
|
|
328
458
|
// LanceDB may not return a count; try a direct query to verify
|
|
@@ -342,6 +472,59 @@ export class LanceDbStore {
|
|
|
342
472
|
return false;
|
|
343
473
|
}
|
|
344
474
|
}
|
|
475
|
+
/**
|
|
476
|
+
* Read and cache the actual fixed-size dimension of the `embedding` column.
|
|
477
|
+
* Called whenever the table is (re)opened so write/search validation uses
|
|
478
|
+
* the store's real schema instead of the constructor's expectation.
|
|
479
|
+
*/
|
|
480
|
+
async cacheTableDimension(table) {
|
|
481
|
+
if (this.knownDimension !== null)
|
|
482
|
+
return this.knownDimension;
|
|
483
|
+
try {
|
|
484
|
+
const schema = await table.schema();
|
|
485
|
+
const dim = extractEmbeddingDimension(schema.fields);
|
|
486
|
+
if (dim !== undefined)
|
|
487
|
+
this.knownDimension = dim;
|
|
488
|
+
return dim;
|
|
489
|
+
}
|
|
490
|
+
catch {
|
|
491
|
+
return undefined;
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
/**
|
|
495
|
+
* Return the actual embedding dimension of the store's vector column, or
|
|
496
|
+
* `undefined` when no table exists yet. Never creates the table.
|
|
497
|
+
*/
|
|
498
|
+
async getVectorDimension() {
|
|
499
|
+
if (this.knownDimension !== null)
|
|
500
|
+
return this.knownDimension;
|
|
501
|
+
try {
|
|
502
|
+
const db = await this.getDb();
|
|
503
|
+
const tableNames = await db.tableNames();
|
|
504
|
+
if (!tableNames.includes(TABLE_NAME))
|
|
505
|
+
return undefined;
|
|
506
|
+
const table = await this.getTable();
|
|
507
|
+
return await this.cacheTableDimension(table);
|
|
508
|
+
}
|
|
509
|
+
catch {
|
|
510
|
+
return undefined;
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
/**
|
|
514
|
+
* Fail loudly when rows carry embeddings whose length differs from the
|
|
515
|
+
* store's vector column. LanceDB would silently pad/truncate them instead,
|
|
516
|
+
* leaving rows that no query can match.
|
|
517
|
+
*/
|
|
518
|
+
assertEmbeddingDimensions(rows) {
|
|
519
|
+
const storeDim = this.knownDimension;
|
|
520
|
+
if (storeDim === null)
|
|
521
|
+
return;
|
|
522
|
+
for (const row of rows) {
|
|
523
|
+
if (row.embedding.length !== storeDim) {
|
|
524
|
+
throw new DimensionMismatchError(storeDim, row.embedding.length);
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
}
|
|
345
528
|
async hasColumn(name) {
|
|
346
529
|
try {
|
|
347
530
|
const schema = await this.table.schema();
|
|
@@ -489,6 +672,7 @@ export class LanceDbStore {
|
|
|
489
672
|
.filter((r) => r !== null);
|
|
490
673
|
if (rows.length === 0)
|
|
491
674
|
return;
|
|
675
|
+
this.assertEmbeddingDimensions(rows);
|
|
492
676
|
// INSERT FIRST: data is safely stored before any delete
|
|
493
677
|
await table.add(rows);
|
|
494
678
|
if (!dedup)
|
|
@@ -530,6 +714,7 @@ export class LanceDbStore {
|
|
|
530
714
|
}
|
|
531
715
|
if (allRows.length === 0)
|
|
532
716
|
return;
|
|
717
|
+
this.assertEmbeddingDimensions(allRows);
|
|
533
718
|
// INSERT FIRST (single add for the whole batch), then per-file dedup
|
|
534
719
|
await table.add(allRows);
|
|
535
720
|
for (const [filePath, ids] of dedupByFile) {
|
|
@@ -551,9 +736,21 @@ export class LanceDbStore {
|
|
|
551
736
|
async searchWithFilter(embedding, topK, filter) {
|
|
552
737
|
try {
|
|
553
738
|
// Guard against dimension mismatch BEFORE the native call — LanceDB
|
|
554
|
-
// throws a cryptic
|
|
555
|
-
|
|
556
|
-
|
|
739
|
+
// throws a cryptic "No vector column found to match with the query
|
|
740
|
+
// vector dimension" error. Compare against the table's *actual* column
|
|
741
|
+
// dimension, not the constructor's expectation: a store built by a
|
|
742
|
+
// different embedding model has a mismatching schema even when the
|
|
743
|
+
// handle was constructed with the current model's dimension.
|
|
744
|
+
const storeDimension = (await this.getVectorDimension()) ?? this.vectorDimension;
|
|
745
|
+
if (embedding.length !== storeDimension) {
|
|
746
|
+
if (storeDimension !== this.vectorDimension) {
|
|
747
|
+
console.warn(`[lancedb] Store vector column is ${storeDimension}-dimensional but this handle expects ` +
|
|
748
|
+
`${this.vectorDimension} — the index was built with a different embedding model. ` +
|
|
749
|
+
"Rebuild it with 'opencode-rag index --force' (a plain 'opencode-rag index' also rebuilds automatically).");
|
|
750
|
+
}
|
|
751
|
+
else {
|
|
752
|
+
console.warn(`[lancedb] searchWithFilter: query embedding dimension ${embedding.length} != store dimension ${storeDimension} — returning empty`);
|
|
753
|
+
}
|
|
557
754
|
return [];
|
|
558
755
|
}
|
|
559
756
|
return await this.searchInternal(embedding, topK, filter);
|
|
@@ -1090,6 +1287,7 @@ export class LanceDbStore {
|
|
|
1090
1287
|
await this.close();
|
|
1091
1288
|
if (newPath)
|
|
1092
1289
|
this.dbPath = newPath;
|
|
1290
|
+
this.knownDimension = null;
|
|
1093
1291
|
}
|
|
1094
1292
|
/**
|
|
1095
1293
|
* Close the database connection and release resources.
|
|
@@ -1153,6 +1351,7 @@ export class LanceDbStore {
|
|
|
1153
1351
|
console.warn(`[lancedb] Backed up chunks.lance to ${backup}`);
|
|
1154
1352
|
await this.table?.close();
|
|
1155
1353
|
this.table = null;
|
|
1354
|
+
this.knownDimension = null;
|
|
1156
1355
|
try {
|
|
1157
1356
|
const db = await this.getDb();
|
|
1158
1357
|
const tableNames = await db.tableNames();
|