opencode-rag-plugin 1.22.1 → 1.22.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -98,6 +98,7 @@ export function registerIndexCommand(program) {
98
98
  }
99
99
  }
100
100
  logCliInfo(logFilePath, "index", `${c.label("Scanning:")} ${c.file(cwd)}`);
101
+ let incomplete = false;
101
102
  const runPass = async (watchTriggered = false, abortSignal, filterPaths) => {
102
103
  const passStarted = Date.now();
103
104
  const stats = await runIndexPass({
@@ -116,6 +117,14 @@ export function registerIndexCommand(program) {
116
117
  });
117
118
  if (!watchTriggered) {
118
119
  logIndexSummary(logFilePath, stats);
120
+ if (stats.embeddingUnavailable) {
121
+ incomplete = true;
122
+ logCliInfo(logFilePath, "index", `\n${c.error("Indexing incomplete:")} the embedding provider was unavailable — nothing was stored. Fix the provider and run the index again.`);
123
+ }
124
+ else if (stats.totalChunks === 0 && stats.newFiles + stats.modifiedFiles > 0) {
125
+ incomplete = true;
126
+ logCliInfo(logFilePath, "index", `\n${c.warn("Indexing incomplete:")} files were processed but no chunks were stored. Check the log for details.`);
127
+ }
119
128
  logCliInfo(logFilePath, "index", `\n${c.success("Indexing complete.")} ${c.num(stats.finalCount)} chunks stored (${formatDuration(Date.now() - passStarted)}).`);
120
129
  }
121
130
  if (sigReceived && !watchTriggered) {
@@ -128,7 +137,7 @@ export function registerIndexCommand(program) {
128
137
  process.removeListener("SIGTERM", handleSigint);
129
138
  if (!options.watch) {
130
139
  await cleanupContext(ctx);
131
- process.exit(sigReceived ? 130 : 0);
140
+ process.exit(sigReceived ? 130 : incomplete ? 1 : 0);
132
141
  }
133
142
  // Only one watcher may run per workspace — a background auto-indexer
134
143
  // in an OpenCode session (or another `index --watch`) may already own
@@ -75,6 +75,20 @@ export function registerStatusCommand(program) {
75
75
  logCliInfo(logFilePath, "status", `${c.label("Store path:")} ${c.file(storePath)}`);
76
76
  logCliInfo(logFilePath, "status", `${c.label("Embedding provider:")} ${c.value(config.embedding.provider)}`);
77
77
  logCliInfo(logFilePath, "status", `${c.label("Embedding model:")} ${c.value(config.embedding.model)}`);
78
+ const storeDimension = await store.getVectorDimension?.();
79
+ const embedderDimension = config.embedding.vectorDimension;
80
+ if (embedderDimension && embedderDimension > 0) {
81
+ logCliInfo(logFilePath, "status", `${c.label("Embedder dim:")} ${c.num(embedderDimension)}`);
82
+ }
83
+ if (storeDimension !== undefined) {
84
+ logCliInfo(logFilePath, "status", `${c.label("Store dimension:")} ${c.num(storeDimension)}`);
85
+ }
86
+ if (storeDimension !== undefined &&
87
+ embedderDimension !== undefined &&
88
+ embedderDimension > 0 &&
89
+ storeDimension !== embedderDimension) {
90
+ logCliInfo(logFilePath, "status", `${c.label("Dimension mismatch:")} ${c.warn("yes")} — the store was built with a different embedding model. Run 'opencode-rag index' to rebuild.`);
91
+ }
78
92
  logCliInfo(logFilePath, "status", `${c.label("File extensions:")} ${config.indexing.includeExtensions.join(", ")}`);
79
93
  logCliInfo(logFilePath, "status", `${c.label("Excluded dirs:")} ${config.indexing.excludeDirs.join(", ")}`);
80
94
  logCliInfo(logFilePath, "status", `${c.label("Default top-K:")} ${c.num(config.retrieval.topK)}`);
@@ -92,7 +92,7 @@ export async function resolveCliContext(opt, logFilePath, bootstrapOpts) {
92
92
  configPath: opt.config,
93
93
  ...bootstrapOpts,
94
94
  });
95
- logCliInfo(logFilePath, "config", `${c.label("Config:")} ${c.file(ctx.logFilePath)}`);
95
+ logCliInfo(logFilePath, "config", `${c.label("Config:")} ${c.file(ctx.configPath ?? "(built-in defaults)")}`);
96
96
  logConfigDetails(logFilePath, ctx.config);
97
97
  return ctx;
98
98
  }
@@ -105,6 +105,9 @@ export async function resolveCliContext(opt, logFilePath, bootstrapOpts) {
105
105
  function logConfigDetails(logFilePath, config) {
106
106
  logCliInfo(logFilePath, "config", ` ${c.label("Embedding provider:")} ${c.value(config.embedding.provider)}`);
107
107
  logCliInfo(logFilePath, "config", ` ${c.label("Embedding model:")} ${c.value(config.embedding.model)}`);
108
+ if (config.embedding.vectorDimension && config.embedding.vectorDimension > 0) {
109
+ logCliInfo(logFilePath, "config", ` ${c.label("Embedding dim:")} ${c.num(config.embedding.vectorDimension)}`);
110
+ }
108
111
  logCliInfo(logFilePath, "config", ` ${c.label("Vector store:")} ${c.file(config.vectorStore.path)}`);
109
112
  }
110
113
  /**
@@ -153,7 +156,13 @@ export function logIndexSummary(logFilePath, stats) {
153
156
  if (stats.descriptionFailedFiles > 0) {
154
157
  logCliInfo(logFilePath, "index", ` ${c.label("Desc failed:")} ${c.num(stats.descriptionFailedFiles)}`);
155
158
  }
159
+ if (stats.embeddingFailures > 0) {
160
+ logCliInfo(logFilePath, "index", ` ${c.label("Embed failed:")} ${c.num(stats.embeddingFailures)} chunk(s) without vectors — retried next pass`);
161
+ }
156
162
  logCliInfo(logFilePath, "index", ` ${c.label("Chunks written:")} ${c.num(stats.totalChunks)}`);
163
+ if (stats.embeddingUnavailable) {
164
+ logCliInfo(logFilePath, "index", ` ${c.error("Embedding provider unavailable during this pass — no chunks were stored.")}`);
165
+ }
157
166
  }
158
167
  /**
159
168
  * Format a millisecond duration into a compact human-readable string.
@@ -35,6 +35,8 @@ export interface RagContext {
35
35
  dimension: number;
36
36
  /** Resolved path to the debug log file. */
37
37
  logFilePath: string;
38
+ /** Resolved path to the config file, or undefined when built-in defaults are used. */
39
+ configPath?: string;
38
40
  }
39
41
  /** Bootstrap the full RAG pipeline context: load config, resolve API keys, create embedder, vector store, keyword index, and description provider. */
40
42
  export declare function resolveRagContext(opts?: BootstrapOptions): Promise<RagContext>;
@@ -6,26 +6,46 @@ import path from "node:path";
6
6
  import { loadConfig, findConfigFile, resolveLogConfig, DEFAULT_CONFIG } from "./config.js";
7
7
  import { resolveApiKey } from "./resolve-api-key.js";
8
8
  import { loadChunkersFromConfig } from "../chunker/loader.js";
9
- import { createEmbedder } from "../embedder/factory.js";
9
+ import { createEmbedder, probeEmbeddingDimension } from "../embedder/factory.js";
10
10
  import { createDescriptionProvider } from "../describer/factory.js";
11
11
  import { createVectorStore } from "../vectorstore/factory.js";
12
+ import { readStoreDimension } from "../vectorstore/lancedb.js";
12
13
  import { KeywordIndex } from "../retriever/keyword-index.js";
13
- /** Probe the embedding provider to determine the vector dimension. Falls back to 384 on failure. */
14
- async function probeDimension(embedder) {
15
- try {
16
- const probe = await embedder.embed(["dimension-probe"], "query");
17
- if (probe && probe[0] && probe[0].length > 0 && typeof probe[0][0] === "number") {
18
- return probe[0].length;
19
- }
14
+ /**
15
+ * Resolve the embedding dimension for the store.
16
+ *
17
+ * Precedence: an explicit `embedding.vectorDimension` in the config wins (it
18
+ * is persisted after the first successful probe), then a live probe of the
19
+ * provider, then the dimension of an existing store table, then the 384
20
+ * fallback. The old behavior always probed and fell back to 384 — which
21
+ * silently created 384-dimensional stores whenever the provider was down,
22
+ * even though the config declared the real dimension.
23
+ *
24
+ * @param embedder - Configured embedding provider (probed only when needed).
25
+ * @param storePath - Vector store path, used to read an existing schema.
26
+ * @param skipProbe - When true, never call the provider (read-only commands).
27
+ * @returns The resolved dimension.
28
+ */
29
+ async function resolveDimension(embedder, storePath, skipProbe, configured) {
30
+ if (configured && configured > 0) {
31
+ return configured;
20
32
  }
21
- catch {
22
- // fallback to 384
33
+ if (!skipProbe) {
34
+ const probe = await probeEmbeddingDimension(embedder);
35
+ if (probe.dimension !== undefined) {
36
+ return probe.dimension;
37
+ }
38
+ const storeDimension = await readStoreDimension(storePath);
39
+ if (storeDimension !== undefined) {
40
+ console.warn(`[bootstrap] Could not probe embedding dimension (${probe.error?.message ?? "unknown error"}) — ` +
41
+ `using the existing store's dimension ${storeDimension}.`);
42
+ return storeDimension;
43
+ }
44
+ console.warn(`[bootstrap] Could not probe embedding dimension (${probe.error?.message ?? "unknown error"}) — ` +
45
+ "falling back to 384. Set embedding.vectorDimension explicitly if the provider produces a different size.");
46
+ return 384;
23
47
  }
24
- // A wrong dimension is only discovered later as cryptic LanceDB errors —
25
- // surface the fallback loudly so misconfigured providers are easy to spot.
26
- console.warn("[bootstrap] Could not probe embedding dimension — falling back to 384. " +
27
- "If indexing later fails with dimension errors, set embedding.vectorDimension explicitly.");
28
- return 384;
48
+ return (await readStoreDimension(storePath)) ?? 384;
29
49
  }
30
50
  /** Load the keyword index from disk, or create a new empty one if loading fails. */
31
51
  async function loadKeywordIndex(storePath) {
@@ -61,8 +81,16 @@ export async function resolveRagContext(opts = {}) {
61
81
  }
62
82
  const logFilePath = path.resolve(workDir, resolveLogConfig(cfg).logFilePath);
63
83
  const embedder = createEmbedder(cfg);
64
- const dimension = opts.skipProbe ? 384 : await probeDimension(embedder);
65
84
  const storePath = path.resolve(workDir, cfg.vectorStore.path);
85
+ const dimension = await resolveDimension(embedder, storePath, opts.skipProbe ?? false, cfg.embedding.vectorDimension);
86
+ // Warn (but do not block) when the existing store schema disagrees with the
87
+ // resolved dimension: the next index pass rebuilds automatically, and search
88
+ // callers would otherwise see only cryptic LanceDB errors.
89
+ const storeDimension = await readStoreDimension(storePath);
90
+ if (storeDimension !== undefined && storeDimension !== dimension) {
91
+ console.warn(`[bootstrap] Store was built with vector dimension ${storeDimension} but the current embedder produces ${dimension} — ` +
92
+ "run 'opencode-rag index' to rebuild the index with the current model.");
93
+ }
66
94
  const store = createVectorStore(cfg, storePath, dimension);
67
95
  const keywordIndex = opts.skipKeywordIndex
68
96
  ? new KeywordIndex(storePath)
@@ -80,6 +108,7 @@ export async function resolveRagContext(opts = {}) {
80
108
  descriptionProvider,
81
109
  dimension,
82
110
  logFilePath,
111
+ configPath,
83
112
  };
84
113
  }
85
114
  //# sourceMappingURL=bootstrap.js.map
@@ -296,6 +296,18 @@ export interface RagConfig {
296
296
  ollamaMaxBatchSize?: number;
297
297
  /** Maximum concurrent description generation requests. */
298
298
  descriptionConcurrency?: number;
299
+ /**
300
+ * Include LLM-generated chunk descriptions in the text that gets embedded.
301
+ *
302
+ * Descriptions help general-purpose embedding models align natural-language
303
+ * queries with raw code. Code-specialized models (e.g. jina-code-embeddings,
304
+ * trained for NL→code retrieval with instruction prefixes) do not need the
305
+ * crutch — and their passage prompt expects a code snippet, not prose.
306
+ * When false, descriptions are still generated and stored (search results,
307
+ * web UI), but only path/meta header/content are embedded.
308
+ * @default true
309
+ */
310
+ embedDescriptions?: boolean;
299
311
  /**
300
312
  * Maximum file size in bytes for SVG/XML files before chunking is skipped.
301
313
  * Large SVGs can cause tree-sitter to hang. Set to 0 for no limit.
@@ -117,6 +117,7 @@ export const DEFAULT_CONFIG = {
117
117
  embedConcurrency: 3,
118
118
  ollamaMaxBatchSize: 100,
119
119
  descriptionConcurrency: 4,
120
+ embedDescriptions: true,
120
121
  maxSvgSizeBytes: 1_048_576,
121
122
  optimizeIntervalWindows: 8,
122
123
  },
@@ -224,6 +224,14 @@ export interface VectorStore {
224
224
  * Used by the index pipeline to detect silent corruption before scanning.
225
225
  */
226
226
  checkIntegrity?(): Promise<boolean>;
227
+ /**
228
+ * Return the actual embedding dimension of the stored vector column, or
229
+ * `undefined` when the store has no table yet. Used to detect a store that
230
+ * was built with a different embedding model than the currently configured
231
+ * one — a mismatch makes every vector search fail and silently corrupts
232
+ * writes, so the pipeline rebuilds the index instead.
233
+ */
234
+ getVectorDimension?(): Promise<number | undefined>;
227
235
  }
228
236
  /** Filter criteria for narrowing search results by file path, language, kind, or extension. */
229
237
  export interface MetadataFilter {
@@ -15,6 +15,26 @@ import type { RagConfig } from "../core/config.js";
15
15
  * @throws If the provider is unsupported or a required apiKey is not set
16
16
  */
17
17
  export declare function createEmbedder(config: RagConfig): EmbeddingProvider;
18
+ /** Result of probing an embedding provider for its output dimension. */
19
+ export interface EmbeddingProbeResult {
20
+ /** Detected vector dimension, or `undefined` when the probe failed. */
21
+ dimension?: number;
22
+ /** Error raised by the provider (or an empty response), when the probe failed. */
23
+ error?: Error;
24
+ }
25
+ /**
26
+ * Probe an embedding provider with a single short text to discover the
27
+ * dimension of the vectors it produces.
28
+ *
29
+ * Used as a health/dimension preflight before expensive index work: a provider
30
+ * that is down (server not running, model not pulled) fails here in
31
+ * milliseconds instead of after chunking and describing every file. Errors are
32
+ * captured in the result rather than thrown.
33
+ *
34
+ * @param embedder - The embedding provider to probe.
35
+ * @returns The detected dimension, or the failure reason.
36
+ */
37
+ export declare function probeEmbeddingDimension(embedder: EmbeddingProvider): Promise<EmbeddingProbeResult>;
18
38
  /**
19
39
  * Embed a list of texts in batches with optional concurrency control and per-batch retry.
20
40
  *
@@ -40,6 +40,30 @@ export function createEmbedder(config) {
40
40
  }
41
41
  throw new Error(`Unknown embedding provider: ${provider}`);
42
42
  }
43
+ /**
44
+ * Probe an embedding provider with a single short text to discover the
45
+ * dimension of the vectors it produces.
46
+ *
47
+ * Used as a health/dimension preflight before expensive index work: a provider
48
+ * that is down (server not running, model not pulled) fails here in
49
+ * milliseconds instead of after chunking and describing every file. Errors are
50
+ * captured in the result rather than thrown.
51
+ *
52
+ * @param embedder - The embedding provider to probe.
53
+ * @returns The detected dimension, or the failure reason.
54
+ */
55
+ export async function probeEmbeddingDimension(embedder) {
56
+ try {
57
+ const probe = await embedder.embed(["dimension-probe"], "query");
58
+ if (probe && probe[0] && probe[0].length > 0 && typeof probe[0][0] === "number") {
59
+ return { dimension: probe[0].length };
60
+ }
61
+ return { error: new Error("Embedding provider returned an empty probe vector") };
62
+ }
63
+ catch (err) {
64
+ return { error: err instanceof Error ? err : new Error(String(err)) };
65
+ }
66
+ }
43
67
  /**
44
68
  * HTTP statuses that indicate a permanent failure — retrying cannot help
45
69
  * (auth errors, bad requests, missing resources). Providers raise these as
@@ -7,7 +7,7 @@ import pLimit from "p-limit";
7
7
  import { scanWorkspaceFiles } from "../content/reader.js";
8
8
  import { loadManifest, saveManifest, normalizeFilePath, computeDescriptionConfigHash } from "../core/manifest.js";
9
9
  import { DescriptionCache } from "../core/desc-cache.js";
10
- import { embedBatch } from "../embedder/factory.js";
10
+ import { embedBatch, probeEmbeddingDimension } from "../embedder/factory.js";
11
11
  import { createVectorStore } from "../vectorstore/factory.js";
12
12
  import { swapStoreDirectories } from "../vectorstore/lancedb.js";
13
13
  import { createIndexStats } from "./stats.js";
@@ -157,46 +157,12 @@ async function runIndexPassInner(options, logger) {
157
157
  // file content IS different — these would be caught by hash comparison below but
158
158
  // this pre-clear ensures the description cache is consulted during re-description.
159
159
  // (Files with same hash but different descHash already fall through in prepareFile.)
160
- let filterPaths;
161
- let gitDeletedPaths = [];
162
- if (!options.force && manifestStatus === "ok" && manifest.lastGitCommit) {
163
- const repoRoot = getRepoRoot(options.cwd);
164
- if (repoRoot) {
165
- const diffResult = getChangedFilesSince(options.cwd, manifest.lastGitCommit);
166
- if (diffResult) {
167
- const untracked = getUntrackedFiles(options.cwd);
168
- const changedSet = new Set();
169
- for (const f of diffResult.changedFiles)
170
- changedSet.add(f);
171
- for (const f of untracked)
172
- changedSet.add(f);
173
- // Git paths are relative to the REPO ROOT. When the workspace is a
174
- // subdirectory of the repo (monorepos), convert them to workspace-
175
- // relative paths before passing them to the scan — resolving repo-root
176
- // paths against `cwd` would silently miss every change.
177
- const toCwdRelative = (p) => {
178
- const abs = path.resolve(repoRoot, p);
179
- const rel = path.relative(options.cwd, abs);
180
- if (rel.startsWith("..") || path.isAbsolute(rel))
181
- return null;
182
- return rel;
183
- };
184
- const cwdRelative = [];
185
- for (const p of changedSet) {
186
- const rel = toCwdRelative(p);
187
- if (rel !== null)
188
- cwdRelative.push(rel);
189
- }
190
- filterPaths = cwdRelative;
191
- gitDeletedPaths = diffResult.deletedFiles;
192
- logger.debug(`Git incremental: ${filterPaths.length} changed/untracked, ${gitDeletedPaths.length} deleted since ${manifest.lastGitCommit.slice(0, 8)}`);
193
- }
194
- }
195
- }
196
- const scanStart = Date.now();
197
- const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger, options.force ? undefined : manifest, filterPaths, options.imageVisionProvider, descCache);
198
- const scanSec = ((Date.now() - scanStart) / 1000).toFixed(1);
199
- logger.info(`Workspace scan complete: ${workspaceFiles.length} files in ${scanSec}s`);
160
+ // ── Store health checks (run BEFORE the scan) ──────────────────────────
161
+ // The checks below can invalidate the manifest and trigger a full rebuild.
162
+ // They must run before scanWorkspaceFiles: the reader skips reading the
163
+ // contents of unchanged files when a manifest entry matches (mtime + size +
164
+ // descHash unchanged), so clearing the manifest after the scan would leave
165
+ // those files with empty content and turn them into bogus "removed" entries.
200
166
  logger.info(`Querying vector store for existing chunk count...`);
201
167
  const storeCountStart = Date.now();
202
168
  const existingCount = await options.store.count();
@@ -236,15 +202,44 @@ async function runIndexPassInner(options, logger) {
236
202
  manifestStatus = "missing";
237
203
  }
238
204
  }
205
+ // ── Detect a store built by a different embedding model ────────────────
206
+ // LanceDB silently zero-pads/truncates mismatched vectors on write, and
207
+ // every vector query then fails with "No vector column found to match with
208
+ // the query vector dimension". Treat a schema dimension mismatch like a
209
+ // corrupt store: clear the manifest so this pass rebuilds everything into a
210
+ // fresh store at the configured dimension.
211
+ let dimensionMismatch = false;
212
+ try {
213
+ const storeDimension = await options.store.getVectorDimension?.();
214
+ if (!options.force &&
215
+ storeDimension !== undefined &&
216
+ options.dimension !== undefined &&
217
+ storeDimension !== options.dimension) {
218
+ dimensionMismatch = true;
219
+ logger.warn(`Store was built with vector dimension ${storeDimension} but the configured embedding model produces ` +
220
+ `${options.dimension} — rebuilding the full index with the current model.`);
221
+ if (manifestStatus === "ok") {
222
+ for (const key of Object.keys(manifest.files))
223
+ delete manifest.files[key];
224
+ manifest.lastIndexedAt = undefined;
225
+ manifestStatus = "missing";
226
+ }
227
+ }
228
+ }
229
+ catch {
230
+ // getVectorDimension is best-effort — proceed without the drift check.
231
+ }
239
232
  // Effective store used throughout the pass — may be a temp store for atomic rebuild.
240
233
  let effectiveStore = options.store;
241
234
  let tempStorePath;
242
- if (options.force || (manifestStatus !== "ok" && existingCount > 0)) {
235
+ /** Set when the temporary rebuild store actually received rows. */
236
+ let tempStoreWroteChunks = false;
237
+ if (options.force || (manifestStatus !== "ok" && (existingCount > 0 || dimensionMismatch))) {
243
238
  for (const key of Object.keys(manifest.files)) {
244
239
  delete manifest.files[key];
245
240
  }
246
241
  manifest.lastIndexedAt = undefined;
247
- rebuildPerformed = existingCount > 0 || !!options.force;
242
+ rebuildPerformed = existingCount > 0 || !!options.force || dimensionMismatch;
248
243
  if (manifestStatus !== "ok" && existingCount > 0) {
249
244
  logger.warn("Manifest missing or corrupt; rebuilding full index.");
250
245
  }
@@ -271,7 +266,7 @@ async function runIndexPassInner(options, logger) {
271
266
  logger.warn("Cannot rebuild safely without embedding dimension — aborting to protect existing data. " +
272
267
  "Run 'opencode-rag index --force' manually to rebuild.");
273
268
  // Restore manifest entries we just deleted so the next pass can retry incrementally
274
- return createIndexStats(workspaceFiles.length, manifestStatus);
269
+ return createIndexStats(0, manifestStatus);
275
270
  }
276
271
  else {
277
272
  // No existing data — safe to proceed with in-place indexing (no clear needed)
@@ -279,8 +274,76 @@ async function runIndexPassInner(options, logger) {
279
274
  logger.debug("No existing data; indexing from scratch.");
280
275
  }
281
276
  }
277
+ let filterPaths;
278
+ let gitDeletedPaths = [];
279
+ if (!options.force && manifestStatus === "ok" && manifest.lastGitCommit) {
280
+ const repoRoot = getRepoRoot(options.cwd);
281
+ if (repoRoot) {
282
+ const diffResult = getChangedFilesSince(options.cwd, manifest.lastGitCommit);
283
+ if (diffResult) {
284
+ const untracked = getUntrackedFiles(options.cwd);
285
+ const changedSet = new Set();
286
+ for (const f of diffResult.changedFiles)
287
+ changedSet.add(f);
288
+ for (const f of untracked)
289
+ changedSet.add(f);
290
+ // Git paths are relative to the REPO ROOT. When the workspace is a
291
+ // subdirectory of the repo (monorepos), convert them to workspace-
292
+ // relative paths before passing them to the scan — resolving repo-root
293
+ // paths against `cwd` would silently miss every change.
294
+ const toCwdRelative = (p) => {
295
+ const abs = path.resolve(repoRoot, p);
296
+ const rel = path.relative(options.cwd, abs);
297
+ if (rel.startsWith("..") || path.isAbsolute(rel))
298
+ return null;
299
+ return rel;
300
+ };
301
+ const cwdRelative = [];
302
+ for (const p of changedSet) {
303
+ const rel = toCwdRelative(p);
304
+ if (rel !== null)
305
+ cwdRelative.push(rel);
306
+ }
307
+ filterPaths = cwdRelative;
308
+ gitDeletedPaths = diffResult.deletedFiles;
309
+ logger.debug(`Git incremental: ${filterPaths.length} changed/untracked, ${gitDeletedPaths.length} deleted since ${manifest.lastGitCommit.slice(0, 8)}`);
310
+ }
311
+ }
312
+ }
313
+ const scanStart = Date.now();
314
+ const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger, options.force ? undefined : manifest, filterPaths, options.imageVisionProvider, descCache);
315
+ const scanSec = ((Date.now() - scanStart) / 1000).toFixed(1);
316
+ logger.info(`Workspace scan complete: ${workspaceFiles.length} files in ${scanSec}s`);
317
+ // ── Preflight: verify the embedding provider before expensive work ─────
318
+ // A provider outage used to burn the entire pass (chunking + describing
319
+ // every file) and then store nothing, reporting success. Probe once here
320
+ // so the pass aborts in milliseconds with an actionable message.
321
+ const pendingEmbeddingWork = options.force || workspaceFiles.some((f) => {
322
+ if (f.isEmpty || f.isTooSmall)
323
+ return false;
324
+ const previous = manifest.files[f.normalizedPath];
325
+ return !previous || previous.hash !== f.hash;
326
+ });
327
+ if (pendingEmbeddingWork && !(options.abortSignal?.aborted ?? false)) {
328
+ const probe = await probeEmbeddingDimension(options.embedder);
329
+ if (probe.dimension === undefined) {
330
+ logger.warn(`Embedding provider unavailable (${probe.error?.message ?? "probe failed"}) — ` +
331
+ "aborting index pass before chunking/description; nothing was stored.");
332
+ const failedStats = createIndexStats(workspaceFiles.length, manifestStatus);
333
+ failedStats.embeddingUnavailable = true;
334
+ return failedStats;
335
+ }
336
+ if (options.dimension !== undefined && probe.dimension !== options.dimension) {
337
+ logger.warn(`Embedding provider produces ${probe.dimension}-dimensional vectors but this index is configured for ` +
338
+ `${options.dimension} — aborting to avoid writing incompatible vectors. ` +
339
+ "Set embedding.vectorDimension to the provider's dimension and run the index again.");
340
+ const failedStats = createIndexStats(workspaceFiles.length, manifestStatus);
341
+ failedStats.embeddingUnavailable = true;
342
+ return failedStats;
343
+ }
344
+ }
282
345
  const stats = createIndexStats(workspaceFiles.length, manifestStatus);
283
- stats.rebuildPerformed = rebuildPerformed;
346
+ stats.rebuildPerformed = rebuildPerformed || dimensionMismatch;
284
347
  for (const file of workspaceFiles) {
285
348
  if (file.extractionStatus === "failed" && file.extractionError) {
286
349
  stats.extractionFailures++;
@@ -416,6 +479,8 @@ async function runIndexPassInner(options, logger) {
416
479
  if (!allEmbedded && (prep.chunks?.length ?? 0) > 0) {
417
480
  options.logger?.warn?.(` ${prep.fileLabel}: ${validChunks.length}/${prep.chunks?.length} chunks embedded — marking for retry on next pass`);
418
481
  }
482
+ if (validChunks.length > 0)
483
+ tempStoreWroteChunks = true;
419
484
  storePayloads.push({ prep, validChunks, allEmbedded });
420
485
  }
421
486
  // One bulk write per window. Dedup (per-modified-file deletes) is skipped
@@ -738,7 +803,7 @@ async function runIndexPassInner(options, logger) {
738
803
  }
739
804
  }
740
805
  for (const prep of deferredPreps) {
741
- prep.textToEmbed = buildTextsToEmbed(prep.chunks, prep.relPath, prep.metaHeader ?? "", prep.docPrefix ?? "", prep.isImageFile ?? false);
806
+ prep.textToEmbed = buildTextsToEmbed(prep.chunks, prep.relPath, prep.metaHeader ?? "", prep.docPrefix ?? "", prep.isImageFile ?? false, options.config.indexing.embedDescriptions !== false);
742
807
  }
743
808
  const totalDescribedChunks = deferredPreps.reduce((s, p) => s + (p.chunks?.length ?? 0), 0);
744
809
  logger.info(`Description phase complete: ${totalDescribedChunks} chunks across ${deferredPreps.length} files`);
@@ -801,6 +866,8 @@ async function runIndexPassInner(options, logger) {
801
866
  }
802
867
  catch (err) {
803
868
  logger.warn(` Global embedding failed: ${err.message}`);
869
+ stats.embeddingFailures += allTexts.length;
870
+ stats.embeddingUnavailable = true;
804
871
  for (const { fileIdx } of embedQueue) {
805
872
  options.progress?.failFile(prepared[fileIdx].fileLabel);
806
873
  earlyWorkerResults.set(fileIdx, {
@@ -814,6 +881,21 @@ async function runIndexPassInner(options, logger) {
814
881
  }
815
882
  embedQueue.length = 0; // prevent double-processing in store phase
816
883
  }
884
+ // `embedBatch` returns empty vectors for batches whose retries were
885
+ // exhausted — count those so the summary and exit code report a failed
886
+ // pass instead of a successful "0 chunks stored".
887
+ const failedVectors = allEmbeddings.filter((v) => !Array.isArray(v) || v.length === 0).length;
888
+ if (failedVectors > 0) {
889
+ stats.embeddingFailures += failedVectors;
890
+ if (failedVectors >= allTexts.length) {
891
+ stats.embeddingUnavailable = true;
892
+ logger.warn(`All ${allTexts.length} embedding requests failed — no vectors were produced, so nothing was stored. ` +
893
+ "Check that the embedding provider is running and the model is available, then run the index again.");
894
+ }
895
+ else {
896
+ logger.warn(` ${failedVectors}/${allTexts.length} embedding requests failed — those chunks are kept for retry on the next pass.`);
897
+ }
898
+ }
817
899
  }
818
900
  // ── Distribute embeddings back to per-file chunks ─────────────────────
819
901
  for (let i = 0; i < embedQueue.length; i++) {
@@ -875,25 +957,43 @@ async function runIndexPassInner(options, logger) {
875
957
  tryUpdateLastGitCommit(options.cwd, manifest);
876
958
  }
877
959
  // ── Atomically promote temp store if a full rebuild was performed ──
960
+ let tempStorePromoted = false;
878
961
  if (tempStorePath) {
879
962
  if (!aborted()) {
880
- try {
881
- await effectiveStore.close();
882
- await options.store.close();
883
- // Swap the newly-built temp directory into the real path
884
- await swapStoreDirectories(tempStorePath, options.storePath);
885
- // Re-open the original store handle so callers can search the new data
886
- await options.store.reopen?.(options.storePath);
887
- logger.debug(`Promoted temporary store ${tempStorePath} → ${options.storePath}`);
888
- }
889
- catch (err) {
890
- logger.warn(`Could not promote temporary store: ${err.message}. ` +
891
- `Original data preserved at ${options.storePath}`);
963
+ if (!tempStoreWroteChunks) {
964
+ // Nothing was written (e.g. every embedding request failed). Keep the
965
+ // existing store and manifest untouched rather than promoting an
966
+ // empty store over good data.
967
+ logger.warn("Rebuild produced no chunks — keeping the existing index and manifest unchanged.");
968
+ try {
969
+ await effectiveStore.close();
970
+ }
971
+ catch { }
892
972
  try {
893
973
  await fs.rm(tempStorePath, { recursive: true, force: true });
894
974
  }
895
975
  catch { }
896
976
  }
977
+ else {
978
+ try {
979
+ await effectiveStore.close();
980
+ await options.store.close();
981
+ // Swap the newly-built temp directory into the real path
982
+ await swapStoreDirectories(tempStorePath, options.storePath);
983
+ // Re-open the original store handle so callers can search the new data
984
+ await options.store.reopen?.(options.storePath);
985
+ tempStorePromoted = true;
986
+ logger.debug(`Promoted temporary store ${tempStorePath} → ${options.storePath}`);
987
+ }
988
+ catch (err) {
989
+ logger.warn(`Could not promote temporary store: ${err.message}. ` +
990
+ `Original data preserved at ${options.storePath}`);
991
+ try {
992
+ await fs.rm(tempStorePath, { recursive: true, force: true });
993
+ }
994
+ catch { }
995
+ }
996
+ }
897
997
  }
898
998
  else {
899
999
  // Aborted — discard temp, keep original data intact.
@@ -912,16 +1012,27 @@ async function runIndexPassInner(options, logger) {
912
1012
  return stats;
913
1013
  }
914
1014
  }
915
- // Save manifest and keyword index (always to the real store path — after
916
- // a successful swap this points to the new data; after an abort it's the old).
917
- await saveManifest(options.storePath, manifest);
918
- await options.keywordIndex?.save(options.storePath);
1015
+ // The in-memory manifest was cleared at rebuild start and only repopulated
1016
+ // for the new store. When the rebuild did not promote (empty temp store or
1017
+ // failed swap), keep the previous manifest/keyword index on disk — saving
1018
+ // the cleared state would orphan the existing store's data.
1019
+ const keepPreviousIndexState = tempStorePath !== undefined && !tempStorePromoted;
1020
+ if (keepPreviousIndexState) {
1021
+ logger.warn("Rebuild did not complete — keeping the previous manifest and keyword index.");
1022
+ }
1023
+ else {
1024
+ // Save manifest and keyword index (always to the real store path — after
1025
+ // a successful swap this points to the new data).
1026
+ await saveManifest(options.storePath, manifest);
1027
+ await options.keywordIndex?.save(options.storePath);
1028
+ }
919
1029
  // Compact fragments and prune old version manifests so countRows() can't
920
1030
  // hang on accumulated versions from many add/delete cycles. After a temp
921
1031
  // store rebuild, optimize the reopened real store handle (the temp handle
922
1032
  // was closed and its directory moved); use aggressive pruning since the
923
- // swapped-in store is private to this process.
924
- if (!aborted()) {
1033
+ // swapped-in store is private to this process. Skipped when the rebuild did
1034
+ // not promote (the old store is untouched and its handle may be closed).
1035
+ if (!aborted() && !keepPreviousIndexState) {
925
1036
  logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
926
1037
  const optimizeStart = Date.now();
927
1038
  try {
@@ -38,6 +38,10 @@ export interface IndexRunStats {
38
38
  }>;
39
39
  /** Number of files where description generation failed. */
40
40
  descriptionFailedFiles: number;
41
+ /** Number of chunks whose embedding request failed (kept for retry on the next pass). */
42
+ embeddingFailures: number;
43
+ /** True when the pass could not embed at all (provider unavailable or dimension mismatch) — nothing was stored. */
44
+ embeddingUnavailable: boolean;
41
45
  /** True when the pass was skipped because another pass holds the lock. */
42
46
  skipped: boolean;
43
47
  }
@@ -24,6 +24,8 @@ export function createIndexStats(totalFiles, manifestStatus) {
24
24
  extractionFailures: 0,
25
25
  extractionErrors: [],
26
26
  descriptionFailedFiles: 0,
27
+ embeddingFailures: 0,
28
+ embeddingUnavailable: false,
27
29
  skipped: false,
28
30
  };
29
31
  }
@@ -61,16 +61,21 @@ export interface PreparedFile {
61
61
  /**
62
62
  * Build the list of text strings that will be sent to the embedding provider.
63
63
  * Each chunk is prefixed with the document prefix, relative path, metadata
64
- * header, and (if available) a description.
64
+ * header, and (if available and enabled) a description.
65
+ *
66
+ * Code-specialized embedding models do not need the description crutch: with
67
+ * `indexing.embedDescriptions: false` the prose description is omitted from
68
+ * the embedded text (it is still stored on the chunk for display).
65
69
  *
66
70
  * @param chunks - Chunks to build embedding texts from.
67
71
  * @param relPath - Relative file path used as context prefix.
68
72
  * @param metaHeader - Assembled metadata header (file type, directory, etc.).
69
73
  * @param docPrefix - Optional document-level prefix from configuration.
70
74
  * @param isImage - Whether the source is an image (uses description only).
75
+ * @param includeDescription - Whether to include non-image descriptions (default true).
71
76
  * @returns An array of formatted text strings, one per chunk.
72
77
  */
73
- export declare function buildTextsToEmbed(chunks: Chunk[], relPath: string, metaHeader: string, docPrefix: string, isImage: boolean): string[];
78
+ export declare function buildTextsToEmbed(chunks: Chunk[], relPath: string, metaHeader: string, docPrefix: string, isImage: boolean, includeDescription?: boolean): string[];
74
79
  interface Logger {
75
80
  info(message: string): void;
76
81
  warn(message: string): void;
@@ -115,6 +120,7 @@ export declare function prepareFile(file: WorkspaceFile, cwd: string, previous:
115
120
  };
116
121
  indexing?: {
117
122
  maxSvgSizeBytes?: number;
123
+ embedDescriptions?: boolean;
118
124
  };
119
125
  }, keywordIndex: KeywordIndex | undefined, descriptionProvider: DescriptionProvider | undefined, logger: Logger, deferDescriptions?: boolean, descHash?: string): Promise<PreparedFile>;
120
126
  /**
@@ -10,23 +10,28 @@ import { generateDescriptions, buildFallbackDescription } from "./description-st
10
10
  /**
11
11
  * Build the list of text strings that will be sent to the embedding provider.
12
12
  * Each chunk is prefixed with the document prefix, relative path, metadata
13
- * header, and (if available) a description.
13
+ * header, and (if available and enabled) a description.
14
+ *
15
+ * Code-specialized embedding models do not need the description crutch: with
16
+ * `indexing.embedDescriptions: false` the prose description is omitted from
17
+ * the embedded text (it is still stored on the chunk for display).
14
18
  *
15
19
  * @param chunks - Chunks to build embedding texts from.
16
20
  * @param relPath - Relative file path used as context prefix.
17
21
  * @param metaHeader - Assembled metadata header (file type, directory, etc.).
18
22
  * @param docPrefix - Optional document-level prefix from configuration.
19
23
  * @param isImage - Whether the source is an image (uses description only).
24
+ * @param includeDescription - Whether to include non-image descriptions (default true).
20
25
  * @returns An array of formatted text strings, one per chunk.
21
26
  */
22
- export function buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage) {
27
+ export function buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage, includeDescription = true) {
23
28
  const textToEmbed = [];
24
29
  for (const chunk of chunks) {
25
30
  if (isImage) {
26
31
  textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description);
27
32
  }
28
33
  else {
29
- const desc = chunk.description ?? "";
34
+ const desc = includeDescription ? chunk.description ?? "" : "";
30
35
  if (desc.trim().length > 0) {
31
36
  textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + desc + "\n\n" + chunk.content);
32
37
  }
@@ -210,7 +215,7 @@ export async function prepareFile(file, cwd, previous, config, keywordIndex, des
210
215
  }
211
216
  }
212
217
  }
213
- const textToEmbed = buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage);
218
+ const textToEmbed = buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage, config.indexing?.embedDescriptions !== false);
214
219
  logger.debug(` ${fileLabel}: textToEmbed ${textToEmbed.length} entries (descProvider: ${descriptionProvider ? "yes" : "no"})`);
215
220
  return {
216
221
  normalizedPath: file.normalizedPath,
package/dist/plugin.js CHANGED
@@ -7,7 +7,8 @@ import { tool } from "@opencode-ai/plugin/tool";
7
7
  import { CODE_SEARCH_FILTER } from "./core/interfaces.js";
8
8
  import { normalizeFileExtensions } from "./core/filters.js";
9
9
  import { loadConfig, findConfigFile, DEFAULT_CONFIG, resolveLogConfig, persistProbedDimension } from "./core/config.js";
10
- import { createEmbedder } from "./embedder/factory.js";
10
+ import { createEmbedder, probeEmbeddingDimension } from "./embedder/factory.js";
11
+ import { readStoreDimension } from "./vectorstore/lancedb.js";
11
12
  import { createDescriptionProvider } from "./describer/factory.js";
12
13
  import { createVectorStore } from "./vectorstore/factory.js";
13
14
  import { retrieve } from "./retriever/retriever.js";
@@ -419,7 +420,8 @@ export function createRagHooks(options) {
419
420
  ...options.dependencies,
420
421
  };
421
422
  const embedder = options.embedder ?? dependencies.createEmbedder(options.cfg);
422
- const store = options.store ?? dependencies.createStore(options.storePath, 384, options.cfg);
423
+ const configuredDimension = options.cfg.embedding.vectorDimension;
424
+ const store = options.store ?? dependencies.createStore(options.storePath, configuredDimension && configuredDimension > 0 ? configuredDimension : 384, options.cfg);
423
425
  const keywordIndex = options.keywordIndex;
424
426
  // Runtime overrides for live config editing from TUI
425
427
  let cachedOverrides = loadRuntimeOverrides(options.storePath);
@@ -1394,6 +1396,8 @@ export const ragPlugin = async (input, _options) => {
1394
1396
  }, logLevel);
1395
1397
  // Use cached dimension from config if available (avoids blocking startup with an API call)
1396
1398
  // If not set, probe the embedding provider once and persist the result.
1399
+ // When the probe fails, prefer the existing store's schema over the 384
1400
+ // default — a transient outage must not downgrade the store's dimension.
1397
1401
  const embedder = createEmbedder(effectiveCfg);
1398
1402
  let vectorDimension = effectiveCfg.embedding.vectorDimension;
1399
1403
  if (vectorDimension && vectorDimension > 0) {
@@ -1403,34 +1407,47 @@ export const ragPlugin = async (input, _options) => {
1403
1407
  }, logLevel);
1404
1408
  }
1405
1409
  else {
1406
- vectorDimension = 384;
1407
- try {
1408
- const probe = await embedder.embed(["dimension-probe"], "query");
1409
- if (probe && probe[0] && probe[0].length > 0 && typeof probe[0][0] === "number") {
1410
- vectorDimension = probe[0].length;
1411
- const configPath = findConfigFile(input.directory);
1412
- if (configPath) {
1413
- try {
1414
- persistProbedDimension(configPath, vectorDimension);
1415
- }
1416
- catch { /* best-effort */ }
1410
+ const probe = await probeEmbeddingDimension(embedder);
1411
+ if (probe.dimension !== undefined) {
1412
+ vectorDimension = probe.dimension;
1413
+ const configPath = findConfigFile(input.directory);
1414
+ if (configPath) {
1415
+ try {
1416
+ persistProbedDimension(configPath, vectorDimension);
1417
1417
  }
1418
+ catch { /* best-effort */ }
1418
1419
  }
1419
1420
  appendDebugLog(logFilePath, {
1420
1421
  scope: "plugin",
1421
1422
  message: `Vector dimension: ${vectorDimension}`,
1422
1423
  }, logLevel);
1423
1424
  }
1424
- catch (err) {
1425
+ else {
1426
+ vectorDimension = (await readStoreDimension(storePath)) ?? 384;
1425
1427
  appendDebugLog(logFilePath, {
1426
1428
  scope: "plugin",
1427
1429
  message: `Dimension probe failed, falling back to ${vectorDimension}`,
1428
- error: err,
1430
+ error: probe.error,
1429
1431
  }, logLevel);
1430
1432
  }
1431
1433
  }
1432
1434
  const store = createVectorStore(effectiveCfg, storePath, vectorDimension);
1433
1435
  ragStores.set(input.directory, store);
1436
+ // Warn when the store was built by a different embedding model — vector
1437
+ // search fails (or silently degrades) until the index is rebuilt.
1438
+ try {
1439
+ const storeDimension = await store.getVectorDimension?.();
1440
+ if (storeDimension !== undefined && storeDimension !== vectorDimension) {
1441
+ appendDebugLog(logFilePath, {
1442
+ scope: "plugin",
1443
+ message: `Store vector dimension is ${storeDimension} but the configured embedder produces ${vectorDimension} — ` +
1444
+ "run 'opencode-rag index' to rebuild the index with the current model.",
1445
+ }, logLevel);
1446
+ }
1447
+ }
1448
+ catch {
1449
+ // best-effort — dimension introspection must never block plugin startup
1450
+ }
1434
1451
  // Load or create keyword index for hybrid search
1435
1452
  const keywordIndex = await loadKeywordIndex(storePath, logFilePath, logLevel);
1436
1453
  // Create description provider (enabled by default)
@@ -53,10 +53,45 @@ export declare function isCorruptionError(err: unknown): boolean;
53
53
  * @returns True if the error matches a transient transaction conflict.
54
54
  */
55
55
  export declare function isTransientConflictError(err: unknown): boolean;
56
+ /**
57
+ * Thrown when a write supplies embeddings whose length differs from the
58
+ * store's vector column dimension. LanceDB itself does NOT validate this —
59
+ * it silently zero-pads or truncates vectors into the fixed-size column,
60
+ * producing rows that can never be found by a query. Failing loudly here
61
+ * keeps that silent corruption out of the store.
62
+ */
63
+ export declare class DimensionMismatchError extends Error {
64
+ /** Dimension of the store's vector column. */
65
+ readonly storeDimension: number;
66
+ /** Dimension of the embedding that was supplied. */
67
+ readonly embeddingDimension: number;
68
+ constructor(storeDimension: number, embeddingDimension: number, action?: string);
69
+ }
70
+ /** Type guard for {@link DimensionMismatchError}. */
71
+ export declare function isDimensionMismatchError(err: unknown): err is DimensionMismatchError;
72
+ /**
73
+ * Extract the fixed-size vector dimension of the `embedding` column from a
74
+ * LanceDB/Arrow schema. Returns `undefined` when the column is missing or is
75
+ * not a fixed-size list (e.g. before the table is created).
76
+ */
77
+ export declare function extractEmbeddingDimension(fields: ReadonlyArray<{
78
+ name: string;
79
+ type: unknown;
80
+ }>): number | undefined;
81
+ /**
82
+ * Read the embedding dimension of an existing store without creating a table
83
+ * or holding a long-lived connection. Returns `undefined` when the store or
84
+ * table does not exist (or cannot be read).
85
+ *
86
+ * Used by the CLI bootstrap to avoid creating a brand-new store with a
87
+ * speculative dimension when the embedding provider cannot be probed.
88
+ */
89
+ export declare function readStoreDimension(storePath: string): Promise<number | undefined>;
56
90
  /**
57
91
  * Atomically replace one LanceDB store directory with another.
58
92
  * Swaps the real directory with a temporary one that was built during a rebuild.
59
- * The old directory is moved to `${realPath}_old` and deleted asynchronously.
93
+ * The old directory is moved to `${realPath}_old` and deleted asynchronously
94
+ * after non-Lance artifacts (quirk memory, caches) have been carried over.
60
95
  *
61
96
  * @param tempPath - Path to the newly built store (source).
62
97
  * @param realPath - Path to the current store (destination, will be replaced).
@@ -94,6 +129,13 @@ export declare class LanceDbStore implements VectorStore {
94
129
  private indexRepairPromise;
95
130
  /** Consecutive failed repair attempts — bounded so a broken store cannot retrain forever. */
96
131
  private indexRepairFailures;
132
+ /**
133
+ * Actual dimension of the `embedding` column in the opened table. Read from
134
+ * the schema on first table access and used to validate writes/searches —
135
+ * the constructor's `vectorDimension` describes what the caller *expects*,
136
+ * which can drift from the store when the embedding model changes.
137
+ */
138
+ private knownDimension;
97
139
  /**
98
140
  * Execute an async function under an exclusive write lock.
99
141
  *
@@ -120,6 +162,23 @@ export declare class LanceDbStore implements VectorStore {
120
162
  private getTable;
121
163
  private initTable;
122
164
  private tableHasDescriptionColumn;
165
+ /**
166
+ * Read and cache the actual fixed-size dimension of the `embedding` column.
167
+ * Called whenever the table is (re)opened so write/search validation uses
168
+ * the store's real schema instead of the constructor's expectation.
169
+ */
170
+ private cacheTableDimension;
171
+ /**
172
+ * Return the actual embedding dimension of the store's vector column, or
173
+ * `undefined` when no table exists yet. Never creates the table.
174
+ */
175
+ getVectorDimension(): Promise<number | undefined>;
176
+ /**
177
+ * Fail loudly when rows carry embeddings whose length differs from the
178
+ * store's vector column. LanceDB would silently pad/truncate them instead,
179
+ * leaving rows that no query can match.
180
+ */
181
+ private assertEmbeddingDimensions;
123
182
  private hasColumn;
124
183
  /** Add kind/quirkType/tags columns if missing from an existing table. */
125
184
  private migrateNewColumns;
@@ -148,10 +148,129 @@ export function isTransientConflictError(err) {
148
148
  }
149
149
  return false;
150
150
  }
151
+ /**
152
+ * Thrown when a write supplies embeddings whose length differs from the
153
+ * store's vector column dimension. LanceDB itself does NOT validate this —
154
+ * it silently zero-pads or truncates vectors into the fixed-size column,
155
+ * producing rows that can never be found by a query. Failing loudly here
156
+ * keeps that silent corruption out of the store.
157
+ */
158
+ export class DimensionMismatchError extends Error {
159
+ /** Dimension of the store's vector column. */
160
+ storeDimension;
161
+ /** Dimension of the embedding that was supplied. */
162
+ embeddingDimension;
163
+ constructor(storeDimension, embeddingDimension, action = "rebuild it with 'opencode-rag index --force'") {
164
+ super(`Embedding dimension mismatch: the store holds ${storeDimension}-dimensional vectors ` +
165
+ `but the embedding model produced ${embeddingDimension}. The index was built with a ` +
166
+ `different embedding model — ${action}.`);
167
+ this.name = "DimensionMismatchError";
168
+ this.storeDimension = storeDimension;
169
+ this.embeddingDimension = embeddingDimension;
170
+ }
171
+ }
172
+ /** Type guard for {@link DimensionMismatchError}. */
173
+ export function isDimensionMismatchError(err) {
174
+ return err instanceof DimensionMismatchError || (err instanceof Error && err.name === "DimensionMismatchError");
175
+ }
176
+ /**
177
+ * Extract the fixed-size vector dimension of the `embedding` column from a
178
+ * LanceDB/Arrow schema. Returns `undefined` when the column is missing or is
179
+ * not a fixed-size list (e.g. before the table is created).
180
+ */
181
+ export function extractEmbeddingDimension(fields) {
182
+ const field = fields.find((f) => f.name === "embedding");
183
+ const type = field?.type;
184
+ if (type && typeof type.listSize === "number" && type.listSize > 0) {
185
+ return type.listSize;
186
+ }
187
+ return undefined;
188
+ }
189
+ /**
190
+ * Read the embedding dimension of an existing store without creating a table
191
+ * or holding a long-lived connection. Returns `undefined` when the store or
192
+ * table does not exist (or cannot be read).
193
+ *
194
+ * Used by the CLI bootstrap to avoid creating a brand-new store with a
195
+ * speculative dimension when the embedding provider cannot be probed.
196
+ */
197
+ export async function readStoreDimension(storePath) {
198
+ if (storePath.startsWith("memory:"))
199
+ return undefined;
200
+ try {
201
+ // Do not create a store directory just to read a schema.
202
+ await fs.access(storePath);
203
+ }
204
+ catch {
205
+ return undefined;
206
+ }
207
+ try {
208
+ const db = await lancedb.connect(storePath);
209
+ const tableNames = await db.tableNames();
210
+ if (!tableNames.includes(TABLE_NAME))
211
+ return undefined;
212
+ const table = await db.openTable(TABLE_NAME);
213
+ const schema = await table.schema();
214
+ return extractEmbeddingDimension(schema.fields);
215
+ }
216
+ catch {
217
+ return undefined;
218
+ }
219
+ }
220
+ /**
221
+ * Non-Lance artifacts that live in the store directory next to the LanceDB
222
+ * data and must survive a rebuild swap. `chunks.lance`/`manifest.json` are
223
+ * rebuilt by the pipeline; the files below are not derivable from the new
224
+ * index and would otherwise be destroyed with the old directory.
225
+ */
226
+ const PRESERVED_STORE_ENTRIES = [
227
+ "quirks.jsonl",
228
+ "runtime-overrides.json",
229
+ "watcher-status.json",
230
+ ".desc-cache.json",
231
+ "keyword-index.json",
232
+ "eval-sessions",
233
+ ];
234
+ /**
235
+ * Carry non-Lance store artifacts (quirk memory, caches, overrides) from the
236
+ * pre-swap directory into the freshly promoted one. Best-effort: a failure to
237
+ * preserve one entry must not abort the swap.
238
+ */
239
+ async function preserveStoreArtifacts(oldPath, realPath) {
240
+ for (const entry of PRESERVED_STORE_ENTRIES) {
241
+ const src = path.join(oldPath, entry);
242
+ const dest = path.join(realPath, entry);
243
+ try {
244
+ await fs.access(src);
245
+ }
246
+ catch {
247
+ continue; // artifact not present in the old store
248
+ }
249
+ try {
250
+ try {
251
+ await fs.access(dest);
252
+ continue; // destination already has this artifact (e.g. re-saved desc cache)
253
+ }
254
+ catch {
255
+ // destination missing — move or copy it over
256
+ }
257
+ await fs.rename(src, dest);
258
+ }
259
+ catch {
260
+ try {
261
+ await fs.cp(src, dest, { recursive: true, force: true });
262
+ }
263
+ catch {
264
+ // best-effort — keep the swap result even if an artifact cannot be carried over
265
+ }
266
+ }
267
+ }
268
+ }
151
269
  /**
152
270
  * Atomically replace one LanceDB store directory with another.
153
271
  * Swaps the real directory with a temporary one that was built during a rebuild.
154
- * The old directory is moved to `${realPath}_old` and deleted asynchronously.
272
+ * The old directory is moved to `${realPath}_old` and deleted asynchronously
273
+ * after non-Lance artifacts (quirk memory, caches) have been carried over.
155
274
  *
156
275
  * @param tempPath - Path to the newly built store (source).
157
276
  * @param realPath - Path to the current store (destination, will be replaced).
@@ -176,7 +295,9 @@ export async function swapStoreDirectories(tempPath, realPath) {
176
295
  catch { }
177
296
  throw err;
178
297
  }
179
- // Best-effort async cleanup of old directory
298
+ // Preserve quirks.jsonl & friends (the temp store contains only Lance data),
299
+ // then clean up the old directory best-effort.
300
+ await preserveStoreArtifacts(oldPath, realPath);
180
301
  fs.rm(oldPath, { recursive: true, force: true }).catch(() => { });
181
302
  }
182
303
  /**
@@ -201,6 +322,13 @@ export class LanceDbStore {
201
322
  indexRepairPromise = null;
202
323
  /** Consecutive failed repair attempts — bounded so a broken store cannot retrain forever. */
203
324
  indexRepairFailures = 0;
325
+ /**
326
+ * Actual dimension of the `embedding` column in the opened table. Read from
327
+ * the schema on first table access and used to validate writes/searches —
328
+ * the constructor's `vectorDimension` describes what the caller *expects*,
329
+ * which can drift from the store when the embedding model changes.
330
+ */
331
+ knownDimension = null;
204
332
  /**
205
333
  * Execute an async function under an exclusive write lock.
206
334
  *
@@ -270,6 +398,7 @@ export class LanceDbStore {
270
398
  const tableNames = await db.tableNames();
271
399
  if (tableNames.includes(TABLE_NAME)) {
272
400
  this.table = await db.openTable(TABLE_NAME);
401
+ await this.cacheTableDimension(this.table);
273
402
  if (await this.tableHasDescriptionColumn()) {
274
403
  await this.migrateNewColumns();
275
404
  return this.table;
@@ -323,6 +452,7 @@ export class LanceDbStore {
323
452
  data: [seedRow],
324
453
  mode: "overwrite",
325
454
  });
455
+ this.knownDimension = this.vectorDimension;
326
456
  const deleted = await this.table.delete('id = "__seed__"');
327
457
  if (deleted === undefined) {
328
458
  // LanceDB may not return a count; try a direct query to verify
@@ -342,6 +472,59 @@ export class LanceDbStore {
342
472
  return false;
343
473
  }
344
474
  }
475
+ /**
476
+ * Read and cache the actual fixed-size dimension of the `embedding` column.
477
+ * Called whenever the table is (re)opened so write/search validation uses
478
+ * the store's real schema instead of the constructor's expectation.
479
+ */
480
+ async cacheTableDimension(table) {
481
+ if (this.knownDimension !== null)
482
+ return this.knownDimension;
483
+ try {
484
+ const schema = await table.schema();
485
+ const dim = extractEmbeddingDimension(schema.fields);
486
+ if (dim !== undefined)
487
+ this.knownDimension = dim;
488
+ return dim;
489
+ }
490
+ catch {
491
+ return undefined;
492
+ }
493
+ }
494
+ /**
495
+ * Return the actual embedding dimension of the store's vector column, or
496
+ * `undefined` when no table exists yet. Never creates the table.
497
+ */
498
+ async getVectorDimension() {
499
+ if (this.knownDimension !== null)
500
+ return this.knownDimension;
501
+ try {
502
+ const db = await this.getDb();
503
+ const tableNames = await db.tableNames();
504
+ if (!tableNames.includes(TABLE_NAME))
505
+ return undefined;
506
+ const table = await this.getTable();
507
+ return await this.cacheTableDimension(table);
508
+ }
509
+ catch {
510
+ return undefined;
511
+ }
512
+ }
513
+ /**
514
+ * Fail loudly when rows carry embeddings whose length differs from the
515
+ * store's vector column. LanceDB would silently pad/truncate them instead,
516
+ * leaving rows that no query can match.
517
+ */
518
+ assertEmbeddingDimensions(rows) {
519
+ const storeDim = this.knownDimension;
520
+ if (storeDim === null)
521
+ return;
522
+ for (const row of rows) {
523
+ if (row.embedding.length !== storeDim) {
524
+ throw new DimensionMismatchError(storeDim, row.embedding.length);
525
+ }
526
+ }
527
+ }
345
528
  async hasColumn(name) {
346
529
  try {
347
530
  const schema = await this.table.schema();
@@ -489,6 +672,7 @@ export class LanceDbStore {
489
672
  .filter((r) => r !== null);
490
673
  if (rows.length === 0)
491
674
  return;
675
+ this.assertEmbeddingDimensions(rows);
492
676
  // INSERT FIRST: data is safely stored before any delete
493
677
  await table.add(rows);
494
678
  if (!dedup)
@@ -530,6 +714,7 @@ export class LanceDbStore {
530
714
  }
531
715
  if (allRows.length === 0)
532
716
  return;
717
+ this.assertEmbeddingDimensions(allRows);
533
718
  // INSERT FIRST (single add for the whole batch), then per-file dedup
534
719
  await table.add(allRows);
535
720
  for (const [filePath, ids] of dedupByFile) {
@@ -551,9 +736,21 @@ export class LanceDbStore {
551
736
  async searchWithFilter(embedding, topK, filter) {
552
737
  try {
553
738
  // Guard against dimension mismatch BEFORE the native call — LanceDB
554
- // throws a cryptic error that used to be swallowed into "no results".
555
- if (embedding.length !== this.vectorDimension) {
556
- console.warn(`[lancedb] searchWithFilter: query embedding dimension ${embedding.length} != store dimension ${this.vectorDimension} — returning empty`);
739
+ // throws a cryptic "No vector column found to match with the query
740
+ // vector dimension" error. Compare against the table's *actual* column
741
+ // dimension, not the constructor's expectation: a store built by a
742
+ // different embedding model has a mismatching schema even when the
743
+ // handle was constructed with the current model's dimension.
744
+ const storeDimension = (await this.getVectorDimension()) ?? this.vectorDimension;
745
+ if (embedding.length !== storeDimension) {
746
+ if (storeDimension !== this.vectorDimension) {
747
+ console.warn(`[lancedb] Store vector column is ${storeDimension}-dimensional but this handle expects ` +
748
+ `${this.vectorDimension} — the index was built with a different embedding model. ` +
749
+ "Rebuild it with 'opencode-rag index --force' (a plain 'opencode-rag index' also rebuilds automatically).");
750
+ }
751
+ else {
752
+ console.warn(`[lancedb] searchWithFilter: query embedding dimension ${embedding.length} != store dimension ${storeDimension} — returning empty`);
753
+ }
557
754
  return [];
558
755
  }
559
756
  return await this.searchInternal(embedding, topK, filter);
@@ -1090,6 +1287,7 @@ export class LanceDbStore {
1090
1287
  await this.close();
1091
1288
  if (newPath)
1092
1289
  this.dbPath = newPath;
1290
+ this.knownDimension = null;
1093
1291
  }
1094
1292
  /**
1095
1293
  * Close the database connection and release resources.
@@ -1153,6 +1351,7 @@ export class LanceDbStore {
1153
1351
  console.warn(`[lancedb] Backed up chunks.lance to ${backup}`);
1154
1352
  await this.table?.close();
1155
1353
  this.table = null;
1354
+ this.knownDimension = null;
1156
1355
  try {
1157
1356
  const db = await this.getDb();
1158
1357
  const tableNames = await db.tableNames();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "opencode-rag-plugin",
3
- "version": "1.22.1",
3
+ "version": "1.22.2",
4
4
  "description": "OpenCode plugin for local-first RAG-based semantic code search",
5
5
  "type": "module",
6
6
  "main": "./dist/plugin-entry.js",