opencode-rag-plugin 1.22.0 → 1.22.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/index-command.js +10 -1
- package/dist/cli/commands/status.js +14 -0
- package/dist/cli/format.js +10 -1
- package/dist/core/bootstrap.d.ts +2 -0
- package/dist/core/bootstrap.js +45 -16
- package/dist/core/config.d.ts +14 -0
- package/dist/core/config.js +2 -0
- package/dist/core/interfaces.d.ts +8 -0
- package/dist/core/runtime-overrides.d.ts +1 -0
- package/dist/core/runtime-overrides.js +1 -0
- package/dist/embedder/factory.d.ts +20 -0
- package/dist/embedder/factory.js +24 -0
- package/dist/indexer/pipeline.js +175 -64
- package/dist/indexer/stats.d.ts +4 -0
- package/dist/indexer/stats.js +2 -0
- package/dist/indexer/worker.d.ts +8 -2
- package/dist/indexer/worker.js +9 -4
- package/dist/plugin.js +32 -15
- package/dist/retriever/context-optimizer.js +9 -2
- package/dist/tui.js +11 -3
- package/dist/vectorstore/lancedb.d.ts +60 -1
- package/dist/vectorstore/lancedb.js +204 -5
- package/package.json +1 -1
package/dist/indexer/pipeline.js
CHANGED
|
@@ -7,7 +7,7 @@ import pLimit from "p-limit";
|
|
|
7
7
|
import { scanWorkspaceFiles } from "../content/reader.js";
|
|
8
8
|
import { loadManifest, saveManifest, normalizeFilePath, computeDescriptionConfigHash } from "../core/manifest.js";
|
|
9
9
|
import { DescriptionCache } from "../core/desc-cache.js";
|
|
10
|
-
import { embedBatch } from "../embedder/factory.js";
|
|
10
|
+
import { embedBatch, probeEmbeddingDimension } from "../embedder/factory.js";
|
|
11
11
|
import { createVectorStore } from "../vectorstore/factory.js";
|
|
12
12
|
import { swapStoreDirectories } from "../vectorstore/lancedb.js";
|
|
13
13
|
import { createIndexStats } from "./stats.js";
|
|
@@ -157,46 +157,12 @@ async function runIndexPassInner(options, logger) {
|
|
|
157
157
|
// file content IS different — these would be caught by hash comparison below but
|
|
158
158
|
// this pre-clear ensures the description cache is consulted during re-description.
|
|
159
159
|
// (Files with same hash but different descHash already fall through in prepareFile.)
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
if (diffResult) {
|
|
167
|
-
const untracked = getUntrackedFiles(options.cwd);
|
|
168
|
-
const changedSet = new Set();
|
|
169
|
-
for (const f of diffResult.changedFiles)
|
|
170
|
-
changedSet.add(f);
|
|
171
|
-
for (const f of untracked)
|
|
172
|
-
changedSet.add(f);
|
|
173
|
-
// Git paths are relative to the REPO ROOT. When the workspace is a
|
|
174
|
-
// subdirectory of the repo (monorepos), convert them to workspace-
|
|
175
|
-
// relative paths before passing them to the scan — resolving repo-root
|
|
176
|
-
// paths against `cwd` would silently miss every change.
|
|
177
|
-
const toCwdRelative = (p) => {
|
|
178
|
-
const abs = path.resolve(repoRoot, p);
|
|
179
|
-
const rel = path.relative(options.cwd, abs);
|
|
180
|
-
if (rel.startsWith("..") || path.isAbsolute(rel))
|
|
181
|
-
return null;
|
|
182
|
-
return rel;
|
|
183
|
-
};
|
|
184
|
-
const cwdRelative = [];
|
|
185
|
-
for (const p of changedSet) {
|
|
186
|
-
const rel = toCwdRelative(p);
|
|
187
|
-
if (rel !== null)
|
|
188
|
-
cwdRelative.push(rel);
|
|
189
|
-
}
|
|
190
|
-
filterPaths = cwdRelative;
|
|
191
|
-
gitDeletedPaths = diffResult.deletedFiles;
|
|
192
|
-
logger.debug(`Git incremental: ${filterPaths.length} changed/untracked, ${gitDeletedPaths.length} deleted since ${manifest.lastGitCommit.slice(0, 8)}`);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
const scanStart = Date.now();
|
|
197
|
-
const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger, options.force ? undefined : manifest, filterPaths, options.imageVisionProvider, descCache);
|
|
198
|
-
const scanSec = ((Date.now() - scanStart) / 1000).toFixed(1);
|
|
199
|
-
logger.info(`Workspace scan complete: ${workspaceFiles.length} files in ${scanSec}s`);
|
|
160
|
+
// ── Store health checks (run BEFORE the scan) ──────────────────────────
|
|
161
|
+
// The checks below can invalidate the manifest and trigger a full rebuild.
|
|
162
|
+
// They must run before scanWorkspaceFiles: the reader skips reading the
|
|
163
|
+
// contents of unchanged files when a manifest entry matches (mtime + size +
|
|
164
|
+
// descHash unchanged), so clearing the manifest after the scan would leave
|
|
165
|
+
// those files with empty content and turn them into bogus "removed" entries.
|
|
200
166
|
logger.info(`Querying vector store for existing chunk count...`);
|
|
201
167
|
const storeCountStart = Date.now();
|
|
202
168
|
const existingCount = await options.store.count();
|
|
@@ -236,15 +202,44 @@ async function runIndexPassInner(options, logger) {
|
|
|
236
202
|
manifestStatus = "missing";
|
|
237
203
|
}
|
|
238
204
|
}
|
|
205
|
+
// ── Detect a store built by a different embedding model ────────────────
|
|
206
|
+
// LanceDB silently zero-pads/truncates mismatched vectors on write, and
|
|
207
|
+
// every vector query then fails with "No vector column found to match with
|
|
208
|
+
// the query vector dimension". Treat a schema dimension mismatch like a
|
|
209
|
+
// corrupt store: clear the manifest so this pass rebuilds everything into a
|
|
210
|
+
// fresh store at the configured dimension.
|
|
211
|
+
let dimensionMismatch = false;
|
|
212
|
+
try {
|
|
213
|
+
const storeDimension = await options.store.getVectorDimension?.();
|
|
214
|
+
if (!options.force &&
|
|
215
|
+
storeDimension !== undefined &&
|
|
216
|
+
options.dimension !== undefined &&
|
|
217
|
+
storeDimension !== options.dimension) {
|
|
218
|
+
dimensionMismatch = true;
|
|
219
|
+
logger.warn(`Store was built with vector dimension ${storeDimension} but the configured embedding model produces ` +
|
|
220
|
+
`${options.dimension} — rebuilding the full index with the current model.`);
|
|
221
|
+
if (manifestStatus === "ok") {
|
|
222
|
+
for (const key of Object.keys(manifest.files))
|
|
223
|
+
delete manifest.files[key];
|
|
224
|
+
manifest.lastIndexedAt = undefined;
|
|
225
|
+
manifestStatus = "missing";
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
catch {
|
|
230
|
+
// getVectorDimension is best-effort — proceed without the drift check.
|
|
231
|
+
}
|
|
239
232
|
// Effective store used throughout the pass — may be a temp store for atomic rebuild.
|
|
240
233
|
let effectiveStore = options.store;
|
|
241
234
|
let tempStorePath;
|
|
242
|
-
|
|
235
|
+
/** Set when the temporary rebuild store actually received rows. */
|
|
236
|
+
let tempStoreWroteChunks = false;
|
|
237
|
+
if (options.force || (manifestStatus !== "ok" && (existingCount > 0 || dimensionMismatch))) {
|
|
243
238
|
for (const key of Object.keys(manifest.files)) {
|
|
244
239
|
delete manifest.files[key];
|
|
245
240
|
}
|
|
246
241
|
manifest.lastIndexedAt = undefined;
|
|
247
|
-
rebuildPerformed = existingCount > 0 || !!options.force;
|
|
242
|
+
rebuildPerformed = existingCount > 0 || !!options.force || dimensionMismatch;
|
|
248
243
|
if (manifestStatus !== "ok" && existingCount > 0) {
|
|
249
244
|
logger.warn("Manifest missing or corrupt; rebuilding full index.");
|
|
250
245
|
}
|
|
@@ -271,7 +266,7 @@ async function runIndexPassInner(options, logger) {
|
|
|
271
266
|
logger.warn("Cannot rebuild safely without embedding dimension — aborting to protect existing data. " +
|
|
272
267
|
"Run 'opencode-rag index --force' manually to rebuild.");
|
|
273
268
|
// Restore manifest entries we just deleted so the next pass can retry incrementally
|
|
274
|
-
return createIndexStats(
|
|
269
|
+
return createIndexStats(0, manifestStatus);
|
|
275
270
|
}
|
|
276
271
|
else {
|
|
277
272
|
// No existing data — safe to proceed with in-place indexing (no clear needed)
|
|
@@ -279,8 +274,76 @@ async function runIndexPassInner(options, logger) {
|
|
|
279
274
|
logger.debug("No existing data; indexing from scratch.");
|
|
280
275
|
}
|
|
281
276
|
}
|
|
277
|
+
let filterPaths;
|
|
278
|
+
let gitDeletedPaths = [];
|
|
279
|
+
if (!options.force && manifestStatus === "ok" && manifest.lastGitCommit) {
|
|
280
|
+
const repoRoot = getRepoRoot(options.cwd);
|
|
281
|
+
if (repoRoot) {
|
|
282
|
+
const diffResult = getChangedFilesSince(options.cwd, manifest.lastGitCommit);
|
|
283
|
+
if (diffResult) {
|
|
284
|
+
const untracked = getUntrackedFiles(options.cwd);
|
|
285
|
+
const changedSet = new Set();
|
|
286
|
+
for (const f of diffResult.changedFiles)
|
|
287
|
+
changedSet.add(f);
|
|
288
|
+
for (const f of untracked)
|
|
289
|
+
changedSet.add(f);
|
|
290
|
+
// Git paths are relative to the REPO ROOT. When the workspace is a
|
|
291
|
+
// subdirectory of the repo (monorepos), convert them to workspace-
|
|
292
|
+
// relative paths before passing them to the scan — resolving repo-root
|
|
293
|
+
// paths against `cwd` would silently miss every change.
|
|
294
|
+
const toCwdRelative = (p) => {
|
|
295
|
+
const abs = path.resolve(repoRoot, p);
|
|
296
|
+
const rel = path.relative(options.cwd, abs);
|
|
297
|
+
if (rel.startsWith("..") || path.isAbsolute(rel))
|
|
298
|
+
return null;
|
|
299
|
+
return rel;
|
|
300
|
+
};
|
|
301
|
+
const cwdRelative = [];
|
|
302
|
+
for (const p of changedSet) {
|
|
303
|
+
const rel = toCwdRelative(p);
|
|
304
|
+
if (rel !== null)
|
|
305
|
+
cwdRelative.push(rel);
|
|
306
|
+
}
|
|
307
|
+
filterPaths = cwdRelative;
|
|
308
|
+
gitDeletedPaths = diffResult.deletedFiles;
|
|
309
|
+
logger.debug(`Git incremental: ${filterPaths.length} changed/untracked, ${gitDeletedPaths.length} deleted since ${manifest.lastGitCommit.slice(0, 8)}`);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
const scanStart = Date.now();
|
|
314
|
+
const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger, options.force ? undefined : manifest, filterPaths, options.imageVisionProvider, descCache);
|
|
315
|
+
const scanSec = ((Date.now() - scanStart) / 1000).toFixed(1);
|
|
316
|
+
logger.info(`Workspace scan complete: ${workspaceFiles.length} files in ${scanSec}s`);
|
|
317
|
+
// ── Preflight: verify the embedding provider before expensive work ─────
|
|
318
|
+
// A provider outage used to burn the entire pass (chunking + describing
|
|
319
|
+
// every file) and then store nothing, reporting success. Probe once here
|
|
320
|
+
// so the pass aborts in milliseconds with an actionable message.
|
|
321
|
+
const pendingEmbeddingWork = options.force || workspaceFiles.some((f) => {
|
|
322
|
+
if (f.isEmpty || f.isTooSmall)
|
|
323
|
+
return false;
|
|
324
|
+
const previous = manifest.files[f.normalizedPath];
|
|
325
|
+
return !previous || previous.hash !== f.hash;
|
|
326
|
+
});
|
|
327
|
+
if (pendingEmbeddingWork && !(options.abortSignal?.aborted ?? false)) {
|
|
328
|
+
const probe = await probeEmbeddingDimension(options.embedder);
|
|
329
|
+
if (probe.dimension === undefined) {
|
|
330
|
+
logger.warn(`Embedding provider unavailable (${probe.error?.message ?? "probe failed"}) — ` +
|
|
331
|
+
"aborting index pass before chunking/description; nothing was stored.");
|
|
332
|
+
const failedStats = createIndexStats(workspaceFiles.length, manifestStatus);
|
|
333
|
+
failedStats.embeddingUnavailable = true;
|
|
334
|
+
return failedStats;
|
|
335
|
+
}
|
|
336
|
+
if (options.dimension !== undefined && probe.dimension !== options.dimension) {
|
|
337
|
+
logger.warn(`Embedding provider produces ${probe.dimension}-dimensional vectors but this index is configured for ` +
|
|
338
|
+
`${options.dimension} — aborting to avoid writing incompatible vectors. ` +
|
|
339
|
+
"Set embedding.vectorDimension to the provider's dimension and run the index again.");
|
|
340
|
+
const failedStats = createIndexStats(workspaceFiles.length, manifestStatus);
|
|
341
|
+
failedStats.embeddingUnavailable = true;
|
|
342
|
+
return failedStats;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
282
345
|
const stats = createIndexStats(workspaceFiles.length, manifestStatus);
|
|
283
|
-
stats.rebuildPerformed = rebuildPerformed;
|
|
346
|
+
stats.rebuildPerformed = rebuildPerformed || dimensionMismatch;
|
|
284
347
|
for (const file of workspaceFiles) {
|
|
285
348
|
if (file.extractionStatus === "failed" && file.extractionError) {
|
|
286
349
|
stats.extractionFailures++;
|
|
@@ -416,6 +479,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
416
479
|
if (!allEmbedded && (prep.chunks?.length ?? 0) > 0) {
|
|
417
480
|
options.logger?.warn?.(` ${prep.fileLabel}: ${validChunks.length}/${prep.chunks?.length} chunks embedded — marking for retry on next pass`);
|
|
418
481
|
}
|
|
482
|
+
if (validChunks.length > 0)
|
|
483
|
+
tempStoreWroteChunks = true;
|
|
419
484
|
storePayloads.push({ prep, validChunks, allEmbedded });
|
|
420
485
|
}
|
|
421
486
|
// One bulk write per window. Dedup (per-modified-file deletes) is skipped
|
|
@@ -738,7 +803,7 @@ async function runIndexPassInner(options, logger) {
|
|
|
738
803
|
}
|
|
739
804
|
}
|
|
740
805
|
for (const prep of deferredPreps) {
|
|
741
|
-
prep.textToEmbed = buildTextsToEmbed(prep.chunks, prep.relPath, prep.metaHeader ?? "", prep.docPrefix ?? "", prep.isImageFile ?? false);
|
|
806
|
+
prep.textToEmbed = buildTextsToEmbed(prep.chunks, prep.relPath, prep.metaHeader ?? "", prep.docPrefix ?? "", prep.isImageFile ?? false, options.config.indexing.embedDescriptions !== false);
|
|
742
807
|
}
|
|
743
808
|
const totalDescribedChunks = deferredPreps.reduce((s, p) => s + (p.chunks?.length ?? 0), 0);
|
|
744
809
|
logger.info(`Description phase complete: ${totalDescribedChunks} chunks across ${deferredPreps.length} files`);
|
|
@@ -801,6 +866,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
801
866
|
}
|
|
802
867
|
catch (err) {
|
|
803
868
|
logger.warn(` Global embedding failed: ${err.message}`);
|
|
869
|
+
stats.embeddingFailures += allTexts.length;
|
|
870
|
+
stats.embeddingUnavailable = true;
|
|
804
871
|
for (const { fileIdx } of embedQueue) {
|
|
805
872
|
options.progress?.failFile(prepared[fileIdx].fileLabel);
|
|
806
873
|
earlyWorkerResults.set(fileIdx, {
|
|
@@ -814,6 +881,21 @@ async function runIndexPassInner(options, logger) {
|
|
|
814
881
|
}
|
|
815
882
|
embedQueue.length = 0; // prevent double-processing in store phase
|
|
816
883
|
}
|
|
884
|
+
// `embedBatch` returns empty vectors for batches whose retries were
|
|
885
|
+
// exhausted — count those so the summary and exit code report a failed
|
|
886
|
+
// pass instead of a successful "0 chunks stored".
|
|
887
|
+
const failedVectors = allEmbeddings.filter((v) => !Array.isArray(v) || v.length === 0).length;
|
|
888
|
+
if (failedVectors > 0) {
|
|
889
|
+
stats.embeddingFailures += failedVectors;
|
|
890
|
+
if (failedVectors >= allTexts.length) {
|
|
891
|
+
stats.embeddingUnavailable = true;
|
|
892
|
+
logger.warn(`All ${allTexts.length} embedding requests failed — no vectors were produced, so nothing was stored. ` +
|
|
893
|
+
"Check that the embedding provider is running and the model is available, then run the index again.");
|
|
894
|
+
}
|
|
895
|
+
else {
|
|
896
|
+
logger.warn(` ${failedVectors}/${allTexts.length} embedding requests failed — those chunks are kept for retry on the next pass.`);
|
|
897
|
+
}
|
|
898
|
+
}
|
|
817
899
|
}
|
|
818
900
|
// ── Distribute embeddings back to per-file chunks ─────────────────────
|
|
819
901
|
for (let i = 0; i < embedQueue.length; i++) {
|
|
@@ -875,25 +957,43 @@ async function runIndexPassInner(options, logger) {
|
|
|
875
957
|
tryUpdateLastGitCommit(options.cwd, manifest);
|
|
876
958
|
}
|
|
877
959
|
// ── Atomically promote temp store if a full rebuild was performed ──
|
|
960
|
+
let tempStorePromoted = false;
|
|
878
961
|
if (tempStorePath) {
|
|
879
962
|
if (!aborted()) {
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
//
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
catch (err) {
|
|
890
|
-
logger.warn(`Could not promote temporary store: ${err.message}. ` +
|
|
891
|
-
`Original data preserved at ${options.storePath}`);
|
|
963
|
+
if (!tempStoreWroteChunks) {
|
|
964
|
+
// Nothing was written (e.g. every embedding request failed). Keep the
|
|
965
|
+
// existing store and manifest untouched rather than promoting an
|
|
966
|
+
// empty store over good data.
|
|
967
|
+
logger.warn("Rebuild produced no chunks — keeping the existing index and manifest unchanged.");
|
|
968
|
+
try {
|
|
969
|
+
await effectiveStore.close();
|
|
970
|
+
}
|
|
971
|
+
catch { }
|
|
892
972
|
try {
|
|
893
973
|
await fs.rm(tempStorePath, { recursive: true, force: true });
|
|
894
974
|
}
|
|
895
975
|
catch { }
|
|
896
976
|
}
|
|
977
|
+
else {
|
|
978
|
+
try {
|
|
979
|
+
await effectiveStore.close();
|
|
980
|
+
await options.store.close();
|
|
981
|
+
// Swap the newly-built temp directory into the real path
|
|
982
|
+
await swapStoreDirectories(tempStorePath, options.storePath);
|
|
983
|
+
// Re-open the original store handle so callers can search the new data
|
|
984
|
+
await options.store.reopen?.(options.storePath);
|
|
985
|
+
tempStorePromoted = true;
|
|
986
|
+
logger.debug(`Promoted temporary store ${tempStorePath} → ${options.storePath}`);
|
|
987
|
+
}
|
|
988
|
+
catch (err) {
|
|
989
|
+
logger.warn(`Could not promote temporary store: ${err.message}. ` +
|
|
990
|
+
`Original data preserved at ${options.storePath}`);
|
|
991
|
+
try {
|
|
992
|
+
await fs.rm(tempStorePath, { recursive: true, force: true });
|
|
993
|
+
}
|
|
994
|
+
catch { }
|
|
995
|
+
}
|
|
996
|
+
}
|
|
897
997
|
}
|
|
898
998
|
else {
|
|
899
999
|
// Aborted — discard temp, keep original data intact.
|
|
@@ -912,16 +1012,27 @@ async function runIndexPassInner(options, logger) {
|
|
|
912
1012
|
return stats;
|
|
913
1013
|
}
|
|
914
1014
|
}
|
|
915
|
-
//
|
|
916
|
-
//
|
|
917
|
-
|
|
918
|
-
|
|
1015
|
+
// The in-memory manifest was cleared at rebuild start and only repopulated
|
|
1016
|
+
// for the new store. When the rebuild did not promote (empty temp store or
|
|
1017
|
+
// failed swap), keep the previous manifest/keyword index on disk — saving
|
|
1018
|
+
// the cleared state would orphan the existing store's data.
|
|
1019
|
+
const keepPreviousIndexState = tempStorePath !== undefined && !tempStorePromoted;
|
|
1020
|
+
if (keepPreviousIndexState) {
|
|
1021
|
+
logger.warn("Rebuild did not complete — keeping the previous manifest and keyword index.");
|
|
1022
|
+
}
|
|
1023
|
+
else {
|
|
1024
|
+
// Save manifest and keyword index (always to the real store path — after
|
|
1025
|
+
// a successful swap this points to the new data).
|
|
1026
|
+
await saveManifest(options.storePath, manifest);
|
|
1027
|
+
await options.keywordIndex?.save(options.storePath);
|
|
1028
|
+
}
|
|
919
1029
|
// Compact fragments and prune old version manifests so countRows() can't
|
|
920
1030
|
// hang on accumulated versions from many add/delete cycles. After a temp
|
|
921
1031
|
// store rebuild, optimize the reopened real store handle (the temp handle
|
|
922
1032
|
// was closed and its directory moved); use aggressive pruning since the
|
|
923
|
-
// swapped-in store is private to this process.
|
|
924
|
-
|
|
1033
|
+
// swapped-in store is private to this process. Skipped when the rebuild did
|
|
1034
|
+
// not promote (the old store is untouched and its handle may be closed).
|
|
1035
|
+
if (!aborted() && !keepPreviousIndexState) {
|
|
925
1036
|
logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
|
|
926
1037
|
const optimizeStart = Date.now();
|
|
927
1038
|
try {
|
package/dist/indexer/stats.d.ts
CHANGED
|
@@ -38,6 +38,10 @@ export interface IndexRunStats {
|
|
|
38
38
|
}>;
|
|
39
39
|
/** Number of files where description generation failed. */
|
|
40
40
|
descriptionFailedFiles: number;
|
|
41
|
+
/** Number of chunks whose embedding request failed (kept for retry on the next pass). */
|
|
42
|
+
embeddingFailures: number;
|
|
43
|
+
/** True when the pass could not embed at all (provider unavailable or dimension mismatch) — nothing was stored. */
|
|
44
|
+
embeddingUnavailable: boolean;
|
|
41
45
|
/** True when the pass was skipped because another pass holds the lock. */
|
|
42
46
|
skipped: boolean;
|
|
43
47
|
}
|
package/dist/indexer/stats.js
CHANGED
package/dist/indexer/worker.d.ts
CHANGED
|
@@ -61,16 +61,21 @@ export interface PreparedFile {
|
|
|
61
61
|
/**
|
|
62
62
|
* Build the list of text strings that will be sent to the embedding provider.
|
|
63
63
|
* Each chunk is prefixed with the document prefix, relative path, metadata
|
|
64
|
-
* header, and (if available) a description.
|
|
64
|
+
* header, and (if available and enabled) a description.
|
|
65
|
+
*
|
|
66
|
+
* Code-specialized embedding models do not need the description crutch: with
|
|
67
|
+
* `indexing.embedDescriptions: false` the prose description is omitted from
|
|
68
|
+
* the embedded text (it is still stored on the chunk for display).
|
|
65
69
|
*
|
|
66
70
|
* @param chunks - Chunks to build embedding texts from.
|
|
67
71
|
* @param relPath - Relative file path used as context prefix.
|
|
68
72
|
* @param metaHeader - Assembled metadata header (file type, directory, etc.).
|
|
69
73
|
* @param docPrefix - Optional document-level prefix from configuration.
|
|
70
74
|
* @param isImage - Whether the source is an image (uses description only).
|
|
75
|
+
* @param includeDescription - Whether to include non-image descriptions (default true).
|
|
71
76
|
* @returns An array of formatted text strings, one per chunk.
|
|
72
77
|
*/
|
|
73
|
-
export declare function buildTextsToEmbed(chunks: Chunk[], relPath: string, metaHeader: string, docPrefix: string, isImage: boolean): string[];
|
|
78
|
+
export declare function buildTextsToEmbed(chunks: Chunk[], relPath: string, metaHeader: string, docPrefix: string, isImage: boolean, includeDescription?: boolean): string[];
|
|
74
79
|
interface Logger {
|
|
75
80
|
info(message: string): void;
|
|
76
81
|
warn(message: string): void;
|
|
@@ -115,6 +120,7 @@ export declare function prepareFile(file: WorkspaceFile, cwd: string, previous:
|
|
|
115
120
|
};
|
|
116
121
|
indexing?: {
|
|
117
122
|
maxSvgSizeBytes?: number;
|
|
123
|
+
embedDescriptions?: boolean;
|
|
118
124
|
};
|
|
119
125
|
}, keywordIndex: KeywordIndex | undefined, descriptionProvider: DescriptionProvider | undefined, logger: Logger, deferDescriptions?: boolean, descHash?: string): Promise<PreparedFile>;
|
|
120
126
|
/**
|
package/dist/indexer/worker.js
CHANGED
|
@@ -10,23 +10,28 @@ import { generateDescriptions, buildFallbackDescription } from "./description-st
|
|
|
10
10
|
/**
|
|
11
11
|
* Build the list of text strings that will be sent to the embedding provider.
|
|
12
12
|
* Each chunk is prefixed with the document prefix, relative path, metadata
|
|
13
|
-
* header, and (if available) a description.
|
|
13
|
+
* header, and (if available and enabled) a description.
|
|
14
|
+
*
|
|
15
|
+
* Code-specialized embedding models do not need the description crutch: with
|
|
16
|
+
* `indexing.embedDescriptions: false` the prose description is omitted from
|
|
17
|
+
* the embedded text (it is still stored on the chunk for display).
|
|
14
18
|
*
|
|
15
19
|
* @param chunks - Chunks to build embedding texts from.
|
|
16
20
|
* @param relPath - Relative file path used as context prefix.
|
|
17
21
|
* @param metaHeader - Assembled metadata header (file type, directory, etc.).
|
|
18
22
|
* @param docPrefix - Optional document-level prefix from configuration.
|
|
19
23
|
* @param isImage - Whether the source is an image (uses description only).
|
|
24
|
+
* @param includeDescription - Whether to include non-image descriptions (default true).
|
|
20
25
|
* @returns An array of formatted text strings, one per chunk.
|
|
21
26
|
*/
|
|
22
|
-
export function buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage) {
|
|
27
|
+
export function buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage, includeDescription = true) {
|
|
23
28
|
const textToEmbed = [];
|
|
24
29
|
for (const chunk of chunks) {
|
|
25
30
|
if (isImage) {
|
|
26
31
|
textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description);
|
|
27
32
|
}
|
|
28
33
|
else {
|
|
29
|
-
const desc = chunk.description ?? "";
|
|
34
|
+
const desc = includeDescription ? chunk.description ?? "" : "";
|
|
30
35
|
if (desc.trim().length > 0) {
|
|
31
36
|
textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + desc + "\n\n" + chunk.content);
|
|
32
37
|
}
|
|
@@ -210,7 +215,7 @@ export async function prepareFile(file, cwd, previous, config, keywordIndex, des
|
|
|
210
215
|
}
|
|
211
216
|
}
|
|
212
217
|
}
|
|
213
|
-
const textToEmbed = buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage);
|
|
218
|
+
const textToEmbed = buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage, config.indexing?.embedDescriptions !== false);
|
|
214
219
|
logger.debug(` ${fileLabel}: textToEmbed ${textToEmbed.length} entries (descProvider: ${descriptionProvider ? "yes" : "no"})`);
|
|
215
220
|
return {
|
|
216
221
|
normalizedPath: file.normalizedPath,
|
package/dist/plugin.js
CHANGED
|
@@ -7,7 +7,8 @@ import { tool } from "@opencode-ai/plugin/tool";
|
|
|
7
7
|
import { CODE_SEARCH_FILTER } from "./core/interfaces.js";
|
|
8
8
|
import { normalizeFileExtensions } from "./core/filters.js";
|
|
9
9
|
import { loadConfig, findConfigFile, DEFAULT_CONFIG, resolveLogConfig, persistProbedDimension } from "./core/config.js";
|
|
10
|
-
import { createEmbedder } from "./embedder/factory.js";
|
|
10
|
+
import { createEmbedder, probeEmbeddingDimension } from "./embedder/factory.js";
|
|
11
|
+
import { readStoreDimension } from "./vectorstore/lancedb.js";
|
|
11
12
|
import { createDescriptionProvider } from "./describer/factory.js";
|
|
12
13
|
import { createVectorStore } from "./vectorstore/factory.js";
|
|
13
14
|
import { retrieve } from "./retriever/retriever.js";
|
|
@@ -419,7 +420,8 @@ export function createRagHooks(options) {
|
|
|
419
420
|
...options.dependencies,
|
|
420
421
|
};
|
|
421
422
|
const embedder = options.embedder ?? dependencies.createEmbedder(options.cfg);
|
|
422
|
-
const
|
|
423
|
+
const configuredDimension = options.cfg.embedding.vectorDimension;
|
|
424
|
+
const store = options.store ?? dependencies.createStore(options.storePath, configuredDimension && configuredDimension > 0 ? configuredDimension : 384, options.cfg);
|
|
423
425
|
const keywordIndex = options.keywordIndex;
|
|
424
426
|
// Runtime overrides for live config editing from TUI
|
|
425
427
|
let cachedOverrides = loadRuntimeOverrides(options.storePath);
|
|
@@ -1394,6 +1396,8 @@ export const ragPlugin = async (input, _options) => {
|
|
|
1394
1396
|
}, logLevel);
|
|
1395
1397
|
// Use cached dimension from config if available (avoids blocking startup with an API call)
|
|
1396
1398
|
// If not set, probe the embedding provider once and persist the result.
|
|
1399
|
+
// When the probe fails, prefer the existing store's schema over the 384
|
|
1400
|
+
// default — a transient outage must not downgrade the store's dimension.
|
|
1397
1401
|
const embedder = createEmbedder(effectiveCfg);
|
|
1398
1402
|
let vectorDimension = effectiveCfg.embedding.vectorDimension;
|
|
1399
1403
|
if (vectorDimension && vectorDimension > 0) {
|
|
@@ -1403,34 +1407,47 @@ export const ragPlugin = async (input, _options) => {
|
|
|
1403
1407
|
}, logLevel);
|
|
1404
1408
|
}
|
|
1405
1409
|
else {
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
try {
|
|
1414
|
-
persistProbedDimension(configPath, vectorDimension);
|
|
1415
|
-
}
|
|
1416
|
-
catch { /* best-effort */ }
|
|
1410
|
+
const probe = await probeEmbeddingDimension(embedder);
|
|
1411
|
+
if (probe.dimension !== undefined) {
|
|
1412
|
+
vectorDimension = probe.dimension;
|
|
1413
|
+
const configPath = findConfigFile(input.directory);
|
|
1414
|
+
if (configPath) {
|
|
1415
|
+
try {
|
|
1416
|
+
persistProbedDimension(configPath, vectorDimension);
|
|
1417
1417
|
}
|
|
1418
|
+
catch { /* best-effort */ }
|
|
1418
1419
|
}
|
|
1419
1420
|
appendDebugLog(logFilePath, {
|
|
1420
1421
|
scope: "plugin",
|
|
1421
1422
|
message: `Vector dimension: ${vectorDimension}`,
|
|
1422
1423
|
}, logLevel);
|
|
1423
1424
|
}
|
|
1424
|
-
|
|
1425
|
+
else {
|
|
1426
|
+
vectorDimension = (await readStoreDimension(storePath)) ?? 384;
|
|
1425
1427
|
appendDebugLog(logFilePath, {
|
|
1426
1428
|
scope: "plugin",
|
|
1427
1429
|
message: `Dimension probe failed, falling back to ${vectorDimension}`,
|
|
1428
|
-
error:
|
|
1430
|
+
error: probe.error,
|
|
1429
1431
|
}, logLevel);
|
|
1430
1432
|
}
|
|
1431
1433
|
}
|
|
1432
1434
|
const store = createVectorStore(effectiveCfg, storePath, vectorDimension);
|
|
1433
1435
|
ragStores.set(input.directory, store);
|
|
1436
|
+
// Warn when the store was built by a different embedding model — vector
|
|
1437
|
+
// search fails (or silently degrades) until the index is rebuilt.
|
|
1438
|
+
try {
|
|
1439
|
+
const storeDimension = await store.getVectorDimension?.();
|
|
1440
|
+
if (storeDimension !== undefined && storeDimension !== vectorDimension) {
|
|
1441
|
+
appendDebugLog(logFilePath, {
|
|
1442
|
+
scope: "plugin",
|
|
1443
|
+
message: `Store vector dimension is ${storeDimension} but the configured embedder produces ${vectorDimension} — ` +
|
|
1444
|
+
"run 'opencode-rag index' to rebuild the index with the current model.",
|
|
1445
|
+
}, logLevel);
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
catch {
|
|
1449
|
+
// best-effort — dimension introspection must never block plugin startup
|
|
1450
|
+
}
|
|
1434
1451
|
// Load or create keyword index for hybrid search
|
|
1435
1452
|
const keywordIndex = await loadKeywordIndex(storePath, logFilePath, logLevel);
|
|
1436
1453
|
// Create description provider (enabled by default)
|
|
@@ -122,8 +122,7 @@ function dedupeSimilar(results, threshold) {
|
|
|
122
122
|
if (sim > threshold) {
|
|
123
123
|
const [keepIdx, removeIdx] = kept[i].score >= kept[j].score ? [i, j] : [j, i];
|
|
124
124
|
const removedId = kept[removeIdx].chunk.id;
|
|
125
|
-
|
|
126
|
-
kept[keepIdx] = {
|
|
125
|
+
const keeper = {
|
|
127
126
|
...kept[keepIdx],
|
|
128
127
|
optimized: {
|
|
129
128
|
...kept[keepIdx].optimized,
|
|
@@ -133,6 +132,14 @@ function dedupeSimilar(results, threshold) {
|
|
|
133
132
|
],
|
|
134
133
|
},
|
|
135
134
|
};
|
|
135
|
+
if (removeIdx < keepIdx) {
|
|
136
|
+
kept.splice(removeIdx, 1);
|
|
137
|
+
kept[keepIdx - 1] = keeper;
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
kept[keepIdx] = keeper;
|
|
141
|
+
kept.splice(removeIdx, 1);
|
|
142
|
+
}
|
|
136
143
|
changed = true;
|
|
137
144
|
break;
|
|
138
145
|
}
|
package/dist/tui.js
CHANGED
|
@@ -162,6 +162,7 @@ function renderSidebar(theme, version, status, tuiConfig, tokenStats) {
|
|
|
162
162
|
: `Watcher idle \u00B7 last ${formatRelativeTime(watcher.lastRunAt)}`;
|
|
163
163
|
const fileListKey = tuiConfig?.fileListKeybinding ?? "ctrl+enter";
|
|
164
164
|
const chunksKey = tuiConfig?.chunksKeybinding ?? "ctrl+alt+enter";
|
|
165
|
+
const settingsKey = tuiConfig?.settingsKeybinding ?? "ctrl+shift+r";
|
|
165
166
|
return box({
|
|
166
167
|
width: "100%",
|
|
167
168
|
flexDirection: "column",
|
|
@@ -186,7 +187,7 @@ function renderSidebar(theme, version, status, tuiConfig, tokenStats) {
|
|
|
186
187
|
text({ fg: theme.text }, [statusLine]),
|
|
187
188
|
text({ fg: theme.textMuted }, [timeLine]),
|
|
188
189
|
text({ fg: watcher.running ? theme.accent : theme.textMuted }, [watcherLine]),
|
|
189
|
-
text({ fg: theme.textMuted }, [
|
|
190
|
+
text({ fg: theme.textMuted }, [`${formatKeybinding(settingsKey)} → Settings`]),
|
|
190
191
|
text({ fg: theme.textMuted }, [`${formatKeybinding(fileListKey)} → Add File List`]),
|
|
191
192
|
text({ fg: theme.textMuted }, [`${formatKeybinding(chunksKey)} → Add Chunks`]),
|
|
192
193
|
...(tokenStats && tokenStats.queries > 0 ? [
|
|
@@ -589,6 +590,12 @@ function buildSettingCategories(cfg, ro, providers) {
|
|
|
589
590
|
label: "Keybindings",
|
|
590
591
|
description: "Configure keyboard shortcuts",
|
|
591
592
|
entries: [
|
|
593
|
+
{
|
|
594
|
+
path: ["tui", "settingsKeybinding"],
|
|
595
|
+
label: "Open settings",
|
|
596
|
+
type: "string",
|
|
597
|
+
currentValue: tuiRo.settingsKeybinding ?? tuiCfg.settingsKeybinding ?? "ctrl+shift+r",
|
|
598
|
+
},
|
|
592
599
|
{
|
|
593
600
|
path: ["tui", "fileListKeybinding"],
|
|
594
601
|
label: "Add file list",
|
|
@@ -951,10 +958,11 @@ const plugin = {
|
|
|
951
958
|
// ignore
|
|
952
959
|
}
|
|
953
960
|
}
|
|
954
|
-
// Register keybinding for settings dialog
|
|
961
|
+
// Register keybinding for settings dialog (configurable)
|
|
955
962
|
try {
|
|
963
|
+
const settingsKey = tuiConfig?.settingsKeybinding ?? "ctrl+shift+r";
|
|
956
964
|
api.keymap.registerLayer({
|
|
957
|
-
bindings: [{ key:
|
|
965
|
+
bindings: [{ key: settingsKey, cmd: "opencode-rag:settings" }],
|
|
958
966
|
commands: [
|
|
959
967
|
{
|
|
960
968
|
name: "opencode-rag:settings",
|