opencode-rag-plugin 1.22.1 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +2 -2
- package/dist/chunker/image.d.ts +14 -0
- package/dist/chunker/image.js +26 -0
- package/dist/cli/commands/describe-image.js +10 -6
- package/dist/cli/commands/index-command.js +10 -1
- package/dist/cli/commands/init.js +9 -1
- package/dist/cli/commands/status.js +14 -0
- package/dist/cli/format.js +10 -1
- package/dist/core/bootstrap.d.ts +2 -0
- package/dist/core/bootstrap.js +45 -16
- package/dist/core/config.d.ts +48 -1
- package/dist/core/config.js +5 -0
- package/dist/core/interfaces.d.ts +8 -0
- package/dist/core/resolve-api-key.js +5 -0
- package/dist/embedder/factory.d.ts +20 -0
- package/dist/embedder/factory.js +24 -0
- package/dist/embedder/health.d.ts +1 -1
- package/dist/embedder/health.js +17 -4
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/indexer/pipeline.js +175 -64
- package/dist/indexer/stats.d.ts +4 -0
- package/dist/indexer/stats.js +2 -0
- package/dist/indexer/worker.d.ts +8 -2
- package/dist/indexer/worker.js +9 -4
- package/dist/mcp/handlers.d.ts +1 -1
- package/dist/mcp/handlers.js +7 -6
- package/dist/opencode/tools.d.ts +3 -0
- package/dist/opencode/tools.js +11 -7
- package/dist/plugin.js +32 -15
- package/dist/vectorstore/lancedb.d.ts +60 -1
- package/dist/vectorstore/lancedb.js +204 -5
- package/package.json +1 -1
package/dist/indexer/pipeline.js
CHANGED
|
@@ -7,7 +7,7 @@ import pLimit from "p-limit";
|
|
|
7
7
|
import { scanWorkspaceFiles } from "../content/reader.js";
|
|
8
8
|
import { loadManifest, saveManifest, normalizeFilePath, computeDescriptionConfigHash } from "../core/manifest.js";
|
|
9
9
|
import { DescriptionCache } from "../core/desc-cache.js";
|
|
10
|
-
import { embedBatch } from "../embedder/factory.js";
|
|
10
|
+
import { embedBatch, probeEmbeddingDimension } from "../embedder/factory.js";
|
|
11
11
|
import { createVectorStore } from "../vectorstore/factory.js";
|
|
12
12
|
import { swapStoreDirectories } from "../vectorstore/lancedb.js";
|
|
13
13
|
import { createIndexStats } from "./stats.js";
|
|
@@ -157,46 +157,12 @@ async function runIndexPassInner(options, logger) {
|
|
|
157
157
|
// file content IS different — these would be caught by hash comparison below but
|
|
158
158
|
// this pre-clear ensures the description cache is consulted during re-description.
|
|
159
159
|
// (Files with same hash but different descHash already fall through in prepareFile.)
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
if (diffResult) {
|
|
167
|
-
const untracked = getUntrackedFiles(options.cwd);
|
|
168
|
-
const changedSet = new Set();
|
|
169
|
-
for (const f of diffResult.changedFiles)
|
|
170
|
-
changedSet.add(f);
|
|
171
|
-
for (const f of untracked)
|
|
172
|
-
changedSet.add(f);
|
|
173
|
-
// Git paths are relative to the REPO ROOT. When the workspace is a
|
|
174
|
-
// subdirectory of the repo (monorepos), convert them to workspace-
|
|
175
|
-
// relative paths before passing them to the scan — resolving repo-root
|
|
176
|
-
// paths against `cwd` would silently miss every change.
|
|
177
|
-
const toCwdRelative = (p) => {
|
|
178
|
-
const abs = path.resolve(repoRoot, p);
|
|
179
|
-
const rel = path.relative(options.cwd, abs);
|
|
180
|
-
if (rel.startsWith("..") || path.isAbsolute(rel))
|
|
181
|
-
return null;
|
|
182
|
-
return rel;
|
|
183
|
-
};
|
|
184
|
-
const cwdRelative = [];
|
|
185
|
-
for (const p of changedSet) {
|
|
186
|
-
const rel = toCwdRelative(p);
|
|
187
|
-
if (rel !== null)
|
|
188
|
-
cwdRelative.push(rel);
|
|
189
|
-
}
|
|
190
|
-
filterPaths = cwdRelative;
|
|
191
|
-
gitDeletedPaths = diffResult.deletedFiles;
|
|
192
|
-
logger.debug(`Git incremental: ${filterPaths.length} changed/untracked, ${gitDeletedPaths.length} deleted since ${manifest.lastGitCommit.slice(0, 8)}`);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
const scanStart = Date.now();
|
|
197
|
-
const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger, options.force ? undefined : manifest, filterPaths, options.imageVisionProvider, descCache);
|
|
198
|
-
const scanSec = ((Date.now() - scanStart) / 1000).toFixed(1);
|
|
199
|
-
logger.info(`Workspace scan complete: ${workspaceFiles.length} files in ${scanSec}s`);
|
|
160
|
+
// ── Store health checks (run BEFORE the scan) ──────────────────────────
|
|
161
|
+
// The checks below can invalidate the manifest and trigger a full rebuild.
|
|
162
|
+
// They must run before scanWorkspaceFiles: the reader skips reading the
|
|
163
|
+
// contents of unchanged files when a manifest entry matches (mtime + size +
|
|
164
|
+
// descHash unchanged), so clearing the manifest after the scan would leave
|
|
165
|
+
// those files with empty content and turn them into bogus "removed" entries.
|
|
200
166
|
logger.info(`Querying vector store for existing chunk count...`);
|
|
201
167
|
const storeCountStart = Date.now();
|
|
202
168
|
const existingCount = await options.store.count();
|
|
@@ -236,15 +202,44 @@ async function runIndexPassInner(options, logger) {
|
|
|
236
202
|
manifestStatus = "missing";
|
|
237
203
|
}
|
|
238
204
|
}
|
|
205
|
+
// ── Detect a store built by a different embedding model ────────────────
|
|
206
|
+
// LanceDB silently zero-pads/truncates mismatched vectors on write, and
|
|
207
|
+
// every vector query then fails with "No vector column found to match with
|
|
208
|
+
// the query vector dimension". Treat a schema dimension mismatch like a
|
|
209
|
+
// corrupt store: clear the manifest so this pass rebuilds everything into a
|
|
210
|
+
// fresh store at the configured dimension.
|
|
211
|
+
let dimensionMismatch = false;
|
|
212
|
+
try {
|
|
213
|
+
const storeDimension = await options.store.getVectorDimension?.();
|
|
214
|
+
if (!options.force &&
|
|
215
|
+
storeDimension !== undefined &&
|
|
216
|
+
options.dimension !== undefined &&
|
|
217
|
+
storeDimension !== options.dimension) {
|
|
218
|
+
dimensionMismatch = true;
|
|
219
|
+
logger.warn(`Store was built with vector dimension ${storeDimension} but the configured embedding model produces ` +
|
|
220
|
+
`${options.dimension} — rebuilding the full index with the current model.`);
|
|
221
|
+
if (manifestStatus === "ok") {
|
|
222
|
+
for (const key of Object.keys(manifest.files))
|
|
223
|
+
delete manifest.files[key];
|
|
224
|
+
manifest.lastIndexedAt = undefined;
|
|
225
|
+
manifestStatus = "missing";
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
catch {
|
|
230
|
+
// getVectorDimension is best-effort — proceed without the drift check.
|
|
231
|
+
}
|
|
239
232
|
// Effective store used throughout the pass — may be a temp store for atomic rebuild.
|
|
240
233
|
let effectiveStore = options.store;
|
|
241
234
|
let tempStorePath;
|
|
242
|
-
|
|
235
|
+
/** Set when the temporary rebuild store actually received rows. */
|
|
236
|
+
let tempStoreWroteChunks = false;
|
|
237
|
+
if (options.force || (manifestStatus !== "ok" && (existingCount > 0 || dimensionMismatch))) {
|
|
243
238
|
for (const key of Object.keys(manifest.files)) {
|
|
244
239
|
delete manifest.files[key];
|
|
245
240
|
}
|
|
246
241
|
manifest.lastIndexedAt = undefined;
|
|
247
|
-
rebuildPerformed = existingCount > 0 || !!options.force;
|
|
242
|
+
rebuildPerformed = existingCount > 0 || !!options.force || dimensionMismatch;
|
|
248
243
|
if (manifestStatus !== "ok" && existingCount > 0) {
|
|
249
244
|
logger.warn("Manifest missing or corrupt; rebuilding full index.");
|
|
250
245
|
}
|
|
@@ -271,7 +266,7 @@ async function runIndexPassInner(options, logger) {
|
|
|
271
266
|
logger.warn("Cannot rebuild safely without embedding dimension — aborting to protect existing data. " +
|
|
272
267
|
"Run 'opencode-rag index --force' manually to rebuild.");
|
|
273
268
|
// Restore manifest entries we just deleted so the next pass can retry incrementally
|
|
274
|
-
return createIndexStats(
|
|
269
|
+
return createIndexStats(0, manifestStatus);
|
|
275
270
|
}
|
|
276
271
|
else {
|
|
277
272
|
// No existing data — safe to proceed with in-place indexing (no clear needed)
|
|
@@ -279,8 +274,76 @@ async function runIndexPassInner(options, logger) {
|
|
|
279
274
|
logger.debug("No existing data; indexing from scratch.");
|
|
280
275
|
}
|
|
281
276
|
}
|
|
277
|
+
let filterPaths;
|
|
278
|
+
let gitDeletedPaths = [];
|
|
279
|
+
if (!options.force && manifestStatus === "ok" && manifest.lastGitCommit) {
|
|
280
|
+
const repoRoot = getRepoRoot(options.cwd);
|
|
281
|
+
if (repoRoot) {
|
|
282
|
+
const diffResult = getChangedFilesSince(options.cwd, manifest.lastGitCommit);
|
|
283
|
+
if (diffResult) {
|
|
284
|
+
const untracked = getUntrackedFiles(options.cwd);
|
|
285
|
+
const changedSet = new Set();
|
|
286
|
+
for (const f of diffResult.changedFiles)
|
|
287
|
+
changedSet.add(f);
|
|
288
|
+
for (const f of untracked)
|
|
289
|
+
changedSet.add(f);
|
|
290
|
+
// Git paths are relative to the REPO ROOT. When the workspace is a
|
|
291
|
+
// subdirectory of the repo (monorepos), convert them to workspace-
|
|
292
|
+
// relative paths before passing them to the scan — resolving repo-root
|
|
293
|
+
// paths against `cwd` would silently miss every change.
|
|
294
|
+
const toCwdRelative = (p) => {
|
|
295
|
+
const abs = path.resolve(repoRoot, p);
|
|
296
|
+
const rel = path.relative(options.cwd, abs);
|
|
297
|
+
if (rel.startsWith("..") || path.isAbsolute(rel))
|
|
298
|
+
return null;
|
|
299
|
+
return rel;
|
|
300
|
+
};
|
|
301
|
+
const cwdRelative = [];
|
|
302
|
+
for (const p of changedSet) {
|
|
303
|
+
const rel = toCwdRelative(p);
|
|
304
|
+
if (rel !== null)
|
|
305
|
+
cwdRelative.push(rel);
|
|
306
|
+
}
|
|
307
|
+
filterPaths = cwdRelative;
|
|
308
|
+
gitDeletedPaths = diffResult.deletedFiles;
|
|
309
|
+
logger.debug(`Git incremental: ${filterPaths.length} changed/untracked, ${gitDeletedPaths.length} deleted since ${manifest.lastGitCommit.slice(0, 8)}`);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
const scanStart = Date.now();
|
|
314
|
+
const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger, options.force ? undefined : manifest, filterPaths, options.imageVisionProvider, descCache);
|
|
315
|
+
const scanSec = ((Date.now() - scanStart) / 1000).toFixed(1);
|
|
316
|
+
logger.info(`Workspace scan complete: ${workspaceFiles.length} files in ${scanSec}s`);
|
|
317
|
+
// ── Preflight: verify the embedding provider before expensive work ─────
|
|
318
|
+
// A provider outage used to burn the entire pass (chunking + describing
|
|
319
|
+
// every file) and then store nothing, reporting success. Probe once here
|
|
320
|
+
// so the pass aborts in milliseconds with an actionable message.
|
|
321
|
+
const pendingEmbeddingWork = options.force || workspaceFiles.some((f) => {
|
|
322
|
+
if (f.isEmpty || f.isTooSmall)
|
|
323
|
+
return false;
|
|
324
|
+
const previous = manifest.files[f.normalizedPath];
|
|
325
|
+
return !previous || previous.hash !== f.hash;
|
|
326
|
+
});
|
|
327
|
+
if (pendingEmbeddingWork && !(options.abortSignal?.aborted ?? false)) {
|
|
328
|
+
const probe = await probeEmbeddingDimension(options.embedder);
|
|
329
|
+
if (probe.dimension === undefined) {
|
|
330
|
+
logger.warn(`Embedding provider unavailable (${probe.error?.message ?? "probe failed"}) — ` +
|
|
331
|
+
"aborting index pass before chunking/description; nothing was stored.");
|
|
332
|
+
const failedStats = createIndexStats(workspaceFiles.length, manifestStatus);
|
|
333
|
+
failedStats.embeddingUnavailable = true;
|
|
334
|
+
return failedStats;
|
|
335
|
+
}
|
|
336
|
+
if (options.dimension !== undefined && probe.dimension !== options.dimension) {
|
|
337
|
+
logger.warn(`Embedding provider produces ${probe.dimension}-dimensional vectors but this index is configured for ` +
|
|
338
|
+
`${options.dimension} — aborting to avoid writing incompatible vectors. ` +
|
|
339
|
+
"Set embedding.vectorDimension to the provider's dimension and run the index again.");
|
|
340
|
+
const failedStats = createIndexStats(workspaceFiles.length, manifestStatus);
|
|
341
|
+
failedStats.embeddingUnavailable = true;
|
|
342
|
+
return failedStats;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
282
345
|
const stats = createIndexStats(workspaceFiles.length, manifestStatus);
|
|
283
|
-
stats.rebuildPerformed = rebuildPerformed;
|
|
346
|
+
stats.rebuildPerformed = rebuildPerformed || dimensionMismatch;
|
|
284
347
|
for (const file of workspaceFiles) {
|
|
285
348
|
if (file.extractionStatus === "failed" && file.extractionError) {
|
|
286
349
|
stats.extractionFailures++;
|
|
@@ -416,6 +479,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
416
479
|
if (!allEmbedded && (prep.chunks?.length ?? 0) > 0) {
|
|
417
480
|
options.logger?.warn?.(` ${prep.fileLabel}: ${validChunks.length}/${prep.chunks?.length} chunks embedded — marking for retry on next pass`);
|
|
418
481
|
}
|
|
482
|
+
if (validChunks.length > 0)
|
|
483
|
+
tempStoreWroteChunks = true;
|
|
419
484
|
storePayloads.push({ prep, validChunks, allEmbedded });
|
|
420
485
|
}
|
|
421
486
|
// One bulk write per window. Dedup (per-modified-file deletes) is skipped
|
|
@@ -738,7 +803,7 @@ async function runIndexPassInner(options, logger) {
|
|
|
738
803
|
}
|
|
739
804
|
}
|
|
740
805
|
for (const prep of deferredPreps) {
|
|
741
|
-
prep.textToEmbed = buildTextsToEmbed(prep.chunks, prep.relPath, prep.metaHeader ?? "", prep.docPrefix ?? "", prep.isImageFile ?? false);
|
|
806
|
+
prep.textToEmbed = buildTextsToEmbed(prep.chunks, prep.relPath, prep.metaHeader ?? "", prep.docPrefix ?? "", prep.isImageFile ?? false, options.config.indexing.embedDescriptions !== false);
|
|
742
807
|
}
|
|
743
808
|
const totalDescribedChunks = deferredPreps.reduce((s, p) => s + (p.chunks?.length ?? 0), 0);
|
|
744
809
|
logger.info(`Description phase complete: ${totalDescribedChunks} chunks across ${deferredPreps.length} files`);
|
|
@@ -801,6 +866,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
801
866
|
}
|
|
802
867
|
catch (err) {
|
|
803
868
|
logger.warn(` Global embedding failed: ${err.message}`);
|
|
869
|
+
stats.embeddingFailures += allTexts.length;
|
|
870
|
+
stats.embeddingUnavailable = true;
|
|
804
871
|
for (const { fileIdx } of embedQueue) {
|
|
805
872
|
options.progress?.failFile(prepared[fileIdx].fileLabel);
|
|
806
873
|
earlyWorkerResults.set(fileIdx, {
|
|
@@ -814,6 +881,21 @@ async function runIndexPassInner(options, logger) {
|
|
|
814
881
|
}
|
|
815
882
|
embedQueue.length = 0; // prevent double-processing in store phase
|
|
816
883
|
}
|
|
884
|
+
// `embedBatch` returns empty vectors for batches whose retries were
|
|
885
|
+
// exhausted — count those so the summary and exit code report a failed
|
|
886
|
+
// pass instead of a successful "0 chunks stored".
|
|
887
|
+
const failedVectors = allEmbeddings.filter((v) => !Array.isArray(v) || v.length === 0).length;
|
|
888
|
+
if (failedVectors > 0) {
|
|
889
|
+
stats.embeddingFailures += failedVectors;
|
|
890
|
+
if (failedVectors >= allTexts.length) {
|
|
891
|
+
stats.embeddingUnavailable = true;
|
|
892
|
+
logger.warn(`All ${allTexts.length} embedding requests failed — no vectors were produced, so nothing was stored. ` +
|
|
893
|
+
"Check that the embedding provider is running and the model is available, then run the index again.");
|
|
894
|
+
}
|
|
895
|
+
else {
|
|
896
|
+
logger.warn(` ${failedVectors}/${allTexts.length} embedding requests failed — those chunks are kept for retry on the next pass.`);
|
|
897
|
+
}
|
|
898
|
+
}
|
|
817
899
|
}
|
|
818
900
|
// ── Distribute embeddings back to per-file chunks ─────────────────────
|
|
819
901
|
for (let i = 0; i < embedQueue.length; i++) {
|
|
@@ -875,25 +957,43 @@ async function runIndexPassInner(options, logger) {
|
|
|
875
957
|
tryUpdateLastGitCommit(options.cwd, manifest);
|
|
876
958
|
}
|
|
877
959
|
// ── Atomically promote temp store if a full rebuild was performed ──
|
|
960
|
+
let tempStorePromoted = false;
|
|
878
961
|
if (tempStorePath) {
|
|
879
962
|
if (!aborted()) {
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
//
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
catch (err) {
|
|
890
|
-
logger.warn(`Could not promote temporary store: ${err.message}. ` +
|
|
891
|
-
`Original data preserved at ${options.storePath}`);
|
|
963
|
+
if (!tempStoreWroteChunks) {
|
|
964
|
+
// Nothing was written (e.g. every embedding request failed). Keep the
|
|
965
|
+
// existing store and manifest untouched rather than promoting an
|
|
966
|
+
// empty store over good data.
|
|
967
|
+
logger.warn("Rebuild produced no chunks — keeping the existing index and manifest unchanged.");
|
|
968
|
+
try {
|
|
969
|
+
await effectiveStore.close();
|
|
970
|
+
}
|
|
971
|
+
catch { }
|
|
892
972
|
try {
|
|
893
973
|
await fs.rm(tempStorePath, { recursive: true, force: true });
|
|
894
974
|
}
|
|
895
975
|
catch { }
|
|
896
976
|
}
|
|
977
|
+
else {
|
|
978
|
+
try {
|
|
979
|
+
await effectiveStore.close();
|
|
980
|
+
await options.store.close();
|
|
981
|
+
// Swap the newly-built temp directory into the real path
|
|
982
|
+
await swapStoreDirectories(tempStorePath, options.storePath);
|
|
983
|
+
// Re-open the original store handle so callers can search the new data
|
|
984
|
+
await options.store.reopen?.(options.storePath);
|
|
985
|
+
tempStorePromoted = true;
|
|
986
|
+
logger.debug(`Promoted temporary store ${tempStorePath} → ${options.storePath}`);
|
|
987
|
+
}
|
|
988
|
+
catch (err) {
|
|
989
|
+
logger.warn(`Could not promote temporary store: ${err.message}. ` +
|
|
990
|
+
`Original data preserved at ${options.storePath}`);
|
|
991
|
+
try {
|
|
992
|
+
await fs.rm(tempStorePath, { recursive: true, force: true });
|
|
993
|
+
}
|
|
994
|
+
catch { }
|
|
995
|
+
}
|
|
996
|
+
}
|
|
897
997
|
}
|
|
898
998
|
else {
|
|
899
999
|
// Aborted — discard temp, keep original data intact.
|
|
@@ -912,16 +1012,27 @@ async function runIndexPassInner(options, logger) {
|
|
|
912
1012
|
return stats;
|
|
913
1013
|
}
|
|
914
1014
|
}
|
|
915
|
-
//
|
|
916
|
-
//
|
|
917
|
-
|
|
918
|
-
|
|
1015
|
+
// The in-memory manifest was cleared at rebuild start and only repopulated
|
|
1016
|
+
// for the new store. When the rebuild did not promote (empty temp store or
|
|
1017
|
+
// failed swap), keep the previous manifest/keyword index on disk — saving
|
|
1018
|
+
// the cleared state would orphan the existing store's data.
|
|
1019
|
+
const keepPreviousIndexState = tempStorePath !== undefined && !tempStorePromoted;
|
|
1020
|
+
if (keepPreviousIndexState) {
|
|
1021
|
+
logger.warn("Rebuild did not complete — keeping the previous manifest and keyword index.");
|
|
1022
|
+
}
|
|
1023
|
+
else {
|
|
1024
|
+
// Save manifest and keyword index (always to the real store path — after
|
|
1025
|
+
// a successful swap this points to the new data).
|
|
1026
|
+
await saveManifest(options.storePath, manifest);
|
|
1027
|
+
await options.keywordIndex?.save(options.storePath);
|
|
1028
|
+
}
|
|
919
1029
|
// Compact fragments and prune old version manifests so countRows() can't
|
|
920
1030
|
// hang on accumulated versions from many add/delete cycles. After a temp
|
|
921
1031
|
// store rebuild, optimize the reopened real store handle (the temp handle
|
|
922
1032
|
// was closed and its directory moved); use aggressive pruning since the
|
|
923
|
-
// swapped-in store is private to this process.
|
|
924
|
-
|
|
1033
|
+
// swapped-in store is private to this process. Skipped when the rebuild did
|
|
1034
|
+
// not promote (the old store is untouched and its handle may be closed).
|
|
1035
|
+
if (!aborted() && !keepPreviousIndexState) {
|
|
925
1036
|
logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
|
|
926
1037
|
const optimizeStart = Date.now();
|
|
927
1038
|
try {
|
package/dist/indexer/stats.d.ts
CHANGED
|
@@ -38,6 +38,10 @@ export interface IndexRunStats {
|
|
|
38
38
|
}>;
|
|
39
39
|
/** Number of files where description generation failed. */
|
|
40
40
|
descriptionFailedFiles: number;
|
|
41
|
+
/** Number of chunks whose embedding request failed (kept for retry on the next pass). */
|
|
42
|
+
embeddingFailures: number;
|
|
43
|
+
/** True when the pass could not embed at all (provider unavailable or dimension mismatch) — nothing was stored. */
|
|
44
|
+
embeddingUnavailable: boolean;
|
|
41
45
|
/** True when the pass was skipped because another pass holds the lock. */
|
|
42
46
|
skipped: boolean;
|
|
43
47
|
}
|
package/dist/indexer/stats.js
CHANGED
package/dist/indexer/worker.d.ts
CHANGED
|
@@ -61,16 +61,21 @@ export interface PreparedFile {
|
|
|
61
61
|
/**
|
|
62
62
|
* Build the list of text strings that will be sent to the embedding provider.
|
|
63
63
|
* Each chunk is prefixed with the document prefix, relative path, metadata
|
|
64
|
-
* header, and (if available) a description.
|
|
64
|
+
* header, and (if available and enabled) a description.
|
|
65
|
+
*
|
|
66
|
+
* Code-specialized embedding models do not need the description crutch: with
|
|
67
|
+
* `indexing.embedDescriptions: false` the prose description is omitted from
|
|
68
|
+
* the embedded text (it is still stored on the chunk for display).
|
|
65
69
|
*
|
|
66
70
|
* @param chunks - Chunks to build embedding texts from.
|
|
67
71
|
* @param relPath - Relative file path used as context prefix.
|
|
68
72
|
* @param metaHeader - Assembled metadata header (file type, directory, etc.).
|
|
69
73
|
* @param docPrefix - Optional document-level prefix from configuration.
|
|
70
74
|
* @param isImage - Whether the source is an image (uses description only).
|
|
75
|
+
* @param includeDescription - Whether to include non-image descriptions (default true).
|
|
71
76
|
* @returns An array of formatted text strings, one per chunk.
|
|
72
77
|
*/
|
|
73
|
-
export declare function buildTextsToEmbed(chunks: Chunk[], relPath: string, metaHeader: string, docPrefix: string, isImage: boolean): string[];
|
|
78
|
+
export declare function buildTextsToEmbed(chunks: Chunk[], relPath: string, metaHeader: string, docPrefix: string, isImage: boolean, includeDescription?: boolean): string[];
|
|
74
79
|
interface Logger {
|
|
75
80
|
info(message: string): void;
|
|
76
81
|
warn(message: string): void;
|
|
@@ -115,6 +120,7 @@ export declare function prepareFile(file: WorkspaceFile, cwd: string, previous:
|
|
|
115
120
|
};
|
|
116
121
|
indexing?: {
|
|
117
122
|
maxSvgSizeBytes?: number;
|
|
123
|
+
embedDescriptions?: boolean;
|
|
118
124
|
};
|
|
119
125
|
}, keywordIndex: KeywordIndex | undefined, descriptionProvider: DescriptionProvider | undefined, logger: Logger, deferDescriptions?: boolean, descHash?: string): Promise<PreparedFile>;
|
|
120
126
|
/**
|
package/dist/indexer/worker.js
CHANGED
|
@@ -10,23 +10,28 @@ import { generateDescriptions, buildFallbackDescription } from "./description-st
|
|
|
10
10
|
/**
|
|
11
11
|
* Build the list of text strings that will be sent to the embedding provider.
|
|
12
12
|
* Each chunk is prefixed with the document prefix, relative path, metadata
|
|
13
|
-
* header, and (if available) a description.
|
|
13
|
+
* header, and (if available and enabled) a description.
|
|
14
|
+
*
|
|
15
|
+
* Code-specialized embedding models do not need the description crutch: with
|
|
16
|
+
* `indexing.embedDescriptions: false` the prose description is omitted from
|
|
17
|
+
* the embedded text (it is still stored on the chunk for display).
|
|
14
18
|
*
|
|
15
19
|
* @param chunks - Chunks to build embedding texts from.
|
|
16
20
|
* @param relPath - Relative file path used as context prefix.
|
|
17
21
|
* @param metaHeader - Assembled metadata header (file type, directory, etc.).
|
|
18
22
|
* @param docPrefix - Optional document-level prefix from configuration.
|
|
19
23
|
* @param isImage - Whether the source is an image (uses description only).
|
|
24
|
+
* @param includeDescription - Whether to include non-image descriptions (default true).
|
|
20
25
|
* @returns An array of formatted text strings, one per chunk.
|
|
21
26
|
*/
|
|
22
|
-
export function buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage) {
|
|
27
|
+
export function buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage, includeDescription = true) {
|
|
23
28
|
const textToEmbed = [];
|
|
24
29
|
for (const chunk of chunks) {
|
|
25
30
|
if (isImage) {
|
|
26
31
|
textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description);
|
|
27
32
|
}
|
|
28
33
|
else {
|
|
29
|
-
const desc = chunk.description ?? "";
|
|
34
|
+
const desc = includeDescription ? chunk.description ?? "" : "";
|
|
30
35
|
if (desc.trim().length > 0) {
|
|
31
36
|
textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + desc + "\n\n" + chunk.content);
|
|
32
37
|
}
|
|
@@ -210,7 +215,7 @@ export async function prepareFile(file, cwd, previous, config, keywordIndex, des
|
|
|
210
215
|
}
|
|
211
216
|
}
|
|
212
217
|
}
|
|
213
|
-
const textToEmbed = buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage);
|
|
218
|
+
const textToEmbed = buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImage, config.indexing?.embedDescriptions !== false);
|
|
214
219
|
logger.debug(` ${fileLabel}: textToEmbed ${textToEmbed.length} entries (descProvider: ${descriptionProvider ? "yes" : "no"})`);
|
|
215
220
|
return {
|
|
216
221
|
normalizedPath: file.normalizedPath,
|
package/dist/mcp/handlers.d.ts
CHANGED
|
@@ -97,7 +97,7 @@ export interface DescribeImageResult {
|
|
|
97
97
|
/** Human-readable formatted output with metadata. */
|
|
98
98
|
formatted: string;
|
|
99
99
|
}
|
|
100
|
-
/** Describe an image file using the configured vision provider (Ollama, OpenAI, Anthropic, or Gemini). */
|
|
100
|
+
/** Describe an image file using the configured vision provider (Ollama, OpenAI, Anthropic, or Gemini). Honors `imageDescription.onDemand` overrides. */
|
|
101
101
|
export declare function handleDescribeImage(params: DescribeImageParams, cfg: RagConfig, worktree: string, visionProvider?: ImageVisionProvider): Promise<DescribeImageResult>;
|
|
102
102
|
/** Find usages and references of a symbol across the indexed codebase using hybrid (keyword + vector) search. */
|
|
103
103
|
export declare function handleFindUsages(params: FindUsagesParams, embedder: EmbeddingProvider, store: VectorStore, cfg: RagConfig, keywordIndex?: KeywordIndex, retrieveFn?: typeof retrieve): Promise<FindUsagesResult>;
|
package/dist/mcp/handlers.js
CHANGED
|
@@ -227,7 +227,7 @@ export async function handleFileSkeleton(params, worktree) {
|
|
|
227
227
|
const formatted = `${skeleton.length} structural elements (${summary || "—"})\n\n${formatSkeleton(skeleton)}`;
|
|
228
228
|
return { elements: skeleton, formatted, summary };
|
|
229
229
|
}
|
|
230
|
-
/** Describe an image file using the configured vision provider (Ollama, OpenAI, Anthropic, or Gemini). */
|
|
230
|
+
/** Describe an image file using the configured vision provider (Ollama, OpenAI, Anthropic, or Gemini). Honors `imageDescription.onDemand` overrides. */
|
|
231
231
|
export async function handleDescribeImage(params, cfg, worktree, visionProvider) {
|
|
232
232
|
const { existsSync, readFileSync } = await import("node:fs");
|
|
233
233
|
const path = await import("node:path");
|
|
@@ -244,21 +244,22 @@ export async function handleDescribeImage(params, cfg, worktree, visionProvider)
|
|
|
244
244
|
if (!imageDescriptionConfig?.enabled) {
|
|
245
245
|
throw new Error("Image description is not enabled in config (imageDescription.enabled)");
|
|
246
246
|
}
|
|
247
|
-
const { getMimeType } = await import("../chunker/image.js");
|
|
247
|
+
const { getMimeType, createImageVisionProvider, resolveOnDemandImageConfig } = await import("../chunker/image.js");
|
|
248
248
|
const { resizeImage } = await import("../content/image.js");
|
|
249
|
+
const effectiveImageConfig = resolveOnDemandImageConfig(imageDescriptionConfig);
|
|
249
250
|
const buffer = readFileSync(resolvedPath);
|
|
250
251
|
const mimeType = getMimeType(ext);
|
|
251
|
-
const maxDimension =
|
|
252
|
+
const maxDimension = effectiveImageConfig.resizeMaxDimension ?? 1024;
|
|
252
253
|
const sized = maxDimension > 0 ? await resizeImage(buffer, resolvedPath, maxDimension) : buffer;
|
|
253
254
|
const b64 = sized.toString("base64");
|
|
254
|
-
const provider = visionProvider ??
|
|
255
|
-
const description = await provider.describeImage(b64, mimeType,
|
|
255
|
+
const provider = visionProvider ?? createImageVisionProvider(effectiveImageConfig);
|
|
256
|
+
const description = await provider.describeImage(b64, mimeType, effectiveImageConfig.prompt, params.systemPrompt);
|
|
256
257
|
const formatted = [
|
|
257
258
|
`**Image description** — ${params.filePath}`,
|
|
258
259
|
"",
|
|
259
260
|
description,
|
|
260
261
|
"",
|
|
261
|
-
`_Generated with ${
|
|
262
|
+
`_Generated with ${effectiveImageConfig.provider}/${effectiveImageConfig.model}_`,
|
|
262
263
|
].join("\n");
|
|
263
264
|
return { description, formatted };
|
|
264
265
|
}
|
package/dist/opencode/tools.d.ts
CHANGED
|
@@ -51,6 +51,9 @@ export interface DescribeImageToolOptions {
|
|
|
51
51
|
* for natural-language description. Supports Ollama, OpenAI, Anthropic, and
|
|
52
52
|
* Google Gemini providers with automatic resizing.
|
|
53
53
|
*
|
|
54
|
+
* On-demand calls honor the optional `imageDescription.onDemand` overrides
|
|
55
|
+
* (a different provider/model than the indexing pipeline).
|
|
56
|
+
*
|
|
54
57
|
* @param options - Tool configuration including workspace root and vision provider.
|
|
55
58
|
* @returns A tool definition suitable for OpenCode plugin registration.
|
|
56
59
|
*/
|
package/dist/opencode/tools.js
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { tool } from "@opencode-ai/plugin/tool";
|
|
16
16
|
import { CODE_SEARCH_FILTER } from "../core/interfaces.js";
|
|
17
|
-
import { SUPPORTED_IMAGE_EXTENSIONS, createImageVisionProvider, getMimeType } from "../chunker/image.js";
|
|
17
|
+
import { SUPPORTED_IMAGE_EXTENSIONS, createImageVisionProvider, resolveOnDemandImageConfig, getMimeType } from "../chunker/image.js";
|
|
18
18
|
import { resizeImage } from "../content/image.js";
|
|
19
19
|
import { retrieve } from "../retriever/retriever.js";
|
|
20
20
|
import { Parser } from "web-tree-sitter";
|
|
@@ -247,6 +247,9 @@ export function createFileSkeletonTool(options) {
|
|
|
247
247
|
* for natural-language description. Supports Ollama, OpenAI, Anthropic, and
|
|
248
248
|
* Google Gemini providers with automatic resizing.
|
|
249
249
|
*
|
|
250
|
+
* On-demand calls honor the optional `imageDescription.onDemand` overrides
|
|
251
|
+
* (a different provider/model than the indexing pipeline).
|
|
252
|
+
*
|
|
250
253
|
* @param options - Tool configuration including workspace root and vision provider.
|
|
251
254
|
* @returns A tool definition suitable for OpenCode plugin registration.
|
|
252
255
|
*/
|
|
@@ -292,22 +295,23 @@ export function createDescribeImageTool(options) {
|
|
|
292
295
|
metadata: { tool: "describe_image", filePath: args.filePath, error: "disabled" },
|
|
293
296
|
};
|
|
294
297
|
}
|
|
298
|
+
const effectiveImageConfig = resolveOnDemandImageConfig(imageDescriptionConfig);
|
|
295
299
|
const buffer = readFileSync(resolvedPath);
|
|
296
300
|
const mimeType = getMimeType(ext);
|
|
297
|
-
const maxDimension =
|
|
301
|
+
const maxDimension = effectiveImageConfig.resizeMaxDimension ?? 1024;
|
|
298
302
|
const sized = maxDimension > 0 ? await resizeImage(buffer, resolvedPath, maxDimension) : buffer;
|
|
299
303
|
const b64 = sized.toString("base64");
|
|
300
|
-
const provider = visionProvider ?? createImageVisionProvider(
|
|
301
|
-
const description = await provider.describeImage(b64, mimeType,
|
|
304
|
+
const provider = visionProvider ?? createImageVisionProvider(effectiveImageConfig);
|
|
305
|
+
const description = await provider.describeImage(b64, mimeType, effectiveImageConfig.prompt, args.systemPrompt);
|
|
302
306
|
return {
|
|
303
307
|
title: `Image description — ${args.filePath}`,
|
|
304
|
-
output: `**${args.filePath}**\n\n${description}\n\n_Generated with ${
|
|
308
|
+
output: `**${args.filePath}**\n\n${description}\n\n_Generated with ${effectiveImageConfig.provider}/${effectiveImageConfig.model}_`,
|
|
305
309
|
metadata: {
|
|
306
310
|
tool: "describe_image",
|
|
307
311
|
filePath: args.filePath,
|
|
308
312
|
description,
|
|
309
|
-
provider:
|
|
310
|
-
model:
|
|
313
|
+
provider: effectiveImageConfig.provider,
|
|
314
|
+
model: effectiveImageConfig.model,
|
|
311
315
|
},
|
|
312
316
|
};
|
|
313
317
|
}
|
package/dist/plugin.js
CHANGED
|
@@ -7,7 +7,8 @@ import { tool } from "@opencode-ai/plugin/tool";
|
|
|
7
7
|
import { CODE_SEARCH_FILTER } from "./core/interfaces.js";
|
|
8
8
|
import { normalizeFileExtensions } from "./core/filters.js";
|
|
9
9
|
import { loadConfig, findConfigFile, DEFAULT_CONFIG, resolveLogConfig, persistProbedDimension } from "./core/config.js";
|
|
10
|
-
import { createEmbedder } from "./embedder/factory.js";
|
|
10
|
+
import { createEmbedder, probeEmbeddingDimension } from "./embedder/factory.js";
|
|
11
|
+
import { readStoreDimension } from "./vectorstore/lancedb.js";
|
|
11
12
|
import { createDescriptionProvider } from "./describer/factory.js";
|
|
12
13
|
import { createVectorStore } from "./vectorstore/factory.js";
|
|
13
14
|
import { retrieve } from "./retriever/retriever.js";
|
|
@@ -419,7 +420,8 @@ export function createRagHooks(options) {
|
|
|
419
420
|
...options.dependencies,
|
|
420
421
|
};
|
|
421
422
|
const embedder = options.embedder ?? dependencies.createEmbedder(options.cfg);
|
|
422
|
-
const
|
|
423
|
+
const configuredDimension = options.cfg.embedding.vectorDimension;
|
|
424
|
+
const store = options.store ?? dependencies.createStore(options.storePath, configuredDimension && configuredDimension > 0 ? configuredDimension : 384, options.cfg);
|
|
423
425
|
const keywordIndex = options.keywordIndex;
|
|
424
426
|
// Runtime overrides for live config editing from TUI
|
|
425
427
|
let cachedOverrides = loadRuntimeOverrides(options.storePath);
|
|
@@ -1394,6 +1396,8 @@ export const ragPlugin = async (input, _options) => {
|
|
|
1394
1396
|
}, logLevel);
|
|
1395
1397
|
// Use cached dimension from config if available (avoids blocking startup with an API call)
|
|
1396
1398
|
// If not set, probe the embedding provider once and persist the result.
|
|
1399
|
+
// When the probe fails, prefer the existing store's schema over the 384
|
|
1400
|
+
// default — a transient outage must not downgrade the store's dimension.
|
|
1397
1401
|
const embedder = createEmbedder(effectiveCfg);
|
|
1398
1402
|
let vectorDimension = effectiveCfg.embedding.vectorDimension;
|
|
1399
1403
|
if (vectorDimension && vectorDimension > 0) {
|
|
@@ -1403,34 +1407,47 @@ export const ragPlugin = async (input, _options) => {
|
|
|
1403
1407
|
}, logLevel);
|
|
1404
1408
|
}
|
|
1405
1409
|
else {
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
try {
|
|
1414
|
-
persistProbedDimension(configPath, vectorDimension);
|
|
1415
|
-
}
|
|
1416
|
-
catch { /* best-effort */ }
|
|
1410
|
+
const probe = await probeEmbeddingDimension(embedder);
|
|
1411
|
+
if (probe.dimension !== undefined) {
|
|
1412
|
+
vectorDimension = probe.dimension;
|
|
1413
|
+
const configPath = findConfigFile(input.directory);
|
|
1414
|
+
if (configPath) {
|
|
1415
|
+
try {
|
|
1416
|
+
persistProbedDimension(configPath, vectorDimension);
|
|
1417
1417
|
}
|
|
1418
|
+
catch { /* best-effort */ }
|
|
1418
1419
|
}
|
|
1419
1420
|
appendDebugLog(logFilePath, {
|
|
1420
1421
|
scope: "plugin",
|
|
1421
1422
|
message: `Vector dimension: ${vectorDimension}`,
|
|
1422
1423
|
}, logLevel);
|
|
1423
1424
|
}
|
|
1424
|
-
|
|
1425
|
+
else {
|
|
1426
|
+
vectorDimension = (await readStoreDimension(storePath)) ?? 384;
|
|
1425
1427
|
appendDebugLog(logFilePath, {
|
|
1426
1428
|
scope: "plugin",
|
|
1427
1429
|
message: `Dimension probe failed, falling back to ${vectorDimension}`,
|
|
1428
|
-
error:
|
|
1430
|
+
error: probe.error,
|
|
1429
1431
|
}, logLevel);
|
|
1430
1432
|
}
|
|
1431
1433
|
}
|
|
1432
1434
|
const store = createVectorStore(effectiveCfg, storePath, vectorDimension);
|
|
1433
1435
|
ragStores.set(input.directory, store);
|
|
1436
|
+
// Warn when the store was built by a different embedding model — vector
|
|
1437
|
+
// search fails (or silently degrades) until the index is rebuilt.
|
|
1438
|
+
try {
|
|
1439
|
+
const storeDimension = await store.getVectorDimension?.();
|
|
1440
|
+
if (storeDimension !== undefined && storeDimension !== vectorDimension) {
|
|
1441
|
+
appendDebugLog(logFilePath, {
|
|
1442
|
+
scope: "plugin",
|
|
1443
|
+
message: `Store vector dimension is ${storeDimension} but the configured embedder produces ${vectorDimension} — ` +
|
|
1444
|
+
"run 'opencode-rag index' to rebuild the index with the current model.",
|
|
1445
|
+
}, logLevel);
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
catch {
|
|
1449
|
+
// best-effort — dimension introspection must never block plugin startup
|
|
1450
|
+
}
|
|
1434
1451
|
// Load or create keyword index for hybrid search
|
|
1435
1452
|
const keywordIndex = await loadKeywordIndex(storePath, logFilePath, logLevel);
|
|
1436
1453
|
// Create description provider (enabled by default)
|