opencode-rag-plugin 1.22.0 → 1.22.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/index-command.js +10 -1
- package/dist/cli/commands/status.js +14 -0
- package/dist/cli/format.js +10 -1
- package/dist/core/bootstrap.d.ts +2 -0
- package/dist/core/bootstrap.js +45 -16
- package/dist/core/config.d.ts +14 -0
- package/dist/core/config.js +2 -0
- package/dist/core/interfaces.d.ts +8 -0
- package/dist/core/runtime-overrides.d.ts +1 -0
- package/dist/core/runtime-overrides.js +1 -0
- package/dist/embedder/factory.d.ts +20 -0
- package/dist/embedder/factory.js +24 -0
- package/dist/indexer/pipeline.js +175 -64
- package/dist/indexer/stats.d.ts +4 -0
- package/dist/indexer/stats.js +2 -0
- package/dist/indexer/worker.d.ts +8 -2
- package/dist/indexer/worker.js +9 -4
- package/dist/plugin.js +32 -15
- package/dist/retriever/context-optimizer.js +9 -2
- package/dist/tui.js +11 -3
- package/dist/vectorstore/lancedb.d.ts +60 -1
- package/dist/vectorstore/lancedb.js +204 -5
- package/package.json +1 -1
|
@@ -98,6 +98,7 @@ export function registerIndexCommand(program) {
|
|
|
98
98
|
}
|
|
99
99
|
}
|
|
100
100
|
logCliInfo(logFilePath, "index", `${c.label("Scanning:")} ${c.file(cwd)}`);
|
|
101
|
+
let incomplete = false;
|
|
101
102
|
const runPass = async (watchTriggered = false, abortSignal, filterPaths) => {
|
|
102
103
|
const passStarted = Date.now();
|
|
103
104
|
const stats = await runIndexPass({
|
|
@@ -116,6 +117,14 @@ export function registerIndexCommand(program) {
|
|
|
116
117
|
});
|
|
117
118
|
if (!watchTriggered) {
|
|
118
119
|
logIndexSummary(logFilePath, stats);
|
|
120
|
+
if (stats.embeddingUnavailable) {
|
|
121
|
+
incomplete = true;
|
|
122
|
+
logCliInfo(logFilePath, "index", `\n${c.error("Indexing incomplete:")} the embedding provider was unavailable — nothing was stored. Fix the provider and run the index again.`);
|
|
123
|
+
}
|
|
124
|
+
else if (stats.totalChunks === 0 && stats.newFiles + stats.modifiedFiles > 0) {
|
|
125
|
+
incomplete = true;
|
|
126
|
+
logCliInfo(logFilePath, "index", `\n${c.warn("Indexing incomplete:")} files were processed but no chunks were stored. Check the log for details.`);
|
|
127
|
+
}
|
|
119
128
|
logCliInfo(logFilePath, "index", `\n${c.success("Indexing complete.")} ${c.num(stats.finalCount)} chunks stored (${formatDuration(Date.now() - passStarted)}).`);
|
|
120
129
|
}
|
|
121
130
|
if (sigReceived && !watchTriggered) {
|
|
@@ -128,7 +137,7 @@ export function registerIndexCommand(program) {
|
|
|
128
137
|
process.removeListener("SIGTERM", handleSigint);
|
|
129
138
|
if (!options.watch) {
|
|
130
139
|
await cleanupContext(ctx);
|
|
131
|
-
process.exit(sigReceived ? 130 : 0);
|
|
140
|
+
process.exit(sigReceived ? 130 : incomplete ? 1 : 0);
|
|
132
141
|
}
|
|
133
142
|
// Only one watcher may run per workspace — a background auto-indexer
|
|
134
143
|
// in an OpenCode session (or another `index --watch`) may already own
|
|
@@ -75,6 +75,20 @@ export function registerStatusCommand(program) {
|
|
|
75
75
|
logCliInfo(logFilePath, "status", `${c.label("Store path:")} ${c.file(storePath)}`);
|
|
76
76
|
logCliInfo(logFilePath, "status", `${c.label("Embedding provider:")} ${c.value(config.embedding.provider)}`);
|
|
77
77
|
logCliInfo(logFilePath, "status", `${c.label("Embedding model:")} ${c.value(config.embedding.model)}`);
|
|
78
|
+
const storeDimension = await store.getVectorDimension?.();
|
|
79
|
+
const embedderDimension = config.embedding.vectorDimension;
|
|
80
|
+
if (embedderDimension && embedderDimension > 0) {
|
|
81
|
+
logCliInfo(logFilePath, "status", `${c.label("Embedder dim:")} ${c.num(embedderDimension)}`);
|
|
82
|
+
}
|
|
83
|
+
if (storeDimension !== undefined) {
|
|
84
|
+
logCliInfo(logFilePath, "status", `${c.label("Store dimension:")} ${c.num(storeDimension)}`);
|
|
85
|
+
}
|
|
86
|
+
if (storeDimension !== undefined &&
|
|
87
|
+
embedderDimension !== undefined &&
|
|
88
|
+
embedderDimension > 0 &&
|
|
89
|
+
storeDimension !== embedderDimension) {
|
|
90
|
+
logCliInfo(logFilePath, "status", `${c.label("Dimension mismatch:")} ${c.warn("yes")} — the store was built with a different embedding model. Run 'opencode-rag index' to rebuild.`);
|
|
91
|
+
}
|
|
78
92
|
logCliInfo(logFilePath, "status", `${c.label("File extensions:")} ${config.indexing.includeExtensions.join(", ")}`);
|
|
79
93
|
logCliInfo(logFilePath, "status", `${c.label("Excluded dirs:")} ${config.indexing.excludeDirs.join(", ")}`);
|
|
80
94
|
logCliInfo(logFilePath, "status", `${c.label("Default top-K:")} ${c.num(config.retrieval.topK)}`);
|
package/dist/cli/format.js
CHANGED
|
@@ -92,7 +92,7 @@ export async function resolveCliContext(opt, logFilePath, bootstrapOpts) {
|
|
|
92
92
|
configPath: opt.config,
|
|
93
93
|
...bootstrapOpts,
|
|
94
94
|
});
|
|
95
|
-
logCliInfo(logFilePath, "config", `${c.label("Config:")} ${c.file(ctx.
|
|
95
|
+
logCliInfo(logFilePath, "config", `${c.label("Config:")} ${c.file(ctx.configPath ?? "(built-in defaults)")}`);
|
|
96
96
|
logConfigDetails(logFilePath, ctx.config);
|
|
97
97
|
return ctx;
|
|
98
98
|
}
|
|
@@ -105,6 +105,9 @@ export async function resolveCliContext(opt, logFilePath, bootstrapOpts) {
|
|
|
105
105
|
function logConfigDetails(logFilePath, config) {
|
|
106
106
|
logCliInfo(logFilePath, "config", ` ${c.label("Embedding provider:")} ${c.value(config.embedding.provider)}`);
|
|
107
107
|
logCliInfo(logFilePath, "config", ` ${c.label("Embedding model:")} ${c.value(config.embedding.model)}`);
|
|
108
|
+
if (config.embedding.vectorDimension && config.embedding.vectorDimension > 0) {
|
|
109
|
+
logCliInfo(logFilePath, "config", ` ${c.label("Embedding dim:")} ${c.num(config.embedding.vectorDimension)}`);
|
|
110
|
+
}
|
|
108
111
|
logCliInfo(logFilePath, "config", ` ${c.label("Vector store:")} ${c.file(config.vectorStore.path)}`);
|
|
109
112
|
}
|
|
110
113
|
/**
|
|
@@ -153,7 +156,13 @@ export function logIndexSummary(logFilePath, stats) {
|
|
|
153
156
|
if (stats.descriptionFailedFiles > 0) {
|
|
154
157
|
logCliInfo(logFilePath, "index", ` ${c.label("Desc failed:")} ${c.num(stats.descriptionFailedFiles)}`);
|
|
155
158
|
}
|
|
159
|
+
if (stats.embeddingFailures > 0) {
|
|
160
|
+
logCliInfo(logFilePath, "index", ` ${c.label("Embed failed:")} ${c.num(stats.embeddingFailures)} chunk(s) without vectors — retried next pass`);
|
|
161
|
+
}
|
|
156
162
|
logCliInfo(logFilePath, "index", ` ${c.label("Chunks written:")} ${c.num(stats.totalChunks)}`);
|
|
163
|
+
if (stats.embeddingUnavailable) {
|
|
164
|
+
logCliInfo(logFilePath, "index", ` ${c.error("Embedding provider unavailable during this pass — no chunks were stored.")}`);
|
|
165
|
+
}
|
|
157
166
|
}
|
|
158
167
|
/**
|
|
159
168
|
* Format a millisecond duration into a compact human-readable string.
|
package/dist/core/bootstrap.d.ts
CHANGED
|
@@ -35,6 +35,8 @@ export interface RagContext {
|
|
|
35
35
|
dimension: number;
|
|
36
36
|
/** Resolved path to the debug log file. */
|
|
37
37
|
logFilePath: string;
|
|
38
|
+
/** Resolved path to the config file, or undefined when built-in defaults are used. */
|
|
39
|
+
configPath?: string;
|
|
38
40
|
}
|
|
39
41
|
/** Bootstrap the full RAG pipeline context: load config, resolve API keys, create embedder, vector store, keyword index, and description provider. */
|
|
40
42
|
export declare function resolveRagContext(opts?: BootstrapOptions): Promise<RagContext>;
|
package/dist/core/bootstrap.js
CHANGED
|
@@ -6,26 +6,46 @@ import path from "node:path";
|
|
|
6
6
|
import { loadConfig, findConfigFile, resolveLogConfig, DEFAULT_CONFIG } from "./config.js";
|
|
7
7
|
import { resolveApiKey } from "./resolve-api-key.js";
|
|
8
8
|
import { loadChunkersFromConfig } from "../chunker/loader.js";
|
|
9
|
-
import { createEmbedder } from "../embedder/factory.js";
|
|
9
|
+
import { createEmbedder, probeEmbeddingDimension } from "../embedder/factory.js";
|
|
10
10
|
import { createDescriptionProvider } from "../describer/factory.js";
|
|
11
11
|
import { createVectorStore } from "../vectorstore/factory.js";
|
|
12
|
+
import { readStoreDimension } from "../vectorstore/lancedb.js";
|
|
12
13
|
import { KeywordIndex } from "../retriever/keyword-index.js";
|
|
13
|
-
/**
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
14
|
+
/**
|
|
15
|
+
* Resolve the embedding dimension for the store.
|
|
16
|
+
*
|
|
17
|
+
* Precedence: an explicit `embedding.vectorDimension` in the config wins (it
|
|
18
|
+
* is persisted after the first successful probe), then a live probe of the
|
|
19
|
+
* provider, then the dimension of an existing store table, then the 384
|
|
20
|
+
* fallback. The old behavior always probed and fell back to 384 — which
|
|
21
|
+
* silently created 384-dimensional stores whenever the provider was down,
|
|
22
|
+
* even though the config declared the real dimension.
|
|
23
|
+
*
|
|
24
|
+
* @param embedder - Configured embedding provider (probed only when needed).
|
|
25
|
+
* @param storePath - Vector store path, used to read an existing schema.
|
|
26
|
+
* @param skipProbe - When true, never call the provider (read-only commands).
|
|
27
|
+
* @returns The resolved dimension.
|
|
28
|
+
*/
|
|
29
|
+
async function resolveDimension(embedder, storePath, skipProbe, configured) {
|
|
30
|
+
if (configured && configured > 0) {
|
|
31
|
+
return configured;
|
|
20
32
|
}
|
|
21
|
-
|
|
22
|
-
|
|
33
|
+
if (!skipProbe) {
|
|
34
|
+
const probe = await probeEmbeddingDimension(embedder);
|
|
35
|
+
if (probe.dimension !== undefined) {
|
|
36
|
+
return probe.dimension;
|
|
37
|
+
}
|
|
38
|
+
const storeDimension = await readStoreDimension(storePath);
|
|
39
|
+
if (storeDimension !== undefined) {
|
|
40
|
+
console.warn(`[bootstrap] Could not probe embedding dimension (${probe.error?.message ?? "unknown error"}) — ` +
|
|
41
|
+
`using the existing store's dimension ${storeDimension}.`);
|
|
42
|
+
return storeDimension;
|
|
43
|
+
}
|
|
44
|
+
console.warn(`[bootstrap] Could not probe embedding dimension (${probe.error?.message ?? "unknown error"}) — ` +
|
|
45
|
+
"falling back to 384. Set embedding.vectorDimension explicitly if the provider produces a different size.");
|
|
46
|
+
return 384;
|
|
23
47
|
}
|
|
24
|
-
|
|
25
|
-
// surface the fallback loudly so misconfigured providers are easy to spot.
|
|
26
|
-
console.warn("[bootstrap] Could not probe embedding dimension — falling back to 384. " +
|
|
27
|
-
"If indexing later fails with dimension errors, set embedding.vectorDimension explicitly.");
|
|
28
|
-
return 384;
|
|
48
|
+
return (await readStoreDimension(storePath)) ?? 384;
|
|
29
49
|
}
|
|
30
50
|
/** Load the keyword index from disk, or create a new empty one if loading fails. */
|
|
31
51
|
async function loadKeywordIndex(storePath) {
|
|
@@ -61,8 +81,16 @@ export async function resolveRagContext(opts = {}) {
|
|
|
61
81
|
}
|
|
62
82
|
const logFilePath = path.resolve(workDir, resolveLogConfig(cfg).logFilePath);
|
|
63
83
|
const embedder = createEmbedder(cfg);
|
|
64
|
-
const dimension = opts.skipProbe ? 384 : await probeDimension(embedder);
|
|
65
84
|
const storePath = path.resolve(workDir, cfg.vectorStore.path);
|
|
85
|
+
const dimension = await resolveDimension(embedder, storePath, opts.skipProbe ?? false, cfg.embedding.vectorDimension);
|
|
86
|
+
// Warn (but do not block) when the existing store schema disagrees with the
|
|
87
|
+
// resolved dimension: the next index pass rebuilds automatically, and search
|
|
88
|
+
// callers would otherwise see only cryptic LanceDB errors.
|
|
89
|
+
const storeDimension = await readStoreDimension(storePath);
|
|
90
|
+
if (storeDimension !== undefined && storeDimension !== dimension) {
|
|
91
|
+
console.warn(`[bootstrap] Store was built with vector dimension ${storeDimension} but the current embedder produces ${dimension} — ` +
|
|
92
|
+
"run 'opencode-rag index' to rebuild the index with the current model.");
|
|
93
|
+
}
|
|
66
94
|
const store = createVectorStore(cfg, storePath, dimension);
|
|
67
95
|
const keywordIndex = opts.skipKeywordIndex
|
|
68
96
|
? new KeywordIndex(storePath)
|
|
@@ -80,6 +108,7 @@ export async function resolveRagContext(opts = {}) {
|
|
|
80
108
|
descriptionProvider,
|
|
81
109
|
dimension,
|
|
82
110
|
logFilePath,
|
|
111
|
+
configPath,
|
|
83
112
|
};
|
|
84
113
|
}
|
|
85
114
|
//# sourceMappingURL=bootstrap.js.map
|
package/dist/core/config.d.ts
CHANGED
|
@@ -183,6 +183,8 @@ export interface TuiConfig {
|
|
|
183
183
|
fileListKeybinding: string;
|
|
184
184
|
/** Keybinding to toggle the chunk viewer panel. */
|
|
185
185
|
chunksKeybinding: string;
|
|
186
|
+
/** Keybinding to open the RAG settings dialog. Must be distinguishable by the terminal (see docs). */
|
|
187
|
+
settingsKeybinding: string;
|
|
186
188
|
}
|
|
187
189
|
/** Configuration for the standalone MCP (Model Context Protocol) server. */
|
|
188
190
|
export interface McpConfig {
|
|
@@ -294,6 +296,18 @@ export interface RagConfig {
|
|
|
294
296
|
ollamaMaxBatchSize?: number;
|
|
295
297
|
/** Maximum concurrent description generation requests. */
|
|
296
298
|
descriptionConcurrency?: number;
|
|
299
|
+
/**
|
|
300
|
+
* Include LLM-generated chunk descriptions in the text that gets embedded.
|
|
301
|
+
*
|
|
302
|
+
* Descriptions help general-purpose embedding models align natural-language
|
|
303
|
+
* queries with raw code. Code-specialized models (e.g. jina-code-embeddings,
|
|
304
|
+
* trained for NL→code retrieval with instruction prefixes) do not need the
|
|
305
|
+
* crutch — and their passage prompt expects a code snippet, not prose.
|
|
306
|
+
* When false, descriptions are still generated and stored (search results,
|
|
307
|
+
* web UI), but only path/meta header/content are embedded.
|
|
308
|
+
* @default true
|
|
309
|
+
*/
|
|
310
|
+
embedDescriptions?: boolean;
|
|
297
311
|
/**
|
|
298
312
|
* Maximum file size in bytes for SVG/XML files before chunking is skipped.
|
|
299
313
|
* Large SVGs can cause tree-sitter to hang. Set to 0 for no limit.
|
package/dist/core/config.js
CHANGED
|
@@ -117,6 +117,7 @@ export const DEFAULT_CONFIG = {
|
|
|
117
117
|
embedConcurrency: 3,
|
|
118
118
|
ollamaMaxBatchSize: 100,
|
|
119
119
|
descriptionConcurrency: 4,
|
|
120
|
+
embedDescriptions: true,
|
|
120
121
|
maxSvgSizeBytes: 1_048_576,
|
|
121
122
|
optimizeIntervalWindows: 8,
|
|
122
123
|
},
|
|
@@ -294,6 +295,7 @@ export const DEFAULT_CONFIG = {
|
|
|
294
295
|
tui: {
|
|
295
296
|
fileListKeybinding: "ctrl+enter",
|
|
296
297
|
chunksKeybinding: "ctrl+alt+enter",
|
|
298
|
+
settingsKeybinding: "ctrl+shift+r",
|
|
297
299
|
},
|
|
298
300
|
logging: {
|
|
299
301
|
level: "info",
|
|
@@ -224,6 +224,14 @@ export interface VectorStore {
|
|
|
224
224
|
* Used by the index pipeline to detect silent corruption before scanning.
|
|
225
225
|
*/
|
|
226
226
|
checkIntegrity?(): Promise<boolean>;
|
|
227
|
+
/**
|
|
228
|
+
* Return the actual embedding dimension of the stored vector column, or
|
|
229
|
+
* `undefined` when the store has no table yet. Used to detect a store that
|
|
230
|
+
* was built with a different embedding model than the currently configured
|
|
231
|
+
* one — a mismatch makes every vector search fail and silently corrupts
|
|
232
|
+
* writes, so the pipeline rebuilds the index instead.
|
|
233
|
+
*/
|
|
234
|
+
getVectorDimension?(): Promise<number | undefined>;
|
|
227
235
|
}
|
|
228
236
|
/** Filter criteria for narrowing search results by file path, language, kind, or extension. */
|
|
229
237
|
export interface MetadataFilter {
|
|
@@ -149,6 +149,7 @@ export function applyRuntimeOverrides(cfg, overrides) {
|
|
|
149
149
|
...(merged.tui ?? {}),
|
|
150
150
|
fileListKeybinding: overrides.tui.fileListKeybinding ?? merged.tui?.fileListKeybinding ?? DEFAULT_CONFIG.tui.fileListKeybinding,
|
|
151
151
|
chunksKeybinding: overrides.tui.chunksKeybinding ?? merged.tui?.chunksKeybinding ?? DEFAULT_CONFIG.tui.chunksKeybinding,
|
|
152
|
+
settingsKeybinding: overrides.tui.settingsKeybinding ?? merged.tui?.settingsKeybinding ?? DEFAULT_CONFIG.tui.settingsKeybinding,
|
|
152
153
|
};
|
|
153
154
|
}
|
|
154
155
|
return merged;
|
|
@@ -15,6 +15,26 @@ import type { RagConfig } from "../core/config.js";
|
|
|
15
15
|
* @throws If the provider is unsupported or a required apiKey is not set
|
|
16
16
|
*/
|
|
17
17
|
export declare function createEmbedder(config: RagConfig): EmbeddingProvider;
|
|
18
|
+
/** Result of probing an embedding provider for its output dimension. */
|
|
19
|
+
export interface EmbeddingProbeResult {
|
|
20
|
+
/** Detected vector dimension, or `undefined` when the probe failed. */
|
|
21
|
+
dimension?: number;
|
|
22
|
+
/** Error raised by the provider (or an empty response), when the probe failed. */
|
|
23
|
+
error?: Error;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Probe an embedding provider with a single short text to discover the
|
|
27
|
+
* dimension of the vectors it produces.
|
|
28
|
+
*
|
|
29
|
+
* Used as a health/dimension preflight before expensive index work: a provider
|
|
30
|
+
* that is down (server not running, model not pulled) fails here in
|
|
31
|
+
* milliseconds instead of after chunking and describing every file. Errors are
|
|
32
|
+
* captured in the result rather than thrown.
|
|
33
|
+
*
|
|
34
|
+
* @param embedder - The embedding provider to probe.
|
|
35
|
+
* @returns The detected dimension, or the failure reason.
|
|
36
|
+
*/
|
|
37
|
+
export declare function probeEmbeddingDimension(embedder: EmbeddingProvider): Promise<EmbeddingProbeResult>;
|
|
18
38
|
/**
|
|
19
39
|
* Embed a list of texts in batches with optional concurrency control and per-batch retry.
|
|
20
40
|
*
|
package/dist/embedder/factory.js
CHANGED
|
@@ -40,6 +40,30 @@ export function createEmbedder(config) {
|
|
|
40
40
|
}
|
|
41
41
|
throw new Error(`Unknown embedding provider: ${provider}`);
|
|
42
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Probe an embedding provider with a single short text to discover the
|
|
45
|
+
* dimension of the vectors it produces.
|
|
46
|
+
*
|
|
47
|
+
* Used as a health/dimension preflight before expensive index work: a provider
|
|
48
|
+
* that is down (server not running, model not pulled) fails here in
|
|
49
|
+
* milliseconds instead of after chunking and describing every file. Errors are
|
|
50
|
+
* captured in the result rather than thrown.
|
|
51
|
+
*
|
|
52
|
+
* @param embedder - The embedding provider to probe.
|
|
53
|
+
* @returns The detected dimension, or the failure reason.
|
|
54
|
+
*/
|
|
55
|
+
export async function probeEmbeddingDimension(embedder) {
|
|
56
|
+
try {
|
|
57
|
+
const probe = await embedder.embed(["dimension-probe"], "query");
|
|
58
|
+
if (probe && probe[0] && probe[0].length > 0 && typeof probe[0][0] === "number") {
|
|
59
|
+
return { dimension: probe[0].length };
|
|
60
|
+
}
|
|
61
|
+
return { error: new Error("Embedding provider returned an empty probe vector") };
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
return { error: err instanceof Error ? err : new Error(String(err)) };
|
|
65
|
+
}
|
|
66
|
+
}
|
|
43
67
|
/**
|
|
44
68
|
* HTTP statuses that indicate a permanent failure — retrying cannot help
|
|
45
69
|
* (auth errors, bad requests, missing resources). Providers raise these as
|