opencode-rag-plugin 1.19.4 → 1.19.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunker/base.js +19 -5
- package/dist/chunker/factory.js +27 -9
- package/dist/chunker/grammar.d.ts +18 -1
- package/dist/chunker/grammar.js +48 -10
- package/dist/chunker/image.js +5 -0
- package/dist/chunker/pdf.js +30 -14
- package/dist/cli/commands/backend-detect.d.ts +61 -0
- package/dist/cli/commands/backend-detect.js +119 -0
- package/dist/cli/commands/index-command.js +11 -0
- package/dist/cli/commands/init-helpers.d.ts +4 -1
- package/dist/cli/commands/init-helpers.js +21 -4
- package/dist/cli/commands/init.js +61 -24
- package/dist/cli/commands/quirk.js +9 -3
- package/dist/cli/commands/setup.js +12 -3
- package/dist/cli/commands/status.js +14 -5
- package/dist/cli/commands/ui.js +23 -9
- package/dist/cli/commands/update.js +4 -5
- package/dist/cli/format.d.ts +5 -2
- package/dist/cli/format.js +14 -5
- package/dist/content/image.js +33 -11
- package/dist/content/reader.js +74 -13
- package/dist/core/bootstrap.js +10 -3
- package/dist/core/config.d.ts +27 -1
- package/dist/core/config.js +41 -2
- package/dist/core/desc-cache.d.ts +8 -2
- package/dist/core/desc-cache.js +10 -3
- package/dist/core/doc-progress.js +5 -2
- package/dist/core/interfaces.d.ts +32 -4
- package/dist/core/manifest.js +1 -1
- package/dist/core/provider-defaults.d.ts +2 -0
- package/dist/core/provider-defaults.js +19 -4
- package/dist/core/runtime-overrides.d.ts +0 -6
- package/dist/core/version-check.d.ts +5 -0
- package/dist/core/version-check.js +8 -2
- package/dist/describer/anthropic.d.ts +2 -2
- package/dist/describer/anthropic.js +19 -5
- package/dist/describer/describer.d.ts +20 -0
- package/dist/describer/describer.js +132 -16
- package/dist/describer/gemini.js +25 -10
- package/dist/describer/shared.d.ts +28 -0
- package/dist/describer/shared.js +60 -0
- package/dist/embedder/factory.d.ts +5 -3
- package/dist/embedder/factory.js +42 -9
- package/dist/embedder/health.js +19 -19
- package/dist/embedder/http.d.ts +14 -1
- package/dist/embedder/http.js +60 -6
- package/dist/embedder/ollama.d.ts +3 -1
- package/dist/embedder/ollama.js +12 -2
- package/dist/eval/session-logger.js +7 -0
- package/dist/eval/storage.js +8 -0
- package/dist/indexer/git-diff.d.ts +1 -1
- package/dist/indexer/git-diff.js +5 -1
- package/dist/indexer/pipeline.js +511 -346
- package/dist/indexer/stats.d.ts +2 -0
- package/dist/indexer/stats.js +1 -0
- package/dist/indexer/watch.js +8 -1
- package/dist/indexer/worker.js +21 -0
- package/dist/mcp/cli.js +4 -0
- package/dist/mcp/handlers.js +16 -5
- package/dist/mcp/server.js +3 -0
- package/dist/opencode/create-read-tool.js +14 -3
- package/dist/opencode/tool-args.js +23 -1
- package/dist/plugin.js +66 -152
- package/dist/quirks/auto-capture.js +5 -0
- package/dist/quirks/quirk-store.d.ts +1 -1
- package/dist/quirks/quirk-store.js +56 -17
- package/dist/retriever/context-optimizer.js +18 -4
- package/dist/retriever/keyword-index.d.ts +2 -0
- package/dist/retriever/keyword-index.js +38 -4
- package/dist/retriever/retriever.js +6 -1
- package/dist/tui.js +41 -4
- package/dist/vectorstore/lancedb.d.ts +104 -8
- package/dist/vectorstore/lancedb.js +345 -71
- package/dist/vectorstore/memory.d.ts +8 -3
- package/dist/vectorstore/memory.js +27 -4
- package/dist/watcher.d.ts +8 -0
- package/dist/watcher.js +237 -85
- package/dist/web/api.d.ts +5 -1
- package/dist/web/api.js +195 -69
- package/dist/web/server.d.ts +2 -0
- package/dist/web/server.js +66 -28
- package/dist/web/static.d.ts +5 -2
- package/dist/web/static.js +9 -5
- package/dist/web/ui/assets/index-BDPYdtA1.js +3 -0
- package/dist/web/ui/index.html +1 -1
- package/package.json +1 -1
- package/dist/web/ui/assets/index-CJBvt6e0.js +0 -3
package/dist/chunker/base.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @fileoverview Abstract base class for tree-sitter based language chunkers.
|
|
3
3
|
*/
|
|
4
4
|
import { Parser } from "web-tree-sitter";
|
|
5
|
-
import { loadLanguage, loadLanguageFromPath, walkTree } from "./grammar.js";
|
|
5
|
+
import { loadLanguage, loadLanguageFromPath, walkTree, buildByteOffsetMap } from "./grammar.js";
|
|
6
6
|
import { uuid } from "./uuid.js";
|
|
7
7
|
/** Module-level parser pool shared across all TreeSitterChunker instances. */
|
|
8
8
|
const parserPool = new Map();
|
|
@@ -45,8 +45,15 @@ export class TreeSitterChunker {
|
|
|
45
45
|
if (!tree) {
|
|
46
46
|
return [];
|
|
47
47
|
}
|
|
48
|
-
|
|
49
|
-
|
|
48
|
+
// tree-sitter ranges are UTF-8 byte offsets; translate for slicing
|
|
49
|
+
const offsetMap = buildByteOffsetMap(content);
|
|
50
|
+
let nodes;
|
|
51
|
+
try {
|
|
52
|
+
nodes = walkTree(tree.rootNode, types, content, 10, 0, offsetMap);
|
|
53
|
+
}
|
|
54
|
+
finally {
|
|
55
|
+
tree.delete();
|
|
56
|
+
}
|
|
50
57
|
return nodes.map((node) => ({
|
|
51
58
|
id: uuid(),
|
|
52
59
|
content: node.text,
|
|
@@ -108,8 +115,15 @@ export class TreeSitterChunker {
|
|
|
108
115
|
if (!tree) {
|
|
109
116
|
return [];
|
|
110
117
|
}
|
|
111
|
-
|
|
112
|
-
|
|
118
|
+
// tree-sitter ranges are UTF-8 byte offsets; translate for slicing
|
|
119
|
+
const offsetMap = buildByteOffsetMap(content);
|
|
120
|
+
let nodes;
|
|
121
|
+
try {
|
|
122
|
+
nodes = walkTree(tree.rootNode, this.nodeTypes, content, 10, 0, offsetMap);
|
|
123
|
+
}
|
|
124
|
+
finally {
|
|
125
|
+
tree.delete();
|
|
126
|
+
}
|
|
113
127
|
return nodes.map((node) => ({
|
|
114
128
|
id: uuid(),
|
|
115
129
|
content: node.text,
|
package/dist/chunker/factory.js
CHANGED
|
@@ -174,14 +174,26 @@ function splitOversized(chunks, filePath) {
|
|
|
174
174
|
}
|
|
175
175
|
return result;
|
|
176
176
|
}
|
|
177
|
-
/**
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
177
|
+
/**
|
|
178
|
+
* Wrap a chunker with a per-file byte limit.
|
|
179
|
+
*
|
|
180
|
+
* Previously the limit was assigned onto the shared chunker singleton
|
|
181
|
+
* (`chunker.maxContentBytes = ...`) — under pLimit concurrency one file's
|
|
182
|
+
* limit could bleed into unrelated concurrent files, and it was never
|
|
183
|
+
* reset, so a later large XML/SVG file was silently skipped.
|
|
184
|
+
*/
|
|
185
|
+
function withByteLimit(chunker, limit) {
|
|
186
|
+
const language = chunker.language;
|
|
187
|
+
return {
|
|
188
|
+
language,
|
|
189
|
+
fileExtensions: chunker.fileExtensions,
|
|
190
|
+
async chunk(filePath, content) {
|
|
191
|
+
if (limit > 0 && Buffer.byteLength(content, "utf-8") > limit) {
|
|
192
|
+
throw new Error(`File exceeds ${limit} byte limit for ${language} chunker`);
|
|
193
|
+
}
|
|
194
|
+
return chunker.chunk(filePath, content);
|
|
195
|
+
},
|
|
196
|
+
};
|
|
185
197
|
}
|
|
186
198
|
/**
|
|
187
199
|
* Chunk a file by looking up its registered chunker, applying node-type
|
|
@@ -203,7 +215,13 @@ export async function chunkFile(filePath, content, nodeTypesOverrides, options)
|
|
|
203
215
|
chunker = chunker.withNodeTypes(new Set(overrideTypes));
|
|
204
216
|
}
|
|
205
217
|
}
|
|
206
|
-
|
|
218
|
+
// Per-file byte limit (SVG/XML/csproj) without mutating the shared singleton
|
|
219
|
+
if (options?.maxSvgSizeBytes && chunker instanceof TreeSitterChunker) {
|
|
220
|
+
const ext = filePath.toLowerCase();
|
|
221
|
+
if (ext.endsWith(".svg") || ext.endsWith(".xml") || ext.endsWith(".csproj")) {
|
|
222
|
+
chunker = withByteLimit(chunker, options.maxSvgSizeBytes);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
207
225
|
const chunks = await chunker.chunk(filePath, content);
|
|
208
226
|
if (chunks.length === 0) {
|
|
209
227
|
return fallbackChunker.chunk(filePath, content);
|
|
@@ -57,6 +57,23 @@ export interface AstNode {
|
|
|
57
57
|
*/
|
|
58
58
|
leadingDoc?: string;
|
|
59
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* Build a byte-offset → UTF-16-index map for a source string.
|
|
62
|
+
*
|
|
63
|
+
* tree-sitter reports node ranges as UTF-8 BYTE offsets, but JavaScript
|
|
64
|
+
* `String.prototype.slice` counts UTF-16 code units. For any file containing
|
|
65
|
+
* multi-byte characters the two disagree, so slicing with raw node indices
|
|
66
|
+
* truncates/garbles chunk content and doc comments. Tree-sitter node
|
|
67
|
+
* boundaries always align to code-point boundaries, so an exact map from
|
|
68
|
+
* byte offset to UTF-16 index is sufficient.
|
|
69
|
+
*
|
|
70
|
+
* Returns `null` for pure-ASCII sources (byte offsets == UTF-16 offsets),
|
|
71
|
+
* which is the common case and avoids the allocation.
|
|
72
|
+
*
|
|
73
|
+
* @param source - The full source text.
|
|
74
|
+
* @returns A Map from UTF-8 byte offset to UTF-16 index, or `null` for ASCII.
|
|
75
|
+
*/
|
|
76
|
+
export declare function buildByteOffsetMap(source: string): Map<number, number> | null;
|
|
60
77
|
/**
|
|
61
78
|
* Recursively walk a tree-sitter AST and collect nodes matching the given
|
|
62
79
|
* type set.
|
|
@@ -75,4 +92,4 @@ export interface AstNode {
|
|
|
75
92
|
* @param depth - Current recursion depth (internal, start at 0).
|
|
76
93
|
* @returns An array of {@link AstNode} objects for matching declarations.
|
|
77
94
|
*/
|
|
78
|
-
export declare function walkTree(node: Node, nodeTypes: Set<string>, source: string, maxDepth?: number, depth?: number): AstNode[];
|
|
95
|
+
export declare function walkTree(node: Node, nodeTypes: Set<string>, source: string, maxDepth?: number, depth?: number, offsetMap?: Map<number, number> | null): AstNode[];
|
package/dist/chunker/grammar.js
CHANGED
|
@@ -120,6 +120,44 @@ const COMMENT_NODE_TYPES = new Set([
|
|
|
120
120
|
"marginalia",
|
|
121
121
|
"Comment",
|
|
122
122
|
]);
|
|
123
|
+
/**
|
|
124
|
+
* Build a byte-offset → UTF-16-index map for a source string.
|
|
125
|
+
*
|
|
126
|
+
* tree-sitter reports node ranges as UTF-8 BYTE offsets, but JavaScript
|
|
127
|
+
* `String.prototype.slice` counts UTF-16 code units. For any file containing
|
|
128
|
+
* multi-byte characters the two disagree, so slicing with raw node indices
|
|
129
|
+
* truncates/garbles chunk content and doc comments. Tree-sitter node
|
|
130
|
+
* boundaries always align to code-point boundaries, so an exact map from
|
|
131
|
+
* byte offset to UTF-16 index is sufficient.
|
|
132
|
+
*
|
|
133
|
+
* Returns `null` for pure-ASCII sources (byte offsets == UTF-16 offsets),
|
|
134
|
+
* which is the common case and avoids the allocation.
|
|
135
|
+
*
|
|
136
|
+
* @param source - The full source text.
|
|
137
|
+
* @returns A Map from UTF-8 byte offset to UTF-16 index, or `null` for ASCII.
|
|
138
|
+
*/
|
|
139
|
+
export function buildByteOffsetMap(source) {
|
|
140
|
+
if (!/[\u0080-\uffff]/u.test(source))
|
|
141
|
+
return null;
|
|
142
|
+
const map = new Map();
|
|
143
|
+
let byteOffset = 0;
|
|
144
|
+
let utf16Index = 0;
|
|
145
|
+
for (const ch of source) {
|
|
146
|
+
map.set(byteOffset, utf16Index);
|
|
147
|
+
byteOffset += Buffer.byteLength(ch, "utf-8");
|
|
148
|
+
utf16Index += ch.length;
|
|
149
|
+
}
|
|
150
|
+
map.set(byteOffset, utf16Index);
|
|
151
|
+
return map;
|
|
152
|
+
}
|
|
153
|
+
/** Slice `source` by tree-sitter byte offsets, translating through the map when present. */
|
|
154
|
+
function sliceByBytes(source, startByte, endByte, offsetMap) {
|
|
155
|
+
if (!offsetMap)
|
|
156
|
+
return source.slice(startByte, endByte);
|
|
157
|
+
const start = offsetMap.get(startByte) ?? startByte;
|
|
158
|
+
const end = offsetMap.get(endByte) ?? endByte;
|
|
159
|
+
return source.slice(start, end);
|
|
160
|
+
}
|
|
123
161
|
function cleanCommentText(text) {
|
|
124
162
|
const lines = text.split("\n");
|
|
125
163
|
if (text.startsWith("/*")) {
|
|
@@ -152,19 +190,19 @@ function cleanCommentText(text) {
|
|
|
152
190
|
let cleaned = lines.map((line) => line.replace(prefix, ""));
|
|
153
191
|
return cleaned.filter((l) => l.trim().length > 0).join("\n");
|
|
154
192
|
}
|
|
155
|
-
function extractLeadingComments(node, source) {
|
|
193
|
+
function extractLeadingComments(node, source, offsetMap) {
|
|
156
194
|
const comments = [];
|
|
157
195
|
let sibling = node.previousSibling;
|
|
158
196
|
while (sibling) {
|
|
159
197
|
if (COMMENT_NODE_TYPES.has(sibling.type)) {
|
|
160
|
-
const raw = source
|
|
198
|
+
const raw = sliceByBytes(source, sibling.startIndex, sibling.endIndex, offsetMap);
|
|
161
199
|
comments.unshift(cleanCommentText(raw));
|
|
162
200
|
sibling = sibling.previousSibling;
|
|
163
201
|
}
|
|
164
202
|
else if (sibling.type === "expression_statement") {
|
|
165
203
|
const firstChild = sibling.namedChildren[0];
|
|
166
204
|
if (firstChild?.type === "string") {
|
|
167
|
-
const raw = source
|
|
205
|
+
const raw = sliceByBytes(source, firstChild.startIndex, firstChild.endIndex, offsetMap);
|
|
168
206
|
comments.unshift(cleanCommentText(raw));
|
|
169
207
|
sibling = sibling.previousSibling;
|
|
170
208
|
}
|
|
@@ -180,7 +218,7 @@ function extractLeadingComments(node, source) {
|
|
|
180
218
|
return undefined;
|
|
181
219
|
return comments.join("\n\n");
|
|
182
220
|
}
|
|
183
|
-
function extractDocstringFromBody(node, source) {
|
|
221
|
+
function extractDocstringFromBody(node, source, offsetMap) {
|
|
184
222
|
if (node.type !== "function_definition" && node.type !== "class_definition") {
|
|
185
223
|
return undefined;
|
|
186
224
|
}
|
|
@@ -193,7 +231,7 @@ function extractDocstringFromBody(node, source) {
|
|
|
193
231
|
const stringNode = firstStmt.namedChildren[0];
|
|
194
232
|
if (!stringNode || stringNode.type !== "string")
|
|
195
233
|
return undefined;
|
|
196
|
-
return cleanCommentText(source
|
|
234
|
+
return cleanCommentText(sliceByBytes(source, stringNode.startIndex, stringNode.endIndex, offsetMap));
|
|
197
235
|
}
|
|
198
236
|
/**
|
|
199
237
|
* Recursively walk a tree-sitter AST and collect nodes matching the given
|
|
@@ -213,16 +251,16 @@ function extractDocstringFromBody(node, source) {
|
|
|
213
251
|
* @param depth - Current recursion depth (internal, start at 0).
|
|
214
252
|
* @returns An array of {@link AstNode} objects for matching declarations.
|
|
215
253
|
*/
|
|
216
|
-
export function walkTree(node, nodeTypes, source, maxDepth = 10, depth = 0) {
|
|
254
|
+
export function walkTree(node, nodeTypes, source, maxDepth = 10, depth = 0, offsetMap) {
|
|
217
255
|
const results = [];
|
|
218
256
|
if (nodeTypes.has(node.type) && depth > 0) {
|
|
219
|
-
const leadingComments = extractLeadingComments(node, source);
|
|
220
|
-
const bodyDocstring = extractDocstringFromBody(node, source);
|
|
257
|
+
const leadingComments = extractLeadingComments(node, source, offsetMap);
|
|
258
|
+
const bodyDocstring = extractDocstringFromBody(node, source, offsetMap);
|
|
221
259
|
const leadingDoc = [leadingComments, bodyDocstring]
|
|
222
260
|
.filter((d) => d !== undefined && d.length > 0)
|
|
223
261
|
.join("\n\n");
|
|
224
262
|
results.push({
|
|
225
|
-
text: source
|
|
263
|
+
text: sliceByBytes(source, node.startIndex, node.endIndex, offsetMap),
|
|
226
264
|
startLine: node.startPosition.row + 1,
|
|
227
265
|
endLine: node.endPosition.row + 1,
|
|
228
266
|
startIndex: node.startIndex,
|
|
@@ -234,7 +272,7 @@ export function walkTree(node, nodeTypes, source, maxDepth = 10, depth = 0) {
|
|
|
234
272
|
}
|
|
235
273
|
if (depth < maxDepth) {
|
|
236
274
|
for (const child of node.children) {
|
|
237
|
-
results.push(...walkTree(child, nodeTypes, source, maxDepth, depth + 1));
|
|
275
|
+
results.push(...walkTree(child, nodeTypes, source, maxDepth, depth + 1, offsetMap));
|
|
238
276
|
}
|
|
239
277
|
}
|
|
240
278
|
return results;
|
package/dist/chunker/image.js
CHANGED
|
@@ -42,6 +42,7 @@ class OllamaImageVisionProvider {
|
|
|
42
42
|
timeoutMs;
|
|
43
43
|
think;
|
|
44
44
|
numCtx;
|
|
45
|
+
keepAlive;
|
|
45
46
|
proxy;
|
|
46
47
|
constructor(config) {
|
|
47
48
|
this.baseUrl = config.baseUrl.replace(/\/+$/, "");
|
|
@@ -49,6 +50,7 @@ class OllamaImageVisionProvider {
|
|
|
49
50
|
this.timeoutMs = config.timeoutMs;
|
|
50
51
|
this.think = config.think ?? false;
|
|
51
52
|
this.numCtx = config.numCtx;
|
|
53
|
+
this.keepAlive = config.keepAlive;
|
|
52
54
|
this.proxy = config.proxy;
|
|
53
55
|
}
|
|
54
56
|
async describeImage(imageBase64, _mimeType, prompt, abort) {
|
|
@@ -65,6 +67,9 @@ class OllamaImageVisionProvider {
|
|
|
65
67
|
think: this.think,
|
|
66
68
|
options: { num_ctx: this.numCtx },
|
|
67
69
|
};
|
|
70
|
+
if (this.keepAlive) {
|
|
71
|
+
body.keep_alive = this.keepAlive;
|
|
72
|
+
}
|
|
68
73
|
let lastError;
|
|
69
74
|
for (let attempt = 0; attempt <= VISION_RETRY_MAX; attempt++) {
|
|
70
75
|
const response = await postJson(`${this.baseUrl}/chat`, body, {}, this.timeoutMs, this.proxy, abort);
|
package/dist/chunker/pdf.js
CHANGED
|
@@ -22,19 +22,18 @@ function getStandardFontsUrl() {
|
|
|
22
22
|
* Create a pdfjs-dist PDF document from a buffer.
|
|
23
23
|
* Uses `@thednp/dommatrix` for DOMMatrix polyfill (lighter alternative to native canvas).
|
|
24
24
|
* @param buffer - Raw buffer of the PDF file.
|
|
25
|
-
* @returns
|
|
25
|
+
* @returns The pdfjs LoadingTask for the document.
|
|
26
26
|
*/
|
|
27
|
-
async function
|
|
27
|
+
async function getPdfLoadingTask(buffer) {
|
|
28
28
|
const { default: CSSMatrix } = await import("@thednp/dommatrix");
|
|
29
29
|
globalThis.DOMMatrix ??= CSSMatrix;
|
|
30
30
|
globalThis.DOMMatrixReadOnly ??= CSSMatrix;
|
|
31
31
|
const { getDocument } = await import("pdfjs-dist/legacy/build/pdf.mjs");
|
|
32
|
-
|
|
32
|
+
return getDocument({
|
|
33
33
|
data: new Uint8Array(buffer),
|
|
34
34
|
standardFontDataUrl: getStandardFontsUrl(),
|
|
35
35
|
verbosity: 0,
|
|
36
36
|
});
|
|
37
|
-
return loadingTask.promise;
|
|
38
37
|
}
|
|
39
38
|
/**
|
|
40
39
|
* Extract the full text content from a PDF file buffer.
|
|
@@ -47,17 +46,34 @@ export async function extractPdfText(buffer) {
|
|
|
47
46
|
if (buffer.length > MAX_PDF_SIZE_BYTES) {
|
|
48
47
|
throw new Error(`PDF too large: ${(buffer.length / 1024 / 1024).toFixed(1)} MB (max 100 MB)`);
|
|
49
48
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
const
|
|
55
|
-
const
|
|
56
|
-
const
|
|
57
|
-
|
|
58
|
-
|
|
49
|
+
// loadingTask.destroy() releases the pdfjs worker/document resources —
|
|
50
|
+
// without it long-running watch/plugin processes leak them per PDF.
|
|
51
|
+
const loadingTask = await getPdfLoadingTask(buffer);
|
|
52
|
+
try {
|
|
53
|
+
const pdf = await loadingTask.promise;
|
|
54
|
+
const texts = [];
|
|
55
|
+
const pageCount = Math.min(pdf.numPages, MAX_PDF_PAGES);
|
|
56
|
+
for (let i = 1; i <= pageCount; i++) {
|
|
57
|
+
const page = await pdf.getPage(i);
|
|
58
|
+
try {
|
|
59
|
+
const content = await page.getTextContent();
|
|
60
|
+
const strings = [];
|
|
61
|
+
for (const item of content.items) {
|
|
62
|
+
if (typeof item === "object" && item !== null && "str" in item) {
|
|
63
|
+
strings.push(item.str);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
texts.push(strings.join(" "));
|
|
67
|
+
}
|
|
68
|
+
finally {
|
|
69
|
+
page.cleanup();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return texts.join("\n\n");
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
await loadingTask.destroy().catch(() => { });
|
|
59
76
|
}
|
|
60
|
-
return texts.join("\n\n");
|
|
61
77
|
}
|
|
62
78
|
/**
|
|
63
79
|
* Chunker for PDF documents (.pdf).
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Auto-detect the Ollama backend (CPU vs GPU) during `init` and
|
|
3
|
+
* pick embedding batch settings tuned for the detected backend.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Auto-detect whether Ollama runs models on the GPU or on the CPU and return
|
|
7
|
+
* matching embedding batch tuning.
|
|
8
|
+
*
|
|
9
|
+
* Detection uses `GET /api/ps`: loaded models report `size_vram` (bytes
|
|
10
|
+
* resident in VRAM). `size_vram > 0` means the model is (at least partially)
|
|
11
|
+
* offloaded to the GPU. If no model is loaded yet, a minimal `/api/embed`
|
|
12
|
+
* warmup loads the default embedding model first.
|
|
13
|
+
*
|
|
14
|
+
* Tuning is derived from benchmarks (see quirk memory):
|
|
15
|
+
* - GPU: batch 40 + concurrency 4 ≈ 86 texts/s (~97% of the ~88 texts/s ceiling)
|
|
16
|
+
* - CPU: flat ~3.5 texts/s regardless of batch size → small batches (20) with
|
|
17
|
+
* concurrency 1 keep each request fast and under the 4096-token context
|
|
18
|
+
* - unreachable/unknown: defaults (100 / 3 / 100)
|
|
19
|
+
*/
|
|
20
|
+
import { type ProxyConfig } from "../../core/config.js";
|
|
21
|
+
/** Detected Ollama backend kind. */
|
|
22
|
+
export type OllamaBackend = "gpu" | "cpu" | "unreachable" | "unknown";
|
|
23
|
+
/** Embedding batch settings written into the generated config. */
|
|
24
|
+
export interface IndexingTuning {
|
|
25
|
+
embedBatchSize: number;
|
|
26
|
+
embedConcurrency: number;
|
|
27
|
+
ollamaMaxBatchSize: number;
|
|
28
|
+
}
|
|
29
|
+
/** Result of the backend detection. */
|
|
30
|
+
export interface OllamaBackendInfo {
|
|
31
|
+
backend: OllamaBackend;
|
|
32
|
+
/** Tuning to write into the generated config. */
|
|
33
|
+
tuning: IndexingTuning;
|
|
34
|
+
/** Human-readable summary for the init output. */
|
|
35
|
+
message: string;
|
|
36
|
+
}
|
|
37
|
+
interface PsModel {
|
|
38
|
+
name: string;
|
|
39
|
+
size_vram?: number;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Classify loaded Ollama models into a backend + tuning profile.
|
|
43
|
+
*
|
|
44
|
+
* Prefers the configured default embedding model when it is loaded, falling
|
|
45
|
+
* back to any loaded model (a loaded GPU model means the host has a working
|
|
46
|
+
* GPU that Ollama will also use for embeddings).
|
|
47
|
+
*
|
|
48
|
+
* @param models - The `models` array from `GET /api/ps`.
|
|
49
|
+
* @returns The backend info with the matching tuning profile.
|
|
50
|
+
*/
|
|
51
|
+
export declare function classifyOllamaModels(models: PsModel[]): OllamaBackendInfo;
|
|
52
|
+
/**
|
|
53
|
+
* Detect the Ollama backend by probing `/api/ps`, warming up the default
|
|
54
|
+
* embedding model when nothing is loaded yet.
|
|
55
|
+
*
|
|
56
|
+
* @param baseUrl - Ollama API base URL (defaults to the config default).
|
|
57
|
+
* @param proxy - Optional proxy configuration.
|
|
58
|
+
* @returns Backend info; never throws.
|
|
59
|
+
*/
|
|
60
|
+
export declare function detectOllamaBackend(baseUrl?: string, proxy?: ProxyConfig): Promise<OllamaBackendInfo>;
|
|
61
|
+
export {};
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Auto-detect the Ollama backend (CPU vs GPU) during `init` and
|
|
3
|
+
* pick embedding batch settings tuned for the detected backend.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Auto-detect whether Ollama runs models on the GPU or on the CPU and return
|
|
7
|
+
* matching embedding batch tuning.
|
|
8
|
+
*
|
|
9
|
+
* Detection uses `GET /api/ps`: loaded models report `size_vram` (bytes
|
|
10
|
+
* resident in VRAM). `size_vram > 0` means the model is (at least partially)
|
|
11
|
+
* offloaded to the GPU. If no model is loaded yet, a minimal `/api/embed`
|
|
12
|
+
* warmup loads the default embedding model first.
|
|
13
|
+
*
|
|
14
|
+
* Tuning is derived from benchmarks (see quirk memory):
|
|
15
|
+
* - GPU: batch 40 + concurrency 4 ≈ 86 texts/s (~97% of the ~88 texts/s ceiling)
|
|
16
|
+
* - CPU: flat ~3.5 texts/s regardless of batch size → small batches (20) with
|
|
17
|
+
* concurrency 1 keep each request fast and under the 4096-token context
|
|
18
|
+
* - unreachable/unknown: defaults (100 / 3 / 100)
|
|
19
|
+
*/
|
|
20
|
+
import { DEFAULT_CONFIG } from "../../core/config.js";
|
|
21
|
+
import { fetchWithProxy, postJson } from "../../embedder/http.js";
|
|
22
|
+
/** Benchmarked optimum on a GPU-backed Ollama (RTX 4090, qwen3-embedding:0.6b). */
|
|
23
|
+
const GPU_TUNING = {
|
|
24
|
+
embedBatchSize: 40,
|
|
25
|
+
embedConcurrency: 4,
|
|
26
|
+
ollamaMaxBatchSize: 40,
|
|
27
|
+
};
|
|
28
|
+
/** CPU-backed Ollama: throughput is flat, so keep batches small and sequential. */
|
|
29
|
+
const CPU_TUNING = {
|
|
30
|
+
embedBatchSize: 20,
|
|
31
|
+
embedConcurrency: 1,
|
|
32
|
+
ollamaMaxBatchSize: 20,
|
|
33
|
+
};
|
|
34
|
+
/** Fallback when Ollama is unreachable or the backend cannot be determined. */
|
|
35
|
+
const DEFAULT_TUNING = {
|
|
36
|
+
embedBatchSize: DEFAULT_CONFIG.indexing.embedBatchSize,
|
|
37
|
+
embedConcurrency: DEFAULT_CONFIG.indexing.embedConcurrency ?? 3,
|
|
38
|
+
ollamaMaxBatchSize: DEFAULT_CONFIG.indexing.ollamaMaxBatchSize ?? 100,
|
|
39
|
+
};
|
|
40
|
+
/**
|
|
41
|
+
* Classify loaded Ollama models into a backend + tuning profile.
|
|
42
|
+
*
|
|
43
|
+
* Prefers the configured default embedding model when it is loaded, falling
|
|
44
|
+
* back to any loaded model (a loaded GPU model means the host has a working
|
|
45
|
+
* GPU that Ollama will also use for embeddings).
|
|
46
|
+
*
|
|
47
|
+
* @param models - The `models` array from `GET /api/ps`.
|
|
48
|
+
* @returns The backend info with the matching tuning profile.
|
|
49
|
+
*/
|
|
50
|
+
export function classifyOllamaModels(models) {
|
|
51
|
+
if (!models || models.length === 0) {
|
|
52
|
+
return {
|
|
53
|
+
backend: "unknown",
|
|
54
|
+
tuning: DEFAULT_TUNING,
|
|
55
|
+
message: "Could not determine the Ollama backend (no models loaded) — using default batch settings.",
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
const embedModel = DEFAULT_CONFIG.embedding.model;
|
|
59
|
+
const probe = models.find((m) => m.name === embedModel) ?? models[0];
|
|
60
|
+
const onGpu = probe ? (probe.size_vram ?? 0) > 0 : false;
|
|
61
|
+
if (onGpu) {
|
|
62
|
+
return {
|
|
63
|
+
backend: "gpu",
|
|
64
|
+
tuning: GPU_TUNING,
|
|
65
|
+
message: `Ollama detected on GPU — tuned embedding for batch 40 / concurrency 4 (${(probe?.size_vram ?? 0) / (1024 * 1024) | 0} MiB in VRAM).`,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
backend: "cpu",
|
|
70
|
+
tuning: CPU_TUNING,
|
|
71
|
+
message: "Ollama detected on CPU — tuned embedding for batch 20 / concurrency 1.",
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/** Fetch `GET /api/ps`, returning null when Ollama is unreachable or errors. */
|
|
75
|
+
async function getOllamaPs(baseUrl, proxy) {
|
|
76
|
+
const url = `${baseUrl.replace(/\/+$/, "")}/ps`;
|
|
77
|
+
try {
|
|
78
|
+
const res = await fetchWithProxy(url, { method: "GET", signal: AbortSignal.timeout(3000) }, proxy);
|
|
79
|
+
if (!res.ok)
|
|
80
|
+
return null;
|
|
81
|
+
const data = (await res.json());
|
|
82
|
+
return data.models ?? null;
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Detect the Ollama backend by probing `/api/ps`, warming up the default
|
|
90
|
+
* embedding model when nothing is loaded yet.
|
|
91
|
+
*
|
|
92
|
+
* @param baseUrl - Ollama API base URL (defaults to the config default).
|
|
93
|
+
* @param proxy - Optional proxy configuration.
|
|
94
|
+
* @returns Backend info; never throws.
|
|
95
|
+
*/
|
|
96
|
+
export async function detectOllamaBackend(baseUrl = DEFAULT_CONFIG.embedding.baseUrl, proxy) {
|
|
97
|
+
let models = await getOllamaPs(baseUrl, proxy);
|
|
98
|
+
if (models === null) {
|
|
99
|
+
return {
|
|
100
|
+
backend: "unreachable",
|
|
101
|
+
tuning: DEFAULT_TUNING,
|
|
102
|
+
message: "Ollama not reachable — using default batch settings.",
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
if (models.length === 0) {
|
|
106
|
+
// Nothing loaded yet: a minimal embed request loads the default
|
|
107
|
+
// embedding model so /api/ps can report its backend.
|
|
108
|
+
const embedUrl = `${baseUrl.replace(/\/+$/, "")}/embed`;
|
|
109
|
+
try {
|
|
110
|
+
await postJson(embedUrl, { model: DEFAULT_CONFIG.embedding.model, input: "warmup" }, {}, 15000, proxy);
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
// Model missing or request failed — classification will stay unknown.
|
|
114
|
+
}
|
|
115
|
+
models = (await getOllamaPs(baseUrl, proxy)) ?? [];
|
|
116
|
+
}
|
|
117
|
+
return classifyOllamaModels(models);
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=backend-detect.js.map
|
|
@@ -13,6 +13,7 @@ import readline from "node:readline";
|
|
|
13
13
|
import chokidar from "chokidar";
|
|
14
14
|
import { appendDebugLog } from "../../core/fileLogger.js";
|
|
15
15
|
import { createWatchPassScheduler, createWatchIgnore, runIndexPass, } from "../../indexer.js";
|
|
16
|
+
import { tryAcquireWatcherLock, releaseWatcherLock } from "../../watcher.js";
|
|
16
17
|
import { c, resolveCliContext, cleanupContext, logCliError, logCliInfo, logIndexSummary, formatDuration, } from "../format.js";
|
|
17
18
|
/**
|
|
18
19
|
* Build a logger that suppresses console output when watchTriggered is true.
|
|
@@ -129,6 +130,15 @@ export function registerIndexCommand(program) {
|
|
|
129
130
|
await cleanupContext(ctx);
|
|
130
131
|
process.exit(sigReceived ? 130 : 0);
|
|
131
132
|
}
|
|
133
|
+
// Only one watcher may run per workspace — a background auto-indexer
|
|
134
|
+
// in an OpenCode session (or another `index --watch`) may already own
|
|
135
|
+
// this store. The initial pass above still ran; just don't start a
|
|
136
|
+
// duplicate watcher.
|
|
137
|
+
if (!tryAcquireWatcherLock(storePath)) {
|
|
138
|
+
logCliInfo(logFilePath, "index", c.warn("Another watcher is already running for this workspace (e.g. an OpenCode session with auto-index enabled) — not starting a second one. The index is up to date."));
|
|
139
|
+
await cleanupContext(ctx);
|
|
140
|
+
process.exit(0);
|
|
141
|
+
}
|
|
132
142
|
logCliInfo(logFilePath, "index", `\n${c.heading("Watching for changes...")}`);
|
|
133
143
|
const scheduler = createWatchPassScheduler(async (changedPaths) => { await runPass(true, undefined, changedPaths); }, (error) => {
|
|
134
144
|
const message = error.message || String(error);
|
|
@@ -154,6 +164,7 @@ export function registerIndexCommand(program) {
|
|
|
154
164
|
watcher.close(),
|
|
155
165
|
new Promise((r) => setTimeout(r, 5000)),
|
|
156
166
|
]);
|
|
167
|
+
releaseWatcherLock(storePath);
|
|
157
168
|
await cleanupContext(ctx);
|
|
158
169
|
process.exit(0);
|
|
159
170
|
};
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* dependency installation, and gitignore merging.
|
|
7
7
|
*/
|
|
8
8
|
import type { PackageMetadata } from "../types.js";
|
|
9
|
+
import type { IndexingTuning } from "./backend-detect.js";
|
|
9
10
|
/**
|
|
10
11
|
* Build the workspace-local `.opencode/package.json` content.
|
|
11
12
|
*
|
|
@@ -115,6 +116,8 @@ export declare function installPluginFromGlobal(opencodeDir: string, packageName
|
|
|
115
116
|
/**
|
|
116
117
|
* Generate the default `opencode-rag.json` configuration content.
|
|
117
118
|
*
|
|
119
|
+
* @param tuning - Optional embedding batch tuning (auto-detected from the
|
|
120
|
+
* Ollama backend). Falls back to `DEFAULT_CONFIG` for any omitted field.
|
|
118
121
|
* @returns A pretty-printed JSON string with all default configuration values.
|
|
119
122
|
*/
|
|
120
|
-
export declare function generateDefaultConfigJson(): string;
|
|
123
|
+
export declare function generateDefaultConfigJson(tuning?: Partial<IndexingTuning>): string;
|
|
@@ -379,10 +379,23 @@ export async function installPluginFromGlobal(opencodeDir, packageName, skipInst
|
|
|
379
379
|
if (existsSync(workspaceTarget)) {
|
|
380
380
|
rmSync(workspaceTarget, { recursive: true, force: true });
|
|
381
381
|
}
|
|
382
|
-
// Create directory junction from workspace → global runtime
|
|
382
|
+
// Create directory junction from workspace → global runtime.
|
|
383
|
+
// Junction creation can throw (EPERM, cross-drive on Windows) — fall back
|
|
384
|
+
// to a copy instead of aborting `init`.
|
|
383
385
|
console.log(` ${c.created("Linking:")} ${packageName} from global cache...`);
|
|
384
386
|
mkdirSync(path.dirname(workspaceTarget), { recursive: true });
|
|
385
|
-
|
|
387
|
+
try {
|
|
388
|
+
createJunction(globalPluginDir, workspaceTarget);
|
|
389
|
+
}
|
|
390
|
+
catch (err) {
|
|
391
|
+
console.log(` ${c.warn("Junction failed, falling back to copy...")}`);
|
|
392
|
+
rmSync(workspaceTarget, { recursive: true, force: true });
|
|
393
|
+
const { cpSync } = await import("node:fs");
|
|
394
|
+
mkdirSync(path.dirname(workspaceTarget), { recursive: true });
|
|
395
|
+
cpSync(globalPluginDir, workspaceTarget, { recursive: true });
|
|
396
|
+
console.log(` ${c.success("Copied:")} ${packageName} from global cache`);
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
386
399
|
const cliEntry = path.join(workspaceTarget, "dist", "cli.js");
|
|
387
400
|
if (!existsSync(cliEntry)) {
|
|
388
401
|
// Junction may not work (e.g. cross-drive on Windows) — fall back to copy
|
|
@@ -401,9 +414,11 @@ export async function installPluginFromGlobal(opencodeDir, packageName, skipInst
|
|
|
401
414
|
/**
|
|
402
415
|
* Generate the default `opencode-rag.json` configuration content.
|
|
403
416
|
*
|
|
417
|
+
* @param tuning - Optional embedding batch tuning (auto-detected from the
|
|
418
|
+
* Ollama backend). Falls back to `DEFAULT_CONFIG` for any omitted field.
|
|
404
419
|
* @returns A pretty-printed JSON string with all default configuration values.
|
|
405
420
|
*/
|
|
406
|
-
export function generateDefaultConfigJson() {
|
|
421
|
+
export function generateDefaultConfigJson(tuning) {
|
|
407
422
|
return JSON.stringify({
|
|
408
423
|
embedding: {
|
|
409
424
|
provider: DEFAULT_CONFIG.embedding.provider,
|
|
@@ -417,7 +432,9 @@ export function generateDefaultConfigJson() {
|
|
|
417
432
|
chunkOverlap: DEFAULT_CONFIG.indexing.chunkOverlap,
|
|
418
433
|
minFileSizeBytes: DEFAULT_CONFIG.indexing.minFileSizeBytes,
|
|
419
434
|
concurrency: DEFAULT_CONFIG.indexing.concurrency,
|
|
420
|
-
embedBatchSize: DEFAULT_CONFIG.indexing.embedBatchSize,
|
|
435
|
+
embedBatchSize: tuning?.embedBatchSize ?? DEFAULT_CONFIG.indexing.embedBatchSize,
|
|
436
|
+
embedConcurrency: tuning?.embedConcurrency ?? DEFAULT_CONFIG.indexing.embedConcurrency ?? 3,
|
|
437
|
+
ollamaMaxBatchSize: tuning?.ollamaMaxBatchSize ?? DEFAULT_CONFIG.indexing.ollamaMaxBatchSize ?? 100,
|
|
421
438
|
},
|
|
422
439
|
vectorStore: {
|
|
423
440
|
path: DEFAULT_CONFIG.vectorStore.path,
|