opencode-rag-plugin 2.4.5 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunker/docx.d.ts +6 -5
- package/dist/chunker/docx.js +117 -5
- package/dist/chunker/factory.d.ts +6 -0
- package/dist/chunker/factory.js +14 -0
- package/dist/chunker/skeleton.js +7 -1
- package/dist/cli/commands/init-helpers.js +7 -0
- package/dist/content/docx.d.ts +1 -1
- package/dist/content/docx.js +2 -2
- package/dist/core/runtime-overrides.d.ts +10 -0
- package/dist/core/runtime-overrides.js +14 -0
- package/dist/eval/fast-index.js +7 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/indexer/metadata.d.ts +6 -1
- package/dist/indexer/metadata.js +8 -2
- package/dist/indexer/worker.js +3 -1
- package/dist/opencode/create-read-tool.d.ts +7 -0
- package/dist/opencode/create-read-tool.js +4 -1
- package/dist/opencode/tools.d.ts +6 -0
- package/dist/opencode/tools.js +2 -1
- package/dist/plugin.js +11 -1
- package/dist/web/api.d.ts +11 -0
- package/dist/web/api.js +50 -2
- package/package.json +4 -3
package/dist/chunker/docx.d.ts
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @fileoverview Text extraction and paragraph-based chunker for Word (.docx) documents using mammoth.
|
|
3
|
-
*/
|
|
4
1
|
import type { Chunker, Chunk } from "../core/interfaces.js";
|
|
5
2
|
/**
|
|
6
3
|
* Extract the raw text from a Word (.docx) file buffer.
|
|
7
|
-
*
|
|
4
|
+
*
|
|
5
|
+
* Unzips the archive and reads `word/document.xml` directly. Only that single
|
|
6
|
+
* entry is decompressed, and an inflated-size cap is enforced, so a crafted
|
|
7
|
+
* archive cannot exhaust memory.
|
|
8
8
|
* @param buffer - Raw buffer of the .docx file.
|
|
9
9
|
* @returns The extracted raw text as a string.
|
|
10
|
+
* @throws If the buffer is not a valid ZIP archive or lacks `word/document.xml`.
|
|
10
11
|
*/
|
|
11
|
-
export declare function extractDocxText(buffer: Buffer):
|
|
12
|
+
export declare function extractDocxText(buffer: Buffer): string;
|
|
12
13
|
/**
|
|
13
14
|
* Chunker for Word (.docx) documents.
|
|
14
15
|
* Splits the extracted plain text by paragraph boundaries, grouping small
|
package/dist/chunker/docx.js
CHANGED
|
@@ -1,17 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Text extraction and paragraph-based chunker for Word (.docx) documents.
|
|
3
|
+
*
|
|
4
|
+
* A `.docx` file is a ZIP archive containing `word/document.xml`, where the text
|
|
5
|
+
* lives in `<w:t>` runs and paragraph boundaries are `<w:p>` elements. Extraction
|
|
6
|
+
* is done directly with fflate's `unzipSync` so the plugin pulls in no document
|
|
7
|
+
* parser (the previous `mammoth` dependency carried six transitive packages,
|
|
8
|
+
* several of them with known prototype-pollution advisories).
|
|
9
|
+
*/
|
|
10
|
+
import { unzipSync } from "fflate";
|
|
1
11
|
import { uuid } from "./uuid.js";
|
|
2
12
|
const MAX_CHUNK_CHARS = 4000;
|
|
3
13
|
const MIN_GROUP_CHARS = 300;
|
|
4
14
|
const PARAGRAPH_SPLIT = /\n\s*\n/;
|
|
15
|
+
/** ZIP entry holding the document body. */
|
|
16
|
+
const DOCUMENT_XML = "word/document.xml";
|
|
17
|
+
/**
|
|
18
|
+
* Upper bound on the decompressed size of `word/document.xml`.
|
|
19
|
+
*
|
|
20
|
+
* `.docx` files are frequently untrusted, and a small archive can inflate to
|
|
21
|
+
* gigabytes (a zip bomb). Anything past this is refused instead of buffered.
|
|
22
|
+
*/
|
|
23
|
+
const MAX_DOCUMENT_XML_BYTES = 64 * 1024 * 1024;
|
|
24
|
+
/**
|
|
25
|
+
* Decode the XML entities Word emits inside text runs.
|
|
26
|
+
* @param text - Raw (escaped) text from a `<w:t>` element.
|
|
27
|
+
* @returns The unescaped text.
|
|
28
|
+
*/
|
|
29
|
+
function decodeXmlEntities(text) {
|
|
30
|
+
return text
|
|
31
|
+
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCodePoint(parseInt(hex, 16)))
|
|
32
|
+
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(parseInt(dec, 10)))
|
|
33
|
+
.replace(/</g, "<")
|
|
34
|
+
.replace(/>/g, ">")
|
|
35
|
+
.replace(/"/g, '"')
|
|
36
|
+
.replace(/'/g, "'")
|
|
37
|
+
.replace(/&/g, "&");
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Convert raw `word/document.xml` markup into plain text.
|
|
41
|
+
*
|
|
42
|
+
* Mirrors what `mammoth.extractRawText` produced for the chunker's purposes:
|
|
43
|
+
* `<w:t>` text is concatenated, `<w:tab>` becomes a tab, `<w:br>` becomes a
|
|
44
|
+
* newline, and each paragraph is terminated by a blank line so the
|
|
45
|
+
* {@link PARAGRAPH_SPLIT} regex still sees paragraph boundaries.
|
|
46
|
+
* @param xml - Contents of `word/document.xml`.
|
|
47
|
+
* @returns Plain text with paragraphs separated by blank lines.
|
|
48
|
+
*/
|
|
49
|
+
function documentXmlToText(xml) {
|
|
50
|
+
let out = "";
|
|
51
|
+
// Text outside a <w:p> is document metadata, not body text.
|
|
52
|
+
let inParagraph = false;
|
|
53
|
+
const OPEN_TAG = /<w:(t|tab|br|p)(?:\s[^>]*)?\/?>/g;
|
|
54
|
+
const CLOSE_P = /<\/w:p>/g;
|
|
55
|
+
// Walk both matchers in document order.
|
|
56
|
+
let open = OPEN_TAG.exec(xml);
|
|
57
|
+
let close = CLOSE_P.exec(xml);
|
|
58
|
+
while (open !== null) {
|
|
59
|
+
// A closing </w:p> before the next opening tag wins.
|
|
60
|
+
if (close !== null && close.index < open.index) {
|
|
61
|
+
if (inParagraph) {
|
|
62
|
+
out += "\n\n";
|
|
63
|
+
inParagraph = false;
|
|
64
|
+
}
|
|
65
|
+
close = CLOSE_P.exec(xml);
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
const [full, name] = open;
|
|
69
|
+
const selfClosing = full.endsWith("/>");
|
|
70
|
+
if (name === "p") {
|
|
71
|
+
if (!selfClosing && !inParagraph) {
|
|
72
|
+
inParagraph = true;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
else if (inParagraph) {
|
|
76
|
+
if (name === "tab") {
|
|
77
|
+
out += "\t";
|
|
78
|
+
}
|
|
79
|
+
else if (name === "br") {
|
|
80
|
+
out += "\n";
|
|
81
|
+
}
|
|
82
|
+
else if (!selfClosing) {
|
|
83
|
+
const endTag = `</w:t>`;
|
|
84
|
+
const end = xml.indexOf(endTag, open.index + full.length);
|
|
85
|
+
const stop = end === -1 ? xml.length : end;
|
|
86
|
+
out += decodeXmlEntities(xml.slice(open.index + full.length, stop));
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
open = OPEN_TAG.exec(xml);
|
|
90
|
+
}
|
|
91
|
+
// Unterminated trailing paragraph.
|
|
92
|
+
if (inParagraph)
|
|
93
|
+
out += "\n\n";
|
|
94
|
+
return out;
|
|
95
|
+
}
|
|
5
96
|
/**
|
|
6
97
|
* Extract the raw text from a Word (.docx) file buffer.
|
|
7
|
-
*
|
|
98
|
+
*
|
|
99
|
+
* Unzips the archive and reads `word/document.xml` directly. Only that single
|
|
100
|
+
* entry is decompressed, and an inflated-size cap is enforced, so a crafted
|
|
101
|
+
* archive cannot exhaust memory.
|
|
8
102
|
* @param buffer - Raw buffer of the .docx file.
|
|
9
103
|
* @returns The extracted raw text as a string.
|
|
104
|
+
* @throws If the buffer is not a valid ZIP archive or lacks `word/document.xml`.
|
|
10
105
|
*/
|
|
11
|
-
export
|
|
12
|
-
|
|
13
|
-
const
|
|
14
|
-
|
|
106
|
+
export function extractDocxText(buffer) {
|
|
107
|
+
let tooLarge = false;
|
|
108
|
+
const entries = unzipSync(new Uint8Array(buffer), {
|
|
109
|
+
filter: (file) => {
|
|
110
|
+
if (file.name !== DOCUMENT_XML)
|
|
111
|
+
return false;
|
|
112
|
+
if (file.originalSize > MAX_DOCUMENT_XML_BYTES) {
|
|
113
|
+
tooLarge = true;
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
return true;
|
|
117
|
+
},
|
|
118
|
+
});
|
|
119
|
+
if (tooLarge) {
|
|
120
|
+
throw new Error(`DOCX too large: "${DOCUMENT_XML}" exceeds ${MAX_DOCUMENT_XML_BYTES} bytes when decompressed`);
|
|
121
|
+
}
|
|
122
|
+
const doc = entries[DOCUMENT_XML];
|
|
123
|
+
if (!doc) {
|
|
124
|
+
throw new Error(`Not a valid DOCX file: missing "${DOCUMENT_XML}"`);
|
|
125
|
+
}
|
|
126
|
+
return documentXmlToText(Buffer.from(doc).toString("utf8"));
|
|
15
127
|
}
|
|
16
128
|
/**
|
|
17
129
|
* Chunker for Word (.docx) documents.
|
|
@@ -62,6 +62,12 @@ export declare function getRegisteredLanguages(): string[];
|
|
|
62
62
|
* extension recipe when an override retargets an extension.
|
|
63
63
|
*/
|
|
64
64
|
export declare function getExtensionsForLanguage(language: string): string[];
|
|
65
|
+
/**
|
|
66
|
+
* The parser language registered for a file extension, or `undefined` when
|
|
67
|
+
* the extension has no registered parser. Used to detect a `chunking.parsers`
|
|
68
|
+
* override that does not actually change the parser.
|
|
69
|
+
*/
|
|
70
|
+
export declare function getLanguageForExtension(ext: string): string | undefined;
|
|
65
71
|
/**
|
|
66
72
|
* Validate a raw `chunking.parsers` map against the registered parsers.
|
|
67
73
|
*
|
package/dist/chunker/factory.js
CHANGED
|
@@ -167,6 +167,20 @@ export function getExtensionsForLanguage(language) {
|
|
|
167
167
|
}
|
|
168
168
|
return out;
|
|
169
169
|
}
|
|
170
|
+
/**
|
|
171
|
+
* The parser language registered for a file extension, or `undefined` when
|
|
172
|
+
* the extension has no registered parser. Used to detect a `chunking.parsers`
|
|
173
|
+
* override that does not actually change the parser.
|
|
174
|
+
*/
|
|
175
|
+
export function getLanguageForExtension(ext) {
|
|
176
|
+
const key = ext.toLowerCase();
|
|
177
|
+
const direct = extensionMap.get(key);
|
|
178
|
+
if (direct)
|
|
179
|
+
return direct.language;
|
|
180
|
+
// Basename-registered parsers (e.g. "dockerfile") are stored without a dot.
|
|
181
|
+
const base = key.startsWith(".") ? key.slice(1) : key;
|
|
182
|
+
return extensionMap.get(base)?.language;
|
|
183
|
+
}
|
|
170
184
|
/**
|
|
171
185
|
* Validate a raw `chunking.parsers` map against the registered parsers.
|
|
172
186
|
*
|
package/dist/chunker/skeleton.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { Parser } from "web-tree-sitter";
|
|
11
11
|
import { initParser, loadLanguage, walkTree } from "./grammar.js";
|
|
12
|
-
import { getExtensionsForLanguage, getRegisteredLanguages } from "./factory.js";
|
|
12
|
+
import { getExtensionsForLanguage, getLanguageForExtension, getRegisteredLanguages } from "./factory.js";
|
|
13
13
|
import { normalizeParserOverrides, } from "../core/parser-overrides.js";
|
|
14
14
|
/** Extension → tree-sitter grammar + node types for skeleton extraction. */
|
|
15
15
|
export const SKELETON_CONFIGS = {
|
|
@@ -86,6 +86,12 @@ function resolveSkeletonKey(ext, overrides) {
|
|
|
86
86
|
}
|
|
87
87
|
return ext;
|
|
88
88
|
}
|
|
89
|
+
// An override that names the extension's existing parser changes nothing
|
|
90
|
+
// for chunking — so keep the extension's own recipe, which can be more
|
|
91
|
+
// specific than the target language's first recipe (".tsx" also collects
|
|
92
|
+
// arrow functions, ".ts" does not).
|
|
93
|
+
if (getLanguageForExtension(ext) === target)
|
|
94
|
+
return ext;
|
|
89
95
|
const candidates = getExtensionsForLanguage(target);
|
|
90
96
|
const hit = candidates.find((c) => SKELETON_CONFIGS[c] || REGEX_SKELETON[c]);
|
|
91
97
|
// Known parser but no skeleton recipe for it (e.g. plain text, PDF): the
|
|
@@ -610,6 +610,13 @@ export function generateDefaultConfigJson(tuning, existingConfigRaw) {
|
|
|
610
610
|
embedConcurrency: tuning?.embedConcurrency ?? DEFAULT_CONFIG.indexing.embedConcurrency ?? 3,
|
|
611
611
|
ollamaMaxBatchSize: tuning?.ollamaMaxBatchSize ?? DEFAULT_CONFIG.indexing.ollamaMaxBatchSize ?? 100,
|
|
612
612
|
},
|
|
613
|
+
chunking: {
|
|
614
|
+
// Emitted (empty) so the extension → parser mapping is discoverable
|
|
615
|
+
// in a freshly generated config. Fill in e.g. { ".c": "cpp", ".cu": "cpp" }
|
|
616
|
+
// to remap a file extension to another parser.
|
|
617
|
+
nodeTypes: {},
|
|
618
|
+
parsers: {},
|
|
619
|
+
},
|
|
613
620
|
vectorStore: {
|
|
614
621
|
path: DEFAULT_CONFIG.vectorStore.path,
|
|
615
622
|
},
|
package/dist/content/docx.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ import type { ExtractResult } from "./types.js";
|
|
|
5
5
|
/** Recognized DOCX file extension set. */
|
|
6
6
|
export declare const DOCX_EXTENSIONS: Set<string>;
|
|
7
7
|
/**
|
|
8
|
-
* Extract text content from a DOCX buffer
|
|
8
|
+
* Extract text content from a DOCX buffer.
|
|
9
9
|
* @param _filePath - Unused, retained for API consistency.
|
|
10
10
|
* @param buffer - Raw DOCX file bytes.
|
|
11
11
|
*/
|
package/dist/content/docx.js
CHANGED
|
@@ -4,14 +4,14 @@
|
|
|
4
4
|
/** Recognized DOCX file extension set. */
|
|
5
5
|
export const DOCX_EXTENSIONS = new Set([".docx"]);
|
|
6
6
|
/**
|
|
7
|
-
* Extract text content from a DOCX buffer
|
|
7
|
+
* Extract text content from a DOCX buffer.
|
|
8
8
|
* @param _filePath - Unused, retained for API consistency.
|
|
9
9
|
* @param buffer - Raw DOCX file bytes.
|
|
10
10
|
*/
|
|
11
11
|
export async function extract(_filePath, buffer) {
|
|
12
12
|
try {
|
|
13
13
|
const { extractDocxText } = await import("../chunker/docx.js");
|
|
14
|
-
const content =
|
|
14
|
+
const content = extractDocxText(buffer);
|
|
15
15
|
return { content, ok: true };
|
|
16
16
|
}
|
|
17
17
|
catch (err) {
|
|
@@ -51,6 +51,16 @@ export interface RuntimeOverrides {
|
|
|
51
51
|
chunksKeybinding?: string;
|
|
52
52
|
settingsKeybinding?: string;
|
|
53
53
|
};
|
|
54
|
+
/**
|
|
55
|
+
* Chunking overrides. The TUI writes these live, so they must be applied
|
|
56
|
+
* here — otherwise the write to `runtime-overrides.json` is silently
|
|
57
|
+
* dropped and the edit only takes effect after an OpenCode restart (the
|
|
58
|
+
* plugin reads the on-disk config file once, at startup).
|
|
59
|
+
*/
|
|
60
|
+
chunking?: {
|
|
61
|
+
nodeTypes?: Record<string, string[]>;
|
|
62
|
+
parsers?: Record<string, string>;
|
|
63
|
+
};
|
|
54
64
|
}
|
|
55
65
|
/** Load runtime overrides from the store directory. Returns empty object if none exist. */
|
|
56
66
|
export declare function loadRuntimeOverrides(storePath: string): RuntimeOverrides;
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs";
|
|
6
6
|
import { dirname, join } from "node:path";
|
|
7
7
|
import { DEFAULT_CONFIG } from "./config.js";
|
|
8
|
+
import { normalizeParserOverrides } from "./parser-overrides.js";
|
|
8
9
|
/** Load runtime overrides from the store directory. Returns empty object if none exist. */
|
|
9
10
|
export function loadRuntimeOverrides(storePath) {
|
|
10
11
|
const overridePath = join(storePath, "runtime-overrides.json");
|
|
@@ -143,6 +144,19 @@ export function applyRuntimeOverrides(cfg, overrides) {
|
|
|
143
144
|
if (overrides.memory.autoInjectTopK !== undefined)
|
|
144
145
|
m.autoInjectTopK = overrides.memory.autoInjectTopK;
|
|
145
146
|
}
|
|
147
|
+
if (overrides.chunking) {
|
|
148
|
+
if (!merged.chunking)
|
|
149
|
+
merged.chunking = {};
|
|
150
|
+
if (overrides.chunking.nodeTypes !== undefined) {
|
|
151
|
+
merged.chunking.nodeTypes = overrides.chunking.nodeTypes;
|
|
152
|
+
}
|
|
153
|
+
if (overrides.chunking.parsers !== undefined) {
|
|
154
|
+
// Normalize exactly like the config loader does, so a hand-edited
|
|
155
|
+
// runtime-overrides.json gets the same "lowercase, leading dot"
|
|
156
|
+
// handling as the on-disk config.
|
|
157
|
+
merged.chunking.parsers = normalizeParserOverrides(overrides.chunking.parsers);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
146
160
|
if (overrides.tui) {
|
|
147
161
|
merged.tui = {
|
|
148
162
|
...DEFAULT_CONFIG.tui,
|
package/dist/eval/fast-index.js
CHANGED
|
@@ -204,7 +204,13 @@ async function main() {
|
|
|
204
204
|
console.log(" empty");
|
|
205
205
|
continue;
|
|
206
206
|
}
|
|
207
|
-
|
|
207
|
+
// Mirror production chunking: the parser override map and the
|
|
208
|
+
// per-language node-type overrides decide the chunk boundaries, so
|
|
209
|
+
// dropping them here would make eval results incomparable to a
|
|
210
|
+
// real index pass.
|
|
211
|
+
const chunks = await chunkFile(absPath, content, cfg.chunking?.nodeTypes, {
|
|
212
|
+
languageByExtension: cfg.chunking?.parsers,
|
|
213
|
+
});
|
|
208
214
|
if (!chunks || chunks.length === 0) {
|
|
209
215
|
console.log(" no chunks");
|
|
210
216
|
continue;
|
package/dist/index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* low-level building blocks (chunkers, embedders, vector stores, retrieval),
|
|
4
4
|
* high-level convenience API (search, index, getContext), and plugin entry.
|
|
5
5
|
*/
|
|
6
|
-
export { chunkFile, getChunker, registerChunker, getRegisteredLanguages, getExtensionsForLanguage, validateParserOverrides } from "./chunker/factory.js";
|
|
6
|
+
export { chunkFile, getChunker, registerChunker, getRegisteredLanguages, getExtensionsForLanguage, getLanguageForExtension, validateParserOverrides } from "./chunker/factory.js";
|
|
7
7
|
export { extractSkeleton } from "./chunker/skeleton.js";
|
|
8
8
|
export { normalizeParserOverrides } from "./core/parser-overrides.js";
|
|
9
9
|
export type { ParserOverrides } from "./core/parser-overrides.js";
|
package/dist/index.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* low-level building blocks (chunkers, embedders, vector stores, retrieval),
|
|
4
4
|
* high-level convenience API (search, index, getContext), and plugin entry.
|
|
5
5
|
*/
|
|
6
|
-
export { chunkFile, getChunker, registerChunker, getRegisteredLanguages, getExtensionsForLanguage, validateParserOverrides } from "./chunker/factory.js";
|
|
6
|
+
export { chunkFile, getChunker, registerChunker, getRegisteredLanguages, getExtensionsForLanguage, getLanguageForExtension, validateParserOverrides } from "./chunker/factory.js";
|
|
7
7
|
export { extractSkeleton } from "./chunker/skeleton.js";
|
|
8
8
|
export { normalizeParserOverrides } from "./core/parser-overrides.js";
|
|
9
9
|
export { createEmbedder, embedBatch } from "./embedder/factory.js";
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type ParserOverrides } from "../core/parser-overrides.js";
|
|
1
2
|
/**
|
|
2
3
|
* Build a metadata header string for embedding context that describes the
|
|
3
4
|
* file type, top-level directory, content classification, and document title.
|
|
@@ -5,6 +6,10 @@
|
|
|
5
6
|
* @param filePath - Absolute path to the file.
|
|
6
7
|
* @param cwd - Workspace root directory (used to compute relative path).
|
|
7
8
|
* @param content - File content used to extract a document title.
|
|
9
|
+
* @param languageByExtension - Optional `chunking.parsers` override map.
|
|
10
|
+
* When an extension is remapped to another parser, the header must name
|
|
11
|
+
* that parser — otherwise the embedded text says `[c]` while the chunk's
|
|
12
|
+
* `metadata.language` (and therefore the keyword filter) says `cpp`.
|
|
8
13
|
* @returns A space-delimited metadata string such as `"[typescript] [src] [source] MyComponent"`.
|
|
9
14
|
*/
|
|
10
|
-
export declare function buildFileMetadataHeader(filePath: string, cwd: string, content: string): string;
|
|
15
|
+
export declare function buildFileMetadataHeader(filePath: string, cwd: string, content: string, languageByExtension?: ParserOverrides): string;
|
package/dist/indexer/metadata.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* @fileoverview Builds metadata headers (file type, content type, document title) for embedding context.
|
|
3
3
|
*/
|
|
4
4
|
import path from "node:path";
|
|
5
|
+
import { normalizeParserOverrides, } from "../core/parser-overrides.js";
|
|
5
6
|
const FILE_TYPE_LABELS = {
|
|
6
7
|
".pdf": "pdf",
|
|
7
8
|
".docx": "docx",
|
|
@@ -86,12 +87,17 @@ function extractDocumentTitle(content, filePath) {
|
|
|
86
87
|
* @param filePath - Absolute path to the file.
|
|
87
88
|
* @param cwd - Workspace root directory (used to compute relative path).
|
|
88
89
|
* @param content - File content used to extract a document title.
|
|
90
|
+
* @param languageByExtension - Optional `chunking.parsers` override map.
|
|
91
|
+
* When an extension is remapped to another parser, the header must name
|
|
92
|
+
* that parser — otherwise the embedded text says `[c]` while the chunk's
|
|
93
|
+
* `metadata.language` (and therefore the keyword filter) says `cpp`.
|
|
89
94
|
* @returns A space-delimited metadata string such as `"[typescript] [src] [source] MyComponent"`.
|
|
90
95
|
*/
|
|
91
|
-
export function buildFileMetadataHeader(filePath, cwd, content) {
|
|
96
|
+
export function buildFileMetadataHeader(filePath, cwd, content, languageByExtension) {
|
|
92
97
|
const ext = path.extname(filePath).toLowerCase();
|
|
93
98
|
const relPath = path.relative(cwd, filePath).replace(/\\/g, "/");
|
|
94
|
-
const
|
|
99
|
+
const overrides = normalizeParserOverrides(languageByExtension);
|
|
100
|
+
const fileType = overrides[ext] ?? FILE_TYPE_LABELS[ext] ?? ext.slice(1);
|
|
95
101
|
const dirParts = relPath.split("/");
|
|
96
102
|
dirParts.pop();
|
|
97
103
|
const topDir = dirParts[0] ?? "";
|
package/dist/indexer/worker.js
CHANGED
|
@@ -171,7 +171,9 @@ export async function prepareFile(file, cwd, previous, config, keywordIndex, des
|
|
|
171
171
|
const docPrefix = config.embedding.documentPrefix ?? "";
|
|
172
172
|
const relPath = path.relative(cwd, file.filePath).replace(/\\/g, "/");
|
|
173
173
|
const isImage = isImageFile(file.filePath);
|
|
174
|
-
const metaHeader = isImage
|
|
174
|
+
const metaHeader = isImage
|
|
175
|
+
? ""
|
|
176
|
+
: buildFileMetadataHeader(file.filePath, cwd, file.content, config.chunking?.parsers);
|
|
175
177
|
if (deferDescriptions) {
|
|
176
178
|
for (const chunk of chunks) {
|
|
177
179
|
if (isImage) {
|
|
@@ -9,6 +9,13 @@ export interface RagReadToolOptions {
|
|
|
9
9
|
worktree: string;
|
|
10
10
|
/** Full RAG configuration. */
|
|
11
11
|
config: RagConfig;
|
|
12
|
+
/**
|
|
13
|
+
* Live configuration accessor. When provided, the read tool re-reads the
|
|
14
|
+
* config on every call, so runtime overrides (e.g. a TUI edit of
|
|
15
|
+
* `chunking.parsers`) reach the code-fence language label without
|
|
16
|
+
* restarting OpenCode. Falls back to the static `config` when omitted.
|
|
17
|
+
*/
|
|
18
|
+
getConfig?: () => RagConfig;
|
|
12
19
|
/** Embedding provider for retrieval. */
|
|
13
20
|
embedder: EmbeddingProvider;
|
|
14
21
|
/** Vector store for retrieval. */
|
|
@@ -105,6 +105,9 @@ export function createRagReadTool(options) {
|
|
|
105
105
|
// Retrieval failed — continue with just the file contents
|
|
106
106
|
}
|
|
107
107
|
// 5. Format output: full file + optional RAG context
|
|
108
|
+
// Resolve the config per call so a live `chunking.parsers` override
|
|
109
|
+
// (TUI edit) is reflected in the code-fence language label.
|
|
110
|
+
const liveConfig = options.getConfig?.() ?? config;
|
|
108
111
|
const output = formatHybridReadOutput({
|
|
109
112
|
filePath: resolvedPath,
|
|
110
113
|
fileContent,
|
|
@@ -113,7 +116,7 @@ export function createRagReadTool(options) {
|
|
|
113
116
|
ragChunks,
|
|
114
117
|
relatedFiles,
|
|
115
118
|
maxChars: maxReadOutputChars,
|
|
116
|
-
languageByExtension:
|
|
119
|
+
languageByExtension: liveConfig.chunking?.parsers,
|
|
117
120
|
});
|
|
118
121
|
// 6. Return
|
|
119
122
|
return {
|
package/dist/opencode/tools.d.ts
CHANGED
|
@@ -12,6 +12,12 @@ export interface FileSkeletonToolOptions {
|
|
|
12
12
|
worktree: string;
|
|
13
13
|
/** Extension → parser overrides from `chunking.parsers`. */
|
|
14
14
|
languageByExtension?: ParserOverrides;
|
|
15
|
+
/**
|
|
16
|
+
* Live accessor for the override map. When provided it is consulted on
|
|
17
|
+
* every call, so a runtime override (e.g. a TUI edit) is picked up without
|
|
18
|
+
* restarting OpenCode. Falls back to the static `languageByExtension`.
|
|
19
|
+
*/
|
|
20
|
+
getParserOverrides?: () => ParserOverrides | undefined;
|
|
15
21
|
}
|
|
16
22
|
/**
|
|
17
23
|
* Create the `get_file_skeleton` tool.
|
package/dist/opencode/tools.js
CHANGED
|
@@ -93,7 +93,8 @@ export function createFileSkeletonTool(options) {
|
|
|
93
93
|
const resolvedPath = resolveFilePath(args.filePath, worktree);
|
|
94
94
|
const content = readFileSync(resolvedPath, "utf-8");
|
|
95
95
|
const ext = getExtension(args.filePath);
|
|
96
|
-
const
|
|
96
|
+
const overrides = options.getParserOverrides?.() ?? languageByExtension;
|
|
97
|
+
const skeleton = await extractSkeleton(content, ext, { languageByExtension: overrides });
|
|
97
98
|
const declCounts = new Map();
|
|
98
99
|
for (const item of skeleton) {
|
|
99
100
|
const baseType = item.type.replace(/_declaration$|_definition$|_specifier$|_item$/, "");
|
package/dist/plugin.js
CHANGED
|
@@ -471,7 +471,11 @@ export function createRagHooks(options) {
|
|
|
471
471
|
// Runtime overrides for live config editing from TUI
|
|
472
472
|
let cachedOverrides = loadRuntimeOverrides(options.storePath);
|
|
473
473
|
let overridesLastCheck = 0;
|
|
474
|
-
|
|
474
|
+
// How often the override file is re-read from disk. The documented value
|
|
475
|
+
// was 5s; the code had drifted to 1h, which made every "live" TUI edit
|
|
476
|
+
// (including `chunking.parsers`) take up to an hour to take effect — the
|
|
477
|
+
// edit looked broken. Override with OPENCODE_RAG_OVERRIDES_TTL_MS.
|
|
478
|
+
const OVERRIDES_TTL_MS = Number(process.env.OPENCODE_RAG_OVERRIDES_TTL_MS) || 5000;
|
|
475
479
|
let lastEffectiveCfg = null;
|
|
476
480
|
let lastEffectiveCfgTime = 0;
|
|
477
481
|
function getEffectiveCfg() {
|
|
@@ -630,6 +634,9 @@ export function createRagHooks(options) {
|
|
|
630
634
|
const fileSkeletonTool = createFileSkeletonTool({
|
|
631
635
|
worktree: options.worktree,
|
|
632
636
|
languageByExtension: effectiveCfg.chunking?.parsers,
|
|
637
|
+
// Live accessor so a TUI edit of `chunking.parsers` is picked up
|
|
638
|
+
// without an OpenCode restart.
|
|
639
|
+
getParserOverrides: () => getEffectiveCfg().chunking?.parsers,
|
|
633
640
|
});
|
|
634
641
|
tools["get_file_skeleton"] = fileSkeletonTool;
|
|
635
642
|
}
|
|
@@ -761,6 +768,9 @@ export function createRagHooks(options) {
|
|
|
761
768
|
const readTool = createRagReadTool({
|
|
762
769
|
worktree: options.worktree,
|
|
763
770
|
config: options.cfg,
|
|
771
|
+
// Live config so a TUI edit of `chunking.parsers` reaches the
|
|
772
|
+
// code-fence language label without an OpenCode restart.
|
|
773
|
+
getConfig: getEffectiveCfg,
|
|
764
774
|
embedder,
|
|
765
775
|
store,
|
|
766
776
|
logFilePath: options.logFilePath,
|
package/dist/web/api.d.ts
CHANGED
|
@@ -32,6 +32,17 @@ interface ApiResponse {
|
|
|
32
32
|
* @returns An async handler that returns `true` when a route matched or `false` otherwise.
|
|
33
33
|
*/
|
|
34
34
|
export declare function createApiHandler(store: LanceDbStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig, getEmbedder?: () => Promise<EmbeddingProvider>, token?: string, configPath?: string): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
|
|
35
|
+
/**
|
|
36
|
+
* Apply a validated config patch (`{ indexing: { includeDirs: [...] } }`)
|
|
37
|
+
* to the on-disk config file and refresh the in-memory config used by
|
|
38
|
+
* subsequent API requests. Only known sections/keys with validated types
|
|
39
|
+
* are accepted; anything else is rejected with a 400.
|
|
40
|
+
*
|
|
41
|
+
* Exported for unit testing.
|
|
42
|
+
*/
|
|
43
|
+
export declare function handleConfigUpdate(configPath: string | undefined, configRef: {
|
|
44
|
+
current?: RagConfig;
|
|
45
|
+
}, body: unknown): ApiResponse;
|
|
35
46
|
/**
|
|
36
47
|
* Perform token-usage analysis for a single evaluation session.
|
|
37
48
|
*
|
package/dist/web/api.js
CHANGED
|
@@ -675,13 +675,27 @@ const INDEXING_STRING_ARRAY_KEYS = new Set([
|
|
|
675
675
|
"excludeFiles",
|
|
676
676
|
"includeDirs",
|
|
677
677
|
]);
|
|
678
|
+
/** True for a plain object whose every value is a string. */
|
|
679
|
+
function isStringMap(value) {
|
|
680
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
681
|
+
return false;
|
|
682
|
+
return Object.values(value).every((v) => typeof v === "string");
|
|
683
|
+
}
|
|
684
|
+
/** True for a plain object whose every value is an array of strings. */
|
|
685
|
+
function isStringArrayMap(value) {
|
|
686
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
687
|
+
return false;
|
|
688
|
+
return Object.values(value).every((v) => Array.isArray(v) && v.every((x) => typeof x === "string"));
|
|
689
|
+
}
|
|
678
690
|
/**
|
|
679
691
|
* Apply a validated config patch (`{ indexing: { includeDirs: [...] } }`)
|
|
680
692
|
* to the on-disk config file and refresh the in-memory config used by
|
|
681
693
|
* subsequent API requests. Only known sections/keys with validated types
|
|
682
694
|
* are accepted; anything else is rejected with a 400.
|
|
695
|
+
*
|
|
696
|
+
* Exported for unit testing.
|
|
683
697
|
*/
|
|
684
|
-
function handleConfigUpdate(configPath, configRef, body) {
|
|
698
|
+
export function handleConfigUpdate(configPath, configRef, body) {
|
|
685
699
|
if (!configPath) {
|
|
686
700
|
return {
|
|
687
701
|
status: 400,
|
|
@@ -692,7 +706,7 @@ function handleConfigUpdate(configPath, configRef, body) {
|
|
|
692
706
|
if (!patch || typeof patch !== "object" || Array.isArray(patch)) {
|
|
693
707
|
return { status: 400, body: { error: "Invalid config patch" } };
|
|
694
708
|
}
|
|
695
|
-
const otherSections = Object.keys(patch).filter((k) => k !== "indexing");
|
|
709
|
+
const otherSections = Object.keys(patch).filter((k) => k !== "indexing" && k !== "chunking");
|
|
696
710
|
if (otherSections.length > 0) {
|
|
697
711
|
return { status: 400, body: { error: `Unsupported config section(s): ${otherSections.join(", ")}` } };
|
|
698
712
|
}
|
|
@@ -710,6 +724,27 @@ function handleConfigUpdate(configPath, configRef, body) {
|
|
|
710
724
|
}
|
|
711
725
|
}
|
|
712
726
|
}
|
|
727
|
+
const chunkingPatch = patch.chunking;
|
|
728
|
+
if (chunkingPatch !== undefined) {
|
|
729
|
+
if (typeof chunkingPatch !== "object" || chunkingPatch === null || Array.isArray(chunkingPatch)) {
|
|
730
|
+
return { status: 400, body: { error: "Invalid 'chunking' section" } };
|
|
731
|
+
}
|
|
732
|
+
for (const [key, value] of Object.entries(chunkingPatch)) {
|
|
733
|
+
if (key === "parsers") {
|
|
734
|
+
if (!isStringMap(value)) {
|
|
735
|
+
return { status: 400, body: { error: "'chunking.parsers' must be an object mapping file extensions to parser names" } };
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
else if (key === "nodeTypes") {
|
|
739
|
+
if (!isStringArrayMap(value)) {
|
|
740
|
+
return { status: 400, body: { error: "'chunking.nodeTypes' must be an object mapping languages to arrays of node type names" } };
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
else {
|
|
744
|
+
return { status: 400, body: { error: `Unsupported chunking key '${key}'` } };
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
}
|
|
713
748
|
if (indexingPatch) {
|
|
714
749
|
for (const [key, value] of Object.entries(indexingPatch)) {
|
|
715
750
|
const ok = updateConfigValue(configPath, ["indexing", key], value);
|
|
@@ -723,6 +758,19 @@ function handleConfigUpdate(configPath, configRef, body) {
|
|
|
723
758
|
indexing: { ...current.indexing, ...indexingPatch },
|
|
724
759
|
};
|
|
725
760
|
}
|
|
761
|
+
if (chunkingPatch) {
|
|
762
|
+
for (const [key, value] of Object.entries(chunkingPatch)) {
|
|
763
|
+
const ok = updateConfigValue(configPath, ["chunking", key], value);
|
|
764
|
+
if (!ok) {
|
|
765
|
+
return { status: 500, body: { error: `Failed to write 'chunking.${key}' to ${configPath}` } };
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
const current = configRef.current ?? {};
|
|
769
|
+
configRef.current = {
|
|
770
|
+
...current,
|
|
771
|
+
chunking: { ...current.chunking, ...chunkingPatch },
|
|
772
|
+
};
|
|
773
|
+
}
|
|
726
774
|
const redacted = JSON.parse(JSON.stringify(configRef.current));
|
|
727
775
|
redactKeys(redacted);
|
|
728
776
|
return { status: 200, body: { config: redacted } };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-rag-plugin",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.5.1",
|
|
4
4
|
"description": "OpenCode plugin for local-first RAG-based semantic code search",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/plugin-entry.js",
|
|
@@ -112,8 +112,8 @@
|
|
|
112
112
|
"apache-arrow": "18.1.0",
|
|
113
113
|
"chokidar": "^4.0.3",
|
|
114
114
|
"commander": "^13.1.0",
|
|
115
|
+
"fflate": "^0.8.3",
|
|
115
116
|
"js-tiktoken": "^1.0.21",
|
|
116
|
-
"mammoth": "^1.12.0",
|
|
117
117
|
"minimatch": "^8.0.7",
|
|
118
118
|
"p-limit": "^7.3.0",
|
|
119
119
|
"pdfjs-dist": "^6.3.289",
|
|
@@ -149,7 +149,8 @@
|
|
|
149
149
|
"uuid": "^11.1.1",
|
|
150
150
|
"web-tree-sitter": "$web-tree-sitter",
|
|
151
151
|
"@babel/core": "^7.29.7",
|
|
152
|
-
"@opentelemetry/core": "^2.8.0"
|
|
152
|
+
"@opentelemetry/core": "^2.8.0",
|
|
153
|
+
"seroval": "^1.6.8"
|
|
153
154
|
},
|
|
154
155
|
"allowScripts": {
|
|
155
156
|
"opencode-rag-plugin": true,
|