opencode-rag-plugin 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +1 -0
- package/dist/chunker/factory.d.ts +24 -2
- package/dist/chunker/factory.js +88 -3
- package/dist/chunker/loader.js +10 -5
- package/dist/chunker/skeleton.d.ts +36 -0
- package/dist/chunker/skeleton.js +183 -0
- package/dist/content/reader.js +8 -0
- package/dist/core/bootstrap.js +20 -0
- package/dist/core/config.d.ts +20 -0
- package/dist/core/config.js +2 -0
- package/dist/core/interfaces.d.ts +2 -0
- package/dist/core/parser-overrides.d.ts +29 -0
- package/dist/core/parser-overrides.js +49 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +3 -1
- package/dist/indexer/pipeline.js +26 -0
- package/dist/indexer/worker.d.ts +2 -0
- package/dist/indexer/worker.js +1 -0
- package/dist/mcp/handlers.d.ts +2 -1
- package/dist/mcp/handlers.js +5 -103
- package/dist/mcp/server.js +1 -1
- package/dist/opencode/create-read-tool.js +1 -0
- package/dist/opencode/read-format.d.ts +5 -0
- package/dist/opencode/read-format.js +12 -5
- package/dist/opencode/tools.d.ts +3 -0
- package/dist/opencode/tools.js +6 -124
- package/dist/plugin.js +22 -1
- package/dist/quirks/quirk-store.d.ts +45 -0
- package/dist/quirks/quirk-store.js +200 -20
- package/dist/retriever/keyword-index.d.ts +1 -0
- package/dist/retriever/keyword-index.js +3 -0
- package/dist/tui.js +6 -0
- package/dist/vectorstore/lancedb.js +1 -0
- package/package.json +1 -1
package/ReadMe.md
CHANGED
|
@@ -35,6 +35,7 @@ opencode-rag query "authentication middleware"
|
|
|
35
35
|
|---|---|
|
|
36
36
|
| **MCP server** | `opencode-rag mcp` - stdio-based MCP server exposing `search_semantic`, `get_file_skeleton`, `find_usages`, and `describe_image` tools for any MCP-compatible client |
|
|
37
37
|
| **AST chunking** | 26 languages via tree-sitter (TS, JS, Python, Java, Go, Rust, C/C++, C#, Ruby, Kotlin, Swift, Bash, PHP, PowerShell, SQL, JSON, HTML, CSS, XML (including SVG), YAML, TOML, INI, Dockerfile, Markdown, LaTeX, Razor) |
|
|
38
|
+
| **Configurable parsers** | Remap file extensions to any parser via `chunking.parsers` (e.g. `.c` → `cpp`, or index a new `.cu` as C++) — applied to chunking, `get_file_skeleton`, and read labels ([docs](doc/chunking.md#parser-overrides-per-extension-chunkingparsers)) |
|
|
38
39
|
| **Document support** | Markdown, LaTeX, PDF, DOCX, DOC, Excel |
|
|
39
40
|
| **Image indexing** | Describe images via vision LLM and store descriptions as searchable vector chunks |
|
|
40
41
|
| **Hybrid search** | Vector similarity + TF×IDF keyword fusion |
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* @fileoverview Chunker registry, extension-to-chunker mapping, and file chunking orchestration.
|
|
3
3
|
*/
|
|
4
4
|
import type { Chunker, Chunk } from "../core/interfaces.js";
|
|
5
|
+
import { type ParserOverrides } from "../core/parser-overrides.js";
|
|
5
6
|
import { typescriptChunker } from "./typescript.js";
|
|
6
7
|
import { pythonChunker } from "./python.js";
|
|
7
8
|
import { javaChunker } from "./java.js";
|
|
@@ -46,9 +47,28 @@ export declare function registerChunker(chunker: Chunker, extensions?: string[])
|
|
|
46
47
|
* Falls back to the fallback chunker when no extension match is found.
|
|
47
48
|
*
|
|
48
49
|
* @param filePath - Path to the file to chunk.
|
|
50
|
+
* @param languageByExtension - Optional `chunking.parsers` override map
|
|
51
|
+
* (extension → parser language). Overrides win over the built-in mapping and
|
|
52
|
+
* are resolved per call, so they never mutate the process-wide registry
|
|
53
|
+
* (which is shared across workspaces).
|
|
49
54
|
* @returns A chunker instance for the file's extension.
|
|
50
55
|
*/
|
|
51
|
-
export declare function getChunker(filePath: string): Chunker;
|
|
56
|
+
export declare function getChunker(filePath: string, languageByExtension?: ParserOverrides): Chunker;
|
|
57
|
+
/** Every parser (chunker language) name accepted as a `chunking.parsers` target. */
|
|
58
|
+
export declare function getRegisteredLanguages(): string[];
|
|
59
|
+
/**
|
|
60
|
+
* All file extensions currently mapped to a given parser language, in
|
|
61
|
+
* registration order. Used by skeleton extraction to pick a representative
|
|
62
|
+
* extension recipe when an override retargets an extension.
|
|
63
|
+
*/
|
|
64
|
+
export declare function getExtensionsForLanguage(language: string): string[];
|
|
65
|
+
/**
|
|
66
|
+
* Validate a raw `chunking.parsers` map against the registered parsers.
|
|
67
|
+
*
|
|
68
|
+
* @param parsers - Raw (unnormalized) override map from the config.
|
|
69
|
+
* @returns Human-readable warnings; empty when the map is valid or absent.
|
|
70
|
+
*/
|
|
71
|
+
export declare function validateParserOverrides(parsers?: ParserOverrides): string[];
|
|
52
72
|
export declare function getRegisteredExtensions(): string[];
|
|
53
73
|
/**
|
|
54
74
|
* Chunk a file by looking up its registered chunker, applying node-type
|
|
@@ -59,7 +79,7 @@ export declare function getRegisteredExtensions(): string[];
|
|
|
59
79
|
* @param content - The full text content of the file.
|
|
60
80
|
* @param nodeTypesOverrides - Optional language-specific node type overrides
|
|
61
81
|
* for tree-sitter based chunkers.
|
|
62
|
-
* @param options - Optional chunking options (maxSvgSizeBytes, etc.).
|
|
82
|
+
* @param options - Optional chunking options (maxSvgSizeBytes, parser overrides, etc.).
|
|
63
83
|
* @returns An array of chunks for the file.
|
|
64
84
|
*/
|
|
65
85
|
export declare function chunkFile(filePath: string, content: string, nodeTypesOverrides?: Record<string, string[]>, options?: {
|
|
@@ -68,5 +88,7 @@ export declare function chunkFile(filePath: string, content: string, nodeTypesOv
|
|
|
68
88
|
maxChunkSize?: number;
|
|
69
89
|
/** Overlapping lines shared between adjacent/split chunks. */
|
|
70
90
|
chunkOverlap?: number;
|
|
91
|
+
/** Extension → parser overrides from `chunking.parsers`. */
|
|
92
|
+
languageByExtension?: ParserOverrides;
|
|
71
93
|
}): Promise<Chunk[]>;
|
|
72
94
|
export { typescriptChunker, pythonChunker, javaChunker, goChunker, markdownChunker, cChunker, cppChunker, csharpChunker, javascriptChunker, razorChunker, jsonChunker, htmlChunker, cssChunker, xmlChunker, slnChunker, rustChunker, rubyChunker, kotlinChunker, swiftChunker, bashChunker, phpChunker, powershellChunker, iniChunker, yamlChunker, tomlChunker, dockerfileChunker, sqlChunker, texChunker, pdfChunker, fallbackChunker };
|
package/dist/chunker/factory.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { normalizeExtensionKey, normalizeParserOverrides, } from "../core/parser-overrides.js";
|
|
1
2
|
import { TreeSitterChunker } from "./base.js";
|
|
2
3
|
import { typescriptChunker } from "./typescript.js";
|
|
3
4
|
import { pythonChunker } from "./python.js";
|
|
@@ -78,6 +79,19 @@ for (const chunker of chunkers) {
|
|
|
78
79
|
}
|
|
79
80
|
}
|
|
80
81
|
}
|
|
82
|
+
/**
|
|
83
|
+
* Parser (chunker) registry keyed by `language` name — the target space of the
|
|
84
|
+
* `chunking.parsers` config (`{ ".c": "cpp" }`). Kept in sync with
|
|
85
|
+
* {@link extensionMap} by {@link registerChunker}.
|
|
86
|
+
*/
|
|
87
|
+
const languageMap = new Map();
|
|
88
|
+
for (const chunker of chunkers) {
|
|
89
|
+
if (chunker.language)
|
|
90
|
+
languageMap.set(chunker.language, chunker);
|
|
91
|
+
}
|
|
92
|
+
languageMap.set(fallbackChunker.language, fallbackChunker);
|
|
93
|
+
/** Override targets already reported as unknown, so a bad config warns once per process. */
|
|
94
|
+
const warnedUnknownParsers = new Set();
|
|
81
95
|
/**
|
|
82
96
|
* Register a pluggable chunker for one or more file extensions.
|
|
83
97
|
* If an extension already has a chunker, the new one is silently skipped.
|
|
@@ -87,6 +101,8 @@ for (const chunker of chunkers) {
|
|
|
87
101
|
* Defaults to the chunker's own `fileExtensions` property.
|
|
88
102
|
*/
|
|
89
103
|
export function registerChunker(chunker, extensions) {
|
|
104
|
+
if (chunker.language)
|
|
105
|
+
languageMap.set(chunker.language, chunker);
|
|
90
106
|
const exts = extensions ?? ("fileExtensions" in chunker
|
|
91
107
|
? chunker.fileExtensions
|
|
92
108
|
: []);
|
|
@@ -104,17 +120,86 @@ export function registerChunker(chunker, extensions) {
|
|
|
104
120
|
* Falls back to the fallback chunker when no extension match is found.
|
|
105
121
|
*
|
|
106
122
|
* @param filePath - Path to the file to chunk.
|
|
123
|
+
* @param languageByExtension - Optional `chunking.parsers` override map
|
|
124
|
+
* (extension → parser language). Overrides win over the built-in mapping and
|
|
125
|
+
* are resolved per call, so they never mutate the process-wide registry
|
|
126
|
+
* (which is shared across workspaces).
|
|
107
127
|
* @returns A chunker instance for the file's extension.
|
|
108
128
|
*/
|
|
109
|
-
export function getChunker(filePath) {
|
|
129
|
+
export function getChunker(filePath, languageByExtension) {
|
|
110
130
|
const dotIdx = filePath.lastIndexOf(".");
|
|
111
131
|
const ext = dotIdx >= 0 ? filePath.slice(dotIdx).toLowerCase() : "";
|
|
132
|
+
const overrides = languageByExtension && Object.keys(languageByExtension).length > 0
|
|
133
|
+
? normalizeParserOverrides(languageByExtension)
|
|
134
|
+
: undefined;
|
|
135
|
+
const target = ext && overrides ? overrides[ext] : undefined;
|
|
136
|
+
if (target) {
|
|
137
|
+
const overridden = languageMap.get(target);
|
|
138
|
+
if (overridden)
|
|
139
|
+
return overridden;
|
|
140
|
+
if (!warnedUnknownParsers.has(target)) {
|
|
141
|
+
warnedUnknownParsers.add(target);
|
|
142
|
+
console.warn(`[opencode-rag] chunking.parsers maps "${ext}" to unknown parser "${target}" ` +
|
|
143
|
+
`— known parsers: ${getRegisteredLanguages().join(", ")}. Using the default mapping.`);
|
|
144
|
+
}
|
|
145
|
+
// Unknown target: fall through to the default mapping for this extension.
|
|
146
|
+
}
|
|
112
147
|
if (ext && extensionMap.has(ext)) {
|
|
113
148
|
return extensionMap.get(ext);
|
|
114
149
|
}
|
|
115
150
|
const basename = filePath.toLowerCase();
|
|
116
151
|
return extensionMap.get(basename) ?? fallbackChunker;
|
|
117
152
|
}
|
|
153
|
+
/** Every parser (chunker language) name accepted as a `chunking.parsers` target. */
|
|
154
|
+
export function getRegisteredLanguages() {
|
|
155
|
+
return [...new Set([...languageMap.values()].map((chunker) => chunker.language))].sort();
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* All file extensions currently mapped to a given parser language, in
|
|
159
|
+
* registration order. Used by skeleton extraction to pick a representative
|
|
160
|
+
* extension recipe when an override retargets an extension.
|
|
161
|
+
*/
|
|
162
|
+
export function getExtensionsForLanguage(language) {
|
|
163
|
+
const out = [];
|
|
164
|
+
for (const [ext, chunker] of extensionMap) {
|
|
165
|
+
if (chunker.language === language)
|
|
166
|
+
out.push(ext);
|
|
167
|
+
}
|
|
168
|
+
return out;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Validate a raw `chunking.parsers` map against the registered parsers.
|
|
172
|
+
*
|
|
173
|
+
* @param parsers - Raw (unnormalized) override map from the config.
|
|
174
|
+
* @returns Human-readable warnings; empty when the map is valid or absent.
|
|
175
|
+
*/
|
|
176
|
+
export function validateParserOverrides(parsers) {
|
|
177
|
+
const warnings = [];
|
|
178
|
+
if (parsers === undefined || parsers === null)
|
|
179
|
+
return warnings;
|
|
180
|
+
if (typeof parsers !== "object" || Array.isArray(parsers)) {
|
|
181
|
+
warnings.push('chunking.parsers must be an object mapping file extensions to parser names (e.g. { ".c": "cpp" })');
|
|
182
|
+
return warnings;
|
|
183
|
+
}
|
|
184
|
+
const known = new Set(getRegisteredLanguages());
|
|
185
|
+
for (const [rawKey, rawValue] of Object.entries(parsers)) {
|
|
186
|
+
const ext = normalizeExtensionKey(rawKey);
|
|
187
|
+
if (!ext) {
|
|
188
|
+
warnings.push(`chunking.parsers key "${rawKey}" is not a valid file extension`);
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
if (typeof rawValue !== "string" || rawValue.trim().length === 0) {
|
|
192
|
+
warnings.push(`chunking.parsers["${rawKey}"] must be a non-empty parser name`);
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
const language = rawValue.trim().toLowerCase();
|
|
196
|
+
if (!known.has(language)) {
|
|
197
|
+
warnings.push(`chunking.parsers["${rawKey}"] → "${language}" is not a registered parser — ` +
|
|
198
|
+
`known parsers: ${[...known].sort().join(", ")}`);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
return warnings;
|
|
202
|
+
}
|
|
118
203
|
export function getRegisteredExtensions() {
|
|
119
204
|
return [...extensionMap.keys()].sort();
|
|
120
205
|
}
|
|
@@ -230,11 +315,11 @@ function withByteLimit(chunker, limit) {
|
|
|
230
315
|
* @param content - The full text content of the file.
|
|
231
316
|
* @param nodeTypesOverrides - Optional language-specific node type overrides
|
|
232
317
|
* for tree-sitter based chunkers.
|
|
233
|
-
* @param options - Optional chunking options (maxSvgSizeBytes, etc.).
|
|
318
|
+
* @param options - Optional chunking options (maxSvgSizeBytes, parser overrides, etc.).
|
|
234
319
|
* @returns An array of chunks for the file.
|
|
235
320
|
*/
|
|
236
321
|
export async function chunkFile(filePath, content, nodeTypesOverrides, options) {
|
|
237
|
-
let chunker = getChunker(filePath);
|
|
322
|
+
let chunker = getChunker(filePath, options?.languageByExtension);
|
|
238
323
|
if (nodeTypesOverrides && chunker instanceof TreeSitterChunker) {
|
|
239
324
|
const overrideTypes = nodeTypesOverrides[chunker.language];
|
|
240
325
|
if (overrideTypes && overrideTypes.length > 0) {
|
package/dist/chunker/loader.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview Dynamic loading and registration of pluggable chunker modules from configuration.
|
|
3
3
|
*/
|
|
4
|
-
import { registerChunker } from "./factory.js";
|
|
4
|
+
import { registerChunker, validateParserOverrides } from "./factory.js";
|
|
5
5
|
import { ImageChunker, SUPPORTED_IMAGE_EXTENSIONS } from "./image.js";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { pathToFileURL } from "node:url";
|
|
@@ -46,10 +46,15 @@ export async function loadChunkersFromConfig(config, configDir) {
|
|
|
46
46
|
const chunker = new ImageChunker([...SUPPORTED_IMAGE_EXTENSIONS]);
|
|
47
47
|
registerChunker(chunker, [...SUPPORTED_IMAGE_EXTENSIONS]);
|
|
48
48
|
}
|
|
49
|
-
if (
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
49
|
+
if (config.chunkers && config.chunkers.length > 0) {
|
|
50
|
+
for (const entry of config.chunkers) {
|
|
51
|
+
await loadSingleChunker(entry, configDir);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
// Validate AFTER registration so overrides targeting a pluggable chunker's
|
|
55
|
+
// language are recognized. (Known-parser lookup needs the full registry.)
|
|
56
|
+
for (const warning of validateParserOverrides(config.chunking?.parsers)) {
|
|
57
|
+
console.warn(`[opencode-rag] ${warning}`);
|
|
53
58
|
}
|
|
54
59
|
}
|
|
55
60
|
//# sourceMappingURL=loader.js.map
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { type ParserOverrides } from "../core/parser-overrides.js";
|
|
2
|
+
/** Skeleton extraction recipe for one file extension. */
|
|
3
|
+
export interface SkeletonConfig {
|
|
4
|
+
/** tree-sitter grammar name (as resolved by {@link loadLanguage}). */
|
|
5
|
+
grammarName: string;
|
|
6
|
+
/** AST node types to collect as structural elements. */
|
|
7
|
+
nodeTypes: string[];
|
|
8
|
+
}
|
|
9
|
+
/** Extension → tree-sitter grammar + node types for skeleton extraction. */
|
|
10
|
+
export declare const SKELETON_CONFIGS: Record<string, SkeletonConfig>;
|
|
11
|
+
/** Regex-based fallback for languages without tree-sitter WASM support. */
|
|
12
|
+
export declare const REGEX_SKELETON: Record<string, RegExp[]>;
|
|
13
|
+
/** Optional skeleton extraction options. */
|
|
14
|
+
export interface SkeletonOptions {
|
|
15
|
+
/** Extension → parser overrides from `chunking.parsers`. */
|
|
16
|
+
languageByExtension?: ParserOverrides;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Return the file extension from a path (lowercase, dot-prefixed when present).
|
|
20
|
+
*/
|
|
21
|
+
export declare function getExtension(filePath: string): string;
|
|
22
|
+
/**
|
|
23
|
+
* Extract structural outline from source code using tree-sitter.
|
|
24
|
+
* Falls back to regex patterns for languages without WASM grammars, and to a
|
|
25
|
+
* plain line count when neither recipe applies.
|
|
26
|
+
*
|
|
27
|
+
* @param content - Full file content.
|
|
28
|
+
* @param ext - Lowercase, dot-prefixed file extension.
|
|
29
|
+
* @param options - Optional `chunking.parsers` override map.
|
|
30
|
+
*/
|
|
31
|
+
export declare function extractSkeleton(content: string, ext: string, options?: SkeletonOptions): Promise<{
|
|
32
|
+
type: string;
|
|
33
|
+
name: string;
|
|
34
|
+
startLine: number;
|
|
35
|
+
endLine: number;
|
|
36
|
+
}[]>;
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Shared file-skeleton extraction: extension → tree-sitter grammar + node types.
|
|
3
|
+
*
|
|
4
|
+
* Historically this table (and {@link extractSkeleton}) was copy-pasted into
|
|
5
|
+
* both the OpenCode plugin tools (`src/opencode/tools.ts`) and the MCP server
|
|
6
|
+
* (`src/mcp/handlers.ts`), and had already drifted from the chunker registry.
|
|
7
|
+
* It now lives here so both surfaces — plus `chunking.parsers` overrides —
|
|
8
|
+
* resolve extensions the same way.
|
|
9
|
+
*/
|
|
10
|
+
import { Parser } from "web-tree-sitter";
|
|
11
|
+
import { initParser, loadLanguage, walkTree } from "./grammar.js";
|
|
12
|
+
import { getExtensionsForLanguage, getRegisteredLanguages } from "./factory.js";
|
|
13
|
+
import { normalizeParserOverrides, } from "../core/parser-overrides.js";
|
|
14
|
+
/** Extension → tree-sitter grammar + node types for skeleton extraction. */
|
|
15
|
+
export const SKELETON_CONFIGS = {
|
|
16
|
+
".ts": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration"] },
|
|
17
|
+
".tsx": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration", "arrow_function"] },
|
|
18
|
+
".mts": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration"] },
|
|
19
|
+
".cts": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration"] },
|
|
20
|
+
".js": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
|
|
21
|
+
".jsx": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
|
|
22
|
+
".mjs": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
|
|
23
|
+
".cjs": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
|
|
24
|
+
".py": { grammarName: "python", nodeTypes: ["function_definition", "class_definition", "decorated_definition"] },
|
|
25
|
+
".java": { grammarName: "java", nodeTypes: ["class_declaration", "method_declaration", "interface_declaration", "enum_declaration"] },
|
|
26
|
+
".go": { grammarName: "go", nodeTypes: ["function_declaration", "method_declaration", "type_declaration", "type_spec"] },
|
|
27
|
+
".rs": { grammarName: "rust", nodeTypes: ["function_item", "struct_item", "enum_item", "impl_item", "trait_item", "type_item"] },
|
|
28
|
+
".c": { grammarName: "c", nodeTypes: ["function_definition", "struct_specifier", "enum_specifier"] },
|
|
29
|
+
".cpp": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
|
|
30
|
+
".cxx": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
|
|
31
|
+
".h": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
|
|
32
|
+
".hpp": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
|
|
33
|
+
".cs": { grammarName: "c-sharp", nodeTypes: ["class_declaration", "method_declaration", "interface_declaration", "enum_declaration", "struct_declaration"] },
|
|
34
|
+
".rb": { grammarName: "ruby", nodeTypes: ["method", "class", "module"] },
|
|
35
|
+
".swift": { grammarName: "swift", nodeTypes: ["function_declaration", "class_declaration", "struct_declaration", "enum_declaration", "protocol_declaration"] },
|
|
36
|
+
".kt": { grammarName: "kotlin", nodeTypes: ["function_declaration", "class_declaration", "interface_declaration", "object_declaration"] },
|
|
37
|
+
".kts": { grammarName: "kotlin", nodeTypes: ["function_declaration", "class_declaration", "interface_declaration", "object_declaration"] },
|
|
38
|
+
".css": { grammarName: "css", nodeTypes: ["rule_set"] },
|
|
39
|
+
".md": { grammarName: "markdown", nodeTypes: ["atx_heading"] },
|
|
40
|
+
};
|
|
41
|
+
/** Regex-based fallback for languages without tree-sitter WASM support. */
|
|
42
|
+
export const REGEX_SKELETON = {
|
|
43
|
+
".json": [/^(\s*)"(\w+)":/gm],
|
|
44
|
+
".yaml": [/^(\w+):/gm],
|
|
45
|
+
".yml": [/^(\w+):/gm],
|
|
46
|
+
".toml": [/^\[(\w+)\]/gm],
|
|
47
|
+
".sh": [/^(function\s+\w+|^\w+\s*\(\)\s*\{)/gm],
|
|
48
|
+
".bash": [/^(function\s+\w+|^\w+\s*\(\)\s*\{)/gm],
|
|
49
|
+
".zsh": [/^(function\s+\w+|^\w+\s*\(\)\s*\{)/gm],
|
|
50
|
+
".sql": [/^(CREATE|ALTER|DROP|SELECT|INSERT|UPDATE|DELETE)\s/gim],
|
|
51
|
+
};
|
|
52
|
+
/**
|
|
53
|
+
* Sentinel lookup key that matches neither config map. Used when a
|
|
54
|
+
* `chunking.parsers` override targets a parser with no skeleton recipe —
|
|
55
|
+
* the extension's own (now wrong) recipe must NOT be used, so fall through
|
|
56
|
+
* to the line-count fallback instead.
|
|
57
|
+
*/
|
|
58
|
+
const NO_SKELETON = "\u0000no-skeleton";
|
|
59
|
+
/** Unknown override targets already reported, so a bad config warns once per process. */
|
|
60
|
+
const warnedUnknownTargets = new Set();
|
|
61
|
+
/**
|
|
62
|
+
* Return the file extension from a path (lowercase, dot-prefixed when present).
|
|
63
|
+
*/
|
|
64
|
+
export function getExtension(filePath) {
|
|
65
|
+
const dot = filePath.lastIndexOf(".");
|
|
66
|
+
return dot >= 0 ? filePath.slice(dot).toLowerCase() : "";
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Resolve the skeleton-config lookup key for an extension, honoring
|
|
70
|
+
* `chunking.parsers` overrides.
|
|
71
|
+
*
|
|
72
|
+
* Non-overridden extensions resolve to themselves, so per-extension recipes
|
|
73
|
+
* that legitimately differ (`.ts` vs `.tsx`) are untouched. Overridden ones
|
|
74
|
+
* jump to a representative extension of the target parser.
|
|
75
|
+
*/
|
|
76
|
+
function resolveSkeletonKey(ext, overrides) {
|
|
77
|
+
const target = overrides[ext];
|
|
78
|
+
if (!target)
|
|
79
|
+
return ext;
|
|
80
|
+
if (!getRegisteredLanguages().includes(target)) {
|
|
81
|
+
if (!warnedUnknownTargets.has(target)) {
|
|
82
|
+
warnedUnknownTargets.add(target);
|
|
83
|
+
const known = getRegisteredLanguages();
|
|
84
|
+
console.warn(`[opencode-rag] chunking.parsers maps "${ext}" to unknown parser "${target}" ` +
|
|
85
|
+
`— known parsers: ${known.join(", ")}. Using the default skeleton for "${ext}".`);
|
|
86
|
+
}
|
|
87
|
+
return ext;
|
|
88
|
+
}
|
|
89
|
+
const candidates = getExtensionsForLanguage(target);
|
|
90
|
+
const hit = candidates.find((c) => SKELETON_CONFIGS[c] || REGEX_SKELETON[c]);
|
|
91
|
+
// Known parser but no skeleton recipe for it (e.g. plain text, PDF): the
|
|
92
|
+
// extension's own recipe would be the wrong parser, so fall through to the
|
|
93
|
+
// line-count fallback instead of using it.
|
|
94
|
+
return hit ?? NO_SKELETON;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Extract structural outline from source code using tree-sitter.
|
|
98
|
+
* Falls back to regex patterns for languages without WASM grammars, and to a
|
|
99
|
+
* plain line count when neither recipe applies.
|
|
100
|
+
*
|
|
101
|
+
* @param content - Full file content.
|
|
102
|
+
* @param ext - Lowercase, dot-prefixed file extension.
|
|
103
|
+
* @param options - Optional `chunking.parsers` override map.
|
|
104
|
+
*/
|
|
105
|
+
export async function extractSkeleton(content, ext, options) {
|
|
106
|
+
const overrides = normalizeParserOverrides(options?.languageByExtension);
|
|
107
|
+
const key = resolveSkeletonKey(ext, overrides);
|
|
108
|
+
const config = SKELETON_CONFIGS[key];
|
|
109
|
+
if (!config) {
|
|
110
|
+
// Fallback: use regex patterns
|
|
111
|
+
const patterns = REGEX_SKELETON[key];
|
|
112
|
+
if (!patterns) {
|
|
113
|
+
// Last fallback: line count
|
|
114
|
+
const lines = content.split("\n");
|
|
115
|
+
return [{ type: "file", name: `${lines.length} lines`, startLine: 1, endLine: lines.length }];
|
|
116
|
+
}
|
|
117
|
+
const items = [];
|
|
118
|
+
for (const pattern of patterns) {
|
|
119
|
+
let match;
|
|
120
|
+
// Reset lastIndex
|
|
121
|
+
pattern.lastIndex = 0;
|
|
122
|
+
while ((match = pattern.exec(content)) !== null) {
|
|
123
|
+
const name = match[1] ?? match[0].trim();
|
|
124
|
+
const lineNum = content.slice(0, match.index).split("\n").length;
|
|
125
|
+
items.push({ type: ext.slice(1), name, startLine: lineNum, endLine: lineNum });
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
const seen = new Set();
|
|
129
|
+
return items.filter((item) => {
|
|
130
|
+
const seenKey = `${item.type}:${item.name}`;
|
|
131
|
+
if (seen.has(seenKey))
|
|
132
|
+
return false;
|
|
133
|
+
seen.add(seenKey);
|
|
134
|
+
return true;
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
// Tree-sitter skeleton extraction
|
|
138
|
+
await initParser();
|
|
139
|
+
const lang = await loadLanguage(config.grammarName);
|
|
140
|
+
const parser = new Parser();
|
|
141
|
+
parser.setLanguage(lang);
|
|
142
|
+
const tree = parser.parse(content);
|
|
143
|
+
if (!tree)
|
|
144
|
+
return [];
|
|
145
|
+
try {
|
|
146
|
+
const nodeTypes = new Set(config.nodeTypes);
|
|
147
|
+
const astNodes = walkTree(tree.rootNode, nodeTypes, content, 15);
|
|
148
|
+
return astNodes.map((node) => ({
|
|
149
|
+
type: node.type,
|
|
150
|
+
name: extractNodeName(node.text, node.type),
|
|
151
|
+
startLine: node.startLine,
|
|
152
|
+
endLine: node.endLine,
|
|
153
|
+
}));
|
|
154
|
+
}
|
|
155
|
+
finally {
|
|
156
|
+
tree.delete();
|
|
157
|
+
parser.delete();
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Extract a human-readable name from a code node.
|
|
162
|
+
* For declarations, this is typically the identifier after the keyword.
|
|
163
|
+
*/
|
|
164
|
+
function extractNodeName(text, _nodeType) {
|
|
165
|
+
// Try first line only for large nodes
|
|
166
|
+
const firstLine = text.split("\n")[0] ?? text;
|
|
167
|
+
// Common patterns: "function foo(...)" or "class Foo ..." or "foo(...)" (arrow, method)
|
|
168
|
+
const nameMatch = firstLine.match(/^(?:export\s+)?(?:async\s+)?(?:function\s+)?(?:\w+\s+)?(?:(\w+))\s*(?:[<(]|$)/);
|
|
169
|
+
if (nameMatch) {
|
|
170
|
+
// Skip keywords
|
|
171
|
+
const keywordSet = new Set(["export", "async", "function", "class", "interface", "type", "enum", "struct", "impl", "trait", "def", "fun", "fn"]);
|
|
172
|
+
if (!keywordSet.has(nameMatch[1])) {
|
|
173
|
+
return nameMatch[1];
|
|
174
|
+
}
|
|
175
|
+
// Try next word
|
|
176
|
+
const secondMatch = firstLine.match(/^(?:export\s+)?(?:async\s+)?(?:function|class|interface|type|enum|struct|trait|impl|def|fun|fn)\s+(\w+)/);
|
|
177
|
+
if (secondMatch)
|
|
178
|
+
return secondMatch[1];
|
|
179
|
+
}
|
|
180
|
+
// Last resort: return first 80 chars of first line
|
|
181
|
+
return firstLine.length > 80 ? firstLine.slice(0, 77) + "..." : firstLine;
|
|
182
|
+
}
|
|
183
|
+
//# sourceMappingURL=skeleton.js.map
|
package/dist/content/reader.js
CHANGED
|
@@ -6,6 +6,7 @@ import path from "node:path";
|
|
|
6
6
|
import pLimit from "p-limit";
|
|
7
7
|
import { computeFileHash, computeDescriptionConfigHash, normalizeFilePath } from "../core/manifest.js";
|
|
8
8
|
import { createExcludeMatcher, createIncludeMatcher } from "../core/exclude.js";
|
|
9
|
+
import { normalizeExtensionKey } from "../core/parser-overrides.js";
|
|
9
10
|
import { DescriptionCache } from "../core/desc-cache.js";
|
|
10
11
|
import { createImageVisionProvider, } from "../chunker/image.js";
|
|
11
12
|
import * as pdfExtractor from "./pdf.js";
|
|
@@ -91,6 +92,13 @@ async function dispatchExtraction(filePath, buffer, imageVisionProvider, imagePr
|
|
|
91
92
|
*/
|
|
92
93
|
export async function scanWorkspaceFiles(cwd, config, logger, manifest, filterPaths, injectedVisionProvider, descCache) {
|
|
93
94
|
const extensions = new Set(config.indexing.includeExtensions);
|
|
95
|
+
// `chunking.parsers` implies the extension is wanted: mapping an extension to
|
|
96
|
+
// a parser is pointless if the scanner never picks the files up.
|
|
97
|
+
for (const rawKey of Object.keys(config.chunking?.parsers ?? {})) {
|
|
98
|
+
const key = normalizeExtensionKey(rawKey);
|
|
99
|
+
if (key)
|
|
100
|
+
extensions.add(key);
|
|
101
|
+
}
|
|
94
102
|
let imageVisionProvider = null;
|
|
95
103
|
let imagePrompt;
|
|
96
104
|
let imageResizeMaxDimension;
|
package/dist/core/bootstrap.js
CHANGED
|
@@ -11,6 +11,7 @@ import { createDescriptionProvider } from "../describer/factory.js";
|
|
|
11
11
|
import { createVectorStore } from "../vectorstore/factory.js";
|
|
12
12
|
import { readStoreDimension } from "../vectorstore/lancedb.js";
|
|
13
13
|
import { KeywordIndex } from "../retriever/keyword-index.js";
|
|
14
|
+
import { reconcileQuirks } from "../quirks/quirk-store.js";
|
|
14
15
|
/**
|
|
15
16
|
* Resolve the embedding dimension for the store.
|
|
16
17
|
*
|
|
@@ -95,6 +96,25 @@ export async function resolveRagContext(opts = {}) {
|
|
|
95
96
|
const keywordIndex = opts.skipKeywordIndex
|
|
96
97
|
? new KeywordIndex(storePath)
|
|
97
98
|
: await loadKeywordIndex(storePath);
|
|
99
|
+
// Reconcile quirks.jsonl into the store + keyword index (best-effort).
|
|
100
|
+
// Full rebuilds rebuild the table from workspace files only — the preserved
|
|
101
|
+
// quirks.jsonl was never re-embedded, so recall would silently miss quirks
|
|
102
|
+
// while `quirk list` still showed them. Skipped for read-only bootstraps
|
|
103
|
+
// (skipProbe/skipKeywordIndex) — those must never write to the store.
|
|
104
|
+
if (!(opts.skipProbe || opts.skipKeywordIndex)) {
|
|
105
|
+
try {
|
|
106
|
+
const rec = await reconcileQuirks({ embedder, store, keywordIndex, cfg, storePath });
|
|
107
|
+
if (rec.restoredToStore > 0 || rec.removedOrphans > 0) {
|
|
108
|
+
// Surface only store-level drift (the split-brain symptom). Keyword-index
|
|
109
|
+
// healing is routine (addQuirk never persisted the index) and stays silent.
|
|
110
|
+
console.warn(`[bootstrap] Quirk reconcile: restored ${rec.restoredToStore} to store, ` +
|
|
111
|
+
`removed ${rec.removedOrphans} orphans (keyword index: +${rec.addedToKeywordIndex})`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
catch (err) {
|
|
115
|
+
console.warn(`[bootstrap] Quirk reconcile failed: ${err.message}`);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
98
118
|
const descriptionConfig = cfg.description;
|
|
99
119
|
const descriptionProvider = descriptionConfig?.enabled
|
|
100
120
|
? createDescriptionProvider(descriptionConfig)
|
package/dist/core/config.d.ts
CHANGED
|
@@ -423,6 +423,26 @@ export interface RagConfig {
|
|
|
423
423
|
chunking?: {
|
|
424
424
|
/** Map of language to allowed/blocked AST node type patterns. */
|
|
425
425
|
nodeTypes?: Record<string, string[]>;
|
|
426
|
+
/**
|
|
427
|
+
* Map of file extension to parser (chunker) language, overriding the
|
|
428
|
+
* built-in mapping. Both new extensions and existing entries can be
|
|
429
|
+
* remapped:
|
|
430
|
+
*
|
|
431
|
+
* ```json
|
|
432
|
+
* { "chunking": { "parsers": { ".c": "cpp", ".cu": "cpp" } } }
|
|
433
|
+
* ```
|
|
434
|
+
*
|
|
435
|
+
* Keys are normalized (lowercased, leading dot added); values must be a
|
|
436
|
+
* registered parser language (`typescript`, `cpp`, `python`, … or `text`
|
|
437
|
+
* for the fallback parser). Applied consistently to indexing/chunking,
|
|
438
|
+
* `get_file_skeleton` (tool + MCP), and read-tool language labels.
|
|
439
|
+
* Mapped extensions are implicitly added to the scanned set — no need to
|
|
440
|
+
* repeat them in `indexing.includeExtensions`.
|
|
441
|
+
*
|
|
442
|
+
* Changing a mapping changes chunk boundaries: re-run
|
|
443
|
+
* `opencode-rag index --force` afterwards.
|
|
444
|
+
*/
|
|
445
|
+
parsers?: Record<string, string>;
|
|
426
446
|
};
|
|
427
447
|
/** LLM-based chunk description generation config. */
|
|
428
448
|
description?: DescriptionConfig;
|
package/dist/core/config.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { env } from "node:process";
|
|
8
|
+
import { normalizeParserOverrides } from "./parser-overrides.js";
|
|
8
9
|
export const DEFAULT_CONFIG = {
|
|
9
10
|
embedding: {
|
|
10
11
|
provider: "ollama",
|
|
@@ -559,6 +560,7 @@ export function loadConfig(filePath, validate = true) {
|
|
|
559
560
|
...(DEFAULT_CONFIG.chunking?.nodeTypes ?? {}),
|
|
560
561
|
...(parsed.chunking?.nodeTypes ?? {}),
|
|
561
562
|
},
|
|
563
|
+
parsers: normalizeParserOverrides(safeObj(parsed.chunking)?.parsers),
|
|
562
564
|
},
|
|
563
565
|
description: {
|
|
564
566
|
...DEFAULT_CONFIG.description,
|
|
@@ -144,6 +144,8 @@ export interface KeywordIndex {
|
|
|
144
144
|
clear(): void;
|
|
145
145
|
/** Return the total number of indexed chunks. */
|
|
146
146
|
count(): number;
|
|
147
|
+
/** Whether a chunk with the given id is indexed. Used by quirk reconcile. */
|
|
148
|
+
hasChunk(id: string): boolean;
|
|
147
149
|
/** Persist the index to disk as JSON. */
|
|
148
150
|
save(filePath?: string): Promise<void>;
|
|
149
151
|
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Pure helpers for the `chunking.parsers` extension → parser override map.
|
|
3
|
+
*
|
|
4
|
+
* Kept free of imports so `core/config`, the chunker factory, and the content
|
|
5
|
+
* scanner can all share it without introducing module cycles.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Map of file extension (e.g. `".cxx"`) to parser/chunker language name
|
|
9
|
+
* (e.g. `"cpp"`). Keys are normalized to lowercase with a leading dot;
|
|
10
|
+
* values to lowercase language names.
|
|
11
|
+
*/
|
|
12
|
+
export type ParserOverrides = Record<string, string>;
|
|
13
|
+
/**
|
|
14
|
+
* Normalize a single file-extension key: trim, lowercase, ensure a leading dot.
|
|
15
|
+
*
|
|
16
|
+
* @returns The normalized key, or `""` when the key cannot be an extension
|
|
17
|
+
* (empty after trimming, or containing a path separator).
|
|
18
|
+
*/
|
|
19
|
+
export declare function normalizeExtensionKey(raw: unknown): string;
|
|
20
|
+
/**
|
|
21
|
+
* Normalize a whole `chunking.parsers` map.
|
|
22
|
+
*
|
|
23
|
+
* Invalid entries (non-string keys/values, empty values, unusable keys) are
|
|
24
|
+
* dropped rather than throwing — validation surfaces them as warnings.
|
|
25
|
+
*
|
|
26
|
+
* @param map - Raw user-supplied map (may be undefined or malformed).
|
|
27
|
+
* @returns A new map with normalized keys and values; `{}` when nothing valid remains.
|
|
28
|
+
*/
|
|
29
|
+
export declare function normalizeParserOverrides(map?: ParserOverrides): ParserOverrides;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Pure helpers for the `chunking.parsers` extension → parser override map.
|
|
3
|
+
*
|
|
4
|
+
* Kept free of imports so `core/config`, the chunker factory, and the content
|
|
5
|
+
* scanner can all share it without introducing module cycles.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Normalize a single file-extension key: trim, lowercase, ensure a leading dot.
|
|
9
|
+
*
|
|
10
|
+
* @returns The normalized key, or `""` when the key cannot be an extension
|
|
11
|
+
* (empty after trimming, or containing a path separator).
|
|
12
|
+
*/
|
|
13
|
+
export function normalizeExtensionKey(raw) {
|
|
14
|
+
if (typeof raw !== "string")
|
|
15
|
+
return "";
|
|
16
|
+
const trimmed = raw.trim().toLowerCase();
|
|
17
|
+
if (trimmed.length === 0)
|
|
18
|
+
return "";
|
|
19
|
+
if (trimmed.includes("/") || trimmed.includes("\\"))
|
|
20
|
+
return "";
|
|
21
|
+
return trimmed.startsWith(".") ? trimmed : `.${trimmed}`;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Normalize a whole `chunking.parsers` map.
|
|
25
|
+
*
|
|
26
|
+
* Invalid entries (non-string keys/values, empty values, unusable keys) are
|
|
27
|
+
* dropped rather than throwing — validation surfaces them as warnings.
|
|
28
|
+
*
|
|
29
|
+
* @param map - Raw user-supplied map (may be undefined or malformed).
|
|
30
|
+
* @returns A new map with normalized keys and values; `{}` when nothing valid remains.
|
|
31
|
+
*/
|
|
32
|
+
export function normalizeParserOverrides(map) {
|
|
33
|
+
const out = {};
|
|
34
|
+
if (!map || typeof map !== "object" || Array.isArray(map))
|
|
35
|
+
return out;
|
|
36
|
+
for (const [key, value] of Object.entries(map)) {
|
|
37
|
+
const ext = normalizeExtensionKey(key);
|
|
38
|
+
if (!ext)
|
|
39
|
+
continue;
|
|
40
|
+
if (typeof value !== "string")
|
|
41
|
+
continue;
|
|
42
|
+
const language = value.trim().toLowerCase();
|
|
43
|
+
if (language.length === 0)
|
|
44
|
+
continue;
|
|
45
|
+
out[ext] = language;
|
|
46
|
+
}
|
|
47
|
+
return out;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=parser-overrides.js.map
|
package/dist/index.d.ts
CHANGED
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
* low-level building blocks (chunkers, embedders, vector stores, retrieval),
|
|
4
4
|
* high-level convenience API (search, index, getContext), and plugin entry.
|
|
5
5
|
*/
|
|
6
|
-
export { chunkFile, getChunker, registerChunker } from "./chunker/factory.js";
|
|
6
|
+
export { chunkFile, getChunker, registerChunker, getRegisteredLanguages, getExtensionsForLanguage, validateParserOverrides } from "./chunker/factory.js";
|
|
7
|
+
export { extractSkeleton } from "./chunker/skeleton.js";
|
|
8
|
+
export { normalizeParserOverrides } from "./core/parser-overrides.js";
|
|
9
|
+
export type { ParserOverrides } from "./core/parser-overrides.js";
|
|
7
10
|
export { createEmbedder, embedBatch } from "./embedder/factory.js";
|
|
8
11
|
export { createDescriptionProvider } from "./describer/factory.js";
|
|
9
12
|
export { createVectorStore } from "./vectorstore/factory.js";
|