opencode-rag-plugin 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/ReadMe.md CHANGED
@@ -35,6 +35,7 @@ opencode-rag query "authentication middleware"
35
35
  |---|---|
36
36
  | **MCP server** | `opencode-rag mcp` - stdio-based MCP server exposing `search_semantic`, `get_file_skeleton`, `find_usages`, and `describe_image` tools for any MCP-compatible client |
37
37
  | **AST chunking** | 26 languages via tree-sitter (TS, JS, Python, Java, Go, Rust, C/C++, C#, Ruby, Kotlin, Swift, Bash, PHP, PowerShell, SQL, JSON, HTML, CSS, XML (including SVG), YAML, TOML, INI, Dockerfile, Markdown, LaTeX, Razor) |
38
+ | **Configurable parsers** | Remap file extensions to any parser via `chunking.parsers` (e.g. `.c` → `cpp`, or index a new `.cu` as C++) — applied to chunking, `get_file_skeleton`, and read labels ([docs](doc/chunking.md#parser-overrides-per-extension-chunkingparsers)) |
38
39
  | **Document support** | Markdown, LaTeX, PDF, DOCX, DOC, Excel |
39
40
  | **Image indexing** | Describe images via vision LLM and store descriptions as searchable vector chunks |
40
41
  | **Hybrid search** | Vector similarity + TF×IDF keyword fusion |
@@ -2,6 +2,7 @@
2
2
  * @fileoverview Chunker registry, extension-to-chunker mapping, and file chunking orchestration.
3
3
  */
4
4
  import type { Chunker, Chunk } from "../core/interfaces.js";
5
+ import { type ParserOverrides } from "../core/parser-overrides.js";
5
6
  import { typescriptChunker } from "./typescript.js";
6
7
  import { pythonChunker } from "./python.js";
7
8
  import { javaChunker } from "./java.js";
@@ -46,9 +47,28 @@ export declare function registerChunker(chunker: Chunker, extensions?: string[])
46
47
  * Falls back to the fallback chunker when no extension match is found.
47
48
  *
48
49
  * @param filePath - Path to the file to chunk.
50
+ * @param languageByExtension - Optional `chunking.parsers` override map
51
+ * (extension → parser language). Overrides win over the built-in mapping and
52
+ * are resolved per call, so they never mutate the process-wide registry
53
+ * (which is shared across workspaces).
49
54
  * @returns A chunker instance for the file's extension.
50
55
  */
51
- export declare function getChunker(filePath: string): Chunker;
56
+ export declare function getChunker(filePath: string, languageByExtension?: ParserOverrides): Chunker;
57
+ /** Every parser (chunker language) name accepted as a `chunking.parsers` target. */
58
+ export declare function getRegisteredLanguages(): string[];
59
+ /**
60
+ * All file extensions currently mapped to a given parser language, in
61
+ * registration order. Used by skeleton extraction to pick a representative
62
+ * extension recipe when an override retargets an extension.
63
+ */
64
+ export declare function getExtensionsForLanguage(language: string): string[];
65
+ /**
66
+ * Validate a raw `chunking.parsers` map against the registered parsers.
67
+ *
68
+ * @param parsers - Raw (unnormalized) override map from the config.
69
+ * @returns Human-readable warnings; empty when the map is valid or absent.
70
+ */
71
+ export declare function validateParserOverrides(parsers?: ParserOverrides): string[];
52
72
  export declare function getRegisteredExtensions(): string[];
53
73
  /**
54
74
  * Chunk a file by looking up its registered chunker, applying node-type
@@ -59,7 +79,7 @@ export declare function getRegisteredExtensions(): string[];
59
79
  * @param content - The full text content of the file.
60
80
  * @param nodeTypesOverrides - Optional language-specific node type overrides
61
81
  * for tree-sitter based chunkers.
62
- * @param options - Optional chunking options (maxSvgSizeBytes, etc.).
82
+ * @param options - Optional chunking options (maxSvgSizeBytes, parser overrides, etc.).
63
83
  * @returns An array of chunks for the file.
64
84
  */
65
85
  export declare function chunkFile(filePath: string, content: string, nodeTypesOverrides?: Record<string, string[]>, options?: {
@@ -68,5 +88,7 @@ export declare function chunkFile(filePath: string, content: string, nodeTypesOv
68
88
  maxChunkSize?: number;
69
89
  /** Overlapping lines shared between adjacent/split chunks. */
70
90
  chunkOverlap?: number;
91
+ /** Extension → parser overrides from `chunking.parsers`. */
92
+ languageByExtension?: ParserOverrides;
71
93
  }): Promise<Chunk[]>;
72
94
  export { typescriptChunker, pythonChunker, javaChunker, goChunker, markdownChunker, cChunker, cppChunker, csharpChunker, javascriptChunker, razorChunker, jsonChunker, htmlChunker, cssChunker, xmlChunker, slnChunker, rustChunker, rubyChunker, kotlinChunker, swiftChunker, bashChunker, phpChunker, powershellChunker, iniChunker, yamlChunker, tomlChunker, dockerfileChunker, sqlChunker, texChunker, pdfChunker, fallbackChunker };
@@ -1,3 +1,4 @@
1
+ import { normalizeExtensionKey, normalizeParserOverrides, } from "../core/parser-overrides.js";
1
2
  import { TreeSitterChunker } from "./base.js";
2
3
  import { typescriptChunker } from "./typescript.js";
3
4
  import { pythonChunker } from "./python.js";
@@ -78,6 +79,19 @@ for (const chunker of chunkers) {
78
79
  }
79
80
  }
80
81
  }
82
+ /**
83
+ * Parser (chunker) registry keyed by `language` name — the target space of the
84
+ * `chunking.parsers` config (`{ ".c": "cpp" }`). Kept in sync with
85
+ * {@link extensionMap} by {@link registerChunker}.
86
+ */
87
+ const languageMap = new Map();
88
+ for (const chunker of chunkers) {
89
+ if (chunker.language)
90
+ languageMap.set(chunker.language, chunker);
91
+ }
92
+ languageMap.set(fallbackChunker.language, fallbackChunker);
93
+ /** Override targets already reported as unknown, so a bad config warns once per process. */
94
+ const warnedUnknownParsers = new Set();
81
95
  /**
82
96
  * Register a pluggable chunker for one or more file extensions.
83
97
  * If an extension already has a chunker, the new one is silently skipped.
@@ -87,6 +101,8 @@ for (const chunker of chunkers) {
87
101
  * Defaults to the chunker's own `fileExtensions` property.
88
102
  */
89
103
  export function registerChunker(chunker, extensions) {
104
+ if (chunker.language)
105
+ languageMap.set(chunker.language, chunker);
90
106
  const exts = extensions ?? ("fileExtensions" in chunker
91
107
  ? chunker.fileExtensions
92
108
  : []);
@@ -104,17 +120,86 @@ export function registerChunker(chunker, extensions) {
104
120
  * Falls back to the fallback chunker when no extension match is found.
105
121
  *
106
122
  * @param filePath - Path to the file to chunk.
123
+ * @param languageByExtension - Optional `chunking.parsers` override map
124
+ * (extension → parser language). Overrides win over the built-in mapping and
125
+ * are resolved per call, so they never mutate the process-wide registry
126
+ * (which is shared across workspaces).
107
127
  * @returns A chunker instance for the file's extension.
108
128
  */
109
- export function getChunker(filePath) {
129
+ export function getChunker(filePath, languageByExtension) {
110
130
  const dotIdx = filePath.lastIndexOf(".");
111
131
  const ext = dotIdx >= 0 ? filePath.slice(dotIdx).toLowerCase() : "";
132
+ const overrides = languageByExtension && Object.keys(languageByExtension).length > 0
133
+ ? normalizeParserOverrides(languageByExtension)
134
+ : undefined;
135
+ const target = ext && overrides ? overrides[ext] : undefined;
136
+ if (target) {
137
+ const overridden = languageMap.get(target);
138
+ if (overridden)
139
+ return overridden;
140
+ if (!warnedUnknownParsers.has(target)) {
141
+ warnedUnknownParsers.add(target);
142
+ console.warn(`[opencode-rag] chunking.parsers maps "${ext}" to unknown parser "${target}" ` +
143
+ `— known parsers: ${getRegisteredLanguages().join(", ")}. Using the default mapping.`);
144
+ }
145
+ // Unknown target: fall through to the default mapping for this extension.
146
+ }
112
147
  if (ext && extensionMap.has(ext)) {
113
148
  return extensionMap.get(ext);
114
149
  }
115
150
  const basename = filePath.toLowerCase();
116
151
  return extensionMap.get(basename) ?? fallbackChunker;
117
152
  }
153
+ /** Every parser (chunker language) name accepted as a `chunking.parsers` target. */
154
+ export function getRegisteredLanguages() {
155
+ return [...new Set([...languageMap.values()].map((chunker) => chunker.language))].sort();
156
+ }
157
+ /**
158
+ * All file extensions currently mapped to a given parser language, in
159
+ * registration order. Used by skeleton extraction to pick a representative
160
+ * extension recipe when an override retargets an extension.
161
+ */
162
+ export function getExtensionsForLanguage(language) {
163
+ const out = [];
164
+ for (const [ext, chunker] of extensionMap) {
165
+ if (chunker.language === language)
166
+ out.push(ext);
167
+ }
168
+ return out;
169
+ }
170
+ /**
171
+ * Validate a raw `chunking.parsers` map against the registered parsers.
172
+ *
173
+ * @param parsers - Raw (unnormalized) override map from the config.
174
+ * @returns Human-readable warnings; empty when the map is valid or absent.
175
+ */
176
+ export function validateParserOverrides(parsers) {
177
+ const warnings = [];
178
+ if (parsers === undefined || parsers === null)
179
+ return warnings;
180
+ if (typeof parsers !== "object" || Array.isArray(parsers)) {
181
+ warnings.push('chunking.parsers must be an object mapping file extensions to parser names (e.g. { ".c": "cpp" })');
182
+ return warnings;
183
+ }
184
+ const known = new Set(getRegisteredLanguages());
185
+ for (const [rawKey, rawValue] of Object.entries(parsers)) {
186
+ const ext = normalizeExtensionKey(rawKey);
187
+ if (!ext) {
188
+ warnings.push(`chunking.parsers key "${rawKey}" is not a valid file extension`);
189
+ continue;
190
+ }
191
+ if (typeof rawValue !== "string" || rawValue.trim().length === 0) {
192
+ warnings.push(`chunking.parsers["${rawKey}"] must be a non-empty parser name`);
193
+ continue;
194
+ }
195
+ const language = rawValue.trim().toLowerCase();
196
+ if (!known.has(language)) {
197
+ warnings.push(`chunking.parsers["${rawKey}"] → "${language}" is not a registered parser — ` +
198
+ `known parsers: ${[...known].sort().join(", ")}`);
199
+ }
200
+ }
201
+ return warnings;
202
+ }
118
203
  export function getRegisteredExtensions() {
119
204
  return [...extensionMap.keys()].sort();
120
205
  }
@@ -230,11 +315,11 @@ function withByteLimit(chunker, limit) {
230
315
  * @param content - The full text content of the file.
231
316
  * @param nodeTypesOverrides - Optional language-specific node type overrides
232
317
  * for tree-sitter based chunkers.
233
- * @param options - Optional chunking options (maxSvgSizeBytes, etc.).
318
+ * @param options - Optional chunking options (maxSvgSizeBytes, parser overrides, etc.).
234
319
  * @returns An array of chunks for the file.
235
320
  */
236
321
  export async function chunkFile(filePath, content, nodeTypesOverrides, options) {
237
- let chunker = getChunker(filePath);
322
+ let chunker = getChunker(filePath, options?.languageByExtension);
238
323
  if (nodeTypesOverrides && chunker instanceof TreeSitterChunker) {
239
324
  const overrideTypes = nodeTypesOverrides[chunker.language];
240
325
  if (overrideTypes && overrideTypes.length > 0) {
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * @fileoverview Dynamic loading and registration of pluggable chunker modules from configuration.
3
3
  */
4
- import { registerChunker } from "./factory.js";
4
+ import { registerChunker, validateParserOverrides } from "./factory.js";
5
5
  import { ImageChunker, SUPPORTED_IMAGE_EXTENSIONS } from "./image.js";
6
6
  import path from "node:path";
7
7
  import { pathToFileURL } from "node:url";
@@ -46,10 +46,15 @@ export async function loadChunkersFromConfig(config, configDir) {
46
46
  const chunker = new ImageChunker([...SUPPORTED_IMAGE_EXTENSIONS]);
47
47
  registerChunker(chunker, [...SUPPORTED_IMAGE_EXTENSIONS]);
48
48
  }
49
- if (!config.chunkers || config.chunkers.length === 0)
50
- return;
51
- for (const entry of config.chunkers) {
52
- await loadSingleChunker(entry, configDir);
49
+ if (config.chunkers && config.chunkers.length > 0) {
50
+ for (const entry of config.chunkers) {
51
+ await loadSingleChunker(entry, configDir);
52
+ }
53
+ }
54
+ // Validate AFTER registration so overrides targeting a pluggable chunker's
55
+ // language are recognized. (Known-parser lookup needs the full registry.)
56
+ for (const warning of validateParserOverrides(config.chunking?.parsers)) {
57
+ console.warn(`[opencode-rag] ${warning}`);
53
58
  }
54
59
  }
55
60
  //# sourceMappingURL=loader.js.map
@@ -0,0 +1,36 @@
1
+ import { type ParserOverrides } from "../core/parser-overrides.js";
2
+ /** Skeleton extraction recipe for one file extension. */
3
+ export interface SkeletonConfig {
4
+ /** tree-sitter grammar name (as resolved by {@link loadLanguage}). */
5
+ grammarName: string;
6
+ /** AST node types to collect as structural elements. */
7
+ nodeTypes: string[];
8
+ }
9
+ /** Extension → tree-sitter grammar + node types for skeleton extraction. */
10
+ export declare const SKELETON_CONFIGS: Record<string, SkeletonConfig>;
11
+ /** Regex-based fallback for languages without tree-sitter WASM support. */
12
+ export declare const REGEX_SKELETON: Record<string, RegExp[]>;
13
+ /** Optional skeleton extraction options. */
14
+ export interface SkeletonOptions {
15
+ /** Extension → parser overrides from `chunking.parsers`. */
16
+ languageByExtension?: ParserOverrides;
17
+ }
18
+ /**
19
+ * Return the file extension from a path (lowercase, dot-prefixed when present).
20
+ */
21
+ export declare function getExtension(filePath: string): string;
22
+ /**
23
+ * Extract structural outline from source code using tree-sitter.
24
+ * Falls back to regex patterns for languages without WASM grammars, and to a
25
+ * plain line count when neither recipe applies.
26
+ *
27
+ * @param content - Full file content.
28
+ * @param ext - Lowercase, dot-prefixed file extension.
29
+ * @param options - Optional `chunking.parsers` override map.
30
+ */
31
+ export declare function extractSkeleton(content: string, ext: string, options?: SkeletonOptions): Promise<{
32
+ type: string;
33
+ name: string;
34
+ startLine: number;
35
+ endLine: number;
36
+ }[]>;
@@ -0,0 +1,183 @@
1
+ /**
2
+ * @fileoverview Shared file-skeleton extraction: extension → tree-sitter grammar + node types.
3
+ *
4
+ * Historically this table (and {@link extractSkeleton}) was copy-pasted into
5
+ * both the OpenCode plugin tools (`src/opencode/tools.ts`) and the MCP server
6
+ * (`src/mcp/handlers.ts`), and had already drifted from the chunker registry.
7
+ * It now lives here so both surfaces — plus `chunking.parsers` overrides —
8
+ * resolve extensions the same way.
9
+ */
10
+ import { Parser } from "web-tree-sitter";
11
+ import { initParser, loadLanguage, walkTree } from "./grammar.js";
12
+ import { getExtensionsForLanguage, getRegisteredLanguages } from "./factory.js";
13
+ import { normalizeParserOverrides, } from "../core/parser-overrides.js";
14
+ /** Extension → tree-sitter grammar + node types for skeleton extraction. */
15
+ export const SKELETON_CONFIGS = {
16
+ ".ts": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration"] },
17
+ ".tsx": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration", "arrow_function"] },
18
+ ".mts": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration"] },
19
+ ".cts": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration"] },
20
+ ".js": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
21
+ ".jsx": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
22
+ ".mjs": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
23
+ ".cjs": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
24
+ ".py": { grammarName: "python", nodeTypes: ["function_definition", "class_definition", "decorated_definition"] },
25
+ ".java": { grammarName: "java", nodeTypes: ["class_declaration", "method_declaration", "interface_declaration", "enum_declaration"] },
26
+ ".go": { grammarName: "go", nodeTypes: ["function_declaration", "method_declaration", "type_declaration", "type_spec"] },
27
+ ".rs": { grammarName: "rust", nodeTypes: ["function_item", "struct_item", "enum_item", "impl_item", "trait_item", "type_item"] },
28
+ ".c": { grammarName: "c", nodeTypes: ["function_definition", "struct_specifier", "enum_specifier"] },
29
+ ".cpp": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
30
+ ".cxx": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
31
+ ".h": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
32
+ ".hpp": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
33
+ ".cs": { grammarName: "c-sharp", nodeTypes: ["class_declaration", "method_declaration", "interface_declaration", "enum_declaration", "struct_declaration"] },
34
+ ".rb": { grammarName: "ruby", nodeTypes: ["method", "class", "module"] },
35
+ ".swift": { grammarName: "swift", nodeTypes: ["function_declaration", "class_declaration", "struct_declaration", "enum_declaration", "protocol_declaration"] },
36
+ ".kt": { grammarName: "kotlin", nodeTypes: ["function_declaration", "class_declaration", "interface_declaration", "object_declaration"] },
37
+ ".kts": { grammarName: "kotlin", nodeTypes: ["function_declaration", "class_declaration", "interface_declaration", "object_declaration"] },
38
+ ".css": { grammarName: "css", nodeTypes: ["rule_set"] },
39
+ ".md": { grammarName: "markdown", nodeTypes: ["atx_heading"] },
40
+ };
41
+ /** Regex-based fallback for languages without tree-sitter WASM support. */
42
+ export const REGEX_SKELETON = {
43
+ ".json": [/^(\s*)"(\w+)":/gm],
44
+ ".yaml": [/^(\w+):/gm],
45
+ ".yml": [/^(\w+):/gm],
46
+ ".toml": [/^\[(\w+)\]/gm],
47
+ ".sh": [/^(function\s+\w+|^\w+\s*\(\)\s*\{)/gm],
48
+ ".bash": [/^(function\s+\w+|^\w+\s*\(\)\s*\{)/gm],
49
+ ".zsh": [/^(function\s+\w+|^\w+\s*\(\)\s*\{)/gm],
50
+ ".sql": [/^(CREATE|ALTER|DROP|SELECT|INSERT|UPDATE|DELETE)\s/gim],
51
+ };
52
+ /**
53
+ * Sentinel lookup key that matches neither config map. Used when a
54
+ * `chunking.parsers` override targets a parser with no skeleton recipe —
55
+ * the extension's own (now wrong) recipe must NOT be used, so fall through
56
+ * to the line-count fallback instead.
57
+ */
58
+ const NO_SKELETON = "\u0000no-skeleton";
59
+ /** Unknown override targets already reported, so a bad config warns once per process. */
60
+ const warnedUnknownTargets = new Set();
61
+ /**
62
+ * Return the file extension from a path (lowercase, dot-prefixed when present).
63
+ */
64
+ export function getExtension(filePath) {
65
+ const dot = filePath.lastIndexOf(".");
66
+ return dot >= 0 ? filePath.slice(dot).toLowerCase() : "";
67
+ }
68
+ /**
69
+ * Resolve the skeleton-config lookup key for an extension, honoring
70
+ * `chunking.parsers` overrides.
71
+ *
72
+ * Non-overridden extensions resolve to themselves, so per-extension recipes
73
+ * that legitimately differ (`.ts` vs `.tsx`) are untouched. Overridden ones
74
+ * jump to a representative extension of the target parser.
75
+ */
76
+ function resolveSkeletonKey(ext, overrides) {
77
+ const target = overrides[ext];
78
+ if (!target)
79
+ return ext;
80
+ if (!getRegisteredLanguages().includes(target)) {
81
+ if (!warnedUnknownTargets.has(target)) {
82
+ warnedUnknownTargets.add(target);
83
+ const known = getRegisteredLanguages();
84
+ console.warn(`[opencode-rag] chunking.parsers maps "${ext}" to unknown parser "${target}" ` +
85
+ `— known parsers: ${known.join(", ")}. Using the default skeleton for "${ext}".`);
86
+ }
87
+ return ext;
88
+ }
89
+ const candidates = getExtensionsForLanguage(target);
90
+ const hit = candidates.find((c) => SKELETON_CONFIGS[c] || REGEX_SKELETON[c]);
91
+ // Known parser but no skeleton recipe for it (e.g. plain text, PDF): the
92
+ // extension's own recipe would be the wrong parser, so fall through to the
93
+ // line-count fallback instead of using it.
94
+ return hit ?? NO_SKELETON;
95
+ }
96
+ /**
97
+ * Extract structural outline from source code using tree-sitter.
98
+ * Falls back to regex patterns for languages without WASM grammars, and to a
99
+ * plain line count when neither recipe applies.
100
+ *
101
+ * @param content - Full file content.
102
+ * @param ext - Lowercase, dot-prefixed file extension.
103
+ * @param options - Optional `chunking.parsers` override map.
104
+ */
105
+ export async function extractSkeleton(content, ext, options) {
106
+ const overrides = normalizeParserOverrides(options?.languageByExtension);
107
+ const key = resolveSkeletonKey(ext, overrides);
108
+ const config = SKELETON_CONFIGS[key];
109
+ if (!config) {
110
+ // Fallback: use regex patterns
111
+ const patterns = REGEX_SKELETON[key];
112
+ if (!patterns) {
113
+ // Last fallback: line count
114
+ const lines = content.split("\n");
115
+ return [{ type: "file", name: `${lines.length} lines`, startLine: 1, endLine: lines.length }];
116
+ }
117
+ const items = [];
118
+ for (const pattern of patterns) {
119
+ let match;
120
+ // Reset lastIndex
121
+ pattern.lastIndex = 0;
122
+ while ((match = pattern.exec(content)) !== null) {
123
+ const name = match[1] ?? match[0].trim();
124
+ const lineNum = content.slice(0, match.index).split("\n").length;
125
+ items.push({ type: ext.slice(1), name, startLine: lineNum, endLine: lineNum });
126
+ }
127
+ }
128
+ const seen = new Set();
129
+ return items.filter((item) => {
130
+ const seenKey = `${item.type}:${item.name}`;
131
+ if (seen.has(seenKey))
132
+ return false;
133
+ seen.add(seenKey);
134
+ return true;
135
+ });
136
+ }
137
+ // Tree-sitter skeleton extraction
138
+ await initParser();
139
+ const lang = await loadLanguage(config.grammarName);
140
+ const parser = new Parser();
141
+ parser.setLanguage(lang);
142
+ const tree = parser.parse(content);
143
+ if (!tree)
144
+ return [];
145
+ try {
146
+ const nodeTypes = new Set(config.nodeTypes);
147
+ const astNodes = walkTree(tree.rootNode, nodeTypes, content, 15);
148
+ return astNodes.map((node) => ({
149
+ type: node.type,
150
+ name: extractNodeName(node.text, node.type),
151
+ startLine: node.startLine,
152
+ endLine: node.endLine,
153
+ }));
154
+ }
155
+ finally {
156
+ tree.delete();
157
+ parser.delete();
158
+ }
159
+ }
160
+ /**
161
+ * Extract a human-readable name from a code node.
162
+ * For declarations, this is typically the identifier after the keyword.
163
+ */
164
+ function extractNodeName(text, _nodeType) {
165
+ // Try first line only for large nodes
166
+ const firstLine = text.split("\n")[0] ?? text;
167
+ // Common patterns: "function foo(...)" or "class Foo ..." or "foo(...)" (arrow, method)
168
+ const nameMatch = firstLine.match(/^(?:export\s+)?(?:async\s+)?(?:function\s+)?(?:\w+\s+)?(?:(\w+))\s*(?:[<(]|$)/);
169
+ if (nameMatch) {
170
+ // Skip keywords
171
+ const keywordSet = new Set(["export", "async", "function", "class", "interface", "type", "enum", "struct", "impl", "trait", "def", "fun", "fn"]);
172
+ if (!keywordSet.has(nameMatch[1])) {
173
+ return nameMatch[1];
174
+ }
175
+ // Try next word
176
+ const secondMatch = firstLine.match(/^(?:export\s+)?(?:async\s+)?(?:function|class|interface|type|enum|struct|trait|impl|def|fun|fn)\s+(\w+)/);
177
+ if (secondMatch)
178
+ return secondMatch[1];
179
+ }
180
+ // Last resort: return first 80 chars of first line
181
+ return firstLine.length > 80 ? firstLine.slice(0, 77) + "..." : firstLine;
182
+ }
183
+ //# sourceMappingURL=skeleton.js.map
@@ -6,6 +6,7 @@ import path from "node:path";
6
6
  import pLimit from "p-limit";
7
7
  import { computeFileHash, computeDescriptionConfigHash, normalizeFilePath } from "../core/manifest.js";
8
8
  import { createExcludeMatcher, createIncludeMatcher } from "../core/exclude.js";
9
+ import { normalizeExtensionKey } from "../core/parser-overrides.js";
9
10
  import { DescriptionCache } from "../core/desc-cache.js";
10
11
  import { createImageVisionProvider, } from "../chunker/image.js";
11
12
  import * as pdfExtractor from "./pdf.js";
@@ -91,6 +92,13 @@ async function dispatchExtraction(filePath, buffer, imageVisionProvider, imagePr
91
92
  */
92
93
  export async function scanWorkspaceFiles(cwd, config, logger, manifest, filterPaths, injectedVisionProvider, descCache) {
93
94
  const extensions = new Set(config.indexing.includeExtensions);
95
+ // `chunking.parsers` implies the extension is wanted: mapping an extension to
96
+ // a parser is pointless if the scanner never picks the files up.
97
+ for (const rawKey of Object.keys(config.chunking?.parsers ?? {})) {
98
+ const key = normalizeExtensionKey(rawKey);
99
+ if (key)
100
+ extensions.add(key);
101
+ }
94
102
  let imageVisionProvider = null;
95
103
  let imagePrompt;
96
104
  let imageResizeMaxDimension;
@@ -11,6 +11,7 @@ import { createDescriptionProvider } from "../describer/factory.js";
11
11
  import { createVectorStore } from "../vectorstore/factory.js";
12
12
  import { readStoreDimension } from "../vectorstore/lancedb.js";
13
13
  import { KeywordIndex } from "../retriever/keyword-index.js";
14
+ import { reconcileQuirks } from "../quirks/quirk-store.js";
14
15
  /**
15
16
  * Resolve the embedding dimension for the store.
16
17
  *
@@ -95,6 +96,25 @@ export async function resolveRagContext(opts = {}) {
95
96
  const keywordIndex = opts.skipKeywordIndex
96
97
  ? new KeywordIndex(storePath)
97
98
  : await loadKeywordIndex(storePath);
99
+ // Reconcile quirks.jsonl into the store + keyword index (best-effort).
100
+ // Full rebuilds rebuild the table from workspace files only — the preserved
101
+ // quirks.jsonl was never re-embedded, so recall would silently miss quirks
102
+ // while `quirk list` still showed them. Skipped for read-only bootstraps
103
+ // (skipProbe/skipKeywordIndex) — those must never write to the store.
104
+ if (!(opts.skipProbe || opts.skipKeywordIndex)) {
105
+ try {
106
+ const rec = await reconcileQuirks({ embedder, store, keywordIndex, cfg, storePath });
107
+ if (rec.restoredToStore > 0 || rec.removedOrphans > 0) {
108
+ // Surface only store-level drift (the split-brain symptom). Keyword-index
109
+ // healing is routine (addQuirk never persisted the index) and stays silent.
110
+ console.warn(`[bootstrap] Quirk reconcile: restored ${rec.restoredToStore} to store, ` +
111
+ `removed ${rec.removedOrphans} orphans (keyword index: +${rec.addedToKeywordIndex})`);
112
+ }
113
+ }
114
+ catch (err) {
115
+ console.warn(`[bootstrap] Quirk reconcile failed: ${err.message}`);
116
+ }
117
+ }
98
118
  const descriptionConfig = cfg.description;
99
119
  const descriptionProvider = descriptionConfig?.enabled
100
120
  ? createDescriptionProvider(descriptionConfig)
@@ -423,6 +423,26 @@ export interface RagConfig {
423
423
  chunking?: {
424
424
  /** Map of language to allowed/blocked AST node type patterns. */
425
425
  nodeTypes?: Record<string, string[]>;
426
+ /**
427
+ * Map of file extension to parser (chunker) language, overriding the
428
+ * built-in mapping. Both new extensions and existing entries can be
429
+ * remapped:
430
+ *
431
+ * ```json
432
+ * { "chunking": { "parsers": { ".c": "cpp", ".cu": "cpp" } } }
433
+ * ```
434
+ *
435
+ * Keys are normalized (lowercased, leading dot added); values must be a
436
+ * registered parser language (`typescript`, `cpp`, `python`, … or `text`
437
+ * for the fallback parser). Applied consistently to indexing/chunking,
438
+ * `get_file_skeleton` (tool + MCP), and read-tool language labels.
439
+ * Mapped extensions are implicitly added to the scanned set — no need to
440
+ * repeat them in `indexing.includeExtensions`.
441
+ *
442
+ * Changing a mapping changes chunk boundaries: re-run
443
+ * `opencode-rag index --force` afterwards.
444
+ */
445
+ parsers?: Record<string, string>;
426
446
  };
427
447
  /** LLM-based chunk description generation config. */
428
448
  description?: DescriptionConfig;
@@ -5,6 +5,7 @@
5
5
  import { existsSync, readFileSync, writeFileSync } from "node:fs";
6
6
  import path from "node:path";
7
7
  import { env } from "node:process";
8
+ import { normalizeParserOverrides } from "./parser-overrides.js";
8
9
  export const DEFAULT_CONFIG = {
9
10
  embedding: {
10
11
  provider: "ollama",
@@ -559,6 +560,7 @@ export function loadConfig(filePath, validate = true) {
559
560
  ...(DEFAULT_CONFIG.chunking?.nodeTypes ?? {}),
560
561
  ...(parsed.chunking?.nodeTypes ?? {}),
561
562
  },
563
+ parsers: normalizeParserOverrides(safeObj(parsed.chunking)?.parsers),
562
564
  },
563
565
  description: {
564
566
  ...DEFAULT_CONFIG.description,
@@ -144,6 +144,8 @@ export interface KeywordIndex {
144
144
  clear(): void;
145
145
  /** Return the total number of indexed chunks. */
146
146
  count(): number;
147
+ /** Whether a chunk with the given id is indexed. Used by quirk reconcile. */
148
+ hasChunk(id: string): boolean;
147
149
  /** Persist the index to disk as JSON. */
148
150
  save(filePath?: string): Promise<void>;
149
151
  }
@@ -0,0 +1,29 @@
1
+ /**
2
+ * @fileoverview Pure helpers for the `chunking.parsers` extension → parser override map.
3
+ *
4
+ * Kept free of imports so `core/config`, the chunker factory, and the content
5
+ * scanner can all share it without introducing module cycles.
6
+ */
7
+ /**
8
+ * Map of file extension (e.g. `".cxx"`) to parser/chunker language name
9
+ * (e.g. `"cpp"`). Keys are normalized to lowercase with a leading dot;
10
+ * values to lowercase language names.
11
+ */
12
+ export type ParserOverrides = Record<string, string>;
13
+ /**
14
+ * Normalize a single file-extension key: trim, lowercase, ensure a leading dot.
15
+ *
16
+ * @returns The normalized key, or `""` when the key cannot be an extension
17
+ * (empty after trimming, or containing a path separator).
18
+ */
19
+ export declare function normalizeExtensionKey(raw: unknown): string;
20
+ /**
21
+ * Normalize a whole `chunking.parsers` map.
22
+ *
23
+ * Invalid entries (non-string keys/values, empty values, unusable keys) are
24
+ * dropped rather than throwing — validation surfaces them as warnings.
25
+ *
26
+ * @param map - Raw user-supplied map (may be undefined or malformed).
27
+ * @returns A new map with normalized keys and values; `{}` when nothing valid remains.
28
+ */
29
+ export declare function normalizeParserOverrides(map?: ParserOverrides): ParserOverrides;
@@ -0,0 +1,49 @@
1
+ /**
2
+ * @fileoverview Pure helpers for the `chunking.parsers` extension → parser override map.
3
+ *
4
+ * Kept free of imports so `core/config`, the chunker factory, and the content
5
+ * scanner can all share it without introducing module cycles.
6
+ */
7
+ /**
8
+ * Normalize a single file-extension key: trim, lowercase, ensure a leading dot.
9
+ *
10
+ * @returns The normalized key, or `""` when the key cannot be an extension
11
+ * (empty after trimming, or containing a path separator).
12
+ */
13
+ export function normalizeExtensionKey(raw) {
14
+ if (typeof raw !== "string")
15
+ return "";
16
+ const trimmed = raw.trim().toLowerCase();
17
+ if (trimmed.length === 0)
18
+ return "";
19
+ if (trimmed.includes("/") || trimmed.includes("\\"))
20
+ return "";
21
+ return trimmed.startsWith(".") ? trimmed : `.${trimmed}`;
22
+ }
23
+ /**
24
+ * Normalize a whole `chunking.parsers` map.
25
+ *
26
+ * Invalid entries (non-string keys/values, empty values, unusable keys) are
27
+ * dropped rather than throwing — validation surfaces them as warnings.
28
+ *
29
+ * @param map - Raw user-supplied map (may be undefined or malformed).
30
+ * @returns A new map with normalized keys and values; `{}` when nothing valid remains.
31
+ */
32
+ export function normalizeParserOverrides(map) {
33
+ const out = {};
34
+ if (!map || typeof map !== "object" || Array.isArray(map))
35
+ return out;
36
+ for (const [key, value] of Object.entries(map)) {
37
+ const ext = normalizeExtensionKey(key);
38
+ if (!ext)
39
+ continue;
40
+ if (typeof value !== "string")
41
+ continue;
42
+ const language = value.trim().toLowerCase();
43
+ if (language.length === 0)
44
+ continue;
45
+ out[ext] = language;
46
+ }
47
+ return out;
48
+ }
49
+ //# sourceMappingURL=parser-overrides.js.map
package/dist/index.d.ts CHANGED
@@ -3,7 +3,10 @@
3
3
  * low-level building blocks (chunkers, embedders, vector stores, retrieval),
4
4
  * high-level convenience API (search, index, getContext), and plugin entry.
5
5
  */
6
- export { chunkFile, getChunker, registerChunker } from "./chunker/factory.js";
6
+ export { chunkFile, getChunker, registerChunker, getRegisteredLanguages, getExtensionsForLanguage, validateParserOverrides } from "./chunker/factory.js";
7
+ export { extractSkeleton } from "./chunker/skeleton.js";
8
+ export { normalizeParserOverrides } from "./core/parser-overrides.js";
9
+ export type { ParserOverrides } from "./core/parser-overrides.js";
7
10
  export { createEmbedder, embedBatch } from "./embedder/factory.js";
8
11
  export { createDescriptionProvider } from "./describer/factory.js";
9
12
  export { createVectorStore } from "./vectorstore/factory.js";