opencode-rag-plugin 2.0.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +10 -2
- package/dist/chunker/factory.d.ts +24 -2
- package/dist/chunker/factory.js +88 -3
- package/dist/chunker/loader.js +10 -5
- package/dist/chunker/skeleton.d.ts +36 -0
- package/dist/chunker/skeleton.js +183 -0
- package/dist/cli/commands/init.d.ts +29 -0
- package/dist/cli/commands/init.js +326 -281
- package/dist/cli/commands/setup.js +95 -9
- package/dist/content/reader.js +8 -0
- package/dist/core/bootstrap.js +20 -0
- package/dist/core/config.d.ts +20 -0
- package/dist/core/config.js +2 -0
- package/dist/core/interfaces.d.ts +2 -0
- package/dist/core/parser-overrides.d.ts +29 -0
- package/dist/core/parser-overrides.js +49 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +3 -1
- package/dist/indexer/pipeline.js +26 -0
- package/dist/indexer/worker.d.ts +2 -0
- package/dist/indexer/worker.js +1 -0
- package/dist/mcp/handlers.d.ts +2 -1
- package/dist/mcp/handlers.js +5 -103
- package/dist/mcp/server.js +1 -1
- package/dist/opencode/create-read-tool.js +1 -0
- package/dist/opencode/read-format.d.ts +5 -0
- package/dist/opencode/read-format.js +12 -5
- package/dist/opencode/tools.d.ts +3 -0
- package/dist/opencode/tools.js +6 -124
- package/dist/plugin.js +22 -1
- package/dist/quirks/quirk-store.d.ts +45 -0
- package/dist/quirks/quirk-store.js +200 -20
- package/dist/retriever/keyword-index.d.ts +1 -0
- package/dist/retriever/keyword-index.js +3 -0
- package/dist/tui.js +6 -0
- package/dist/vectorstore/lancedb.js +1 -0
- package/package.json +1 -1
|
@@ -4,6 +4,8 @@ import { execSync } from "node:child_process";
|
|
|
4
4
|
import { c } from "../format.js";
|
|
5
5
|
import { getPackageMetadata } from "../helpers.js";
|
|
6
6
|
import { getRuntimeDir, getVersionFile, readVersionFile, setupRuntime, } from "../../core/setup-runtime.js";
|
|
7
|
+
import { compareVersions, getCurrentVersion, installLatestUpdate } from "../../core/version-check.js";
|
|
8
|
+
import { isLikelyProjectRoot, isWorkspaceInitialized, runWorkspaceInit } from "./init.js";
|
|
7
9
|
const PLUGIN_NAME = "opencode-rag-plugin";
|
|
8
10
|
function removeIfExists(targetPath) {
|
|
9
11
|
if (existsSync(targetPath)) {
|
|
@@ -36,13 +38,16 @@ function checkOpenCodeRunning() {
|
|
|
36
38
|
export function registerSetupCommand(program) {
|
|
37
39
|
program
|
|
38
40
|
.command("setup")
|
|
39
|
-
.description("Install/update the OpenCodeRAG runtime
|
|
41
|
+
.description("Install/update the OpenCodeRAG runtime and initialize the current workspace")
|
|
40
42
|
.addHelpText("after", "\nUse cases:\n" +
|
|
41
43
|
" - First-time install: run once per machine to install the plugin runtime\n" +
|
|
42
|
-
" into ~/.opencode/ so OpenCode can discover the RAG plugin
|
|
43
|
-
"
|
|
44
|
+
" into ~/.opencode/ so OpenCode can discover the RAG plugin. When the\n" +
|
|
45
|
+
" plugin is not installed yet, installs the latest version from npm.\n" +
|
|
46
|
+
" - Workspace setup: run inside a project to also initialize .opencode/,\n" +
|
|
47
|
+
" the skill file and opencode-rag.json for this workspace.\n" +
|
|
48
|
+
" - Updating: use 'opencode-rag update' to install a newer release.\n" +
|
|
44
49
|
" - Troubleshooting: use --check to inspect the runtime, or --force to reinstall.\n" +
|
|
45
|
-
"\nMachine
|
|
50
|
+
"\nMachine + workspace step: run 'opencode-rag init' only when you need to re-sync\nan already initialized workspace.\n")
|
|
46
51
|
.option("--uninstall", "remove the runtime and cleanup")
|
|
47
52
|
.option("-f, --force", "force re-setup even if up-to-date")
|
|
48
53
|
.option("--check", "check whether the runtime is correctly installed")
|
|
@@ -103,9 +108,11 @@ export function registerSetupCommand(program) {
|
|
|
103
108
|
const runtimeDist = path.join(runtimePluginDir, "dist");
|
|
104
109
|
const alreadyInstalled = existsSync(runtimeDist);
|
|
105
110
|
if (alreadyInstalled && installedVersion === pluginVersion && !options.force) {
|
|
106
|
-
console.log(` ${c.success("Already up-to-date.")} (${c.value(pluginVersion)}) at ${c.file(runtimeDir)}
|
|
107
|
-
console.log(` ${c.dim("Run `opencode-rag setup --force` to re-install
|
|
108
|
-
|
|
111
|
+
console.log(` ${c.success("Already up-to-date.")} (${c.value(pluginVersion)}) at ${c.file(runtimeDir)}`);
|
|
112
|
+
console.log(` ${c.dim("Run `opencode-rag setup --force` to re-install the runtime.\n")}`);
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
await ensureGlobalPlugin();
|
|
109
116
|
}
|
|
110
117
|
const result = await setupRuntime({ force: options.force, version: pluginVersion });
|
|
111
118
|
if (result.success) {
|
|
@@ -114,8 +121,7 @@ export function registerSetupCommand(program) {
|
|
|
114
121
|
console.log(`\n${c.success("Setup complete.")}`);
|
|
115
122
|
console.log(`\n ${c.dim("Next steps:")}`);
|
|
116
123
|
console.log(` ${c.dim(" 1. Restart OpenCode if it is running")}`);
|
|
117
|
-
console.log(` ${c.dim(" 2. Run `opencode-rag
|
|
118
|
-
console.log(` ${c.dim(" 3. Run `opencode-rag index` to build the search index")}`);
|
|
124
|
+
console.log(` ${c.dim(" 2. Run `opencode-rag index` to build the search index")}`);
|
|
119
125
|
}
|
|
120
126
|
else {
|
|
121
127
|
for (const err of result.errors) {
|
|
@@ -124,8 +130,88 @@ export function registerSetupCommand(program) {
|
|
|
124
130
|
console.error(`\n ${c.error("Setup failed. Please check the errors above or run with --force.\n")}`);
|
|
125
131
|
process.exit(1);
|
|
126
132
|
}
|
|
133
|
+
// --- workspace step: automatically initialize the current workspace ---
|
|
134
|
+
const cwd = process.cwd();
|
|
135
|
+
console.log(`\n${c.heading("Workspace")}`);
|
|
136
|
+
if (isLikelyProjectRoot(cwd)) {
|
|
137
|
+
if (isWorkspaceInitialized(cwd)) {
|
|
138
|
+
console.log(` ${c.exists("Exists:")} OpenCodeRAG already initialized in ${c.file(cwd)}`);
|
|
139
|
+
console.log(` ${c.dim("Run `opencode-rag init` to re-sync workspace files when needed.\n")}`);
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
await runWorkspaceInit({});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
console.log(` ${c.dim("Current directory is not a project root - skipping workspace initialization.")}`);
|
|
147
|
+
console.log(` ${c.dim("Run `opencode-rag init` inside the workspace when needed.\n")}`);
|
|
148
|
+
}
|
|
127
149
|
checkOpenCodeRunning();
|
|
128
150
|
console.log();
|
|
129
151
|
});
|
|
130
152
|
}
|
|
153
|
+
/**
|
|
154
|
+
* Ensure the plugin package is installed globally.
|
|
155
|
+
*
|
|
156
|
+
* When it is missing (e.g. fresh machine), queries the npm registry for the
|
|
157
|
+
* latest OpenCodeRAG version and installs it, so a bare `opencode-rag setup`
|
|
158
|
+
* bootstraps the whole stack. When the plugin exists but a newer version is
|
|
159
|
+
* published, prints a hint pointing at `opencode-rag update`.
|
|
160
|
+
*/
|
|
161
|
+
async function ensureGlobalPlugin() {
|
|
162
|
+
let globalPluginDir = "";
|
|
163
|
+
try {
|
|
164
|
+
const globalRoot = execSync("npm root -g", { encoding: "utf-8", timeout: 10_000 }).trim();
|
|
165
|
+
globalPluginDir = path.join(globalRoot, PLUGIN_NAME);
|
|
166
|
+
}
|
|
167
|
+
catch {
|
|
168
|
+
// npm unavailable -> bootstrap cannot run
|
|
169
|
+
}
|
|
170
|
+
const distOk = globalPluginDir !== "" &&
|
|
171
|
+
existsSync(path.join(globalPluginDir, "dist", "plugin-entry.js")) &&
|
|
172
|
+
existsSync(path.join(globalPluginDir, "dist", "cli.js"));
|
|
173
|
+
if (distOk) {
|
|
174
|
+
const latest = getLatestNpmVersion();
|
|
175
|
+
if (latest && compareVersions(latest, getCurrentVersion()) > 0) {
|
|
176
|
+
console.log(` ${c.warn("Newer version available on npm:")} ${c.value(latest)}`);
|
|
177
|
+
console.log(` ${c.dim("Run `opencode-rag update` to install it.\n")}`);
|
|
178
|
+
}
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
console.log(` ${c.warn("Plugin not installed globally.")} Looking up the latest version on npm...\n`);
|
|
182
|
+
const latest = getLatestNpmVersion();
|
|
183
|
+
if (!latest) {
|
|
184
|
+
console.error(` ${c.error("✗-")} Could not determine the latest version from npm. Install manually with:`);
|
|
185
|
+
console.error(` ${c.file("npm install -g opencode-rag-plugin")}`);
|
|
186
|
+
console.error(` ${c.dim("then re-run `opencode-rag setup`.\n")}`);
|
|
187
|
+
process.exit(1);
|
|
188
|
+
}
|
|
189
|
+
console.log(` ${c.created("Found:")} latest npm release ${c.value(latest)}`);
|
|
190
|
+
console.log(` ${c.dim("Installing...")}\n`);
|
|
191
|
+
const updated = await installLatestUpdate({ verbose: false });
|
|
192
|
+
if (!updated.success) {
|
|
193
|
+
console.error(` ${c.error("✗-")} ${updated.message}`);
|
|
194
|
+
process.exit(1);
|
|
195
|
+
}
|
|
196
|
+
console.log(` ${c.success("✓")} ${updated.message}`);
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Query the npm registry for the latest published version of the plugin.
|
|
200
|
+
*
|
|
201
|
+
* Best-effort: returns `null` when npm is unavailable or the lookup fails.
|
|
202
|
+
*
|
|
203
|
+
* @returns The latest version string, or `null` on failure.
|
|
204
|
+
*/
|
|
205
|
+
function getLatestNpmVersion() {
|
|
206
|
+
try {
|
|
207
|
+
const out = execSync(`npm view ${PLUGIN_NAME} dist-tags.latest`, {
|
|
208
|
+
encoding: "utf-8",
|
|
209
|
+
timeout: 15_000,
|
|
210
|
+
}).trim();
|
|
211
|
+
return out || null;
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
131
217
|
//# sourceMappingURL=setup.js.map
|
package/dist/content/reader.js
CHANGED
|
@@ -6,6 +6,7 @@ import path from "node:path";
|
|
|
6
6
|
import pLimit from "p-limit";
|
|
7
7
|
import { computeFileHash, computeDescriptionConfigHash, normalizeFilePath } from "../core/manifest.js";
|
|
8
8
|
import { createExcludeMatcher, createIncludeMatcher } from "../core/exclude.js";
|
|
9
|
+
import { normalizeExtensionKey } from "../core/parser-overrides.js";
|
|
9
10
|
import { DescriptionCache } from "../core/desc-cache.js";
|
|
10
11
|
import { createImageVisionProvider, } from "../chunker/image.js";
|
|
11
12
|
import * as pdfExtractor from "./pdf.js";
|
|
@@ -91,6 +92,13 @@ async function dispatchExtraction(filePath, buffer, imageVisionProvider, imagePr
|
|
|
91
92
|
*/
|
|
92
93
|
export async function scanWorkspaceFiles(cwd, config, logger, manifest, filterPaths, injectedVisionProvider, descCache) {
|
|
93
94
|
const extensions = new Set(config.indexing.includeExtensions);
|
|
95
|
+
// `chunking.parsers` implies the extension is wanted: mapping an extension to
|
|
96
|
+
// a parser is pointless if the scanner never picks the files up.
|
|
97
|
+
for (const rawKey of Object.keys(config.chunking?.parsers ?? {})) {
|
|
98
|
+
const key = normalizeExtensionKey(rawKey);
|
|
99
|
+
if (key)
|
|
100
|
+
extensions.add(key);
|
|
101
|
+
}
|
|
94
102
|
let imageVisionProvider = null;
|
|
95
103
|
let imagePrompt;
|
|
96
104
|
let imageResizeMaxDimension;
|
package/dist/core/bootstrap.js
CHANGED
|
@@ -11,6 +11,7 @@ import { createDescriptionProvider } from "../describer/factory.js";
|
|
|
11
11
|
import { createVectorStore } from "../vectorstore/factory.js";
|
|
12
12
|
import { readStoreDimension } from "../vectorstore/lancedb.js";
|
|
13
13
|
import { KeywordIndex } from "../retriever/keyword-index.js";
|
|
14
|
+
import { reconcileQuirks } from "../quirks/quirk-store.js";
|
|
14
15
|
/**
|
|
15
16
|
* Resolve the embedding dimension for the store.
|
|
16
17
|
*
|
|
@@ -95,6 +96,25 @@ export async function resolveRagContext(opts = {}) {
|
|
|
95
96
|
const keywordIndex = opts.skipKeywordIndex
|
|
96
97
|
? new KeywordIndex(storePath)
|
|
97
98
|
: await loadKeywordIndex(storePath);
|
|
99
|
+
// Reconcile quirks.jsonl into the store + keyword index (best-effort).
|
|
100
|
+
// Full rebuilds rebuild the table from workspace files only — the preserved
|
|
101
|
+
// quirks.jsonl was never re-embedded, so recall would silently miss quirks
|
|
102
|
+
// while `quirk list` still showed them. Skipped for read-only bootstraps
|
|
103
|
+
// (skipProbe/skipKeywordIndex) — those must never write to the store.
|
|
104
|
+
if (!(opts.skipProbe || opts.skipKeywordIndex)) {
|
|
105
|
+
try {
|
|
106
|
+
const rec = await reconcileQuirks({ embedder, store, keywordIndex, cfg, storePath });
|
|
107
|
+
if (rec.restoredToStore > 0 || rec.removedOrphans > 0) {
|
|
108
|
+
// Surface only store-level drift (the split-brain symptom). Keyword-index
|
|
109
|
+
// healing is routine (addQuirk never persisted the index) and stays silent.
|
|
110
|
+
console.warn(`[bootstrap] Quirk reconcile: restored ${rec.restoredToStore} to store, ` +
|
|
111
|
+
`removed ${rec.removedOrphans} orphans (keyword index: +${rec.addedToKeywordIndex})`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
catch (err) {
|
|
115
|
+
console.warn(`[bootstrap] Quirk reconcile failed: ${err.message}`);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
98
118
|
const descriptionConfig = cfg.description;
|
|
99
119
|
const descriptionProvider = descriptionConfig?.enabled
|
|
100
120
|
? createDescriptionProvider(descriptionConfig)
|
package/dist/core/config.d.ts
CHANGED
|
@@ -423,6 +423,26 @@ export interface RagConfig {
|
|
|
423
423
|
chunking?: {
|
|
424
424
|
/** Map of language to allowed/blocked AST node type patterns. */
|
|
425
425
|
nodeTypes?: Record<string, string[]>;
|
|
426
|
+
/**
|
|
427
|
+
* Map of file extension to parser (chunker) language, overriding the
|
|
428
|
+
* built-in mapping. Both new extensions and existing entries can be
|
|
429
|
+
* remapped:
|
|
430
|
+
*
|
|
431
|
+
* ```json
|
|
432
|
+
* { "chunking": { "parsers": { ".c": "cpp", ".cu": "cpp" } } }
|
|
433
|
+
* ```
|
|
434
|
+
*
|
|
435
|
+
* Keys are normalized (lowercased, leading dot added); values must be a
|
|
436
|
+
* registered parser language (`typescript`, `cpp`, `python`, … or `text`
|
|
437
|
+
* for the fallback parser). Applied consistently to indexing/chunking,
|
|
438
|
+
* `get_file_skeleton` (tool + MCP), and read-tool language labels.
|
|
439
|
+
* Mapped extensions are implicitly added to the scanned set — no need to
|
|
440
|
+
* repeat them in `indexing.includeExtensions`.
|
|
441
|
+
*
|
|
442
|
+
* Changing a mapping changes chunk boundaries: re-run
|
|
443
|
+
* `opencode-rag index --force` afterwards.
|
|
444
|
+
*/
|
|
445
|
+
parsers?: Record<string, string>;
|
|
426
446
|
};
|
|
427
447
|
/** LLM-based chunk description generation config. */
|
|
428
448
|
description?: DescriptionConfig;
|
package/dist/core/config.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { env } from "node:process";
|
|
8
|
+
import { normalizeParserOverrides } from "./parser-overrides.js";
|
|
8
9
|
export const DEFAULT_CONFIG = {
|
|
9
10
|
embedding: {
|
|
10
11
|
provider: "ollama",
|
|
@@ -559,6 +560,7 @@ export function loadConfig(filePath, validate = true) {
|
|
|
559
560
|
...(DEFAULT_CONFIG.chunking?.nodeTypes ?? {}),
|
|
560
561
|
...(parsed.chunking?.nodeTypes ?? {}),
|
|
561
562
|
},
|
|
563
|
+
parsers: normalizeParserOverrides(safeObj(parsed.chunking)?.parsers),
|
|
562
564
|
},
|
|
563
565
|
description: {
|
|
564
566
|
...DEFAULT_CONFIG.description,
|
|
@@ -144,6 +144,8 @@ export interface KeywordIndex {
|
|
|
144
144
|
clear(): void;
|
|
145
145
|
/** Return the total number of indexed chunks. */
|
|
146
146
|
count(): number;
|
|
147
|
+
/** Whether a chunk with the given id is indexed. Used by quirk reconcile. */
|
|
148
|
+
hasChunk(id: string): boolean;
|
|
147
149
|
/** Persist the index to disk as JSON. */
|
|
148
150
|
save(filePath?: string): Promise<void>;
|
|
149
151
|
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Pure helpers for the `chunking.parsers` extension → parser override map.
|
|
3
|
+
*
|
|
4
|
+
* Kept free of imports so `core/config`, the chunker factory, and the content
|
|
5
|
+
* scanner can all share it without introducing module cycles.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Map of file extension (e.g. `".cxx"`) to parser/chunker language name
|
|
9
|
+
* (e.g. `"cpp"`). Keys are normalized to lowercase with a leading dot;
|
|
10
|
+
* values to lowercase language names.
|
|
11
|
+
*/
|
|
12
|
+
export type ParserOverrides = Record<string, string>;
|
|
13
|
+
/**
|
|
14
|
+
* Normalize a single file-extension key: trim, lowercase, ensure a leading dot.
|
|
15
|
+
*
|
|
16
|
+
* @returns The normalized key, or `""` when the key cannot be an extension
|
|
17
|
+
* (empty after trimming, or containing a path separator).
|
|
18
|
+
*/
|
|
19
|
+
export declare function normalizeExtensionKey(raw: unknown): string;
|
|
20
|
+
/**
|
|
21
|
+
* Normalize a whole `chunking.parsers` map.
|
|
22
|
+
*
|
|
23
|
+
* Invalid entries (non-string keys/values, empty values, unusable keys) are
|
|
24
|
+
* dropped rather than throwing — validation surfaces them as warnings.
|
|
25
|
+
*
|
|
26
|
+
* @param map - Raw user-supplied map (may be undefined or malformed).
|
|
27
|
+
* @returns A new map with normalized keys and values; `{}` when nothing valid remains.
|
|
28
|
+
*/
|
|
29
|
+
export declare function normalizeParserOverrides(map?: ParserOverrides): ParserOverrides;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Pure helpers for the `chunking.parsers` extension → parser override map.
|
|
3
|
+
*
|
|
4
|
+
* Kept free of imports so `core/config`, the chunker factory, and the content
|
|
5
|
+
* scanner can all share it without introducing module cycles.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Normalize a single file-extension key: trim, lowercase, ensure a leading dot.
|
|
9
|
+
*
|
|
10
|
+
* @returns The normalized key, or `""` when the key cannot be an extension
|
|
11
|
+
* (empty after trimming, or containing a path separator).
|
|
12
|
+
*/
|
|
13
|
+
export function normalizeExtensionKey(raw) {
|
|
14
|
+
if (typeof raw !== "string")
|
|
15
|
+
return "";
|
|
16
|
+
const trimmed = raw.trim().toLowerCase();
|
|
17
|
+
if (trimmed.length === 0)
|
|
18
|
+
return "";
|
|
19
|
+
if (trimmed.includes("/") || trimmed.includes("\\"))
|
|
20
|
+
return "";
|
|
21
|
+
return trimmed.startsWith(".") ? trimmed : `.${trimmed}`;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Normalize a whole `chunking.parsers` map.
|
|
25
|
+
*
|
|
26
|
+
* Invalid entries (non-string keys/values, empty values, unusable keys) are
|
|
27
|
+
* dropped rather than throwing — validation surfaces them as warnings.
|
|
28
|
+
*
|
|
29
|
+
* @param map - Raw user-supplied map (may be undefined or malformed).
|
|
30
|
+
* @returns A new map with normalized keys and values; `{}` when nothing valid remains.
|
|
31
|
+
*/
|
|
32
|
+
export function normalizeParserOverrides(map) {
|
|
33
|
+
const out = {};
|
|
34
|
+
if (!map || typeof map !== "object" || Array.isArray(map))
|
|
35
|
+
return out;
|
|
36
|
+
for (const [key, value] of Object.entries(map)) {
|
|
37
|
+
const ext = normalizeExtensionKey(key);
|
|
38
|
+
if (!ext)
|
|
39
|
+
continue;
|
|
40
|
+
if (typeof value !== "string")
|
|
41
|
+
continue;
|
|
42
|
+
const language = value.trim().toLowerCase();
|
|
43
|
+
if (language.length === 0)
|
|
44
|
+
continue;
|
|
45
|
+
out[ext] = language;
|
|
46
|
+
}
|
|
47
|
+
return out;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=parser-overrides.js.map
|
package/dist/index.d.ts
CHANGED
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
* low-level building blocks (chunkers, embedders, vector stores, retrieval),
|
|
4
4
|
* high-level convenience API (search, index, getContext), and plugin entry.
|
|
5
5
|
*/
|
|
6
|
-
export { chunkFile, getChunker, registerChunker } from "./chunker/factory.js";
|
|
6
|
+
export { chunkFile, getChunker, registerChunker, getRegisteredLanguages, getExtensionsForLanguage, validateParserOverrides } from "./chunker/factory.js";
|
|
7
|
+
export { extractSkeleton } from "./chunker/skeleton.js";
|
|
8
|
+
export { normalizeParserOverrides } from "./core/parser-overrides.js";
|
|
9
|
+
export type { ParserOverrides } from "./core/parser-overrides.js";
|
|
7
10
|
export { createEmbedder, embedBatch } from "./embedder/factory.js";
|
|
8
11
|
export { createDescriptionProvider } from "./describer/factory.js";
|
|
9
12
|
export { createVectorStore } from "./vectorstore/factory.js";
|
package/dist/index.js
CHANGED
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
* low-level building blocks (chunkers, embedders, vector stores, retrieval),
|
|
4
4
|
* high-level convenience API (search, index, getContext), and plugin entry.
|
|
5
5
|
*/
|
|
6
|
-
export { chunkFile, getChunker, registerChunker } from "./chunker/factory.js";
|
|
6
|
+
export { chunkFile, getChunker, registerChunker, getRegisteredLanguages, getExtensionsForLanguage, validateParserOverrides } from "./chunker/factory.js";
|
|
7
|
+
export { extractSkeleton } from "./chunker/skeleton.js";
|
|
8
|
+
export { normalizeParserOverrides } from "./core/parser-overrides.js";
|
|
7
9
|
export { createEmbedder, embedBatch } from "./embedder/factory.js";
|
|
8
10
|
export { createDescriptionProvider } from "./describer/factory.js";
|
|
9
11
|
export { createVectorStore } from "./vectorstore/factory.js";
|
package/dist/indexer/pipeline.js
CHANGED
|
@@ -10,6 +10,7 @@ import { DescriptionCache } from "../core/desc-cache.js";
|
|
|
10
10
|
import { embedBatch, probeEmbeddingDimension } from "../embedder/factory.js";
|
|
11
11
|
import { createVectorStore } from "../vectorstore/factory.js";
|
|
12
12
|
import { swapStoreDirectories } from "../vectorstore/lancedb.js";
|
|
13
|
+
import { reconcileQuirks } from "../quirks/quirk-store.js";
|
|
13
14
|
import { createIndexStats } from "./stats.js";
|
|
14
15
|
import { prepareFile, buildTextsToEmbed } from "./worker.js";
|
|
15
16
|
import { tagChunksRole } from "../core/chunk-role.js";
|
|
@@ -1027,6 +1028,31 @@ async function runIndexPassInner(options, logger) {
|
|
|
1027
1028
|
logger.warn("Rebuild did not complete — keeping the previous manifest and keyword index.");
|
|
1028
1029
|
}
|
|
1029
1030
|
else {
|
|
1031
|
+
// Reconcile quirks.jsonl into the store + keyword index BEFORE the
|
|
1032
|
+
// pass-end save. Two failure modes this heals:
|
|
1033
|
+
// (a) rebuild swaps rebuild the table from workspace files only — the
|
|
1034
|
+
// preserved quirks.jsonl was never re-embedded, so recall silently
|
|
1035
|
+
// missed every quirk added before the rebuild;
|
|
1036
|
+
// (b) saving our in-memory keyword index without the jsonl quirks
|
|
1037
|
+
// clobbers entries that other processes (plugin/CLI) added since we
|
|
1038
|
+
// loaded it.
|
|
1039
|
+
// Best-effort: a reconcile failure must never fail the pass.
|
|
1040
|
+
try {
|
|
1041
|
+
const rec = await reconcileQuirks({
|
|
1042
|
+
embedder: options.embedder,
|
|
1043
|
+
store: options.store,
|
|
1044
|
+
keywordIndex: options.keywordIndex,
|
|
1045
|
+
cfg: options.config,
|
|
1046
|
+
storePath: options.storePath,
|
|
1047
|
+
});
|
|
1048
|
+
if (rec.restoredToStore > 0 || rec.addedToKeywordIndex > 0 || rec.removedOrphans > 0) {
|
|
1049
|
+
logger.info(`Quirk reconcile: restored ${rec.restoredToStore} to store, ` +
|
|
1050
|
+
`added ${rec.addedToKeywordIndex} to keyword index, removed ${rec.removedOrphans} orphans`);
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
catch (err) {
|
|
1054
|
+
logger.warn(`Quirk reconcile failed: ${err.message}`);
|
|
1055
|
+
}
|
|
1030
1056
|
// Save manifest and keyword index (always to the real store path — after
|
|
1031
1057
|
// a successful swap this points to the new data).
|
|
1032
1058
|
await saveManifest(options.storePath, manifest);
|
package/dist/indexer/worker.d.ts
CHANGED
|
@@ -114,6 +114,7 @@ export declare function prepareFile(file: WorkspaceFile, cwd: string, previous:
|
|
|
114
114
|
};
|
|
115
115
|
chunking?: {
|
|
116
116
|
nodeTypes?: Record<string, string[]>;
|
|
117
|
+
parsers?: Record<string, string>;
|
|
117
118
|
};
|
|
118
119
|
description?: {
|
|
119
120
|
maxContentChars?: number;
|
|
@@ -163,6 +164,7 @@ export declare function processFile(file: WorkspaceFile, cwd: string, previous:
|
|
|
163
164
|
};
|
|
164
165
|
chunking?: {
|
|
165
166
|
nodeTypes?: Record<string, string[]>;
|
|
167
|
+
parsers?: Record<string, string>;
|
|
166
168
|
};
|
|
167
169
|
description?: {
|
|
168
170
|
maxContentChars?: number;
|
package/dist/indexer/worker.js
CHANGED
|
@@ -119,6 +119,7 @@ export async function prepareFile(file, cwd, previous, config, keywordIndex, des
|
|
|
119
119
|
maxSvgSizeBytes: config.indexing?.maxSvgSizeBytes,
|
|
120
120
|
maxChunkSize: config.indexing?.maxChunkSize,
|
|
121
121
|
chunkOverlap: config.indexing?.chunkOverlap,
|
|
122
|
+
languageByExtension: config.chunking?.parsers,
|
|
122
123
|
}).catch((err) => {
|
|
123
124
|
logger.warn(` ${fileLabel} (chunking failed: ${err.message})`);
|
|
124
125
|
return null;
|
package/dist/mcp/handlers.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { type EmbeddingProvider, type VectorStore, type KeywordIndex, type Searc
|
|
|
5
5
|
import type { RagConfig } from "../core/config.js";
|
|
6
6
|
import { type ImageVisionProvider } from "../chunker/image.js";
|
|
7
7
|
import { retrieve } from "../retriever/retriever.js";
|
|
8
|
+
import type { ParserOverrides } from "../core/parser-overrides.js";
|
|
8
9
|
/**
|
|
9
10
|
* Resolve a user-supplied file path against the worktree root, refusing to
|
|
10
11
|
* escape the worktree. Absolute paths are accepted only if they already reside
|
|
@@ -56,7 +57,7 @@ export interface FileSkeletonResult {
|
|
|
56
57
|
summary: string;
|
|
57
58
|
}
|
|
58
59
|
/** Extract the structural skeleton (functions, classes, interfaces) from a source file. */
|
|
59
|
-
export declare function handleFileSkeleton(params: FileSkeletonParams, worktree: string): Promise<FileSkeletonResult>;
|
|
60
|
+
export declare function handleFileSkeleton(params: FileSkeletonParams, worktree: string, languageByExtension?: ParserOverrides): Promise<FileSkeletonResult>;
|
|
60
61
|
/** Parameters for the find_usages MCP tool. */
|
|
61
62
|
export interface FindUsagesParams {
|
|
62
63
|
/** Symbol name to search for. */
|
package/dist/mcp/handlers.js
CHANGED
|
@@ -6,8 +6,7 @@ import { normalizeFileExtensions } from "../core/filters.js";
|
|
|
6
6
|
import { SUPPORTED_IMAGE_EXTENSIONS } from "../chunker/image.js";
|
|
7
7
|
import { retrieve } from "../retriever/retriever.js";
|
|
8
8
|
import { optimizeContext, DEFAULT_CONTEXT_OPTIMIZATION } from "../retriever/context-optimizer.js";
|
|
9
|
-
import {
|
|
10
|
-
import { initParser, loadLanguage, walkTree } from "../chunker/grammar.js";
|
|
9
|
+
import { extractSkeleton, getExtension } from "../chunker/skeleton.js";
|
|
11
10
|
import { readFileSync } from "node:fs";
|
|
12
11
|
import { resolve, isAbsolute, relative, sep as pathSep } from "node:path";
|
|
13
12
|
/**
|
|
@@ -31,105 +30,8 @@ export function resolveFilePath(filePath, worktree) {
|
|
|
31
30
|
}
|
|
32
31
|
return candidate;
|
|
33
32
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
".tsx": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration", "arrow_function"] },
|
|
37
|
-
".mts": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration"] },
|
|
38
|
-
".cts": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration"] },
|
|
39
|
-
".js": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
|
|
40
|
-
".jsx": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
|
|
41
|
-
".mjs": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
|
|
42
|
-
".cjs": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
|
|
43
|
-
".py": { grammarName: "python", nodeTypes: ["function_definition", "class_definition", "decorated_definition"] },
|
|
44
|
-
".java": { grammarName: "java", nodeTypes: ["class_declaration", "method_declaration", "interface_declaration", "enum_declaration"] },
|
|
45
|
-
".go": { grammarName: "go", nodeTypes: ["function_declaration", "method_declaration", "type_declaration", "type_spec"] },
|
|
46
|
-
".rs": { grammarName: "rust", nodeTypes: ["function_item", "struct_item", "enum_item", "impl_item", "trait_item", "type_item"] },
|
|
47
|
-
".c": { grammarName: "c", nodeTypes: ["function_definition", "struct_specifier", "enum_specifier"] },
|
|
48
|
-
".cpp": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
|
|
49
|
-
".cxx": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
|
|
50
|
-
".h": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
|
|
51
|
-
".hpp": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
|
|
52
|
-
".cs": { grammarName: "c-sharp", nodeTypes: ["class_declaration", "method_declaration", "interface_declaration", "enum_declaration", "struct_declaration"] },
|
|
53
|
-
".rb": { grammarName: "ruby", nodeTypes: ["method", "class", "module"] },
|
|
54
|
-
".swift": { grammarName: "swift", nodeTypes: ["function_declaration", "class_declaration", "struct_declaration", "enum_declaration", "protocol_declaration"] },
|
|
55
|
-
".kt": { grammarName: "kotlin", nodeTypes: ["function_declaration", "class_declaration", "interface_declaration", "object_declaration"] },
|
|
56
|
-
".kts": { grammarName: "kotlin", nodeTypes: ["function_declaration", "class_declaration", "interface_declaration", "object_declaration"] },
|
|
57
|
-
".css": { grammarName: "css", nodeTypes: ["rule_set"] },
|
|
58
|
-
".md": { grammarName: "markdown", nodeTypes: ["atx_heading"] },
|
|
59
|
-
};
|
|
60
|
-
const REGEX_SKELETON = {
|
|
61
|
-
".json": [/^(\s*)"(\w+)":/gm],
|
|
62
|
-
".yaml": [/^(\w+):/gm],
|
|
63
|
-
".yml": [/^(\w+):/gm],
|
|
64
|
-
".toml": [/^\[(\w+)\]/gm],
|
|
65
|
-
".sh": [/^(function\s+\w+|^\w+\s*\(\)\s*\{)/gm],
|
|
66
|
-
".bash": [/^(function\s+\w+|^\w+\s*\(\)\s*\{)/gm],
|
|
67
|
-
".zsh": [/^(function\s+\w+|^\w+\s*\(\)\s*\{)/gm],
|
|
68
|
-
".sql": [/^(CREATE|ALTER|DROP|SELECT|INSERT|UPDATE|DELETE)\s/gim],
|
|
69
|
-
};
|
|
70
|
-
function getExtension(filePath) {
|
|
71
|
-
const dot = filePath.lastIndexOf(".");
|
|
72
|
-
return dot >= 0 ? filePath.slice(dot).toLowerCase() : "";
|
|
73
|
-
}
|
|
74
|
-
async function extractSkeleton(content, ext) {
|
|
75
|
-
const config = SKELETON_CONFIGS[ext];
|
|
76
|
-
if (!config) {
|
|
77
|
-
const patterns = REGEX_SKELETON[ext];
|
|
78
|
-
if (!patterns) {
|
|
79
|
-
const lines = content.split("\n");
|
|
80
|
-
return [{ type: "file", name: `${lines.length} lines`, startLine: 1, endLine: lines.length }];
|
|
81
|
-
}
|
|
82
|
-
const items = [];
|
|
83
|
-
for (const pattern of patterns) {
|
|
84
|
-
let match;
|
|
85
|
-
pattern.lastIndex = 0;
|
|
86
|
-
while ((match = pattern.exec(content)) !== null) {
|
|
87
|
-
const name = match[1] ?? match[0].trim();
|
|
88
|
-
const lineNum = content.slice(0, match.index).split("\n").length;
|
|
89
|
-
items.push({ type: ext.slice(1), name, startLine: lineNum, endLine: lineNum });
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
const seen = new Set();
|
|
93
|
-
return items.filter((item) => {
|
|
94
|
-
const key = `${item.type}:${item.name}`;
|
|
95
|
-
if (seen.has(key))
|
|
96
|
-
return false;
|
|
97
|
-
seen.add(key);
|
|
98
|
-
return true;
|
|
99
|
-
});
|
|
100
|
-
}
|
|
101
|
-
await initParser();
|
|
102
|
-
const lang = await loadLanguage(config.grammarName);
|
|
103
|
-
const parser = new Parser();
|
|
104
|
-
parser.setLanguage(lang);
|
|
105
|
-
const tree = parser.parse(content);
|
|
106
|
-
if (!tree)
|
|
107
|
-
return [];
|
|
108
|
-
const nodeTypes = new Set(config.nodeTypes);
|
|
109
|
-
const astNodes = walkTree(tree.rootNode, nodeTypes, content, 15);
|
|
110
|
-
tree.delete();
|
|
111
|
-
parser.delete();
|
|
112
|
-
return astNodes.map((node) => ({
|
|
113
|
-
type: node.type,
|
|
114
|
-
name: extractNodeName(node.text, node.type),
|
|
115
|
-
startLine: node.startLine,
|
|
116
|
-
endLine: node.endLine,
|
|
117
|
-
}));
|
|
118
|
-
}
|
|
119
|
-
function extractNodeName(text, _nodeType) {
|
|
120
|
-
const firstLine = text.split("\n")[0] ?? text;
|
|
121
|
-
const nameMatch = firstLine.match(/^(?:export\s+)?(?:async\s+)?(?:function\s+)?(?:\w+\s+)?(?:(\w+))\s*(?:[<(]|$)/);
|
|
122
|
-
if (nameMatch) {
|
|
123
|
-
const keywordSet = new Set(["export", "async", "function", "class", "interface", "type", "enum", "struct", "impl", "trait", "def", "fun", "fn"]);
|
|
124
|
-
if (!keywordSet.has(nameMatch[1])) {
|
|
125
|
-
return nameMatch[1];
|
|
126
|
-
}
|
|
127
|
-
const secondMatch = firstLine.match(/^(?:export\s+)?(?:async\s+)?(?:function|class|interface|type|enum|struct|trait|impl|def|fun|fn)\s+(\w+)/);
|
|
128
|
-
if (secondMatch)
|
|
129
|
-
return secondMatch[1];
|
|
130
|
-
}
|
|
131
|
-
return firstLine.length > 80 ? firstLine.slice(0, 77) + "..." : firstLine;
|
|
132
|
-
}
|
|
33
|
+
// Skeleton configuration and extraction live in ../chunker/skeleton.ts —
|
|
34
|
+
// shared with the OpenCode plugin tools so both honor `chunking.parsers`.
|
|
133
35
|
function typeIcon(type) {
|
|
134
36
|
if (type.includes("class") || type === "struct_specifier" || type === "struct_item" || type === "struct_declaration")
|
|
135
37
|
return "□";
|
|
@@ -211,11 +113,11 @@ export async function handleSearchSemantic(params, embedder, store, cfg, keyword
|
|
|
211
113
|
return { chunks: results, formatted };
|
|
212
114
|
}
|
|
213
115
|
/** Extract the structural skeleton (functions, classes, interfaces) from a source file. */
|
|
214
|
-
export async function handleFileSkeleton(params, worktree) {
|
|
116
|
+
export async function handleFileSkeleton(params, worktree, languageByExtension) {
|
|
215
117
|
const resolvedPath = resolveFilePath(params.filePath, worktree);
|
|
216
118
|
const content = readFileSync(resolvedPath, "utf-8");
|
|
217
119
|
const ext = getExtension(params.filePath);
|
|
218
|
-
const skeleton = await extractSkeleton(content, ext);
|
|
120
|
+
const skeleton = await extractSkeleton(content, ext, { languageByExtension });
|
|
219
121
|
const declCounts = new Map();
|
|
220
122
|
for (const item of skeleton) {
|
|
221
123
|
const baseType = item.type.replace(/_declaration$|_definition$|_specifier$|_item$/, "");
|
package/dist/mcp/server.js
CHANGED
|
@@ -43,7 +43,7 @@ export async function createMcpServer(options) {
|
|
|
43
43
|
filePath: z.string().min(1, "A file path is required."),
|
|
44
44
|
}, async (args) => {
|
|
45
45
|
try {
|
|
46
|
-
const result = await handleFileSkeleton(args, cwd);
|
|
46
|
+
const result = await handleFileSkeleton(args, cwd, ctx.config.chunking?.parsers);
|
|
47
47
|
return {
|
|
48
48
|
content: [{ type: "text", text: result.formatted }],
|
|
49
49
|
};
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* @fileoverview Formats RAG retrieval output, hybrid file-content output, file fallbacks, and related-file suggestions for the read tool.
|
|
3
3
|
*/
|
|
4
4
|
import type { SearchResult } from "../core/interfaces.js";
|
|
5
|
+
import { type ParserOverrides } from "../core/parser-overrides.js";
|
|
5
6
|
/**
|
|
6
7
|
* Options for formatting read tool output.
|
|
7
8
|
*/
|
|
@@ -44,6 +45,8 @@ export interface FormatHybridReadOutputOptions {
|
|
|
44
45
|
relatedFiles: RelatedFileEntry[];
|
|
45
46
|
/** Maximum output character count. */
|
|
46
47
|
maxChars: number;
|
|
48
|
+
/** Extension → parser overrides from `chunking.parsers` (for the code-fence language). */
|
|
49
|
+
languageByExtension?: ParserOverrides;
|
|
47
50
|
}
|
|
48
51
|
/**
|
|
49
52
|
* Format hybrid read output: full file contents followed by optional RAG context.
|
|
@@ -76,6 +79,8 @@ export interface FormatFileFallbackOptions {
|
|
|
76
79
|
reason: string;
|
|
77
80
|
/** Maximum output character count. */
|
|
78
81
|
maxChars?: number;
|
|
82
|
+
/** Extension → parser overrides from `chunking.parsers` (for the code-fence language). */
|
|
83
|
+
languageByExtension?: ParserOverrides;
|
|
79
84
|
}
|
|
80
85
|
/**
|
|
81
86
|
* Format raw file contents as a fallback when no RAG chunks are available.
|