opencode-rag-plugin 2.0.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +10 -2
- package/dist/chunker/factory.d.ts +24 -2
- package/dist/chunker/factory.js +88 -3
- package/dist/chunker/loader.js +10 -5
- package/dist/chunker/skeleton.d.ts +36 -0
- package/dist/chunker/skeleton.js +183 -0
- package/dist/cli/commands/init.d.ts +29 -0
- package/dist/cli/commands/init.js +326 -281
- package/dist/cli/commands/setup.js +95 -9
- package/dist/content/reader.js +8 -0
- package/dist/core/bootstrap.js +20 -0
- package/dist/core/config.d.ts +20 -0
- package/dist/core/config.js +2 -0
- package/dist/core/interfaces.d.ts +2 -0
- package/dist/core/parser-overrides.d.ts +29 -0
- package/dist/core/parser-overrides.js +49 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +3 -1
- package/dist/indexer/pipeline.js +26 -0
- package/dist/indexer/worker.d.ts +2 -0
- package/dist/indexer/worker.js +1 -0
- package/dist/mcp/handlers.d.ts +2 -1
- package/dist/mcp/handlers.js +5 -103
- package/dist/mcp/server.js +1 -1
- package/dist/opencode/create-read-tool.js +1 -0
- package/dist/opencode/read-format.d.ts +5 -0
- package/dist/opencode/read-format.js +12 -5
- package/dist/opencode/tools.d.ts +3 -0
- package/dist/opencode/tools.js +6 -124
- package/dist/plugin.js +22 -1
- package/dist/quirks/quirk-store.d.ts +45 -0
- package/dist/quirks/quirk-store.js +200 -20
- package/dist/retriever/keyword-index.d.ts +1 -0
- package/dist/retriever/keyword-index.js +3 -0
- package/dist/tui.js +6 -0
- package/dist/vectorstore/lancedb.js +1 -0
- package/package.json +1 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview Formats RAG retrieval output, hybrid file-content output, file fallbacks, and related-file suggestions for the read tool.
|
|
3
3
|
*/
|
|
4
|
+
import { normalizeParserOverrides } from "../core/parser-overrides.js";
|
|
4
5
|
/**
|
|
5
6
|
* Format RAG retrieval results for the search_semantic tool output.
|
|
6
7
|
*
|
|
@@ -45,8 +46,8 @@ export function formatReadOutput(options) {
|
|
|
45
46
|
* - Enforces maxChars limit (truncates RAG section first).
|
|
46
47
|
*/
|
|
47
48
|
export function formatHybridReadOutput(options) {
|
|
48
|
-
const { filePath, fileContent, startLine, endLine, ragChunks, relatedFiles, maxChars } = options;
|
|
49
|
-
const lang = guessLanguage(filePath);
|
|
49
|
+
const { filePath, fileContent, startLine, endLine, ragChunks, relatedFiles, maxChars, languageByExtension } = options;
|
|
50
|
+
const lang = guessLanguage(filePath, languageByExtension);
|
|
50
51
|
// Build the full file code block
|
|
51
52
|
const lines = fileContent.split("\n");
|
|
52
53
|
const sliceStart = startLine !== undefined ? startLine - 1 : 0;
|
|
@@ -139,12 +140,12 @@ function formatChunk(index, result) {
|
|
|
139
140
|
* Applies optional line-range slicing and enforces maxChars limit.
|
|
140
141
|
*/
|
|
141
142
|
export function formatFileFallback(options) {
|
|
142
|
-
const { filePath, content, startLine, endLine, maxChars } = options;
|
|
143
|
+
const { filePath, content, startLine, endLine, maxChars, languageByExtension } = options;
|
|
143
144
|
const lines = content.split("\n");
|
|
144
145
|
const sliceStart = startLine !== undefined ? startLine - 1 : 0;
|
|
145
146
|
const sliceEnd = endLine !== undefined ? endLine : lines.length;
|
|
146
147
|
const sliced = lines.slice(sliceStart, sliceEnd);
|
|
147
|
-
const lang = guessLanguage(filePath);
|
|
148
|
+
const lang = guessLanguage(filePath, languageByExtension);
|
|
148
149
|
const codeBlock = "```" + lang + "\n" + sliced.join("\n") + "\n```";
|
|
149
150
|
let output = codeBlock;
|
|
150
151
|
if (maxChars && output.length > maxChars) {
|
|
@@ -152,8 +153,14 @@ export function formatFileFallback(options) {
|
|
|
152
153
|
}
|
|
153
154
|
return output;
|
|
154
155
|
}
|
|
155
|
-
function guessLanguage(filePath) {
|
|
156
|
+
function guessLanguage(filePath, languageByExtension) {
|
|
156
157
|
const ext = filePath.split(".").pop()?.toLowerCase() ?? "";
|
|
158
|
+
const overrides = normalizeParserOverrides(languageByExtension);
|
|
159
|
+
const target = ext ? overrides[`.${ext}`] : undefined;
|
|
160
|
+
// A `chunking.parsers` override wins: it says which parser reads this file,
|
|
161
|
+
// so it is also the right label for the code fence.
|
|
162
|
+
if (target)
|
|
163
|
+
return target;
|
|
157
164
|
const map = {
|
|
158
165
|
ts: "typescript", tsx: "typescript", js: "javascript", jsx: "javascript",
|
|
159
166
|
py: "python", rb: "ruby", go: "go", rs: "rust", java: "java",
|
package/dist/opencode/tools.d.ts
CHANGED
|
@@ -6,9 +6,12 @@ import { type EmbeddingProvider, type VectorStore, type KeywordIndex } from "../
|
|
|
6
6
|
import type { RagConfig } from "../core/config.js";
|
|
7
7
|
import { type ImageVisionProvider } from "../chunker/image.js";
|
|
8
8
|
import { retrieve } from "../retriever/retriever.js";
|
|
9
|
+
import type { ParserOverrides } from "../core/parser-overrides.js";
|
|
9
10
|
/** Options for creating the `get_file_skeleton` tool. */
|
|
10
11
|
export interface FileSkeletonToolOptions {
|
|
11
12
|
worktree: string;
|
|
13
|
+
/** Extension → parser overrides from `chunking.parsers`. */
|
|
14
|
+
languageByExtension?: ParserOverrides;
|
|
12
15
|
}
|
|
13
16
|
/**
|
|
14
17
|
* Create the `get_file_skeleton` tool.
|
package/dist/opencode/tools.js
CHANGED
|
@@ -17,137 +17,19 @@ import { CODE_SEARCH_FILTER } from "../core/interfaces.js";
|
|
|
17
17
|
import { SUPPORTED_IMAGE_EXTENSIONS, createImageVisionProvider, resolveOnDemandImageConfig, getMimeType } from "../chunker/image.js";
|
|
18
18
|
import { resizeImage } from "../content/image.js";
|
|
19
19
|
import { retrieve } from "../retriever/retriever.js";
|
|
20
|
-
import {
|
|
21
|
-
import { initParser, loadLanguage, walkTree } from "../chunker/grammar.js";
|
|
20
|
+
import { extractSkeleton, getExtension } from "../chunker/skeleton.js";
|
|
22
21
|
import { readFileSync } from "node:fs";
|
|
23
22
|
import { resolveWorkspacePath } from "./tool-args.js";
|
|
24
23
|
import { addQuirk, updateQuirk, removeQuirk, recallQuirks } from "../quirks/quirk-store.js";
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
".mts": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration"] },
|
|
29
|
-
".cts": { grammarName: "typescript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "interface_declaration", "type_alias_declaration", "enum_declaration"] },
|
|
30
|
-
".js": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
|
|
31
|
-
".jsx": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
|
|
32
|
-
".mjs": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
|
|
33
|
-
".cjs": { grammarName: "javascript", nodeTypes: ["function_declaration", "method_definition", "class_declaration", "arrow_function"] },
|
|
34
|
-
".py": { grammarName: "python", nodeTypes: ["function_definition", "class_definition", "decorated_definition"] },
|
|
35
|
-
".java": { grammarName: "java", nodeTypes: ["class_declaration", "method_declaration", "interface_declaration", "enum_declaration"] },
|
|
36
|
-
".go": { grammarName: "go", nodeTypes: ["function_declaration", "method_declaration", "type_declaration", "type_spec"] },
|
|
37
|
-
".rs": { grammarName: "rust", nodeTypes: ["function_item", "struct_item", "enum_item", "impl_item", "trait_item", "type_item"] },
|
|
38
|
-
".c": { grammarName: "c", nodeTypes: ["function_definition", "struct_specifier", "enum_specifier"] },
|
|
39
|
-
".cpp": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
|
|
40
|
-
".cxx": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
|
|
41
|
-
".h": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
|
|
42
|
-
".hpp": { grammarName: "cpp", nodeTypes: ["function_definition", "class_specifier", "struct_specifier", "enum_specifier"] },
|
|
43
|
-
".cs": { grammarName: "c-sharp", nodeTypes: ["class_declaration", "method_declaration", "interface_declaration", "enum_declaration", "struct_declaration"] },
|
|
44
|
-
".rb": { grammarName: "ruby", nodeTypes: ["method", "class", "module"] },
|
|
45
|
-
".swift": { grammarName: "swift", nodeTypes: ["function_declaration", "class_declaration", "struct_declaration", "enum_declaration", "protocol_declaration"] },
|
|
46
|
-
".kt": { grammarName: "kotlin", nodeTypes: ["function_declaration", "class_declaration", "interface_declaration", "object_declaration"] },
|
|
47
|
-
".kts": { grammarName: "kotlin", nodeTypes: ["function_declaration", "class_declaration", "interface_declaration", "object_declaration"] },
|
|
48
|
-
".css": { grammarName: "css", nodeTypes: ["rule_set"] },
|
|
49
|
-
".md": { grammarName: "markdown", nodeTypes: ["atx_heading"] },
|
|
50
|
-
};
|
|
51
|
-
/** Regex-based fallback for languages without tree-sitter WASM support. */
|
|
52
|
-
const REGEX_SKELETON = {
|
|
53
|
-
".json": [/^(\s*)"(\w+)":/gm],
|
|
54
|
-
".yaml": [/^(\w+):/gm],
|
|
55
|
-
".yml": [/^(\w+):/gm],
|
|
56
|
-
".toml": [/^\[(\w+)\]/gm],
|
|
57
|
-
".sh": [/^(function\s+\w+|^\w+\s*\(\)\s*\{)/gm],
|
|
58
|
-
".bash": [/^(function\s+\w+|^\w+\s*\(\)\s*\{)/gm],
|
|
59
|
-
".zsh": [/^(function\s+\w+|^\w+\s*\(\)\s*\{)/gm],
|
|
60
|
-
".sql": [/^(CREATE|ALTER|DROP|SELECT|INSERT|UPDATE|DELETE)\s/gim],
|
|
61
|
-
};
|
|
62
|
-
/**
|
|
63
|
-
* Return the file extension from a path (lowercase).
|
|
64
|
-
*/
|
|
65
|
-
function getExtension(filePath) {
|
|
66
|
-
const dot = filePath.lastIndexOf(".");
|
|
67
|
-
return dot >= 0 ? filePath.slice(dot).toLowerCase() : "";
|
|
68
|
-
}
|
|
24
|
+
// Skeleton configuration (extension → grammar + node types) and extraction
|
|
25
|
+
// live in ../chunker/skeleton.ts, shared with the MCP server so both surfaces
|
|
26
|
+
// honor `chunking.parsers` overrides identically.
|
|
69
27
|
/**
|
|
70
28
|
* Resolve a file path relative to the workspace root.
|
|
71
29
|
*/
|
|
72
30
|
function resolveFilePath(filePath, worktree) {
|
|
73
31
|
return resolveWorkspacePath(worktree, filePath);
|
|
74
32
|
}
|
|
75
|
-
/**
|
|
76
|
-
* Extract structural outline from source code using tree-sitter.
|
|
77
|
-
* Falls back to regex patterns for languages without WASM grammars.
|
|
78
|
-
*/
|
|
79
|
-
async function extractSkeleton(content, ext) {
|
|
80
|
-
const config = SKELETON_CONFIGS[ext];
|
|
81
|
-
if (!config) {
|
|
82
|
-
// Fallback: use regex patterns
|
|
83
|
-
const patterns = REGEX_SKELETON[ext];
|
|
84
|
-
if (!patterns) {
|
|
85
|
-
// Last fallback: line count
|
|
86
|
-
const lines = content.split("\n");
|
|
87
|
-
return [{ type: "file", name: `${lines.length} lines`, startLine: 1, endLine: lines.length }];
|
|
88
|
-
}
|
|
89
|
-
const items = [];
|
|
90
|
-
for (const pattern of patterns) {
|
|
91
|
-
let match;
|
|
92
|
-
// Reset lastIndex
|
|
93
|
-
pattern.lastIndex = 0;
|
|
94
|
-
while ((match = pattern.exec(content)) !== null) {
|
|
95
|
-
const name = match[1] ?? match[0].trim();
|
|
96
|
-
const lineNum = content.slice(0, match.index).split("\n").length;
|
|
97
|
-
items.push({ type: ext.slice(1), name, startLine: lineNum, endLine: lineNum });
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
const seen = new Set();
|
|
101
|
-
return items.filter((item) => {
|
|
102
|
-
const key = `${item.type}:${item.name}`;
|
|
103
|
-
if (seen.has(key))
|
|
104
|
-
return false;
|
|
105
|
-
seen.add(key);
|
|
106
|
-
return true;
|
|
107
|
-
});
|
|
108
|
-
}
|
|
109
|
-
// Tree-sitter skeleton extraction
|
|
110
|
-
await initParser();
|
|
111
|
-
const lang = await loadLanguage(config.grammarName);
|
|
112
|
-
const parser = new Parser();
|
|
113
|
-
parser.setLanguage(lang);
|
|
114
|
-
const tree = parser.parse(content);
|
|
115
|
-
if (!tree)
|
|
116
|
-
return [];
|
|
117
|
-
const nodeTypes = new Set(config.nodeTypes);
|
|
118
|
-
const astNodes = walkTree(tree.rootNode, nodeTypes, content, 15);
|
|
119
|
-
tree.delete();
|
|
120
|
-
parser.delete();
|
|
121
|
-
return astNodes.map((node) => ({
|
|
122
|
-
type: node.type,
|
|
123
|
-
name: extractNodeName(node.text, node.type),
|
|
124
|
-
startLine: node.startLine,
|
|
125
|
-
endLine: node.endLine,
|
|
126
|
-
}));
|
|
127
|
-
}
|
|
128
|
-
/**
|
|
129
|
-
* Extract a human-readable name from a code node.
|
|
130
|
-
* For declarations, this is typically the identifier after the keyword.
|
|
131
|
-
*/
|
|
132
|
-
function extractNodeName(text, _nodeType) {
|
|
133
|
-
// Try first line only for large nodes
|
|
134
|
-
const firstLine = text.split("\n")[0] ?? text;
|
|
135
|
-
// Common patterns: "function foo(...)" or "class Foo ..." or "foo(...)" (arrow, method)
|
|
136
|
-
const nameMatch = firstLine.match(/^(?:export\s+)?(?:async\s+)?(?:function\s+)?(?:\w+\s+)?(?:(\w+))\s*(?:[<(]|$)/);
|
|
137
|
-
if (nameMatch) {
|
|
138
|
-
// Skip keywords
|
|
139
|
-
const keywordSet = new Set(["export", "async", "function", "class", "interface", "type", "enum", "struct", "impl", "trait", "def", "fun", "fn"]);
|
|
140
|
-
if (!keywordSet.has(nameMatch[1])) {
|
|
141
|
-
return nameMatch[1];
|
|
142
|
-
}
|
|
143
|
-
// Try next word
|
|
144
|
-
const secondMatch = firstLine.match(/^(?:export\s+)?(?:async\s+)?(?:function|class|interface|type|enum|struct|trait|impl|def|fun|fn)\s+(\w+)/);
|
|
145
|
-
if (secondMatch)
|
|
146
|
-
return secondMatch[1];
|
|
147
|
-
}
|
|
148
|
-
// Last resort: return first 80 chars of first line
|
|
149
|
-
return firstLine.length > 80 ? firstLine.slice(0, 77) + "..." : firstLine;
|
|
150
|
-
}
|
|
151
33
|
/**
|
|
152
34
|
* Format a skeleton outline as a compact markdown string.
|
|
153
35
|
*/
|
|
@@ -193,7 +75,7 @@ function typeIcon(type) {
|
|
|
193
75
|
* with line numbers, using tree-sitter for parsing.
|
|
194
76
|
*/
|
|
195
77
|
export function createFileSkeletonTool(options) {
|
|
196
|
-
const { worktree } = options;
|
|
78
|
+
const { worktree, languageByExtension } = options;
|
|
197
79
|
return tool({
|
|
198
80
|
description: "Get a quick structural overview of a source file — functions, classes, " +
|
|
199
81
|
"interfaces, methods, and other top-level declarations with their line numbers. " +
|
|
@@ -207,7 +89,7 @@ export function createFileSkeletonTool(options) {
|
|
|
207
89
|
const resolvedPath = resolveFilePath(args.filePath, worktree);
|
|
208
90
|
const content = readFileSync(resolvedPath, "utf-8");
|
|
209
91
|
const ext = getExtension(args.filePath);
|
|
210
|
-
const skeleton = await extractSkeleton(content, ext);
|
|
92
|
+
const skeleton = await extractSkeleton(content, ext, { languageByExtension });
|
|
211
93
|
const declCounts = new Map();
|
|
212
94
|
for (const item of skeleton) {
|
|
213
95
|
const baseType = item.type.replace(/_declaration$|_definition$|_specifier$|_item$/, "");
|
package/dist/plugin.js
CHANGED
|
@@ -28,7 +28,7 @@ import { countTokens } from "./eval/token-counter.js";
|
|
|
28
28
|
import { checkForUpdate, getCurrentVersion, installLatestUpdate } from "./core/version-check.js";
|
|
29
29
|
import { loadAutoUpdateState, saveAutoUpdateState, shouldAttemptInstall } from "./core/auto-update-state.js";
|
|
30
30
|
import { destroyAllPooledConnections } from "./embedder/http.js";
|
|
31
|
-
import { listQuirks, lintQuirks, recallQuirks, sharedWords } from "./quirks/quirk-store.js";
|
|
31
|
+
import { listQuirks, lintQuirks, recallQuirks, sharedWords, reconcileQuirks } from "./quirks/quirk-store.js";
|
|
32
32
|
import { autoCaptureQuirks } from "./quirks/auto-capture.js";
|
|
33
33
|
import { buildSystemGuidanceLines } from "./opencode/system-guidance.js";
|
|
34
34
|
import { existsSync, readFileSync, readdirSync, unlinkSync } from "node:fs";
|
|
@@ -580,6 +580,7 @@ export function createRagHooks(options) {
|
|
|
580
580
|
try {
|
|
581
581
|
const fileSkeletonTool = createFileSkeletonTool({
|
|
582
582
|
worktree: options.worktree,
|
|
583
|
+
languageByExtension: effectiveCfg.chunking?.parsers,
|
|
583
584
|
});
|
|
584
585
|
tools["get_file_skeleton"] = fileSkeletonTool;
|
|
585
586
|
}
|
|
@@ -1452,6 +1453,26 @@ export const ragPlugin = async (input, _options) => {
|
|
|
1452
1453
|
}
|
|
1453
1454
|
// Load or create keyword index for hybrid search
|
|
1454
1455
|
const keywordIndex = await loadKeywordIndex(storePath, logFilePath, logLevel);
|
|
1456
|
+
// Restore quirks that are in quirks.jsonl but missing from the vector store
|
|
1457
|
+
// or keyword index (full rebuilds drop quirk chunks — see reconcileQuirks).
|
|
1458
|
+
// Fire-and-forget: must never block plugin startup, and the embedder may be
|
|
1459
|
+
// offline. The background watcher re-runs this at every pass end.
|
|
1460
|
+
void reconcileQuirks({ embedder, store, keywordIndex, cfg: effectiveCfg, storePath })
|
|
1461
|
+
.then((r) => {
|
|
1462
|
+
if (r.restoredToStore > 0 || r.addedToKeywordIndex > 0 || r.removedOrphans > 0) {
|
|
1463
|
+
appendDebugLog(logFilePath, {
|
|
1464
|
+
scope: "plugin",
|
|
1465
|
+
message: `Quirk reconcile: restored ${r.restoredToStore} to store, added ${r.addedToKeywordIndex} to keyword index, removed ${r.removedOrphans} orphans`,
|
|
1466
|
+
}, logLevel);
|
|
1467
|
+
}
|
|
1468
|
+
})
|
|
1469
|
+
.catch((err) => {
|
|
1470
|
+
appendDebugLog(logFilePath, {
|
|
1471
|
+
scope: "plugin",
|
|
1472
|
+
message: "Quirk reconcile failed",
|
|
1473
|
+
error: err,
|
|
1474
|
+
}, logLevel);
|
|
1475
|
+
});
|
|
1455
1476
|
// Create description provider (enabled by default)
|
|
1456
1477
|
const descriptionConfig = effectiveCfg.description ?? { enabled: true, provider: "ollama", baseUrl: "http://127.0.0.1:11434/api", model: "qwen2.5:3b", systemPrompt: "" };
|
|
1457
1478
|
const descriptionProvider = descriptionConfig.enabled
|
|
@@ -28,6 +28,51 @@ export declare function addQuirk(deps: QuirkStoreDeps, input: QuirkInput): Promi
|
|
|
28
28
|
export declare function removeQuirk(deps: QuirkStoreDeps, id: string): Promise<void>;
|
|
29
29
|
/** List all quirks sorted by lastObserved descending. */
|
|
30
30
|
export declare function listQuirks(deps: QuirkStoreDeps): Promise<Quirk[]>;
|
|
31
|
+
/** Dependencies for {@link reconcileQuirks} — like {@link QuirkStoreDeps}, but
|
|
32
|
+
* the keyword index is optional (index passes may run without one). */
|
|
33
|
+
export interface QuirkReconcileDeps extends Omit<QuirkStoreDeps, "keywordIndex"> {
|
|
34
|
+
keywordIndex?: KeywordIndex;
|
|
35
|
+
}
|
|
36
|
+
/** Outcome of a {@link reconcileQuirks} run. Zeroes mean "already consistent". */
|
|
37
|
+
export interface QuirkReconcileResult {
|
|
38
|
+
/** Quirks re-embedded and restored into the vector store (were in quirks.jsonl but not the table). */
|
|
39
|
+
restoredToStore: number;
|
|
40
|
+
/** Quirks added to the in-memory keyword index (lexical only — no embedding involved). */
|
|
41
|
+
addedToKeywordIndex: number;
|
|
42
|
+
/** Store quirk chunks deleted because their quirks.jsonl entry is gone (past the grace window). */
|
|
43
|
+
removedOrphans: number;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Reconcile `quirks.jsonl` (the source of truth for quirk memory) into the
|
|
47
|
+
* vector store and the keyword index, in both directions.
|
|
48
|
+
*
|
|
49
|
+
* Why this exists: full store rebuilds rebuild the table from **workspace
|
|
50
|
+
* files only** — `swapStoreDirectories` carries `quirks.jsonl` across the swap,
|
|
51
|
+
* but nothing ever re-embeds those entries, so every quirk added before a
|
|
52
|
+
* rebuild silently vanishes from recall while `quirk list`/`lint` (which read
|
|
53
|
+
* the jsonl) still show it. The keyword index has the same gap: it only gains
|
|
54
|
+
* quirks via in-process `addQuirk`, and any process that saves its in-memory
|
|
55
|
+
* index clobbers entries added elsewhere.
|
|
56
|
+
*
|
|
57
|
+
* Behavior:
|
|
58
|
+
* - Restores jsonl quirks missing from the store (one batched embed call).
|
|
59
|
+
* - Adds jsonl quirks missing from the keyword index (no embedding needed).
|
|
60
|
+
* - Deletes store quirk chunks whose jsonl entry is gone — only when the jsonl
|
|
61
|
+
* exists and parses non-empty (a missing/unreadable file means "unknown",
|
|
62
|
+
* never "empty") and only after the orphan was observed twice across
|
|
63
|
+
* {@link QUIRK_ORPHAN_GRACE_MS} (see QUIRK_ORPHAN_STATE_FILE — the table has
|
|
64
|
+
* no row timestamps, and addQuirk writes the store before the jsonl append).
|
|
65
|
+
* - Persists the keyword index when it changed (best-effort).
|
|
66
|
+
*
|
|
67
|
+
* Idempotent and safe under concurrent runs: `store.addChunks` dedups by id and
|
|
68
|
+
* `KeywordIndex.addChunks` overwrites by id. Never applied to memory stores
|
|
69
|
+
* (per-process lifetime, nothing on disk to drift). Failures propagate to the
|
|
70
|
+
* caller, which is expected to treat reconcile as best-effort.
|
|
71
|
+
*
|
|
72
|
+
* @param deps - Embedder, store, optional keyword index, config, store path.
|
|
73
|
+
* @returns Counts of what changed (all zero when already consistent).
|
|
74
|
+
*/
|
|
75
|
+
export declare function reconcileQuirks(deps: QuirkReconcileDeps): Promise<QuirkReconcileResult>;
|
|
31
76
|
/** Recall quirks matching a query, with confidence re-weighting. */
|
|
32
77
|
export declare function recallQuirks(deps: QuirkStoreDeps, query: string, options?: {
|
|
33
78
|
topK?: number;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { appendFileSync, existsSync, readFileSync, writeFileSync, mkdirSync, renameSync } from "node:fs";
|
|
1
|
+
import { appendFileSync, existsSync, readFileSync, writeFileSync, mkdirSync, renameSync, rmSync } from "node:fs";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { retrieve } from "../retriever/retriever.js";
|
|
@@ -201,23 +201,7 @@ export async function addQuirk(deps, input) {
|
|
|
201
201
|
if (!embedding || embedding.length === 0) {
|
|
202
202
|
throw new Error("Embedding returned empty vector for quirk content");
|
|
203
203
|
}
|
|
204
|
-
const chunk =
|
|
205
|
-
id,
|
|
206
|
-
content: input.content,
|
|
207
|
-
description: "",
|
|
208
|
-
embedding,
|
|
209
|
-
metadata: {
|
|
210
|
-
filePath: QUIRK_FILE_PREFIX + id,
|
|
211
|
-
startLine: 0,
|
|
212
|
-
endLine: 0,
|
|
213
|
-
language: "quirk",
|
|
214
|
-
kind: "quirk",
|
|
215
|
-
quirkType: input.quirkType,
|
|
216
|
-
tags: input.tags ?? [],
|
|
217
|
-
confidence,
|
|
218
|
-
lastObserved,
|
|
219
|
-
},
|
|
220
|
-
};
|
|
204
|
+
const chunk = quirkChunk(quirk, embedding);
|
|
221
205
|
await deps.store.addChunks([chunk]);
|
|
222
206
|
deps.keywordIndex.addChunks([chunk]);
|
|
223
207
|
if (!isMemoryStore(deps.storePath)) {
|
|
@@ -228,6 +212,33 @@ export async function addQuirk(deps, input) {
|
|
|
228
212
|
}
|
|
229
213
|
return quirk;
|
|
230
214
|
}
|
|
215
|
+
/**
|
|
216
|
+
* Build the store/keyword-index chunk shape for a quirk. Shared by `addQuirk`
|
|
217
|
+
* and `reconcileQuirks` — a rebuild-restore must produce the exact same row
|
|
218
|
+
* shape (metadata.filePath in particular drives list/remove/reconcile id maps).
|
|
219
|
+
*
|
|
220
|
+
* `embedding` may be `[]` for keyword-index-only adds: `chunkToRow` skips rows
|
|
221
|
+
* with an empty embedding, and the keyword index only tokenizes content.
|
|
222
|
+
*/
|
|
223
|
+
function quirkChunk(q, embedding) {
|
|
224
|
+
return {
|
|
225
|
+
id: q.id,
|
|
226
|
+
content: q.content,
|
|
227
|
+
description: "",
|
|
228
|
+
embedding,
|
|
229
|
+
metadata: {
|
|
230
|
+
filePath: QUIRK_FILE_PREFIX + q.id,
|
|
231
|
+
startLine: 0,
|
|
232
|
+
endLine: 0,
|
|
233
|
+
language: "quirk",
|
|
234
|
+
kind: "quirk",
|
|
235
|
+
quirkType: q.quirkType,
|
|
236
|
+
tags: q.tags ?? [],
|
|
237
|
+
confidence: q.confidence,
|
|
238
|
+
lastObserved: q.lastObserved,
|
|
239
|
+
},
|
|
240
|
+
};
|
|
241
|
+
}
|
|
231
242
|
/** Remove a quirk by its ID. Throws if no quirk with the given ID exists. */
|
|
232
243
|
export async function removeQuirk(deps, id) {
|
|
233
244
|
const existing = await getQuirk(deps, id);
|
|
@@ -255,9 +266,10 @@ export async function listQuirks(deps) {
|
|
|
255
266
|
all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
|
|
256
267
|
return all;
|
|
257
268
|
}
|
|
258
|
-
// Fallback: scan the store
|
|
269
|
+
// Fallback: scan the store. Stored filePaths are workspace-absolute
|
|
270
|
+
// (chunkToRow normalizes), so match quirks on basename, not prefix.
|
|
259
271
|
const filePaths = await deps.store.getFilePaths();
|
|
260
|
-
const quirkPaths = filePaths.filter((fp) => fp.startsWith(QUIRK_FILE_PREFIX));
|
|
272
|
+
const quirkPaths = filePaths.filter((fp) => path.basename(fp).startsWith(QUIRK_FILE_PREFIX));
|
|
261
273
|
const result = [];
|
|
262
274
|
for (const fp of quirkPaths) {
|
|
263
275
|
const chunks = await deps.store.getChunksByFilePath(fp);
|
|
@@ -280,6 +292,174 @@ export async function listQuirks(deps) {
|
|
|
280
292
|
all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
|
|
281
293
|
return all;
|
|
282
294
|
}
|
|
295
|
+
/**
|
|
296
|
+
* Store-dir artifact recording when each orphaned store quirk (in the table but
|
|
297
|
+
* absent from quirks.jsonl) was FIRST observed as an orphan. Deletion requires
|
|
298
|
+
* the orphan to be seen again at least {@link QUIRK_ORPHAN_GRACE_MS} later.
|
|
299
|
+
*
|
|
300
|
+
* Two-sighting instead of a row timestamp because the table has no
|
|
301
|
+
* `lastObserved` column (ChunkRow never stored it), and because `addQuirk`
|
|
302
|
+
* writes the store BEFORE appending to quirks.jsonl — a quirk that is
|
|
303
|
+
* momentarily store-only is an in-flight add and must never be race-deleted.
|
|
304
|
+
* The file makes the grace window work across processes (plugin/CLI/watcher
|
|
305
|
+
* each run their own reconcile).
|
|
306
|
+
*/
|
|
307
|
+
const QUIRK_ORPHAN_STATE_FILE = "quirk-orphans.json";
|
|
308
|
+
const QUIRK_ORPHAN_GRACE_MS = 60 * 60 * 1000;
|
|
309
|
+
/**
|
|
310
|
+
* Reconcile `quirks.jsonl` (the source of truth for quirk memory) into the
|
|
311
|
+
* vector store and the keyword index, in both directions.
|
|
312
|
+
*
|
|
313
|
+
* Why this exists: full store rebuilds rebuild the table from **workspace
|
|
314
|
+
* files only** — `swapStoreDirectories` carries `quirks.jsonl` across the swap,
|
|
315
|
+
* but nothing ever re-embeds those entries, so every quirk added before a
|
|
316
|
+
* rebuild silently vanishes from recall while `quirk list`/`lint` (which read
|
|
317
|
+
* the jsonl) still show it. The keyword index has the same gap: it only gains
|
|
318
|
+
* quirks via in-process `addQuirk`, and any process that saves its in-memory
|
|
319
|
+
* index clobbers entries added elsewhere.
|
|
320
|
+
*
|
|
321
|
+
* Behavior:
|
|
322
|
+
* - Restores jsonl quirks missing from the store (one batched embed call).
|
|
323
|
+
* - Adds jsonl quirks missing from the keyword index (no embedding needed).
|
|
324
|
+
* - Deletes store quirk chunks whose jsonl entry is gone — only when the jsonl
|
|
325
|
+
* exists and parses non-empty (a missing/unreadable file means "unknown",
|
|
326
|
+
* never "empty") and only after the orphan was observed twice across
|
|
327
|
+
* {@link QUIRK_ORPHAN_GRACE_MS} (see QUIRK_ORPHAN_STATE_FILE — the table has
|
|
328
|
+
* no row timestamps, and addQuirk writes the store before the jsonl append).
|
|
329
|
+
* - Persists the keyword index when it changed (best-effort).
|
|
330
|
+
*
|
|
331
|
+
* Idempotent and safe under concurrent runs: `store.addChunks` dedups by id and
|
|
332
|
+
* `KeywordIndex.addChunks` overwrites by id. Never applied to memory stores
|
|
333
|
+
* (per-process lifetime, nothing on disk to drift). Failures propagate to the
|
|
334
|
+
* caller, which is expected to treat reconcile as best-effort.
|
|
335
|
+
*
|
|
336
|
+
* @param deps - Embedder, store, optional keyword index, config, store path.
|
|
337
|
+
* @returns Counts of what changed (all zero when already consistent).
|
|
338
|
+
*/
|
|
339
|
+
export async function reconcileQuirks(deps) {
|
|
340
|
+
const result = { restoredToStore: 0, addedToKeywordIndex: 0, removedOrphans: 0 };
|
|
341
|
+
if (isMemoryStore(deps.storePath))
|
|
342
|
+
return result;
|
|
343
|
+
const jsonlFile = jsonlPath(deps.storePath);
|
|
344
|
+
const jsonlExists = existsSync(jsonlFile);
|
|
345
|
+
const quirks = jsonlExists ? readJsonl(jsonlFile) : [];
|
|
346
|
+
const jsonlIds = new Set(quirks.map((q) => q.id));
|
|
347
|
+
// Exact enumeration of quirk chunks in the table. The store normalizes
|
|
348
|
+
// filePaths to absolute on write (chunkToRow → normalizeFilePath), so quirk
|
|
349
|
+
// rows read back as "<root>/quirk:<id>" — match on the basename, which is
|
|
350
|
+
// always QUIRK_FILE_PREFIX + id, and keep the stored path for lookups/deletes
|
|
351
|
+
// (deleteByFilePath normalizes its arg; passing the stored absolute path
|
|
352
|
+
// matches regardless of this process's cwd).
|
|
353
|
+
const quirkRows = [];
|
|
354
|
+
for (const p of await deps.store.getFilePaths()) {
|
|
355
|
+
const base = path.basename(p);
|
|
356
|
+
if (base.startsWith(QUIRK_FILE_PREFIX)) {
|
|
357
|
+
quirkRows.push({ filePath: p, id: base.slice(QUIRK_FILE_PREFIX.length) });
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
const storeIds = new Set(quirkRows.map((r) => r.id));
|
|
361
|
+
// 1. Restore jsonl quirks the table is missing (the rebuild-wipe direction).
|
|
362
|
+
const missing = quirks.filter((q) => !storeIds.has(q.id));
|
|
363
|
+
if (missing.length > 0) {
|
|
364
|
+
const prefix = deps.cfg.embedding.documentPrefix ?? "";
|
|
365
|
+
const embeddings = await deps.embedder.embed(missing.map((q) => prefix + q.content), "document");
|
|
366
|
+
if (embeddings.length !== missing.length || embeddings.some((e) => !e || e.length === 0)) {
|
|
367
|
+
throw new Error("Embedding returned empty vector(s) while reconciling quirks");
|
|
368
|
+
}
|
|
369
|
+
// Single batched write = single transaction; addChunks dedups by id, so a
|
|
370
|
+
// concurrent reconcile adding the same quirks converges instead of duplicating.
|
|
371
|
+
await deps.store.addChunks(missing.map((q, i) => quirkChunk(q, embeddings[i])));
|
|
372
|
+
result.restoredToStore = missing.length;
|
|
373
|
+
for (const q of missing)
|
|
374
|
+
storeIds.add(q.id); // keep orphan pass below consistent
|
|
375
|
+
}
|
|
376
|
+
// 2. Ensure every jsonl quirk is in the keyword index. Lexical only — the
|
|
377
|
+
// keyword index never looks at embeddings, so this needs no provider.
|
|
378
|
+
let kiDirty = false;
|
|
379
|
+
if (deps.keywordIndex) {
|
|
380
|
+
for (const q of quirks) {
|
|
381
|
+
if (deps.keywordIndex.hasChunk(q.id))
|
|
382
|
+
continue;
|
|
383
|
+
deps.keywordIndex.addChunks([quirkChunk(q, [])]);
|
|
384
|
+
result.addedToKeywordIndex++;
|
|
385
|
+
kiDirty = true;
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
// 3. Orphan direction: store quirks whose jsonl entry is gone. Runs only when
|
|
389
|
+
// the jsonl exists AND parses to at least one quirk (a missing or wholly
|
|
390
|
+
// unreadable jsonl means "unknown", never "empty" — never mass-delete on
|
|
391
|
+
// it). Each orphan must be observed twice, spanning the grace window,
|
|
392
|
+
// before deletion — see QUIRK_ORPHAN_STATE_FILE.
|
|
393
|
+
if (jsonlExists && quirks.length > 0) {
|
|
394
|
+
const suspectsPath = path.join(deps.storePath, QUIRK_ORPHAN_STATE_FILE);
|
|
395
|
+
const suspects = (() => {
|
|
396
|
+
try {
|
|
397
|
+
const parsed = JSON.parse(readFileSync(suspectsPath, "utf-8"));
|
|
398
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
|
399
|
+
}
|
|
400
|
+
catch {
|
|
401
|
+
return {};
|
|
402
|
+
}
|
|
403
|
+
})();
|
|
404
|
+
let suspectsDirty = false;
|
|
405
|
+
const now = Date.now();
|
|
406
|
+
for (const { filePath, id } of quirkRows) {
|
|
407
|
+
if (jsonlIds.has(id)) {
|
|
408
|
+
if (id in suspects) {
|
|
409
|
+
delete suspects[id];
|
|
410
|
+
suspectsDirty = true;
|
|
411
|
+
} // regained its jsonl entry
|
|
412
|
+
continue;
|
|
413
|
+
}
|
|
414
|
+
const firstSeen = suspects[id];
|
|
415
|
+
if (firstSeen === undefined) {
|
|
416
|
+
suspects[id] = now; // first sighting — keep (may be an in-flight add)
|
|
417
|
+
suspectsDirty = true;
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
420
|
+
if (now - firstSeen < QUIRK_ORPHAN_GRACE_MS)
|
|
421
|
+
continue;
|
|
422
|
+
await deps.store.deleteByFilePath(filePath);
|
|
423
|
+
deps.keywordIndex?.removeByFilePath(filePath);
|
|
424
|
+
delete suspects[id];
|
|
425
|
+
result.removedOrphans++;
|
|
426
|
+
kiDirty = true;
|
|
427
|
+
suspectsDirty = true;
|
|
428
|
+
}
|
|
429
|
+
// Prune suspects that no longer exist in the store (jsonl regained them,
|
|
430
|
+
// a rebuild dropped them, or a previous reconcile deleted them).
|
|
431
|
+
for (const id of Object.keys(suspects)) {
|
|
432
|
+
if (!storeIds.has(id)) {
|
|
433
|
+
delete suspects[id];
|
|
434
|
+
suspectsDirty = true;
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
if (suspectsDirty) {
|
|
438
|
+
try {
|
|
439
|
+
if (Object.keys(suspects).length === 0)
|
|
440
|
+
rmSync(suspectsPath, { force: true });
|
|
441
|
+
else
|
|
442
|
+
writeFileSync(suspectsPath, JSON.stringify(suspects, null, 2));
|
|
443
|
+
}
|
|
444
|
+
catch {
|
|
445
|
+
// best-effort — worst case the grace restarts for unmarked orphans
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
// 4. Persist the keyword index so the NEXT process to load it inherits the
|
|
450
|
+
// reconciled state (this is what stops pass-end saves from clobbering
|
|
451
|
+
// quirks added by other processes). Best-effort: a failed save leaves the
|
|
452
|
+
// in-memory index correct; the next reconcile re-saves.
|
|
453
|
+
if (kiDirty && deps.keywordIndex) {
|
|
454
|
+
try {
|
|
455
|
+
await deps.keywordIndex.save(deps.storePath);
|
|
456
|
+
}
|
|
457
|
+
catch {
|
|
458
|
+
// best-effort
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
return result;
|
|
462
|
+
}
|
|
283
463
|
/** Recall quirks matching a query, with confidence re-weighting. */
|
|
284
464
|
export async function recallQuirks(deps, query, options) {
|
|
285
465
|
const topK = options?.topK ?? 10;
|
|
@@ -43,6 +43,7 @@ export declare class KeywordIndex {
|
|
|
43
43
|
close(): void;
|
|
44
44
|
clear(): void;
|
|
45
45
|
count(): number;
|
|
46
|
+
hasChunk(id: string): boolean;
|
|
46
47
|
save(storePath?: string): Promise<void>;
|
|
47
48
|
static load(storePath: string): Promise<KeywordIndex>;
|
|
48
49
|
static clearFile(storePath: string): Promise<void>;
|
|
@@ -273,6 +273,9 @@ export class KeywordIndex {
|
|
|
273
273
|
count() {
|
|
274
274
|
return this.chunkMap.size;
|
|
275
275
|
}
|
|
276
|
+
hasChunk(id) {
|
|
277
|
+
return this.chunkMap.has(id);
|
|
278
|
+
}
|
|
276
279
|
async save(storePath) {
|
|
277
280
|
const effectiveStorePath = storePath ?? this.storePath;
|
|
278
281
|
if (!effectiveStorePath)
|
package/dist/tui.js
CHANGED
|
@@ -458,6 +458,12 @@ function buildSettingCategories(cfg, ro, providers) {
|
|
|
458
458
|
type: "json",
|
|
459
459
|
currentValue: chunkingRo.nodeTypes ?? chunkingCfg.nodeTypes ?? {},
|
|
460
460
|
},
|
|
461
|
+
{
|
|
462
|
+
path: ["chunking", "parsers"],
|
|
463
|
+
label: "Parser overrides — extension → language (JSON)",
|
|
464
|
+
type: "json",
|
|
465
|
+
currentValue: chunkingRo.parsers ?? chunkingCfg.parsers ?? {},
|
|
466
|
+
},
|
|
461
467
|
],
|
|
462
468
|
},
|
|
463
469
|
{
|