@maxgfr/codeindex 2.17.1 → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -3
- package/docs/MIGRATION.md +127 -0
- package/package.json +1 -1
- package/scripts/engine.d.mts +242 -18
- package/scripts/engine.mjs +2274 -1084
- package/src/ast/extract.ts +180 -142
- package/src/coupling.ts +0 -0
- package/src/delta.ts +417 -0
- package/src/derived.ts +17 -0
- package/src/engine-cli.ts +145 -24
- package/src/engine.ts +31 -4
- package/src/extract/code.ts +4 -1
- package/src/graph.ts +0 -0
- package/src/mcp/protocol.ts +176 -0
- package/src/mcp/session.ts +287 -0
- package/src/mcp/tools.ts +541 -0
- package/src/mcp.ts +176 -668
- package/src/pool.ts +271 -0
- package/src/preload.ts +159 -0
- package/src/render/scip.ts +80 -20
- package/src/render/symbols-json.ts +3 -2
- package/src/scan.ts +142 -17
- package/src/traverse.ts +164 -0
- package/src/types.ts +1 -1
- package/src/viz.ts +91 -1
package/src/mcp.ts
CHANGED
|
@@ -1,20 +1,21 @@
|
|
|
1
1
|
// MCP (Model Context Protocol) server over stdio — hand-rolled JSON-RPC 2.0 so
|
|
2
2
|
// the engine stays zero-dependency. Newline-delimited JSON messages, protocol
|
|
3
3
|
// 2024-11-05 (compatible with later revisions' initialize handshake). Exposes
|
|
4
|
-
// the engine's
|
|
5
|
-
//
|
|
4
|
+
// the engine's indexing capabilities as MCP tools; every tool takes a `repo`
|
|
5
|
+
// path and returns text content — JSON, except repo_map, mermaid and
|
|
6
|
+
// read_memory, which return their own formats.
|
|
6
7
|
//
|
|
7
|
-
// Register in an MCP client as:
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
// Register in an MCP client as: codeindex mcp
|
|
9
|
+
// (NOT `node scripts/engine.mjs mcp`: engine.mjs is a side-effect-free library
|
|
10
|
+
// with no main-module guard — see src/engine.ts — so that command does nothing.
|
|
11
|
+
// The entrypoint is the `codeindex` bin, i.e. scripts/cli.mjs.)
|
|
12
|
+
import { readFileSync } from "node:fs";
|
|
13
|
+
import { isAbsolute, join } from "node:path";
|
|
10
14
|
import { createInterface } from "node:readline";
|
|
11
|
-
import { ENGINE_VERSION
|
|
12
|
-
import { ensureGrammars, grammarKeysForExts } from "./ast/loader.js";
|
|
13
|
-
import { buildArtifactsFromScan, type IndexArtifacts } from "./pipeline.js";
|
|
15
|
+
import { ENGINE_VERSION } from "./types.js";
|
|
14
16
|
import { renderGraphJson } from "./render/graph-json.js";
|
|
15
|
-
import { scanRepo, type RepoScan, type ScanOptions } from "./scan.js";
|
|
16
|
-
import { walk } from "./walk.js";
|
|
17
17
|
import { buildCallerIndex } from "./callers.js";
|
|
18
|
+
import { callerIndexFor } from "./derived.js";
|
|
18
19
|
import { detectWorkspaces } from "./workspaces.js";
|
|
19
20
|
import { gitChurn } from "./git.js";
|
|
20
21
|
import { grepRepo } from "./grep.js";
|
|
@@ -28,11 +29,59 @@ import { replaceSymbolBody, insertAfterSymbol, insertBeforeSymbol } from "./edit
|
|
|
28
29
|
import { writeMemory, readMemory, deleteMemory, listMemories } from "./memory.js";
|
|
29
30
|
import { searchIndex } from "./bm25.js";
|
|
30
31
|
import { checkRules, parseRules } from "./rules.js";
|
|
31
|
-
import { EMBED_VERSION, resolveEmbedModelDir
|
|
32
|
-
import { buildEmbeddingIndex
|
|
32
|
+
import { EMBED_VERSION, resolveEmbedModelDir } from "./embed/model.js";
|
|
33
|
+
import { buildEmbeddingIndex } from "./embed/index.js";
|
|
33
34
|
import { searchSemantic } from "./embed/search.js";
|
|
34
35
|
import { resolveEmbedEndpoint, buildEndpointIndex, encodeQueryViaEndpoint, probeEndpoint } from "./embed/endpoint.js";
|
|
35
|
-
import {
|
|
36
|
+
import { walk, type WalkResult } from "./walk.js";
|
|
37
|
+
import { toolsFor, OUTPUT_SCHEMAS } from "./mcp/tools.js";
|
|
38
|
+
import {
|
|
39
|
+
DEFAULT_MAX_RESPONSE_BYTES,
|
|
40
|
+
RICH_TOOLS_SINCE,
|
|
41
|
+
PROTOCOL_VERSIONS,
|
|
42
|
+
capResponse,
|
|
43
|
+
negotiateProtocol,
|
|
44
|
+
resourceLinkFor,
|
|
45
|
+
structuredContentFor,
|
|
46
|
+
validateArgs,
|
|
47
|
+
} from "./mcp/protocol.js";
|
|
48
|
+
import {
|
|
49
|
+
getArtifacts,
|
|
50
|
+
getScan,
|
|
51
|
+
getScanSummary,
|
|
52
|
+
memoizedEmbeddingIndex,
|
|
53
|
+
memoizedEmbedModel,
|
|
54
|
+
scanFingerprint,
|
|
55
|
+
sessionClear,
|
|
56
|
+
warmGrammarsForWalk,
|
|
57
|
+
type SessionScanOptions,
|
|
58
|
+
} from "./mcp/session.js";
|
|
59
|
+
|
|
60
|
+
// The public surface of this module is unchanged: everything that used to live
|
|
61
|
+
// here is re-exported, so `src/engine.ts`, the tests and any consumer importing
|
|
62
|
+
// from "./mcp.js" keep working exactly as before.
|
|
63
|
+
export { toolsFor, TOOLS, TOOL_META, OUTPUT_SCHEMAS, annotationsFor } from "./mcp/tools.js";
|
|
64
|
+
export {
|
|
65
|
+
DEFAULT_MAX_RESPONSE_BYTES,
|
|
66
|
+
PROTOCOL_VERSIONS,
|
|
67
|
+
capResponse,
|
|
68
|
+
negotiateProtocol,
|
|
69
|
+
resourceLinkFor,
|
|
70
|
+
structuredContentFor,
|
|
71
|
+
validateArgs,
|
|
72
|
+
} from "./mcp/protocol.js";
|
|
73
|
+
export {
|
|
74
|
+
getArtifacts,
|
|
75
|
+
getScan,
|
|
76
|
+
getScanSummary,
|
|
77
|
+
memoizedEmbeddingIndex,
|
|
78
|
+
memoizedEmbedModel,
|
|
79
|
+
scanFingerprint,
|
|
80
|
+
toCacheMap,
|
|
81
|
+
warmGrammarsForRepo,
|
|
82
|
+
warmGrammarsForWalk,
|
|
83
|
+
} from "./mcp/session.js";
|
|
84
|
+
export type { EmbeddingIndexCacheKey, SessionScanOptions } from "./mcp/session.js";
|
|
36
85
|
|
|
37
86
|
interface RpcRequest {
|
|
38
87
|
jsonrpc: "2.0";
|
|
@@ -41,622 +90,23 @@ interface RpcRequest {
|
|
|
41
90
|
params?: Record<string, unknown>;
|
|
42
91
|
}
|
|
43
92
|
|
|
44
|
-
const repoProp = { repo: { type: "string", description: "Absolute path to the repository root" } };
|
|
45
|
-
const scopeProps = {
|
|
46
|
-
scope: { type: "string", description: "Restrict to one directory (repo-relative)" },
|
|
47
|
-
include: { type: "array", items: { type: "string" }, description: "Include globs" },
|
|
48
|
-
exclude: { type: "array", items: { type: "string" }, description: "Exclude globs" },
|
|
49
|
-
};
|
|
50
|
-
|
|
51
|
-
const TOOLS = [
|
|
52
|
-
{
|
|
53
|
-
name: "scan_summary",
|
|
54
|
-
description:
|
|
55
|
-
"Deterministically scan a repository: file count, per-language file histogram, HEAD commit, and whether the walk was capped. Fast first look at any codebase.",
|
|
56
|
-
inputSchema: { type: "object", properties: { ...repoProp, ...scopeProps }, required: ["repo"] },
|
|
57
|
-
},
|
|
58
|
-
{
|
|
59
|
-
name: "graph",
|
|
60
|
-
description:
|
|
61
|
-
"Build the full typed cross-file link-graph (import/call/use/doc-link/mention edges, module grouping, PageRank centrality, Louvain communities, tests-map). Returns graph.json. Large on big repos — prefer scan_summary/symbols/callers for targeted questions.",
|
|
62
|
-
inputSchema: { type: "object", properties: { ...repoProp, ...scopeProps }, required: ["repo"] },
|
|
63
|
-
},
|
|
64
|
-
{
|
|
65
|
-
name: "symbols",
|
|
66
|
-
description:
|
|
67
|
-
"Where is a symbol defined and which files reference it? Returns the definition sites (file, line, kind, exported) and referencing files. Omit `name` for the full symbol index.",
|
|
68
|
-
inputSchema: {
|
|
69
|
-
type: "object",
|
|
70
|
-
properties: { ...repoProp, name: { type: "string", description: "Symbol name to look up" } },
|
|
71
|
-
required: ["repo"],
|
|
72
|
-
},
|
|
73
|
-
},
|
|
74
|
-
{
|
|
75
|
-
name: "callers",
|
|
76
|
-
description:
|
|
77
|
-
"Who calls a function? Per-symbol caller index: each defined symbol with the exact (file, line) call sites that bind to it. Omit `name` for the full index.",
|
|
78
|
-
inputSchema: {
|
|
79
|
-
type: "object",
|
|
80
|
-
properties: { ...repoProp, name: { type: "string", description: "Symbol name to look up" } },
|
|
81
|
-
required: ["repo"],
|
|
82
|
-
},
|
|
83
|
-
},
|
|
84
|
-
{
|
|
85
|
-
name: "workspaces",
|
|
86
|
-
description:
|
|
87
|
-
"Detect monorepo packages (npm/pnpm/yarn/lerna/nx/cargo/go.work/maven) with the workspace dependency graph, one cycle if present, and a topological build order.",
|
|
88
|
-
inputSchema: { type: "object", properties: { ...repoProp }, required: ["repo"] },
|
|
89
|
-
},
|
|
90
|
-
{
|
|
91
|
-
name: "churn",
|
|
92
|
-
description: "Per-file git commit counts (whole history, or since a ref) — the churn half of hotspot analysis.",
|
|
93
|
-
inputSchema: {
|
|
94
|
-
type: "object",
|
|
95
|
-
properties: { ...repoProp, since: { type: "string", description: "Only count commits after this ref" } },
|
|
96
|
-
required: ["repo"],
|
|
97
|
-
},
|
|
98
|
-
},
|
|
99
|
-
{
|
|
100
|
-
name: "symbols_overview",
|
|
101
|
-
description:
|
|
102
|
-
"All symbols declared in ONE file (name, kind, line span, exported, parent), in declaration order — the fastest way to understand a file without reading it.",
|
|
103
|
-
inputSchema: {
|
|
104
|
-
type: "object",
|
|
105
|
-
properties: { ...repoProp, file: { type: "string", description: "Repo-relative file path" } },
|
|
106
|
-
required: ["repo", "file"],
|
|
107
|
-
},
|
|
108
|
-
},
|
|
109
|
-
{
|
|
110
|
-
name: "find_symbol",
|
|
111
|
-
description:
|
|
112
|
-
"Find symbol declarations by name or name path ('Class/method' matches a method inside Class). Options: substring matching, includeBody to return the declaration's source. Exact-name matches rank first.",
|
|
113
|
-
inputSchema: {
|
|
114
|
-
type: "object",
|
|
115
|
-
properties: {
|
|
116
|
-
...repoProp,
|
|
117
|
-
namePath: { type: "string", description: "Symbol name or Parent/child path" },
|
|
118
|
-
substring: { type: "boolean" },
|
|
119
|
-
includeBody: { type: "boolean" },
|
|
120
|
-
},
|
|
121
|
-
required: ["repo", "namePath"],
|
|
122
|
-
},
|
|
123
|
-
},
|
|
124
|
-
{
|
|
125
|
-
name: "find_references",
|
|
126
|
-
description:
|
|
127
|
-
"Who references a symbol? Three labeled tiers: defs (declarations), callSites (line-precise, import-corroborated call bindings), referencingFiles (file-level identifier/doc mentions — may include homonyms). Confidence decreases across tiers; the labels let you decide what to trust.",
|
|
128
|
-
inputSchema: {
|
|
129
|
-
type: "object",
|
|
130
|
-
properties: { ...repoProp, name: { type: "string", description: "Symbol name" } },
|
|
131
|
-
required: ["repo", "name"],
|
|
132
|
-
},
|
|
133
|
-
},
|
|
134
|
-
{
|
|
135
|
-
name: "repo_map",
|
|
136
|
-
description:
|
|
137
|
-
"Token-budgeted map of the repository: the highest-PageRank files with their key exported signatures, deterministically rendered to fit `budgetTokens` (default 1024). The densest single read to understand an unfamiliar codebase.",
|
|
138
|
-
inputSchema: {
|
|
139
|
-
type: "object",
|
|
140
|
-
properties: { ...repoProp, budgetTokens: { type: "number", description: "Approximate token budget (default 1024)" } },
|
|
141
|
-
required: ["repo"],
|
|
142
|
-
},
|
|
143
|
-
},
|
|
144
|
-
{
|
|
145
|
-
name: "hotspots",
|
|
146
|
-
description:
|
|
147
|
-
"Where does work concentrate? Files ranked by git churn × size (commits × log2 lines). High-scoring files are where changes and defects cluster.",
|
|
148
|
-
inputSchema: {
|
|
149
|
-
type: "object",
|
|
150
|
-
properties: { ...repoProp, since: { type: "string", description: "Only count commits after this ref" } },
|
|
151
|
-
required: ["repo"],
|
|
152
|
-
},
|
|
153
|
-
},
|
|
154
|
-
{
|
|
155
|
-
name: "coupling",
|
|
156
|
-
description:
|
|
157
|
-
"Change coupling: pairs of files that repeatedly change in the same commits — hidden dependencies no import shows. strength 1.0 = every change to one touched the other.",
|
|
158
|
-
inputSchema: {
|
|
159
|
-
type: "object",
|
|
160
|
-
properties: { ...repoProp, since: { type: "string", description: "Only mine commits after this ref" } },
|
|
161
|
-
required: ["repo"],
|
|
162
|
-
},
|
|
163
|
-
},
|
|
164
|
-
{
|
|
165
|
-
name: "replace_symbol_body",
|
|
166
|
-
description:
|
|
167
|
-
"WRITE: replace a symbol's whole declaration with `body` (verbatim, supply full indentation). The symbol is resolved by name path ('Class/method'); ambiguity errors list the candidates — qualify with `file`. Line spans come from the AST index.",
|
|
168
|
-
inputSchema: {
|
|
169
|
-
type: "object",
|
|
170
|
-
properties: {
|
|
171
|
-
...repoProp,
|
|
172
|
-
namePath: { type: "string" },
|
|
173
|
-
body: { type: "string" },
|
|
174
|
-
file: { type: "string", description: "Disambiguate: repo-relative file containing the symbol" },
|
|
175
|
-
},
|
|
176
|
-
required: ["repo", "namePath", "body"],
|
|
177
|
-
},
|
|
178
|
-
},
|
|
179
|
-
{
|
|
180
|
-
name: "insert_after_symbol",
|
|
181
|
-
description:
|
|
182
|
-
"WRITE: insert `body` after a symbol's declaration (blank-line separation preserved for definition-like kinds). Resolved like replace_symbol_body.",
|
|
183
|
-
inputSchema: {
|
|
184
|
-
type: "object",
|
|
185
|
-
properties: { ...repoProp, namePath: { type: "string" }, body: { type: "string" }, file: { type: "string" } },
|
|
186
|
-
required: ["repo", "namePath", "body"],
|
|
187
|
-
},
|
|
188
|
-
},
|
|
189
|
-
{
|
|
190
|
-
name: "insert_before_symbol",
|
|
191
|
-
description:
|
|
192
|
-
"WRITE: insert `body` before a symbol's declaration (blank-line separation preserved). Resolved like replace_symbol_body.",
|
|
193
|
-
inputSchema: {
|
|
194
|
-
type: "object",
|
|
195
|
-
properties: { ...repoProp, namePath: { type: "string" }, body: { type: "string" }, file: { type: "string" } },
|
|
196
|
-
required: ["repo", "namePath", "body"],
|
|
197
|
-
},
|
|
198
|
-
},
|
|
199
|
-
{
|
|
200
|
-
name: "write_memory",
|
|
201
|
-
description:
|
|
202
|
-
"Persist a named markdown note under <repo>/.codeindex/memories/ (names may use topic/name form). Write small, focused notes: project map, build commands, conventions.",
|
|
203
|
-
inputSchema: {
|
|
204
|
-
type: "object",
|
|
205
|
-
properties: { ...repoProp, name: { type: "string" }, content: { type: "string" } },
|
|
206
|
-
required: ["repo", "name", "content"],
|
|
207
|
-
},
|
|
208
|
-
},
|
|
209
|
-
{
|
|
210
|
-
name: "read_memory",
|
|
211
|
-
description: "Read one persisted memory by name.",
|
|
212
|
-
inputSchema: {
|
|
213
|
-
type: "object",
|
|
214
|
-
properties: { ...repoProp, name: { type: "string" } },
|
|
215
|
-
required: ["repo", "name"],
|
|
216
|
-
},
|
|
217
|
-
},
|
|
218
|
-
{
|
|
219
|
-
name: "list_memories",
|
|
220
|
-
description: "List persisted memory names — load this first, then read individual memories on relevance.",
|
|
221
|
-
inputSchema: { type: "object", properties: { ...repoProp }, required: ["repo"] },
|
|
222
|
-
},
|
|
223
|
-
{
|
|
224
|
-
name: "delete_memory",
|
|
225
|
-
description: "Delete one persisted memory by name.",
|
|
226
|
-
inputSchema: {
|
|
227
|
-
type: "object",
|
|
228
|
-
properties: { ...repoProp, name: { type: "string" } },
|
|
229
|
-
required: ["repo", "name"],
|
|
230
|
-
},
|
|
231
|
-
},
|
|
232
|
-
{
|
|
233
|
-
name: "dead_code",
|
|
234
|
-
description:
|
|
235
|
-
"Dead-code candidates in two labeled tiers: 'unreferenced' (no call site binds AND nothing references the name) and 'uncalled' (referenced somewhere — re-export, type position — but never called). Exported symbols only; test files and entrypoint-looking files excluded as roots.",
|
|
236
|
-
inputSchema: { type: "object", properties: { ...repoProp, ...scopeProps }, required: ["repo"] },
|
|
237
|
-
},
|
|
238
|
-
{
|
|
239
|
-
name: "complexity",
|
|
240
|
-
description:
|
|
241
|
-
"Cyclomatic-complexity estimates (branch-token counting over AST line spans), most-complex first. Pass `file` for one file's symbols, omit for the repo-wide top. Combine with hotspots: the `risk` field of this tool's sibling ranks complexity × churn.",
|
|
242
|
-
inputSchema: {
|
|
243
|
-
type: "object",
|
|
244
|
-
properties: { ...repoProp, file: { type: "string" }, risk: { type: "boolean", description: "Return complexity × git-churn risk ranking instead" } },
|
|
245
|
-
required: ["repo"],
|
|
246
|
-
},
|
|
247
|
-
},
|
|
248
|
-
{
|
|
249
|
-
name: "mermaid",
|
|
250
|
-
description:
|
|
251
|
-
"Mermaid diagram of the module graph (renders inline in Claude/GitHub — no graph database). Optionally scoped to one module's neighborhood.",
|
|
252
|
-
inputSchema: {
|
|
253
|
-
type: "object",
|
|
254
|
-
properties: { ...repoProp, module: { type: "string", description: "Module slug to focus on" } },
|
|
255
|
-
required: ["repo"],
|
|
256
|
-
},
|
|
257
|
-
},
|
|
258
|
-
{
|
|
259
|
-
name: "grep",
|
|
260
|
-
description:
|
|
261
|
-
"Search file contents (ripgrep when available, deterministic JS fallback otherwise). Returns sorted (file, line, text) hits.",
|
|
262
|
-
inputSchema: {
|
|
263
|
-
type: "object",
|
|
264
|
-
properties: {
|
|
265
|
-
...repoProp,
|
|
266
|
-
pattern: { type: "string", description: "Regular expression to search for" },
|
|
267
|
-
globs: { type: "array", items: { type: "string" }, description: "Restrict to matching paths" },
|
|
268
|
-
ignoreCase: { type: "boolean" },
|
|
269
|
-
maxHits: { type: "number" },
|
|
270
|
-
},
|
|
271
|
-
required: ["repo", "pattern"],
|
|
272
|
-
},
|
|
273
|
-
},
|
|
274
|
-
{
|
|
275
|
-
name: "search",
|
|
276
|
-
description:
|
|
277
|
-
'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings by default — deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols. Query terms with zero document frequency get a deterministic trigram-fuzzy fallback (typo-tolerant) unless `fuzzy: false`. Set `semantic: true` to RRF-fuse an embedding tier (HTTP endpoint, else a local static model) with lexical — the response then wraps the ranked list as `{ results, tier, degradedReason? }`, `tier` being "endpoint"/"static" when fusion happened or "lexical" (with `degradedReason`) when it did not (see embed_status). Without `semantic`, the response is the bare ranked array, unchanged.',
|
|
278
|
-
inputSchema: {
|
|
279
|
-
type: "object",
|
|
280
|
-
properties: {
|
|
281
|
-
...repoProp,
|
|
282
|
-
...scopeProps,
|
|
283
|
-
query: { type: "string", description: "Natural-language or identifier query" },
|
|
284
|
-
limit: { type: "number", description: "Max results (default 20)" },
|
|
285
|
-
fuzzy: {
|
|
286
|
-
type: "boolean",
|
|
287
|
-
description:
|
|
288
|
-
"Trigram fuzzy fallback for query terms with zero document frequency (default true)",
|
|
289
|
-
},
|
|
290
|
-
semantic: {
|
|
291
|
-
type: "boolean",
|
|
292
|
-
description:
|
|
293
|
-
'RRF-fuse an embedding tier with lexical (default false). Precedence: the HTTP endpoint (CODEINDEX_EMBED_ENDPOINT) if set, else a local static model. The response reports the effective tier as a top-level `tier` field ("endpoint"/"static" on success, "lexical" plus `degradedReason` when neither is available/reachable) instead of degrading silently — see embed_status.',
|
|
294
|
-
},
|
|
295
|
-
},
|
|
296
|
-
required: ["repo", "query"],
|
|
297
|
-
},
|
|
298
|
-
},
|
|
299
|
-
{
|
|
300
|
-
name: "embed_status",
|
|
301
|
-
description:
|
|
302
|
-
"Report the embedding tier: the effective mode (none/static/endpoint; endpoint > static model), the resolved model (opt-in, never shipped in the package) with its modelId/dim, EMBED_VERSION, and the configured HTTP endpoint with its reachability. Use to check whether `search` with semantic:true will fuse embeddings or degrade to lexical.",
|
|
303
|
-
inputSchema: { type: "object", properties: { ...repoProp }, required: ["repo"] },
|
|
304
|
-
},
|
|
305
|
-
{
|
|
306
|
-
name: "check_rules",
|
|
307
|
-
description:
|
|
308
|
-
'Validate dependency-cruiser-style architecture rules against the link-graph. Rules (inline JSON array): forbidden edges {name, from, to, kind?, severity?, comment?} with glob paths, plus builtins {name, builtin: "cycles"|"orphans"} (module-level import cycles; edge-less code files). Returns deterministic violations with severity error|warn — a CI gate.',
|
|
309
|
-
inputSchema: {
|
|
310
|
-
type: "object",
|
|
311
|
-
properties: {
|
|
312
|
-
...repoProp,
|
|
313
|
-
...scopeProps,
|
|
314
|
-
rules: { type: "array", description: "Rules array (inline JSON — see description)" },
|
|
315
|
-
},
|
|
316
|
-
required: ["repo", "rules"],
|
|
317
|
-
},
|
|
318
|
-
},
|
|
319
|
-
] as const;
|
|
320
|
-
|
|
321
93
|
function str(v: unknown): string | undefined {
|
|
322
94
|
return typeof v === "string" && v ? v : undefined;
|
|
323
95
|
}
|
|
324
96
|
function strArray(v: unknown): string[] | undefined {
|
|
325
97
|
return Array.isArray(v) && v.every((x) => typeof x === "string") && v.length ? (v as string[]) : undefined;
|
|
326
98
|
}
|
|
99
|
+
// A positive numeric argument. Also accepts the numeric STRING a JSON-Schema-less
|
|
100
|
+
// client may send: `"50"` used to fall through to the default in silence, which
|
|
101
|
+
// reads to the caller as the option being ignored.
|
|
102
|
+
function num(v: unknown): number | undefined {
|
|
103
|
+
const n = typeof v === "number" ? v : typeof v === "string" && v.trim() !== "" ? Number(v) : NaN;
|
|
104
|
+
return Number.isFinite(n) && n > 0 ? n : undefined;
|
|
105
|
+
}
|
|
327
106
|
function errMessage(e: unknown): string {
|
|
328
107
|
return e instanceof Error ? e.message : String(e);
|
|
329
108
|
}
|
|
330
109
|
|
|
331
|
-
// --- embedding index memoization --------------------------------------------
|
|
332
|
-
// The MCP server process is long-lived, but every `search` call used to redo
|
|
333
|
-
// the FULL corpus embedding build — N `buildEndpointIndex` HTTP round-trips,
|
|
334
|
-
// or a full `buildEmbeddingIndex` re-encode pass — even when nothing in the
|
|
335
|
-
// repo changed between requests. Memoize the last build behind a fingerprint
|
|
336
|
-
// of the scan contents plus the tier's identity, so an unchanged repo reuses
|
|
337
|
-
// the cached index and any file add/edit/remove (or a switch of endpoint/model)
|
|
338
|
-
// still rebuilds. RepoScan carries no fingerprint of its own (checked
|
|
339
|
-
// scan.ts/types.ts) — every FileRecord already carries a content hash, so
|
|
340
|
-
// hashing the (rel, hash) pairs is the staleness oracle scan.ts itself uses.
|
|
341
|
-
export function scanFingerprint(scan: RepoScan): string {
|
|
342
|
-
return sha1(scan.files.map((f) => `${f.rel}:${f.hash}`).join("\n"));
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
export interface EmbeddingIndexCacheKey {
|
|
346
|
-
mode: "endpoint" | "static";
|
|
347
|
-
// Distinguishes cache entries across configs sharing the same scan: the
|
|
348
|
-
// endpoint URL, or the model dir + modelId for the static tier.
|
|
349
|
-
identity: string;
|
|
350
|
-
scan: RepoScan;
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
// A SINGLE entry — never an unbounded map — holding the most recent build.
|
|
354
|
-
let embeddingIndexCache: { key: string; index: EmbeddingIndex } | undefined;
|
|
355
|
-
|
|
356
|
-
// Reuse the cached index when (mode, identity, scanFingerprint) matches the
|
|
357
|
-
// last build; otherwise call `build` and cache its result. A failed build is
|
|
358
|
-
// NEVER cached (matches today's per-call error behavior: the next request
|
|
359
|
-
// retries from scratch, and a still-valid previous entry — under a different
|
|
360
|
-
// key — is left untouched).
|
|
361
|
-
export async function memoizedEmbeddingIndex(
|
|
362
|
-
key: EmbeddingIndexCacheKey,
|
|
363
|
-
build: () => Promise<EmbeddingIndex> | EmbeddingIndex,
|
|
364
|
-
): Promise<EmbeddingIndex> {
|
|
365
|
-
const cacheKey = `${key.mode}:${key.identity}:${scanFingerprint(key.scan)}`;
|
|
366
|
-
if (embeddingIndexCache && embeddingIndexCache.key === cacheKey) return embeddingIndexCache.index;
|
|
367
|
-
const index = await build();
|
|
368
|
-
embeddingIndexCache = { key: cacheKey, index };
|
|
369
|
-
return index;
|
|
370
|
-
}
|
|
371
|
-
|
|
372
|
-
// A SINGLE entry — never an unbounded map — holding the most recent parse.
|
|
373
|
-
let embedModelCache: { key: string; model: StaticEmbedModel } | undefined;
|
|
374
|
-
|
|
375
|
-
// model.json is 10-30 MB with the real asset; reading + JSON.parsing it on
|
|
376
|
-
// EVERY request dominates static-tier latency, so the parsed model is memoized
|
|
377
|
-
// across requests. One statSync per request keys the cache on
|
|
378
|
-
// (dir, mtimeMs, size) so an in-place re-pull invalidates on the next call.
|
|
379
|
-
// Same discipline as memoizedEmbeddingIndex: a failed load is NEVER cached —
|
|
380
|
-
// the throw propagates and the cache is left as it was, so the next request
|
|
381
|
-
// retries from scratch. A missing model.json returns undefined (the
|
|
382
|
-
// not-present case, exactly like loadEmbedModel).
|
|
383
|
-
export function memoizedEmbedModel(modelDir: string): StaticEmbedModel | undefined {
|
|
384
|
-
let stat;
|
|
385
|
-
try {
|
|
386
|
-
stat = statSync(join(modelDir, "model.json"));
|
|
387
|
-
} catch {
|
|
388
|
-
return undefined;
|
|
389
|
-
}
|
|
390
|
-
const key = `${modelDir}:${stat.mtimeMs}:${stat.size}`;
|
|
391
|
-
if (embedModelCache && embedModelCache.key === key) return embedModelCache.model;
|
|
392
|
-
const model = loadEmbedModel(modelDir);
|
|
393
|
-
if (model) embedModelCache = { key, model };
|
|
394
|
-
return model;
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
// --- session-level scan + artifacts memoization ------------------------------
|
|
398
|
-
// Same single-entry discipline as the embedding caches above: the MCP server
|
|
399
|
-
// process is long-lived, but every tool call used to redo a FULL scanRepo walk
|
|
400
|
-
// + read + hash + extraction pass (and, for graph-shaped tools, the whole
|
|
401
|
-
// pipeline) even when nothing in the repo changed between requests. Cache the
|
|
402
|
-
// last (repo, scan-opts) scan and feed its records back to scanRepo as `cache`
|
|
403
|
-
// on the next call — scan.ts's EXISTING stat-fastpath + exact-hash machinery
|
|
404
|
-
// is the freshness oracle, so a cache hit costs one walk + per-file stats, not
|
|
405
|
-
// reads. When the oracle proves the content unchanged the SAME RepoScan object
|
|
406
|
-
// is returned, which keeps the per-scan WeakMap of derived structures
|
|
407
|
-
// (src/derived.ts) warm across requests. Artifacts are memoized on scan object
|
|
408
|
-
// identity. Rendered strings are NEVER memoized — a big repo's graph.json runs
|
|
409
|
-
// tens of MB, so renders stay per-call while the expensive structures behind
|
|
410
|
-
// them are reused.
|
|
411
|
-
//
|
|
412
|
-
// Determinism: reused records come from scan.ts's own reuse paths (stat
|
|
413
|
-
// fastpath / exact content-hash match), which produce records value-identical
|
|
414
|
-
// to a from-scratch scan — artifacts stay byte-identical; only repeated work
|
|
415
|
-
// disappears.
|
|
416
|
-
|
|
417
|
-
// The scan options a session entry is keyed on. `cache`/`precomputedWalk` are
|
|
418
|
-
// excluded from the contract: the session cache OWNS the cache it feeds back,
|
|
419
|
-
// and a caller-supplied stale walk would desynchronize the freshness oracle.
|
|
420
|
-
export type SessionScanOptions = Omit<ScanOptions, "cache" | "precomputedWalk">;
|
|
421
|
-
|
|
422
|
-
type SessionCacheEntry = { hash: string; record: FileRecord; size?: number; mtimeMs?: number };
|
|
423
|
-
type SessionCacheMap = Map<string, SessionCacheEntry>;
|
|
424
|
-
|
|
425
|
-
// A SINGLE entry — never an unbounded map — holding the most recent scan.
|
|
426
|
-
let sessionCache:
|
|
427
|
-
| { key: string; scan: RepoScan; cacheMap: SessionCacheMap; arts?: IndexArtifacts }
|
|
428
|
-
| undefined;
|
|
429
|
-
|
|
430
|
-
// Fixed property order (and JSON.stringify dropping undefined) keeps the key
|
|
431
|
-
// deterministic regardless of how the caller assembled the options object.
|
|
432
|
-
function sessionKey(repo: string, opts: SessionScanOptions): string {
|
|
433
|
-
return (
|
|
434
|
-
repo +
|
|
435
|
-
"\0" +
|
|
436
|
-
JSON.stringify({
|
|
437
|
-
scope: opts.scope,
|
|
438
|
-
include: opts.include,
|
|
439
|
-
exclude: opts.exclude,
|
|
440
|
-
gitignore: opts.gitignore,
|
|
441
|
-
ignoreDirs: opts.ignoreDirs,
|
|
442
|
-
maxBytes: opts.maxBytes,
|
|
443
|
-
maxFiles: opts.maxFiles,
|
|
444
|
-
maxCallsPerFile: opts.maxCallsPerFile,
|
|
445
|
-
out: opts.out,
|
|
446
|
-
fullHash: opts.fullHash,
|
|
447
|
-
})
|
|
448
|
-
);
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
// A scan re-expressed as the `ScanOptions.cache` shape (the exact map the CLI
|
|
452
|
-
// persists as cache.json): rel → (hash, record, size, mtimeMs), so the next
|
|
453
|
-
// scanRepo can take the stat fastpath / hash-match reuse paths against it.
|
|
454
|
-
// Exported for tests.
|
|
455
|
-
export function toCacheMap(scan: RepoScan): SessionCacheMap {
|
|
456
|
-
const m: SessionCacheMap = new Map();
|
|
457
|
-
for (const f of scan.files) m.set(f.rel, { hash: f.hash, record: f, size: f.size, mtimeMs: scan.mtimes.get(f.rel) });
|
|
458
|
-
return m;
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
// --- persisted-index preload -------------------------------------------------
|
|
462
|
-
// On the FIRST tool call for a repo, a committed .codeindex/ index (written by
|
|
463
|
-
// `codeindex index`) lets the session skip work TWO ways. cache.json seeds the
|
|
464
|
-
// session scan, so every unchanged file takes scan.ts's stat fastpath instead
|
|
465
|
-
// of a read + hash + extraction; and — only when the T4 freshness guard holds —
|
|
466
|
-
// the persisted graph.json/symbols.json are deserialized straight into the
|
|
467
|
-
// session, so the first graph/symbols/mermaid/repo_map/check_rules call skips
|
|
468
|
-
// the whole downstream pipeline (buildArtifactsFromScan). Both are pure
|
|
469
|
-
// optimizations: the seeded scan's reused records are value-identical to a cold
|
|
470
|
-
// scan's (T3/T4 determinism), and the guard is the SAME oracle the CLI's index
|
|
471
|
-
// fastpath uses to prove the on-disk artifacts equal a fresh build here. Absent,
|
|
472
|
-
// stale, corrupt, or any version/commit/sha mismatch → every step falls back to
|
|
473
|
-
// today's cold path EXACTLY (a fresh scanRepo / buildArtifactsFromScan), never a
|
|
474
|
-
// throw.
|
|
475
|
-
|
|
476
|
-
// ADDITIVE cache.json meta describing the artifacts a prior `index` run wrote
|
|
477
|
-
// (see engine-cli.ts's CacheMeta). Old caches lacking these keys simply never
|
|
478
|
-
// pass the guard below — their per-file records are still reused to seed the
|
|
479
|
-
// scan. Only the graph/symbols shas matter here; the embed sidecar has its own
|
|
480
|
-
// memoization path.
|
|
481
|
-
interface PersistedMeta {
|
|
482
|
-
engineVersion?: string;
|
|
483
|
-
commit?: string;
|
|
484
|
-
graphSha1?: string;
|
|
485
|
-
symbolsSha1?: string;
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
// Read <repo>/.codeindex/cache.json into the (cacheMap, meta) the preload needs.
|
|
489
|
-
// Per-file records are reusable ONLY when (schemaVersion, extractorVersion)
|
|
490
|
-
// match this engine — the exact gate the CLI applies before trusting a cache —
|
|
491
|
-
// otherwise the whole cache is discarded (cold scan). Any read/parse failure (no
|
|
492
|
-
// index yet, unreadable, malformed) returns undefined: the cold path.
|
|
493
|
-
function readPersistedIndex(repo: string): { cacheMap: SessionCacheMap; meta: PersistedMeta } | undefined {
|
|
494
|
-
let parsed:
|
|
495
|
-
| ({ schemaVersion?: number; extractorVersion?: number; files?: Record<string, SessionCacheEntry> } & PersistedMeta)
|
|
496
|
-
| undefined;
|
|
497
|
-
try {
|
|
498
|
-
parsed = JSON.parse(readFileSync(join(repo, ".codeindex", "cache.json"), "utf8")) as typeof parsed;
|
|
499
|
-
} catch {
|
|
500
|
-
return undefined;
|
|
501
|
-
}
|
|
502
|
-
if (!parsed || parsed.schemaVersion !== SCHEMA_VERSION || parsed.extractorVersion !== EXTRACTOR_VERSION || !parsed.files) {
|
|
503
|
-
return undefined;
|
|
504
|
-
}
|
|
505
|
-
const cacheMap: SessionCacheMap = new Map(Object.entries(parsed.files));
|
|
506
|
-
const meta: PersistedMeta = {
|
|
507
|
-
engineVersion: parsed.engineVersion,
|
|
508
|
-
commit: parsed.commit,
|
|
509
|
-
graphSha1: parsed.graphSha1,
|
|
510
|
-
symbolsSha1: parsed.symbolsSha1,
|
|
511
|
-
};
|
|
512
|
-
return { cacheMap, meta };
|
|
513
|
-
}
|
|
514
|
-
|
|
515
|
-
// The T4 freshness guard, applied to a session scan seeded from cache.json:
|
|
516
|
-
// contentUnchanged proves this scan's records are the ones that built the
|
|
517
|
-
// on-disk artifacts; engineVersion pins the version stamp graph.json embeds and
|
|
518
|
-
// commit the HEAD it embeds; the sha checks prove the on-disk bytes ARE that
|
|
519
|
-
// build's output. All true ⇒ graph.json/symbols.json are byte-equal to
|
|
520
|
-
// buildArtifactsFromScan(scan) run here, so deserialize them instead of
|
|
521
|
-
// rebuilding. Graph/SymbolIndex are pure JSON POJOs (no Map/Set/typed fields),
|
|
522
|
-
// so JSON.parse is a lossless round-trip — a schemaVersion assert is the only
|
|
523
|
-
// reconstruction needed (see the round-trip test). ANY failure — a stale scan,
|
|
524
|
-
// a version/commit/sha mismatch, a missing/corrupt/partial artifact, an
|
|
525
|
-
// unexpected schemaVersion — returns undefined so the caller rebuilds. NEVER
|
|
526
|
-
// throws (a corrupt artifact must degrade, not crash the session).
|
|
527
|
-
function preloadArtifacts(repo: string, scan: RepoScan, meta: PersistedMeta): IndexArtifacts | undefined {
|
|
528
|
-
if (
|
|
529
|
-
!scan.contentUnchanged ||
|
|
530
|
-
meta.engineVersion !== ENGINE_VERSION ||
|
|
531
|
-
meta.commit !== scan.commit ||
|
|
532
|
-
meta.graphSha1 === undefined ||
|
|
533
|
-
meta.symbolsSha1 === undefined
|
|
534
|
-
) {
|
|
535
|
-
return undefined;
|
|
536
|
-
}
|
|
537
|
-
const dir = join(repo, ".codeindex");
|
|
538
|
-
let graphBytes: Buffer;
|
|
539
|
-
let symbolsBytes: Buffer;
|
|
540
|
-
try {
|
|
541
|
-
graphBytes = readFileSync(join(dir, "graph.json"));
|
|
542
|
-
symbolsBytes = readFileSync(join(dir, "symbols.json"));
|
|
543
|
-
} catch {
|
|
544
|
-
return undefined; // a sha'd artifact went missing since cache.json — rebuild
|
|
545
|
-
}
|
|
546
|
-
// sha over the raw bytes; sha1(string) hashes the same UTF-8 bytes writeFileSync
|
|
547
|
-
// put on disk, so this equals the meta sha the CLI computed over the render.
|
|
548
|
-
if (sha1(graphBytes) !== meta.graphSha1 || sha1(symbolsBytes) !== meta.symbolsSha1) {
|
|
549
|
-
return undefined; // tampered / partial / corrupt on-disk bytes — rebuild
|
|
550
|
-
}
|
|
551
|
-
try {
|
|
552
|
-
const graph = JSON.parse(graphBytes.toString("utf8")) as Graph;
|
|
553
|
-
const symbols = JSON.parse(symbolsBytes.toString("utf8")) as SymbolIndex;
|
|
554
|
-
if (graph.schemaVersion !== SCHEMA_VERSION || symbols.schemaVersion !== SCHEMA_VERSION) return undefined;
|
|
555
|
-
return { scan, graph, symbols };
|
|
556
|
-
} catch {
|
|
557
|
-
// Unreachable once the shas matched (the bytes are valid JSON this engine
|
|
558
|
-
// wrote), but the contract is "never throw" — degrade to a rebuild.
|
|
559
|
-
return undefined;
|
|
560
|
-
}
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
// First-touch preload: seed the session scan from cache.json and, when the guard
|
|
564
|
-
// holds, the artifacts from graph.json/symbols.json. undefined ⇒ no persisted
|
|
565
|
-
// index ⇒ the caller takes the cold scanRepo path unchanged.
|
|
566
|
-
function preloadSession(
|
|
567
|
-
repo: string,
|
|
568
|
-
opts: SessionScanOptions,
|
|
569
|
-
): { scan: RepoScan; cacheMap: SessionCacheMap; arts?: IndexArtifacts } | undefined {
|
|
570
|
-
const persisted = readPersistedIndex(repo);
|
|
571
|
-
if (!persisted) return undefined;
|
|
572
|
-
// Seed the scan from the persisted records — scan.ts's stat fastpath + exact
|
|
573
|
-
// content-hash reuse make this value-identical to a cold scan (T3/T4), only
|
|
574
|
-
// cheaper, and it computes the contentUnchanged the artifact guard reads. When
|
|
575
|
-
// the on-disk content drifted from cache.json, changed files are re-read/
|
|
576
|
-
// extracted here exactly as a cold scan would, so the scan stays correct and
|
|
577
|
-
// the guard simply fails (arts undefined → rebuild on demand).
|
|
578
|
-
const scan = scanRepo(repo, { ...opts, cache: persisted.cacheMap });
|
|
579
|
-
const arts = preloadArtifacts(repo, scan, persisted.meta);
|
|
580
|
-
return { scan, cacheMap: toCacheMap(scan), arts };
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
// The memoizing replacement for scanRepo inside callTool. Exported for tests.
|
|
584
|
-
export function getScan(repo: string, opts: SessionScanOptions = {}): RepoScan {
|
|
585
|
-
const key = sessionKey(repo, opts);
|
|
586
|
-
if (sessionCache && sessionCache.key === key) {
|
|
587
|
-
const fresh = scanRepo(repo, { ...opts, cache: sessionCache.cacheMap });
|
|
588
|
-
if (fresh.contentUnchanged) {
|
|
589
|
-
// Content proven identical → return the SAME object (object identity is
|
|
590
|
-
// what keeps derived.ts's WeakMap and the memoized artifacts warm). A
|
|
591
|
-
// stat-only drift (e.g. a bare touch) still refreshes the cache map so
|
|
592
|
-
// the next call's stat fastpath keys on the new (size, mtimeMs).
|
|
593
|
-
if (fresh.cacheDirty) sessionCache.cacheMap = toCacheMap(fresh);
|
|
594
|
-
// `commit` (headCommit(root)) is NOT part of the stat/hash freshness
|
|
595
|
-
// oracle: a git HEAD move that leaves the worktree untouched — commit /
|
|
596
|
-
// commit --amend / reset --soft / checkout to an identical-tree branch —
|
|
597
|
-
// changes headCommit without altering any file's size or mtime, so
|
|
598
|
-
// contentUnchanged stays true while the cached scan's commit went stale.
|
|
599
|
-
// `fresh` recomputed it just now (exactly what a cold process reports), so
|
|
600
|
-
// sync it onto the returned object; otherwise scan_summary would emit the
|
|
601
|
-
// OLD commit a from-scratch scanRepo never would. Mutate the SAME object
|
|
602
|
-
// rather than clone — cloning would forfeit the identity the artifacts and
|
|
603
|
-
// derived.ts WeakMap key on. Safe: no artifact carries commit (graph /
|
|
604
|
-
// symbols render byte-identically regardless), so nothing memoized here
|
|
605
|
-
// depends on this field.
|
|
606
|
-
if (sessionCache.scan.commit !== fresh.commit) sessionCache.scan.commit = fresh.commit;
|
|
607
|
-
return sessionCache.scan;
|
|
608
|
-
}
|
|
609
|
-
sessionCache = { key, scan: fresh, cacheMap: toCacheMap(fresh) };
|
|
610
|
-
return fresh;
|
|
611
|
-
}
|
|
612
|
-
// First touch of this (repo, opts): try the persisted-index preload before a
|
|
613
|
-
// cold scan. A present, version-compatible .codeindex/cache.json seeds the
|
|
614
|
-
// scan (and, when the guard holds, the artifacts); absent it, fall through to
|
|
615
|
-
// the cold path EXACTLY as before.
|
|
616
|
-
const preloaded = preloadSession(repo, opts);
|
|
617
|
-
if (preloaded) {
|
|
618
|
-
sessionCache = { key, scan: preloaded.scan, cacheMap: preloaded.cacheMap, arts: preloaded.arts };
|
|
619
|
-
return preloaded.scan;
|
|
620
|
-
}
|
|
621
|
-
const scan = scanRepo(repo, opts);
|
|
622
|
-
sessionCache = { key, scan, cacheMap: toCacheMap(scan) };
|
|
623
|
-
return scan;
|
|
624
|
-
}
|
|
625
|
-
|
|
626
|
-
// Lazy pipeline memoized on scan OBJECT IDENTITY: graph-shaped tools reuse the
|
|
627
|
-
// artifacts exactly as long as getScan keeps returning the same scan object.
|
|
628
|
-
// Exported for tests.
|
|
629
|
-
export function getArtifacts(repo: string, opts: SessionScanOptions = {}): IndexArtifacts {
|
|
630
|
-
const scan = getScan(repo, opts);
|
|
631
|
-
if (sessionCache && sessionCache.scan === scan) {
|
|
632
|
-
return (sessionCache.arts ??= buildArtifactsFromScan(scan, opts));
|
|
633
|
-
}
|
|
634
|
-
// Defensive fallback (getScan always leaves sessionCache holding `scan`).
|
|
635
|
-
return buildArtifactsFromScan(scan, opts);
|
|
636
|
-
}
|
|
637
|
-
|
|
638
|
-
// Warm the grammars for the languages CURRENTLY present in `repo`, re-derived on
|
|
639
|
-
// EVERY scan-needing call — never frozen on first touch. The server no longer
|
|
640
|
-
// warms every committed grammar at startup; most sessions touch one repo and a
|
|
641
|
-
// handful of languages, so each scan-needing tool warms the walk-derived set
|
|
642
|
-
// itself. It MUST re-derive per call because the session cache (getScan) is built
|
|
643
|
-
// to pick up mid-session file adds/edits/removes: a language whose first file
|
|
644
|
-
// appears only AFTER the initial scan-needing call must still get its grammar
|
|
645
|
-
// warmed, or that file falls to the regex tier and its symbols diverge from a
|
|
646
|
-
// cold build on the identical on-disk state — a byte-identity break. (A per-
|
|
647
|
-
// repo-path memo froze the grammar set at first touch and silently missed
|
|
648
|
-
// exactly this case.) ensureGrammars is idempotent and near-free once a grammar
|
|
649
|
-
// is loaded — it warms only newly-seen keys — so the sole repeated cost is the
|
|
650
|
-
// walk; the wasm for a given language loads at most once. Determinism: the walk's
|
|
651
|
-
// extension set is a superset of what scanRepo keeps (scope/include/exclude only
|
|
652
|
-
// filter further), so every extracted file has its grammar loaded; Language.load
|
|
653
|
-
// calls are independent, so warming fewer grammars cannot alter the parse of a
|
|
654
|
-
// loaded one.
|
|
655
|
-
export async function warmGrammarsForRepo(repo: string): Promise<void> {
|
|
656
|
-
const { files } = walk(repo, {});
|
|
657
|
-
await ensureGrammars(grammarKeysForExts(files.map((f) => f.ext)));
|
|
658
|
-
}
|
|
659
|
-
|
|
660
110
|
// Tools that never scan the file tree (git/grep/memory/embed-status only) — they
|
|
661
111
|
// must not trigger a grammar warm. Every other tool is scan-needing and warms
|
|
662
112
|
// the repo's grammars first; defaulting to "warm" keeps a newly added scan tool
|
|
@@ -665,6 +115,10 @@ const SCANLESS_TOOLS = new Set([
|
|
|
665
115
|
"workspaces", "churn", "coupling", "grep",
|
|
666
116
|
"write_memory", "read_memory", "list_memories", "delete_memory",
|
|
667
117
|
"embed_status",
|
|
118
|
+
// scan_summary counts and classifies by path only — it never parses, so the
|
|
119
|
+
// grammar warm (a whole extra walk) would be pure overhead. When a scan is
|
|
120
|
+
// already cached getScanSummary reuses it, warm grammars included.
|
|
121
|
+
"scan_summary",
|
|
668
122
|
]);
|
|
669
123
|
|
|
670
124
|
async function callTool(name: string, args: Record<string, unknown>, defaultRepo?: string): Promise<string> {
|
|
@@ -676,21 +130,26 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
|
|
|
676
130
|
const scanOpts = { scope: str(args.scope), include: strArray(args.include), exclude: strArray(args.exclude) };
|
|
677
131
|
// Scan-needing tools warm the present-language grammars (re-derived per call)
|
|
678
132
|
// before any scan so extraction takes the AST tier; scan-less tools skip it.
|
|
679
|
-
|
|
133
|
+
// ONE walk feeds both the warm and the scan below — see warmGrammarsForWalk.
|
|
134
|
+
let walked: WalkResult | undefined;
|
|
135
|
+
if (!SCANLESS_TOOLS.has(name)) {
|
|
136
|
+
walked = walk(repo, {});
|
|
137
|
+
await warmGrammarsForWalk(walked);
|
|
138
|
+
}
|
|
680
139
|
|
|
681
140
|
if (name === "scan_summary") {
|
|
682
|
-
const
|
|
141
|
+
const s = getScanSummary(repo, scanOpts, walked);
|
|
683
142
|
return JSON.stringify(
|
|
684
|
-
{ engineVersion: ENGINE_VERSION, commit:
|
|
143
|
+
{ engineVersion: ENGINE_VERSION, commit: s.commit, fileCount: s.fileCount, languages: s.languages, capped: s.capped },
|
|
685
144
|
null,
|
|
686
145
|
2,
|
|
687
146
|
);
|
|
688
147
|
}
|
|
689
148
|
if (name === "graph") {
|
|
690
|
-
return renderGraphJson(getArtifacts(repo, scanOpts).graph);
|
|
149
|
+
return renderGraphJson(getArtifacts(repo, scanOpts, walked).graph);
|
|
691
150
|
}
|
|
692
151
|
if (name === "symbols") {
|
|
693
|
-
const { symbols } = getArtifacts(repo, scanOpts);
|
|
152
|
+
const { symbols } = getArtifacts(repo, scanOpts, walked);
|
|
694
153
|
const lookup = str(args.name);
|
|
695
154
|
if (lookup) {
|
|
696
155
|
return JSON.stringify({ name: lookup, defs: symbols.defs[lookup] ?? [], refs: symbols.refs[lookup] ?? [] }, null, 2);
|
|
@@ -698,7 +157,13 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
|
|
|
698
157
|
return JSON.stringify(symbols, null, 2);
|
|
699
158
|
}
|
|
700
159
|
if (name === "callers") {
|
|
701
|
-
|
|
160
|
+
// callerIndexFor, not buildCallerIndex: the public builder is unmemoized, so
|
|
161
|
+
// this rebuilt the whole index on EVERY request (318ms per call on a 20k-file
|
|
162
|
+
// repo in the project's own benchmark). The memoized one is keyed on scan
|
|
163
|
+
// object identity, which the session cache preserves across calls.
|
|
164
|
+
// Recall mode is option-dependent, so it cannot use the memoized index.
|
|
165
|
+
const scan = getScan(repo, scanOpts, walked);
|
|
166
|
+
const index = args.recall === true ? buildCallerIndex(scan, undefined, { recall: true }) : callerIndexFor(scan);
|
|
702
167
|
const lookup = str(args.name);
|
|
703
168
|
if (lookup) {
|
|
704
169
|
const entry = index.get(lookup);
|
|
@@ -721,27 +186,28 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
|
|
|
721
186
|
if (name === "symbols_overview") {
|
|
722
187
|
const file = str(args.file);
|
|
723
188
|
if (!file) throw new Error("`file` is required");
|
|
724
|
-
return JSON.stringify(symbolsOverview(getScan(repo, scanOpts), file), null, 2);
|
|
189
|
+
return JSON.stringify(symbolsOverview(getScan(repo, scanOpts, walked), file), null, 2);
|
|
725
190
|
}
|
|
726
191
|
if (name === "find_symbol") {
|
|
727
192
|
const namePath = str(args.namePath);
|
|
728
193
|
if (!namePath) throw new Error("`namePath` is required");
|
|
729
|
-
const matches = findSymbol(getScan(repo, scanOpts), namePath, {
|
|
194
|
+
const matches = findSymbol(getScan(repo, scanOpts, walked), namePath, {
|
|
730
195
|
substring: args.substring === true,
|
|
731
196
|
includeBody: args.includeBody === true,
|
|
197
|
+
maxResults: num(args.maxResults),
|
|
732
198
|
});
|
|
733
199
|
return JSON.stringify(matches, null, 2);
|
|
734
200
|
}
|
|
735
201
|
if (name === "find_references") {
|
|
736
202
|
const symName = str(args.name);
|
|
737
203
|
if (!symName) throw new Error("`name` is required");
|
|
738
|
-
return JSON.stringify(findReferences(getScan(repo, scanOpts), symName), null, 2);
|
|
204
|
+
return JSON.stringify(findReferences(getScan(repo, scanOpts, walked), symName), null, 2);
|
|
739
205
|
}
|
|
740
206
|
if (name === "replace_symbol_body" || name === "insert_after_symbol" || name === "insert_before_symbol") {
|
|
741
207
|
const namePath = str(args.namePath);
|
|
742
208
|
const body = typeof args.body === "string" ? args.body : undefined;
|
|
743
209
|
if (!namePath || body === undefined) throw new Error("`namePath` and `body` are required");
|
|
744
|
-
const scan = getScan(repo, scanOpts);
|
|
210
|
+
const scan = getScan(repo, scanOpts, walked);
|
|
745
211
|
const fn = name === "replace_symbol_body" ? replaceSymbolBody : name === "insert_after_symbol" ? insertAfterSymbol : insertBeforeSymbol;
|
|
746
212
|
const result = fn(scan, namePath, body, str(args.file));
|
|
747
213
|
// A write WE just performed must not be trusted to the stat oracle: an
|
|
@@ -750,7 +216,7 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
|
|
|
750
216
|
// session entry unconditionally — the next call rescans from scratch.
|
|
751
217
|
// (write_memory needs no invalidation: .codeindex/ is excluded from the
|
|
752
218
|
// walk, so memories never enter a scan.)
|
|
753
|
-
|
|
219
|
+
sessionClear();
|
|
754
220
|
return JSON.stringify(result, null, 2);
|
|
755
221
|
}
|
|
756
222
|
if (name === "write_memory") {
|
|
@@ -775,26 +241,31 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
|
|
|
775
241
|
return JSON.stringify({ deleted: deleteMemory(repo, memName) }, null, 2);
|
|
776
242
|
}
|
|
777
243
|
if (name === "dead_code") {
|
|
778
|
-
|
|
244
|
+
const all = findDeadCode(getScan(repo, scanOpts, walked));
|
|
245
|
+
const limit = num(args.limit);
|
|
246
|
+
// Additive: without `limit` the payload is exactly what it always was.
|
|
247
|
+
if (limit === undefined || all.length <= limit) return JSON.stringify(all, null, 2);
|
|
248
|
+
return JSON.stringify({ total: all.length, shown: limit, truncated: true, candidates: all.slice(0, limit) }, null, 2);
|
|
779
249
|
}
|
|
780
250
|
if (name === "complexity") {
|
|
781
|
-
const scan = getScan(repo, scanOpts);
|
|
251
|
+
const scan = getScan(repo, scanOpts, walked);
|
|
782
252
|
if (args.risk === true) {
|
|
783
|
-
|
|
784
|
-
|
|
253
|
+
// `since` was accepted by the CLI's `risk` but silently dropped here.
|
|
254
|
+
const { churn, ok } = gitChurn(repo, { since: str(args.since) });
|
|
255
|
+
return JSON.stringify({ churnOk: ok, risks: riskHotspots(scan, churn, num(args.top)) }, null, 2);
|
|
785
256
|
}
|
|
786
|
-
return JSON.stringify(symbolComplexity(scan, str(args.file)), null, 2);
|
|
257
|
+
return JSON.stringify(symbolComplexity(scan, str(args.file), num(args.top)), null, 2);
|
|
787
258
|
}
|
|
788
259
|
if (name === "mermaid") {
|
|
789
|
-
const { graph } = getArtifacts(repo, scanOpts);
|
|
790
|
-
return renderMermaid(graph, { module: str(args.module) });
|
|
260
|
+
const { graph } = getArtifacts(repo, scanOpts, walked);
|
|
261
|
+
return renderMermaid(graph, { module: str(args.module), maxEdges: num(args.maxEdges) });
|
|
791
262
|
}
|
|
792
263
|
if (name === "repo_map") {
|
|
793
|
-
const { scan, graph } = getArtifacts(repo, scanOpts);
|
|
264
|
+
const { scan, graph } = getArtifacts(repo, scanOpts, walked);
|
|
794
265
|
return renderRepoMap(scan, graph, { budgetTokens: typeof args.budgetTokens === "number" ? args.budgetTokens : undefined });
|
|
795
266
|
}
|
|
796
267
|
if (name === "hotspots") {
|
|
797
|
-
const scan = getScan(repo, scanOpts);
|
|
268
|
+
const scan = getScan(repo, scanOpts, walked);
|
|
798
269
|
const { churn, ok } = gitChurn(repo, { since: str(args.since) });
|
|
799
270
|
return JSON.stringify({ churnOk: ok, hotspots: rankHotspots(scan, churn) }, null, 2);
|
|
800
271
|
}
|
|
@@ -805,8 +276,12 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
|
|
|
805
276
|
if (name === "grep") {
|
|
806
277
|
const pattern = str(args.pattern);
|
|
807
278
|
if (!pattern) throw new Error("`pattern` is required");
|
|
279
|
+
// `scope` was CLI-only: the a205c34 fix folded it into the CLI's glob list
|
|
280
|
+
// but this handler ignored scanOpts entirely.
|
|
281
|
+
const scope = str(args.scope);
|
|
282
|
+
const globs = strArray(args.globs);
|
|
808
283
|
const hits = grepRepo(repo, pattern, {
|
|
809
|
-
globs:
|
|
284
|
+
globs: scope ? [...(globs ?? []), `${scope.replace(/\/+$/, "")}/**`] : globs,
|
|
810
285
|
ignoreCase: args.ignoreCase === true,
|
|
811
286
|
maxHits: typeof args.maxHits === "number" ? args.maxHits : undefined,
|
|
812
287
|
});
|
|
@@ -815,7 +290,7 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
|
|
|
815
290
|
if (name === "search") {
|
|
816
291
|
const query = str(args.query);
|
|
817
292
|
if (!query) throw new Error("`query` is required");
|
|
818
|
-
const scan = getScan(repo, scanOpts);
|
|
293
|
+
const scan = getScan(repo, scanOpts, walked);
|
|
819
294
|
const limit = typeof args.limit === "number" ? args.limit : undefined;
|
|
820
295
|
const fuzzy = typeof args.fuzzy === "boolean" ? args.fuzzy : undefined;
|
|
821
296
|
if (args.semantic === true) {
|
|
@@ -882,33 +357,27 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
|
|
|
882
357
|
return JSON.stringify(status, null, 2);
|
|
883
358
|
}
|
|
884
359
|
if (name === "check_rules") {
|
|
885
|
-
|
|
886
|
-
|
|
360
|
+
// Inline `rules` stays the primary form; `configPath` is the CLI's --config,
|
|
361
|
+
// which had no MCP equivalent, so a repo with a committed rules file had to
|
|
362
|
+
// have it re-pasted into every call.
|
|
363
|
+
const configPath = str(args.configPath);
|
|
364
|
+
let payload: unknown = args.rules;
|
|
365
|
+
if (payload === undefined && configPath) {
|
|
366
|
+
const abs = isAbsolute(configPath) ? configPath : join(repo, configPath);
|
|
367
|
+
try {
|
|
368
|
+
payload = JSON.parse(readFileSync(abs, "utf8"));
|
|
369
|
+
} catch (e) {
|
|
370
|
+
throw new Error(`cannot read rules from ${abs}: ${errMessage(e)}`);
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
if (payload === undefined) throw new Error("`rules` (or `configPath`) is required");
|
|
374
|
+
const rules = parseRules(payload); // throws a descriptive error on a malformed payload
|
|
375
|
+
const { graph } = getArtifacts(repo, scanOpts, walked);
|
|
887
376
|
return JSON.stringify(checkRules(graph, rules), null, 2);
|
|
888
377
|
}
|
|
889
378
|
throw new Error(`unknown tool: ${name}`);
|
|
890
379
|
}
|
|
891
380
|
|
|
892
|
-
// The advertised tool list. Without a server-level repo pin this is TOOLS
|
|
893
|
-
// verbatim (byte-compat for existing consumers). With one, every tool drops
|
|
894
|
-
// `repo` from its `required` set and documents the default — so a client that
|
|
895
|
-
// omits it is spec-correct rather than relying on the server being lenient.
|
|
896
|
-
// The pin is reflected in the schema, not just honoured at call time.
|
|
897
|
-
function toolsFor(defaultRepo?: string): readonly unknown[] {
|
|
898
|
-
if (!defaultRepo) return TOOLS;
|
|
899
|
-
return TOOLS.map((t) => ({
|
|
900
|
-
...t,
|
|
901
|
-
inputSchema: {
|
|
902
|
-
...t.inputSchema,
|
|
903
|
-
properties: {
|
|
904
|
-
...t.inputSchema.properties,
|
|
905
|
-
repo: { type: "string", description: `Absolute path to the repository root (optional — defaults to ${defaultRepo})` },
|
|
906
|
-
},
|
|
907
|
-
required: (t.inputSchema.required as readonly string[]).filter((r) => r !== "repo"),
|
|
908
|
-
},
|
|
909
|
-
}));
|
|
910
|
-
}
|
|
911
|
-
|
|
912
381
|
export interface McpServerOptions {
|
|
913
382
|
// Override the serverInfo announced in the initialize response — for
|
|
914
383
|
// downstream consumers embedding this server under their own identity.
|
|
@@ -919,6 +388,9 @@ export interface McpServerOptions {
|
|
|
919
388
|
// spawn one server per workspace — `codeindex mcp --repo <dir>` — instead of
|
|
920
389
|
// requiring the agent to thread an absolute path through every single call.
|
|
921
390
|
defaultRepo?: string;
|
|
391
|
+
// Cap on a single tool response, in bytes (default DEFAULT_MAX_RESPONSE_BYTES).
|
|
392
|
+
// Responses under it are untouched; see capResponse for what happens above it.
|
|
393
|
+
maxResponseBytes?: number;
|
|
922
394
|
}
|
|
923
395
|
|
|
924
396
|
export async function runMcpServer(opts: McpServerOptions = {}): Promise<void> {
|
|
@@ -926,8 +398,13 @@ export async function runMcpServer(opts: McpServerOptions = {}): Promise<void> {
|
|
|
926
398
|
name: opts.serverInfo?.name ?? "codeindex",
|
|
927
399
|
version: opts.serverInfo?.version ?? ENGINE_VERSION,
|
|
928
400
|
};
|
|
929
|
-
//
|
|
930
|
-
|
|
401
|
+
// The negotiated protocol version, settled by `initialize` and fixed for the
|
|
402
|
+
// session. Until a client says otherwise we assume the oldest revision, so a
|
|
403
|
+
// client that skips the handshake sees exactly the pre-negotiation server.
|
|
404
|
+
let protocolVersion: string = PROTOCOL_VERSIONS[0];
|
|
405
|
+
// Rebuilt when negotiation lands: the pin cannot change mid-session, but the
|
|
406
|
+
// fields we are allowed to advertise depend on the version.
|
|
407
|
+
let tools = toolsFor(opts.defaultRepo, protocolVersion);
|
|
931
408
|
// No startup warm: each scan-needing tool warms the present-language grammars
|
|
932
409
|
// for its repo before it runs (warmGrammarsForRepo re-derives them per call),
|
|
933
410
|
// so a session that never scans — or only touches one language — loads no
|
|
@@ -959,10 +436,12 @@ export async function runMcpServer(opts: McpServerOptions = {}): Promise<void> {
|
|
|
959
436
|
|
|
960
437
|
try {
|
|
961
438
|
if (req.method === "initialize") {
|
|
439
|
+
protocolVersion = negotiateProtocol(req.params?.protocolVersion);
|
|
440
|
+
tools = toolsFor(opts.defaultRepo, protocolVersion);
|
|
962
441
|
send({
|
|
963
442
|
id: req.id,
|
|
964
443
|
result: {
|
|
965
|
-
protocolVersion
|
|
444
|
+
protocolVersion,
|
|
966
445
|
capabilities: { tools: {} },
|
|
967
446
|
serverInfo,
|
|
968
447
|
},
|
|
@@ -976,8 +455,37 @@ export async function runMcpServer(opts: McpServerOptions = {}): Promise<void> {
|
|
|
976
455
|
const name = str(params.name) ?? "";
|
|
977
456
|
const args = (params.arguments ?? {}) as Record<string, unknown>;
|
|
978
457
|
try {
|
|
979
|
-
const
|
|
980
|
-
|
|
458
|
+
const decl = (tools as { name: string; inputSchema: { properties?: Record<string, unknown> } }[]).find(
|
|
459
|
+
(t) => t.name === name,
|
|
460
|
+
);
|
|
461
|
+
const invalid = decl ? validateArgs(decl.inputSchema, args) : undefined;
|
|
462
|
+
if (invalid) throw new Error(invalid);
|
|
463
|
+
const raw = await callTool(name, args, opts.defaultRepo);
|
|
464
|
+
const repo = str(args.repo) ?? opts.defaultRepo ?? "";
|
|
465
|
+
const text = capResponse(raw, name, repo, opts.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES);
|
|
466
|
+
// A capped whole-repo response points at an artifact already on disk.
|
|
467
|
+
// From 2025-06-18 the protocol has a content type that says exactly
|
|
468
|
+
// that, so the client can fetch the bytes instead of re-asking.
|
|
469
|
+
//
|
|
470
|
+
// Gated on `text !== raw` — i.e. capResponse actually replaced the
|
|
471
|
+
// payload. Otherwise a normal 900 KB graph would be JSON.parsed on
|
|
472
|
+
// every single call just to discover it was not truncated.
|
|
473
|
+
const capped = text !== raw;
|
|
474
|
+
const link = capped && protocolVersion >= RICH_TOOLS_SINCE ? resourceLinkFor(text, name) : undefined;
|
|
475
|
+
// Typed, validatable result alongside the text block — for the tools
|
|
476
|
+
// that declare an outputSchema, and never when the guard replaced the
|
|
477
|
+
// payload (see structuredContentFor).
|
|
478
|
+
const structured =
|
|
479
|
+
protocolVersion >= RICH_TOOLS_SINCE
|
|
480
|
+
? structuredContentFor(text, capped, OUTPUT_SCHEMAS[name] !== undefined)
|
|
481
|
+
: undefined;
|
|
482
|
+
send({
|
|
483
|
+
id: req.id,
|
|
484
|
+
result: {
|
|
485
|
+
content: link ? [{ type: "text", text }, link] : [{ type: "text", text }],
|
|
486
|
+
...(structured ? { structuredContent: structured } : {}),
|
|
487
|
+
},
|
|
488
|
+
});
|
|
981
489
|
} catch (e) {
|
|
982
490
|
send({
|
|
983
491
|
id: req.id,
|