@maxgfr/codeindex 2.15.0 → 2.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -4
- package/package.json +2 -2
- package/scripts/engine.d.mts +38 -3
- package/scripts/engine.mjs +332 -105
- package/src/ast/grammars-pull.ts +95 -1
- package/src/ast/warm.ts +74 -0
- package/src/engine-cli.ts +83 -85
- package/src/engine.ts +14 -1
- package/src/mcp.ts +34 -4
- package/src/rewrite.ts +169 -0
- package/src/types.ts +1 -1
package/src/ast/grammars-pull.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { dirname, join, resolve, sep } from "node:path";
|
|
4
4
|
import { gunzipSync } from "node:zlib";
|
|
5
5
|
import { ENGINE_VERSION } from "../types.js";
|
|
@@ -176,3 +176,97 @@ export function extractGrammarsTarball(bytes: Uint8Array, destDir: string): stri
|
|
|
176
176
|
const raw = b.length >= 2 && b[0] === 0x1f && b[1] === 0x8b ? gunzipSync(b) : b;
|
|
177
177
|
return extractTarInto(raw, destDir);
|
|
178
178
|
}
|
|
179
|
+
|
|
180
|
+
// What a pull did. `status` distinguishes the three terminal states so a caller
|
|
181
|
+
// can react without parsing prose: nothing to do, wasms installed, or failed
|
|
182
|
+
// (and on failure NOTHING was written — the cache is exactly as it was).
|
|
183
|
+
export interface GrammarsPullResult {
|
|
184
|
+
ok: boolean;
|
|
185
|
+
status: "up-to-date" | "pulled" | "failed";
|
|
186
|
+
cacheDir: string;
|
|
187
|
+
/** One human-readable line; the CLI writes it to stderr, a library caller may log or drop it. */
|
|
188
|
+
message: string;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// The whole `grammars pull` mechanic — resolve target, fetch the sha256 sidecar,
|
|
192
|
+
// skip when the cache already holds that exact digest, download, then install
|
|
193
|
+
// ATOMICALLY (extract into a tmp sibling, verify the runtime wasm landed, swap
|
|
194
|
+
// by rename). Extracted from the CLI so `warmGrammars` and `codeindex grammars
|
|
195
|
+
// pull` share ONE implementation: the CLI maps the result onto stderr +
|
|
196
|
+
// process.exitCode, the warm path folds it into its own report.
|
|
197
|
+
//
|
|
198
|
+
// NEVER throws: every failure becomes `{ ok: false, status: "failed" }` with a
|
|
199
|
+
// message, because both callers must survive an offline machine — the engine
|
|
200
|
+
// simply stays on the regex tier. `onNote` receives progress lines (checksum
|
|
201
|
+
// degradation, download start) that are not the terminal message.
|
|
202
|
+
export async function pullGrammars(
|
|
203
|
+
cacheDir: string,
|
|
204
|
+
opts: { onNote?: (msg: string) => void } = {},
|
|
205
|
+
): Promise<GrammarsPullResult> {
|
|
206
|
+
const note = opts.onNote ?? (() => {});
|
|
207
|
+
const target = resolveGrammarsPullTarget();
|
|
208
|
+
let expected: string | undefined;
|
|
209
|
+
if (target.sha256Url) {
|
|
210
|
+
try {
|
|
211
|
+
expected = await fetchExpectedSha256(target.sha256Url);
|
|
212
|
+
} catch (e) {
|
|
213
|
+
note(`codeindex: could not fetch checksum (${e instanceof Error ? e.message : String(e)}) — proceeding unverified\n`);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
// Idempotent: the marker sibling records the digest of the tarball that
|
|
217
|
+
// populated cacheDir. Runtime wasm present AND marker === freshly-fetched
|
|
218
|
+
// digest ⇒ already up to date, skip the ~22 MB download.
|
|
219
|
+
const runtime = join(cacheDir, "web-tree-sitter.wasm");
|
|
220
|
+
const markerPath = join(dirname(cacheDir), `${ENGINE_VERSION}.sha256`);
|
|
221
|
+
if (existsSync(runtime) && expected && existsSync(markerPath)) {
|
|
222
|
+
let marker = "";
|
|
223
|
+
try {
|
|
224
|
+
marker = readFileSync(markerPath, "utf8").trim();
|
|
225
|
+
} catch {
|
|
226
|
+
// unreadable marker → fall through and re-pull
|
|
227
|
+
}
|
|
228
|
+
if (marker === expected) {
|
|
229
|
+
return { ok: true, status: "up-to-date", cacheDir, message: `codeindex: grammars already present at ${cacheDir} (up to date)\n` };
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
note(`codeindex: fetching grammars from ${target.url} → ${cacheDir}\n`);
|
|
233
|
+
let bytes: Uint8Array;
|
|
234
|
+
try {
|
|
235
|
+
bytes = await fetchGrammarsTarball(target.url, expected);
|
|
236
|
+
} catch (e) {
|
|
237
|
+
return {
|
|
238
|
+
ok: false,
|
|
239
|
+
status: "failed",
|
|
240
|
+
cacheDir,
|
|
241
|
+
message: `codeindex: pull failed — ${e instanceof Error ? e.message : String(e)} (nothing written)\n`,
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
let tmp: string | undefined;
|
|
245
|
+
try {
|
|
246
|
+
mkdirSync(dirname(cacheDir), { recursive: true });
|
|
247
|
+
tmp = mkdtempSync(join(dirname(cacheDir), ".grammars-tmp-"));
|
|
248
|
+
extractGrammarsTarball(bytes, tmp);
|
|
249
|
+
if (!existsSync(join(tmp, "web-tree-sitter.wasm"))) {
|
|
250
|
+
throw new Error("archive is missing web-tree-sitter.wasm");
|
|
251
|
+
}
|
|
252
|
+
if (existsSync(cacheDir)) rmSync(cacheDir, { recursive: true, force: true });
|
|
253
|
+
renameSync(tmp, cacheDir);
|
|
254
|
+
tmp = undefined;
|
|
255
|
+
if (expected) writeFileSync(markerPath, expected + "\n");
|
|
256
|
+
} catch (e) {
|
|
257
|
+
if (tmp) {
|
|
258
|
+
try {
|
|
259
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
260
|
+
} catch {
|
|
261
|
+
// best-effort cleanup
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
return {
|
|
265
|
+
ok: false,
|
|
266
|
+
status: "failed",
|
|
267
|
+
cacheDir,
|
|
268
|
+
message: `codeindex: pull failed — ${e instanceof Error ? e.message : String(e)} (nothing written)\n`,
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
return { ok: true, status: "pulled", cacheDir, message: `codeindex: grammars extracted → ${cacheDir}\n` };
|
|
272
|
+
}
|
package/src/ast/warm.ts
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { allGrammarKeys, ensureGrammars, grammarReady, resolveGrammarsTier, sharedGrammarsCacheDir } from "./loader.js";
|
|
2
|
+
import type { GrammarsTierName } from "./loader.js";
|
|
3
|
+
import { pullGrammars } from "./grammars-pull.js";
|
|
4
|
+
|
|
5
|
+
// The one-call AST warm-up every consumer needs, and which every consumer but
|
|
6
|
+
// ultraindex was missing.
|
|
7
|
+
//
|
|
8
|
+
// WHY THIS EXISTS. Extraction is AST-preferred with a regex fallback
|
|
9
|
+
// (src/extract/code.ts), but `extractAst` returns undefined unless
|
|
10
|
+
// `grammarReady(key)` — and that is only true after an `await ensureGrammars()`.
|
|
11
|
+
// `scanRepo` is deliberately synchronous, so it CANNOT warm anything itself.
|
|
12
|
+
// A consumer that never awaits the warm-up therefore runs on the regex tier
|
|
13
|
+
// forever, on every machine, no matter what the grammar cache holds — silently,
|
|
14
|
+
// since the fallback is by design invisible. That is a capability quietly left
|
|
15
|
+
// on the table, not a crash, which is exactly why it survived unnoticed.
|
|
16
|
+
//
|
|
17
|
+
// So: call this ONCE at your CLI entry, before the synchronous pipeline. It is
|
|
18
|
+
// idempotent, offline-safe, and never throws.
|
|
19
|
+
export interface WarmGrammarsResult {
|
|
20
|
+
/** Tier AFTER the warm-up (a successful pull moves "none" → "cache"). */
|
|
21
|
+
tier: GrammarsTierName;
|
|
22
|
+
/** True when at least one requested grammar is loaded ⇒ the AST tier is live. */
|
|
23
|
+
ready: boolean;
|
|
24
|
+
/** True when this call populated the shared cache over the network. */
|
|
25
|
+
pulled: boolean;
|
|
26
|
+
/** Everything written to `onNote`, in order — so a caller can persist the trail in its run artifacts. */
|
|
27
|
+
notes: string[];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface WarmGrammarsOptions {
|
|
31
|
+
/** Grammars to load. Default: every shipped grammar. Narrow it with `grammarKeysForExts` when the repo's languages are known. */
|
|
32
|
+
keys?: Iterable<string>;
|
|
33
|
+
/** Fetch the wasms into the shared cache when nothing is resolvable. Default true; `CODEINDEX_NO_GRAMMARS_PULL=1` forces false. */
|
|
34
|
+
pull?: boolean;
|
|
35
|
+
/** Prefix for the diagnostics ("ultrasec: …"). Default "codeindex". */
|
|
36
|
+
label?: string;
|
|
37
|
+
/** Where diagnostics go. Default: process.stderr. Pass a sink to keep stdout/stderr clean. */
|
|
38
|
+
onNote?: (msg: string) => void;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export async function warmGrammars(opts: WarmGrammarsOptions = {}): Promise<WarmGrammarsResult> {
|
|
42
|
+
const label = opts.label ?? "codeindex";
|
|
43
|
+
const notes: string[] = [];
|
|
44
|
+
const note = (msg: string): void => {
|
|
45
|
+
notes.push(msg);
|
|
46
|
+
if (opts.onNote) opts.onNote(msg);
|
|
47
|
+
else process.stderr.write(msg);
|
|
48
|
+
};
|
|
49
|
+
const noPull = process.env.CODEINDEX_NO_GRAMMARS_PULL;
|
|
50
|
+
const mayPull = (opts.pull ?? true) && !(noPull && noPull.trim() && noPull !== "0");
|
|
51
|
+
const keys = [...(opts.keys ?? allGrammarKeys())];
|
|
52
|
+
|
|
53
|
+
let pulled = false;
|
|
54
|
+
if (resolveGrammarsTier().tier === "none" && mayPull) {
|
|
55
|
+
note(`${label}: tree-sitter grammars not found locally — pulling them into the shared cache (once per machine)…\n`);
|
|
56
|
+
const res = await pullGrammars(sharedGrammarsCacheDir(), { onNote: note });
|
|
57
|
+
note(res.message);
|
|
58
|
+
pulled = res.ok && res.status === "pulled";
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
await ensureGrammars(keys);
|
|
62
|
+
|
|
63
|
+
const tier = resolveGrammarsTier().tier;
|
|
64
|
+
const ready = keys.some((k) => grammarReady(k));
|
|
65
|
+
if (!ready) {
|
|
66
|
+
// Never silent: a degraded run must say so, and stay a SUCCESSFUL run —
|
|
67
|
+
// the regex tier still produces a complete, searchable result.
|
|
68
|
+
note(
|
|
69
|
+
`${label}: no tree-sitter grammars available (offline?) — extracting with the regex tier, so symbols and call sites are less precise. ` +
|
|
70
|
+
"Run `codeindex grammars pull` once online to enable AST precision.\n",
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
return { tier, ready, pulled, notes };
|
|
74
|
+
}
|
package/src/engine-cli.ts
CHANGED
|
@@ -3,12 +3,7 @@ import { dirname, join, resolve } from "node:path";
|
|
|
3
3
|
import { SCHEMA_VERSION, EXTRACTOR_VERSION, type FileRecord } from "./types.js";
|
|
4
4
|
import { ENGINE_VERSION } from "./types.js";
|
|
5
5
|
import { ensureGrammars, grammarKeysForExts, resolveGrammarsTier, sharedGrammarsCacheDir } from "./ast/loader.js";
|
|
6
|
-
import {
|
|
7
|
-
resolveGrammarsPullTarget,
|
|
8
|
-
fetchGrammarsTarball,
|
|
9
|
-
fetchExpectedSha256,
|
|
10
|
-
extractGrammarsTarball,
|
|
11
|
-
} from "./ast/grammars-pull.js";
|
|
6
|
+
import { resolveGrammarsPullTarget, pullGrammars } from "./ast/grammars-pull.js";
|
|
12
7
|
import { buildIndexArtifacts, buildArtifactsFromScan, type BuildIndexOptions } from "./pipeline.js";
|
|
13
8
|
import { sha1 } from "./hash.js";
|
|
14
9
|
import { renderGraphJson } from "./render/graph-json.js";
|
|
@@ -82,8 +77,27 @@ Commands:
|
|
|
82
77
|
repomap Token-budgeted map of the highest-PageRank files (--budget-tokens)
|
|
83
78
|
hotspots Churn × size ranking of the files where work concentrates (JSON)
|
|
84
79
|
coupling Change coupling: files that change together (JSON; --since <ref>)
|
|
85
|
-
|
|
86
|
-
|
|
80
|
+
deadcode Dead-code candidates in two labeled tiers: 'unreferenced' (no
|
|
81
|
+
call site binds AND nothing references the name) and 'uncalled'
|
|
82
|
+
(referenced — re-export, type position — but never called)
|
|
83
|
+
complexity Cyclomatic-complexity estimates, most-complex first. Pass a file
|
|
84
|
+
positional for one file; omit for the repo-wide top
|
|
85
|
+
risk Complexity × git-churn ranking (JSON; --since <ref> to bound)
|
|
86
|
+
mermaid Mermaid diagram of the module graph; pass a module positional to
|
|
87
|
+
focus on one neighborhood
|
|
88
|
+
rewrite Map an expensive tree-wide search onto its indexed equivalent:
|
|
89
|
+
cli.mjs rewrite '<command line>'. Prints the replacement command
|
|
90
|
+
and exits 0, or exits 1 when it has no opinion (run the original).
|
|
91
|
+
Deliberately conservative — any shell metacharacter or unknown
|
|
92
|
+
flag refuses the rewrite
|
|
93
|
+
mcp Run as an MCP server over stdio (26 tools: scan_summary, graph,
|
|
94
|
+
symbols, callers, workspaces, churn, symbols_overview,
|
|
95
|
+
find_symbol, find_references, repo_map, hotspots, coupling,
|
|
96
|
+
dead_code, complexity, mermaid, grep, search, embed_status,
|
|
97
|
+
check_rules, the memory quartet and the three symbolic-edit
|
|
98
|
+
writes). Flags: --repo <dir> pins ONE repository so the per-tool
|
|
99
|
+
repo argument becomes optional (an explicit per-call repo still
|
|
100
|
+
wins); --server-name <name> overrides the announced serverInfo
|
|
87
101
|
version Print the engine version
|
|
88
102
|
|
|
89
103
|
Flags:
|
|
@@ -113,6 +127,8 @@ Flags:
|
|
|
113
127
|
--recall \`callers\`: recall-oriented binding (issue #7) — relaxes
|
|
114
128
|
the JS/TS import gate to unique repo-wide names and labels
|
|
115
129
|
each site corroborated|unique-name
|
|
130
|
+
--ignore-case \`grep\`: case-insensitive matching
|
|
131
|
+
--max-hits <n> \`grep\`: cap returned hits (default 200)
|
|
116
132
|
`;
|
|
117
133
|
|
|
118
134
|
interface CliFlags {
|
|
@@ -217,6 +233,31 @@ function scanOptions(flags: CliFlags, precomputedWalk?: WalkResult): BuildIndexO
|
|
|
217
233
|
// version/help/mcp return before we get there.
|
|
218
234
|
const SCANLESS_COMMANDS = new Set(["grep", "churn", "coupling", "workspaces", "grammars"]);
|
|
219
235
|
|
|
236
|
+
// Flags for `codeindex mcp`. Kept separate from parseFlags on purpose (see the
|
|
237
|
+
// dispatch site). `--repo` is resolved to an absolute path and must exist: a
|
|
238
|
+
// server pinned to a typo'd directory would otherwise answer every tool call
|
|
239
|
+
// with the same confusing per-call error instead of failing at startup.
|
|
240
|
+
export function parseMcpFlags(argv: string[]): { defaultRepo?: string; serverInfo?: { name?: string } } {
|
|
241
|
+
let defaultRepo: string | undefined;
|
|
242
|
+
let name: string | undefined;
|
|
243
|
+
for (let i = 0; i < argv.length; i++) {
|
|
244
|
+
const a = argv[i];
|
|
245
|
+
if (a === "--repo") {
|
|
246
|
+
const v = argv[++i];
|
|
247
|
+
if (!v) throw new Error("--repo requires a directory");
|
|
248
|
+
defaultRepo = resolve(v);
|
|
249
|
+
} else if (a === "--server-name") {
|
|
250
|
+
const v = argv[++i];
|
|
251
|
+
if (!v) throw new Error("--server-name requires a value");
|
|
252
|
+
name = v;
|
|
253
|
+
} else {
|
|
254
|
+
throw new Error(`unknown flag for \`mcp\`: ${a}`);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
if (defaultRepo && !existsSync(defaultRepo)) throw new Error(`--repo path does not exist: ${defaultRepo}`);
|
|
258
|
+
return { defaultRepo, serverInfo: name ? { name } : undefined };
|
|
259
|
+
}
|
|
260
|
+
|
|
220
261
|
export async function runCli(argv: string[]): Promise<void> {
|
|
221
262
|
const [cmd, ...rest] = argv;
|
|
222
263
|
if (!cmd || cmd === "help" || cmd === "--help" || cmd === "-h") {
|
|
@@ -227,9 +268,28 @@ export async function runCli(argv: string[]): Promise<void> {
|
|
|
227
268
|
process.stdout.write(ENGINE_VERSION + "\n");
|
|
228
269
|
return;
|
|
229
270
|
}
|
|
271
|
+
if (cmd === "rewrite") {
|
|
272
|
+
// Host contract (iterion's `rewriters` kind, rtk's generalization): stdin
|
|
273
|
+
// is nothing, argv is ONE full command line, stdout is the command to run
|
|
274
|
+
// instead, and the exit code says whether to use it. Exit 1 = "no opinion,
|
|
275
|
+
// run the original" — the overwhelmingly common, deliberately cheap case.
|
|
276
|
+
const { rewriteCommand } = await import("./rewrite.js");
|
|
277
|
+
const rewritten = rewriteCommand(rest.join(" "));
|
|
278
|
+
if (!rewritten) {
|
|
279
|
+
process.exitCode = 1;
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
process.stdout.write(rewritten + "\n");
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
230
285
|
if (cmd === "mcp") {
|
|
286
|
+
// `mcp` takes a deliberately tiny, self-contained flag set rather than
|
|
287
|
+
// going through parseFlags: the shared parser owns positional/scope
|
|
288
|
+
// semantics that mean nothing to a long-lived server, and every unknown
|
|
289
|
+
// flag there is fatal. Pinning is OPT-IN — a bare `codeindex mcp` keeps
|
|
290
|
+
// the historical contract where each tool call carries its own `repo`.
|
|
231
291
|
const { runMcpServer } = await import("./mcp.js");
|
|
232
|
-
await runMcpServer();
|
|
292
|
+
await runMcpServer(parseMcpFlags(rest));
|
|
233
293
|
return;
|
|
234
294
|
}
|
|
235
295
|
|
|
@@ -593,81 +653,14 @@ export async function runCli(argv: string[]): Promise<void> {
|
|
|
593
653
|
};
|
|
594
654
|
emit(JSON.stringify(status, null, 2) + "\n", flags.out);
|
|
595
655
|
} else if (sub === "pull") {
|
|
596
|
-
//
|
|
597
|
-
//
|
|
598
|
-
//
|
|
599
|
-
//
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
expected = await fetchExpectedSha256(target.sha256Url);
|
|
605
|
-
} catch (e) {
|
|
606
|
-
process.stderr.write(
|
|
607
|
-
`codeindex: could not fetch checksum (${e instanceof Error ? e.message : String(e)}) — proceeding unverified\n`,
|
|
608
|
-
);
|
|
609
|
-
}
|
|
610
|
-
}
|
|
611
|
-
// Idempotent: the marker sibling records the digest of the tarball that
|
|
612
|
-
// populated cacheDir. If the runtime wasm is present AND the marker matches
|
|
613
|
-
// the freshly-fetched digest, the cache is already up to date — skip the
|
|
614
|
-
// ~22 MB download entirely. (Keeps cacheDir itself byte-identical to the
|
|
615
|
-
// tarball; the marker lives next to it, never inside it.)
|
|
616
|
-
const runtime = join(cacheDir, "web-tree-sitter.wasm");
|
|
617
|
-
const markerPath = join(dirname(cacheDir), `${ENGINE_VERSION}.sha256`);
|
|
618
|
-
if (existsSync(runtime) && expected && existsSync(markerPath)) {
|
|
619
|
-
let marker = "";
|
|
620
|
-
try {
|
|
621
|
-
marker = readFileSync(markerPath, "utf8").trim();
|
|
622
|
-
} catch {
|
|
623
|
-
// unreadable marker → fall through and re-pull
|
|
624
|
-
}
|
|
625
|
-
if (marker === expected) {
|
|
626
|
-
process.stderr.write(`codeindex: grammars already present at ${cacheDir} (up to date)\n`);
|
|
627
|
-
return;
|
|
628
|
-
}
|
|
629
|
-
}
|
|
630
|
-
process.stderr.write(`codeindex: fetching grammars from ${target.url} → ${cacheDir}\n`);
|
|
631
|
-
let bytes: Uint8Array;
|
|
632
|
-
try {
|
|
633
|
-
// Follows redirects (GitHub → CDN) and verifies sha256 when known.
|
|
634
|
-
bytes = await fetchGrammarsTarball(target.url, expected);
|
|
635
|
-
} catch (e) {
|
|
636
|
-
process.stderr.write(`codeindex: pull failed — ${e instanceof Error ? e.message : String(e)} (nothing written)\n`);
|
|
637
|
-
process.exitCode = 1;
|
|
638
|
-
return;
|
|
639
|
-
}
|
|
640
|
-
// Atomic install: extract into a tmp dir SIBLING to cacheDir (same
|
|
641
|
-
// filesystem → rename is atomic), sanity-check the runtime wasm landed,
|
|
642
|
-
// then swap it into place. A failure at any step discards the tmp dir and
|
|
643
|
-
// leaves any existing cache untouched — a half-populated cache never shows.
|
|
644
|
-
let tmp: string | undefined;
|
|
645
|
-
try {
|
|
646
|
-
mkdirSync(dirname(cacheDir), { recursive: true });
|
|
647
|
-
tmp = mkdtempSync(join(dirname(cacheDir), ".grammars-tmp-"));
|
|
648
|
-
extractGrammarsTarball(bytes, tmp);
|
|
649
|
-
if (!existsSync(join(tmp, "web-tree-sitter.wasm"))) {
|
|
650
|
-
throw new Error("archive is missing web-tree-sitter.wasm");
|
|
651
|
-
}
|
|
652
|
-
if (existsSync(cacheDir)) rmSync(cacheDir, { recursive: true, force: true });
|
|
653
|
-
renameSync(tmp, cacheDir);
|
|
654
|
-
tmp = undefined;
|
|
655
|
-
if (expected) writeFileSync(markerPath, expected + "\n");
|
|
656
|
-
} catch (e) {
|
|
657
|
-
if (tmp) {
|
|
658
|
-
try {
|
|
659
|
-
rmSync(tmp, { recursive: true, force: true });
|
|
660
|
-
} catch {
|
|
661
|
-
// best-effort cleanup
|
|
662
|
-
}
|
|
663
|
-
}
|
|
664
|
-
process.stderr.write(
|
|
665
|
-
`codeindex: pull failed — ${e instanceof Error ? e.message : String(e)} (nothing written)\n`,
|
|
666
|
-
);
|
|
667
|
-
process.exitCode = 1;
|
|
668
|
-
return;
|
|
669
|
-
}
|
|
670
|
-
process.stderr.write(`codeindex: grammars extracted → ${cacheDir}\n`);
|
|
656
|
+
// The mechanic itself (sidecar checksum, idempotent skip, atomic install)
|
|
657
|
+
// lives in pullGrammars — shared verbatim with warmGrammars, so the CLI
|
|
658
|
+
// and the library warm-up can never drift apart. Here we only map its
|
|
659
|
+
// result onto the CLI contract: progress notes and the terminal message to
|
|
660
|
+
// stderr, non-zero exit on failure (which wrote nothing).
|
|
661
|
+
const res = await pullGrammars(cacheDir, { onNote: (m) => process.stderr.write(m) });
|
|
662
|
+
process.stderr.write(res.message);
|
|
663
|
+
if (!res.ok) process.exitCode = 1;
|
|
671
664
|
} else {
|
|
672
665
|
throw new Error("grammars needs a subcommand: status | pull");
|
|
673
666
|
}
|
|
@@ -718,7 +711,12 @@ export async function runCli(argv: string[]): Promise<void> {
|
|
|
718
711
|
emit(renderMermaid(graph, { module: flags.positional }), flags.out);
|
|
719
712
|
} else if (cmd === "grep") {
|
|
720
713
|
if (!flags.positional) throw new Error("grep needs a pattern: cli.mjs grep <pattern> --repo <dir>");
|
|
721
|
-
|
|
714
|
+
// `--scope <dir>` is documented as global sugar for `--include '<dir>/**'`;
|
|
715
|
+
// every other command gets it via scanOptions, but grep bypasses the scan
|
|
716
|
+
// and builds its own glob list — so it has to fold the sugar in itself, or
|
|
717
|
+
// the flag would be silently ignored here alone.
|
|
718
|
+
const scopeGlobs = flags.scope ? [`${flags.scope.replace(/\/+$/, "")}/**`] : [];
|
|
719
|
+
const globs = [...scopeGlobs, ...flags.include, ...flags.exclude.map((g) => `!${g}`)];
|
|
722
720
|
const hits = grepRepo(flags.repo, flags.positional, {
|
|
723
721
|
globs: globs.length ? globs : undefined,
|
|
724
722
|
ignoreCase: flags.ignoreCase,
|
package/src/engine.ts
CHANGED
|
@@ -60,8 +60,14 @@ export {
|
|
|
60
60
|
fetchExpectedSha256,
|
|
61
61
|
extractTarInto,
|
|
62
62
|
extractGrammarsTarball,
|
|
63
|
+
pullGrammars,
|
|
63
64
|
} from "./ast/grammars-pull.js";
|
|
64
|
-
export type { GrammarsPullTarget } from "./ast/grammars-pull.js";
|
|
65
|
+
export type { GrammarsPullTarget, GrammarsPullResult } from "./ast/grammars-pull.js";
|
|
66
|
+
// The one-call AST warm-up for consumers. `scanRepo` is synchronous and cannot
|
|
67
|
+
// warm the grammars itself, so a consumer that never awaits this stays on the
|
|
68
|
+
// regex tier forever — silently. Call it once at the CLI entry.
|
|
69
|
+
export { warmGrammars } from "./ast/warm.js";
|
|
70
|
+
export type { WarmGrammarsResult, WarmGrammarsOptions } from "./ast/warm.js";
|
|
65
71
|
|
|
66
72
|
// Resolution + modules + graph tier.
|
|
67
73
|
export { buildResolveContext, resolveImport, resolveDocLink } from "./resolve.js";
|
|
@@ -179,6 +185,13 @@ export type { RepoMapOptions } from "./repomap.js";
|
|
|
179
185
|
export { runMcpServer } from "./mcp.js";
|
|
180
186
|
export type { McpServerOptions } from "./mcp.js";
|
|
181
187
|
|
|
188
|
+
// Command rewriting: an expensive tree-wide search → its indexed equivalent.
|
|
189
|
+
// Exported so a host can decide for itself rather than shelling out to
|
|
190
|
+
// `engine.mjs rewrite` (same conservative refusal semantics either way).
|
|
191
|
+
// Only the decision function is public — the tokenizer/quoter are internal
|
|
192
|
+
// details, and `tokenize` is already taken here by the wordpiece tokenizer.
|
|
193
|
+
export { rewriteCommand } from "./rewrite.js";
|
|
194
|
+
|
|
182
195
|
// General-purpose helpers shared by consumers (deterministic, dependency-free).
|
|
183
196
|
export { sha1, shortHash } from "./hash.js";
|
|
184
197
|
export { byStr, byKey } from "./sort.js";
|
package/src/mcp.ts
CHANGED
|
@@ -667,8 +667,11 @@ const SCANLESS_TOOLS = new Set([
|
|
|
667
667
|
"embed_status",
|
|
668
668
|
]);
|
|
669
669
|
|
|
670
|
-
async function callTool(name: string, args: Record<string, unknown
|
|
671
|
-
|
|
670
|
+
async function callTool(name: string, args: Record<string, unknown>, defaultRepo?: string): Promise<string> {
|
|
671
|
+
// An explicit per-call `repo` always wins; `defaultRepo` is the server-level
|
|
672
|
+
// pin (`codeindex mcp --repo <dir>`) that lets a host bind one server process
|
|
673
|
+
// to one workspace, so agents need not know — or restate — the absolute path.
|
|
674
|
+
const repo = str(args.repo) ?? defaultRepo;
|
|
672
675
|
if (!repo) throw new Error("`repo` is required (absolute path to the repository root)");
|
|
673
676
|
const scanOpts = { scope: str(args.scope), include: strArray(args.include), exclude: strArray(args.exclude) };
|
|
674
677
|
// Scan-needing tools warm the present-language grammars (re-derived per call)
|
|
@@ -886,11 +889,36 @@ async function callTool(name: string, args: Record<string, unknown>): Promise<st
|
|
|
886
889
|
throw new Error(`unknown tool: ${name}`);
|
|
887
890
|
}
|
|
888
891
|
|
|
892
|
+
// The advertised tool list. Without a server-level repo pin this is TOOLS
|
|
893
|
+
// verbatim (byte-compat for existing consumers). With one, every tool drops
|
|
894
|
+
// `repo` from its `required` set and documents the default — so a client that
|
|
895
|
+
// omits it is spec-correct rather than relying on the server being lenient.
|
|
896
|
+
// The pin is reflected in the schema, not just honoured at call time.
|
|
897
|
+
function toolsFor(defaultRepo?: string): readonly unknown[] {
|
|
898
|
+
if (!defaultRepo) return TOOLS;
|
|
899
|
+
return TOOLS.map((t) => ({
|
|
900
|
+
...t,
|
|
901
|
+
inputSchema: {
|
|
902
|
+
...t.inputSchema,
|
|
903
|
+
properties: {
|
|
904
|
+
...t.inputSchema.properties,
|
|
905
|
+
repo: { type: "string", description: `Absolute path to the repository root (optional — defaults to ${defaultRepo})` },
|
|
906
|
+
},
|
|
907
|
+
required: (t.inputSchema.required as readonly string[]).filter((r) => r !== "repo"),
|
|
908
|
+
},
|
|
909
|
+
}));
|
|
910
|
+
}
|
|
911
|
+
|
|
889
912
|
export interface McpServerOptions {
|
|
890
913
|
// Override the serverInfo announced in the initialize response — for
|
|
891
914
|
// downstream consumers embedding this server under their own identity.
|
|
892
915
|
// Omitted fields keep the defaults (name "codeindex", ENGINE_VERSION).
|
|
893
916
|
serverInfo?: { name?: string; version?: string };
|
|
917
|
+
// Bind the server to ONE repository, so `repo` becomes optional on every
|
|
918
|
+
// tool (an explicit per-call `repo` still wins). This is what lets a host
|
|
919
|
+
// spawn one server per workspace — `codeindex mcp --repo <dir>` — instead of
|
|
920
|
+
// requiring the agent to thread an absolute path through every single call.
|
|
921
|
+
defaultRepo?: string;
|
|
894
922
|
}
|
|
895
923
|
|
|
896
924
|
export async function runMcpServer(opts: McpServerOptions = {}): Promise<void> {
|
|
@@ -898,6 +926,8 @@ export async function runMcpServer(opts: McpServerOptions = {}): Promise<void> {
|
|
|
898
926
|
name: opts.serverInfo?.name ?? "codeindex",
|
|
899
927
|
version: opts.serverInfo?.version ?? ENGINE_VERSION,
|
|
900
928
|
};
|
|
929
|
+
// Derived ONCE: the pin cannot change mid-session, so neither can the list.
|
|
930
|
+
const tools = toolsFor(opts.defaultRepo);
|
|
901
931
|
// No startup warm: each scan-needing tool warms the present-language grammars
|
|
902
932
|
// for its repo before it runs (warmGrammarsForRepo re-derives them per call),
|
|
903
933
|
// so a session that never scans — or only touches one language — loads no
|
|
@@ -940,13 +970,13 @@ export async function runMcpServer(opts: McpServerOptions = {}): Promise<void> {
|
|
|
940
970
|
} else if (req.method === "ping") {
|
|
941
971
|
send({ id: req.id, result: {} });
|
|
942
972
|
} else if (req.method === "tools/list") {
|
|
943
|
-
send({ id: req.id, result: { tools
|
|
973
|
+
send({ id: req.id, result: { tools } });
|
|
944
974
|
} else if (req.method === "tools/call") {
|
|
945
975
|
const params = req.params ?? {};
|
|
946
976
|
const name = str(params.name) ?? "";
|
|
947
977
|
const args = (params.arguments ?? {}) as Record<string, unknown>;
|
|
948
978
|
try {
|
|
949
|
-
const text = await callTool(name, args);
|
|
979
|
+
const text = await callTool(name, args, opts.defaultRepo);
|
|
950
980
|
send({ id: req.id, result: { content: [{ type: "text", text }] } });
|
|
951
981
|
} catch (e) {
|
|
952
982
|
send({
|