@maxgfr/codeindex 2.15.0 → 2.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -4
- package/package.json +2 -2
- package/scripts/engine.d.mts +38 -3
- package/scripts/engine.mjs +332 -105
- package/src/ast/grammars-pull.ts +95 -1
- package/src/ast/warm.ts +74 -0
- package/src/engine-cli.ts +83 -85
- package/src/engine.ts +14 -1
- package/src/mcp.ts +34 -4
- package/src/rewrite.ts +169 -0
- package/src/types.ts +1 -1
package/scripts/engine.mjs
CHANGED
|
@@ -14,7 +14,7 @@ var ENGINE_VERSION, SCHEMA_VERSION, EXTRACTOR_VERSION;
|
|
|
14
14
|
var init_types = __esm({
|
|
15
15
|
"src/types.ts"() {
|
|
16
16
|
"use strict";
|
|
17
|
-
ENGINE_VERSION = "2.
|
|
17
|
+
ENGINE_VERSION = "2.17.0";
|
|
18
18
|
SCHEMA_VERSION = 4;
|
|
19
19
|
EXTRACTOR_VERSION = 10;
|
|
20
20
|
}
|
|
@@ -8259,7 +8259,7 @@ var init_query = __esm({
|
|
|
8259
8259
|
});
|
|
8260
8260
|
|
|
8261
8261
|
// src/edit.ts
|
|
8262
|
-
import { readFileSync as
|
|
8262
|
+
import { readFileSync as readFileSync4, writeFileSync as writeFileSync2 } from "fs";
|
|
8263
8263
|
import { join as join9 } from "path";
|
|
8264
8264
|
function resolveUniqueSymbol(scan2, namePath, file) {
|
|
8265
8265
|
let matches = findSymbol(scan2, namePath);
|
|
@@ -8273,7 +8273,7 @@ function resolveUniqueSymbol(scan2, namePath, file) {
|
|
|
8273
8273
|
throw new Error(`"${namePath}" is ambiguous (${matches.length} matches: ${list}) \u2014 qualify with \`file\` or a Parent/name path`);
|
|
8274
8274
|
}
|
|
8275
8275
|
function readLines(abs) {
|
|
8276
|
-
return
|
|
8276
|
+
return readFileSync4(abs, "utf8").split("\n");
|
|
8277
8277
|
}
|
|
8278
8278
|
function replaceSymbolBody(scan2, namePath, body2, file) {
|
|
8279
8279
|
const sym = resolveUniqueSymbol(scan2, namePath, file);
|
|
@@ -8317,7 +8317,7 @@ var init_edit = __esm({
|
|
|
8317
8317
|
});
|
|
8318
8318
|
|
|
8319
8319
|
// src/memory.ts
|
|
8320
|
-
import { mkdirSync as mkdirSync2, readdirSync as readdirSync2, readFileSync as
|
|
8320
|
+
import { mkdirSync as mkdirSync2, readdirSync as readdirSync2, readFileSync as readFileSync5, rmSync as rmSync2, statSync as statSync2, writeFileSync as writeFileSync3 } from "fs";
|
|
8321
8321
|
import { dirname as dirname3, join as join10 } from "path";
|
|
8322
8322
|
function sanitize(name2) {
|
|
8323
8323
|
const clean = name2.replace(/^mem:/, "").replace(/\.md$/, "");
|
|
@@ -8342,7 +8342,7 @@ function writeMemory(repo, name2, content) {
|
|
|
8342
8342
|
}
|
|
8343
8343
|
function readMemory(repo, name2) {
|
|
8344
8344
|
try {
|
|
8345
|
-
return
|
|
8345
|
+
return readFileSync5(memoryPath(repo, name2), "utf8");
|
|
8346
8346
|
} catch {
|
|
8347
8347
|
return void 0;
|
|
8348
8348
|
}
|
|
@@ -8354,7 +8354,7 @@ function deleteMemory(repo, name2) {
|
|
|
8354
8354
|
} catch {
|
|
8355
8355
|
return false;
|
|
8356
8356
|
}
|
|
8357
|
-
|
|
8357
|
+
rmSync2(path);
|
|
8358
8358
|
return true;
|
|
8359
8359
|
}
|
|
8360
8360
|
function listMemories(repo) {
|
|
@@ -8384,7 +8384,7 @@ var init_memory = __esm({
|
|
|
8384
8384
|
});
|
|
8385
8385
|
|
|
8386
8386
|
// src/workspaces.ts
|
|
8387
|
-
import { existsSync as
|
|
8387
|
+
import { existsSync as existsSync3, readdirSync as readdirSync3, statSync as statSync3 } from "fs";
|
|
8388
8388
|
import { join as join11 } from "path";
|
|
8389
8389
|
function readJson(path, label, warnings) {
|
|
8390
8390
|
const raw = readText(path);
|
|
@@ -8437,7 +8437,7 @@ function wsGlobToRegExp(pat) {
|
|
|
8437
8437
|
}
|
|
8438
8438
|
function probeNodePkg(root, dir, kind, warnings) {
|
|
8439
8439
|
const path = join11(root, dir, "package.json");
|
|
8440
|
-
if (!
|
|
8440
|
+
if (!existsSync3(path)) return void 0;
|
|
8441
8441
|
const manifest = `${dir}/package.json`;
|
|
8442
8442
|
const pkg = readJson(path, manifest, warnings);
|
|
8443
8443
|
const out2 = {
|
|
@@ -8451,7 +8451,7 @@ function probeNodePkg(root, dir, kind, warnings) {
|
|
|
8451
8451
|
}
|
|
8452
8452
|
function probeCargo(root, dir) {
|
|
8453
8453
|
const path = join11(root, dir, "Cargo.toml");
|
|
8454
|
-
if (!
|
|
8454
|
+
if (!existsSync3(path)) return void 0;
|
|
8455
8455
|
const body2 = tomlSectionBody(readText(path), "package");
|
|
8456
8456
|
const out2 = {
|
|
8457
8457
|
name: tomlString(body2, "name") ?? dir,
|
|
@@ -8465,18 +8465,18 @@ function probeCargo(root, dir) {
|
|
|
8465
8465
|
}
|
|
8466
8466
|
function probeGoMod(root, dir) {
|
|
8467
8467
|
const path = join11(root, dir, "go.mod");
|
|
8468
|
-
if (!
|
|
8468
|
+
if (!existsSync3(path)) return void 0;
|
|
8469
8469
|
const name2 = readText(path).match(/^module\s+(\S+)/m)?.[1] ?? dir;
|
|
8470
8470
|
return { name: name2, dir, kind: "go", manifest: `${dir}/go.mod` };
|
|
8471
8471
|
}
|
|
8472
8472
|
function probeMaven(root, dir) {
|
|
8473
8473
|
const path = join11(root, dir, "pom.xml");
|
|
8474
|
-
if (!
|
|
8474
|
+
if (!existsSync3(path)) return void 0;
|
|
8475
8475
|
return { name: ownArtifactId(readText(path)) ?? dir, dir, kind: "maven", manifest: `${dir}/pom.xml` };
|
|
8476
8476
|
}
|
|
8477
8477
|
function probePyproject(root, dir) {
|
|
8478
8478
|
const path = join11(root, dir, "pyproject.toml");
|
|
8479
|
-
if (!
|
|
8479
|
+
if (!existsSync3(path)) return void 0;
|
|
8480
8480
|
const toml = readText(path);
|
|
8481
8481
|
const project = tomlSectionBody(toml, "project");
|
|
8482
8482
|
const poetry = tomlSectionBody(toml, "tool.poetry");
|
|
@@ -8492,7 +8492,7 @@ function probePyproject(root, dir) {
|
|
|
8492
8492
|
}
|
|
8493
8493
|
function probeComposer(root, dir, warnings) {
|
|
8494
8494
|
const path = join11(root, dir, "composer.json");
|
|
8495
|
-
if (!
|
|
8495
|
+
if (!existsSync3(path)) return void 0;
|
|
8496
8496
|
const manifest = `${dir}/composer.json`;
|
|
8497
8497
|
const pkg = readJson(path, manifest, warnings);
|
|
8498
8498
|
const out2 = {
|
|
@@ -8506,7 +8506,7 @@ function probeComposer(root, dir, warnings) {
|
|
|
8506
8506
|
}
|
|
8507
8507
|
function probeNxProject(root, dir, warnings) {
|
|
8508
8508
|
const path = join11(root, dir, "project.json");
|
|
8509
|
-
if (!
|
|
8509
|
+
if (!existsSync3(path)) return void 0;
|
|
8510
8510
|
const manifest = `${dir}/project.json`;
|
|
8511
8511
|
const proj = readJson(path, manifest, warnings);
|
|
8512
8512
|
return {
|
|
@@ -8518,7 +8518,7 @@ function probeNxProject(root, dir, warnings) {
|
|
|
8518
8518
|
}
|
|
8519
8519
|
function probeGradle(root, dir) {
|
|
8520
8520
|
for (const f of ["build.gradle", "build.gradle.kts"]) {
|
|
8521
|
-
if (
|
|
8521
|
+
if (existsSync3(join11(root, dir, f))) {
|
|
8522
8522
|
return { name: dir, dir, kind: "gradle", manifest: `${dir}/${f}` };
|
|
8523
8523
|
}
|
|
8524
8524
|
}
|
|
@@ -9565,7 +9565,7 @@ var init_grep = __esm({
|
|
|
9565
9565
|
|
|
9566
9566
|
// src/embed/model.ts
|
|
9567
9567
|
import { createHash as createHash3 } from "crypto";
|
|
9568
|
-
import { existsSync as
|
|
9568
|
+
import { existsSync as existsSync4, readFileSync as readFileSync6 } from "fs";
|
|
9569
9569
|
import { join as join13 } from "path";
|
|
9570
9570
|
function resolveEmbedModelDir(repo) {
|
|
9571
9571
|
const env = process.env.CODEINDEX_EMBED_DIR;
|
|
@@ -9574,7 +9574,7 @@ function resolveEmbedModelDir(repo) {
|
|
|
9574
9574
|
if (repo) candidates.push(join13(repo, ".codeindex", DEFAULT_EMBED_DIRNAME));
|
|
9575
9575
|
candidates.push(join13(process.cwd(), ".codeindex", DEFAULT_EMBED_DIRNAME));
|
|
9576
9576
|
for (const c2 of candidates) {
|
|
9577
|
-
if (
|
|
9577
|
+
if (existsSync4(join13(c2, "model.json"))) return c2;
|
|
9578
9578
|
}
|
|
9579
9579
|
return void 0;
|
|
9580
9580
|
}
|
|
@@ -9608,8 +9608,8 @@ function parseEmbedModel(raw, source) {
|
|
|
9608
9608
|
function loadEmbedModel(dir) {
|
|
9609
9609
|
if (!dir) return void 0;
|
|
9610
9610
|
const path = join13(dir, "model.json");
|
|
9611
|
-
if (!
|
|
9612
|
-
const raw = JSON.parse(
|
|
9611
|
+
if (!existsSync4(path)) return void 0;
|
|
9612
|
+
const raw = JSON.parse(readFileSync6(path, "utf8"));
|
|
9613
9613
|
return parseEmbedModel(raw, path);
|
|
9614
9614
|
}
|
|
9615
9615
|
function resolveEmbedPullUrl() {
|
|
@@ -10303,7 +10303,7 @@ __export(mcp_exports, {
|
|
|
10303
10303
|
toCacheMap: () => toCacheMap,
|
|
10304
10304
|
warmGrammarsForRepo: () => warmGrammarsForRepo
|
|
10305
10305
|
});
|
|
10306
|
-
import { readFileSync as
|
|
10306
|
+
import { readFileSync as readFileSync7, statSync as statSync4 } from "fs";
|
|
10307
10307
|
import { join as join14 } from "path";
|
|
10308
10308
|
import { createInterface } from "readline";
|
|
10309
10309
|
function str(v) {
|
|
@@ -10360,7 +10360,7 @@ function toCacheMap(scan2) {
|
|
|
10360
10360
|
function readPersistedIndex(repo) {
|
|
10361
10361
|
let parsed;
|
|
10362
10362
|
try {
|
|
10363
|
-
parsed = JSON.parse(
|
|
10363
|
+
parsed = JSON.parse(readFileSync7(join14(repo, ".codeindex", "cache.json"), "utf8"));
|
|
10364
10364
|
} catch {
|
|
10365
10365
|
return void 0;
|
|
10366
10366
|
}
|
|
@@ -10384,8 +10384,8 @@ function preloadArtifacts(repo, scan2, meta) {
|
|
|
10384
10384
|
let graphBytes;
|
|
10385
10385
|
let symbolsBytes;
|
|
10386
10386
|
try {
|
|
10387
|
-
graphBytes =
|
|
10388
|
-
symbolsBytes =
|
|
10387
|
+
graphBytes = readFileSync7(join14(dir, "graph.json"));
|
|
10388
|
+
symbolsBytes = readFileSync7(join14(dir, "symbols.json"));
|
|
10389
10389
|
} catch {
|
|
10390
10390
|
return void 0;
|
|
10391
10391
|
}
|
|
@@ -10440,8 +10440,8 @@ async function warmGrammarsForRepo(repo) {
|
|
|
10440
10440
|
const { files } = walk(repo, {});
|
|
10441
10441
|
await ensureGrammars(grammarKeysForExts(files.map((f) => f.ext)));
|
|
10442
10442
|
}
|
|
10443
|
-
async function callTool(name2, args2) {
|
|
10444
|
-
const repo = str(args2.repo);
|
|
10443
|
+
async function callTool(name2, args2, defaultRepo) {
|
|
10444
|
+
const repo = str(args2.repo) ?? defaultRepo;
|
|
10445
10445
|
if (!repo) throw new Error("`repo` is required (absolute path to the repository root)");
|
|
10446
10446
|
const scanOpts = { scope: str(args2.scope), include: strArray(args2.include), exclude: strArray(args2.exclude) };
|
|
10447
10447
|
if (!SCANLESS_TOOLS.has(name2)) await warmGrammarsForRepo(repo);
|
|
@@ -10636,11 +10636,26 @@ async function callTool(name2, args2) {
|
|
|
10636
10636
|
}
|
|
10637
10637
|
throw new Error(`unknown tool: ${name2}`);
|
|
10638
10638
|
}
|
|
10639
|
+
function toolsFor(defaultRepo) {
|
|
10640
|
+
if (!defaultRepo) return TOOLS;
|
|
10641
|
+
return TOOLS.map((t) => ({
|
|
10642
|
+
...t,
|
|
10643
|
+
inputSchema: {
|
|
10644
|
+
...t.inputSchema,
|
|
10645
|
+
properties: {
|
|
10646
|
+
...t.inputSchema.properties,
|
|
10647
|
+
repo: { type: "string", description: `Absolute path to the repository root (optional \u2014 defaults to ${defaultRepo})` }
|
|
10648
|
+
},
|
|
10649
|
+
required: t.inputSchema.required.filter((r) => r !== "repo")
|
|
10650
|
+
}
|
|
10651
|
+
}));
|
|
10652
|
+
}
|
|
10639
10653
|
async function runMcpServer(opts = {}) {
|
|
10640
10654
|
const serverInfo = {
|
|
10641
10655
|
name: opts.serverInfo?.name ?? "codeindex",
|
|
10642
10656
|
version: opts.serverInfo?.version ?? ENGINE_VERSION
|
|
10643
10657
|
};
|
|
10658
|
+
const tools = toolsFor(opts.defaultRepo);
|
|
10644
10659
|
const send = (msg) => {
|
|
10645
10660
|
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...msg }) + "\n");
|
|
10646
10661
|
};
|
|
@@ -10673,13 +10688,13 @@ async function runMcpServer(opts = {}) {
|
|
|
10673
10688
|
} else if (req.method === "ping") {
|
|
10674
10689
|
send({ id: req.id, result: {} });
|
|
10675
10690
|
} else if (req.method === "tools/list") {
|
|
10676
|
-
send({ id: req.id, result: { tools
|
|
10691
|
+
send({ id: req.id, result: { tools } });
|
|
10677
10692
|
} else if (req.method === "tools/call") {
|
|
10678
10693
|
const params = req.params ?? {};
|
|
10679
10694
|
const name2 = str(params.name) ?? "";
|
|
10680
10695
|
const args2 = params.arguments ?? {};
|
|
10681
10696
|
try {
|
|
10682
|
-
const text = await callTool(name2, args2);
|
|
10697
|
+
const text = await callTool(name2, args2, opts.defaultRepo);
|
|
10683
10698
|
send({ id: req.id, result: { content: [{ type: "text", text }] } });
|
|
10684
10699
|
} catch (e) {
|
|
10685
10700
|
send({
|
|
@@ -10989,6 +11004,121 @@ var init_mcp = __esm({
|
|
|
10989
11004
|
}
|
|
10990
11005
|
});
|
|
10991
11006
|
|
|
11007
|
+
// src/rewrite.ts
|
|
11008
|
+
var rewrite_exports = {};
|
|
11009
|
+
__export(rewrite_exports, {
|
|
11010
|
+
rewriteCommand: () => rewriteCommand,
|
|
11011
|
+
shellQuote: () => shellQuote,
|
|
11012
|
+
tokenize: () => tokenize2
|
|
11013
|
+
});
|
|
11014
|
+
function tokenize2(line) {
|
|
11015
|
+
const out2 = [];
|
|
11016
|
+
let cur = "";
|
|
11017
|
+
let quote;
|
|
11018
|
+
let started = false;
|
|
11019
|
+
for (let i2 = 0; i2 < line.length; i2++) {
|
|
11020
|
+
const c2 = line[i2];
|
|
11021
|
+
if (quote) {
|
|
11022
|
+
if (c2 === quote) quote = void 0;
|
|
11023
|
+
else cur += c2;
|
|
11024
|
+
continue;
|
|
11025
|
+
}
|
|
11026
|
+
if (c2 === '"' || c2 === "'") {
|
|
11027
|
+
quote = c2;
|
|
11028
|
+
started = true;
|
|
11029
|
+
continue;
|
|
11030
|
+
}
|
|
11031
|
+
if (c2 === " " || c2 === " ") {
|
|
11032
|
+
if (started || cur) out2.push(cur);
|
|
11033
|
+
cur = "";
|
|
11034
|
+
started = false;
|
|
11035
|
+
continue;
|
|
11036
|
+
}
|
|
11037
|
+
cur += c2;
|
|
11038
|
+
}
|
|
11039
|
+
if (quote) return void 0;
|
|
11040
|
+
if (started || cur) out2.push(cur);
|
|
11041
|
+
return out2;
|
|
11042
|
+
}
|
|
11043
|
+
function shellQuote(s) {
|
|
11044
|
+
if (s !== "" && !/[^A-Za-z0-9_\-./=@:]/.test(s)) return s;
|
|
11045
|
+
return "'" + s.replace(/'/g, `'\\''`) + "'";
|
|
11046
|
+
}
|
|
11047
|
+
function parseSearch(bin, args2) {
|
|
11048
|
+
const p = { ignoreCase: false, includes: [], recursive: bin !== "grep" && bin !== "egrep" };
|
|
11049
|
+
const positionals = [];
|
|
11050
|
+
for (let i2 = 0; i2 < args2.length; i2++) {
|
|
11051
|
+
const a = args2[i2];
|
|
11052
|
+
if (a === void 0) continue;
|
|
11053
|
+
if (a === "--") {
|
|
11054
|
+
positionals.push(...args2.slice(i2 + 1));
|
|
11055
|
+
break;
|
|
11056
|
+
}
|
|
11057
|
+
if (!a.startsWith("-") || a === "-") {
|
|
11058
|
+
positionals.push(a);
|
|
11059
|
+
continue;
|
|
11060
|
+
}
|
|
11061
|
+
if (a === "-i" || a === "--ignore-case") {
|
|
11062
|
+
p.ignoreCase = true;
|
|
11063
|
+
} else if (a === "-r" || a === "-R" || a === "--recursive") {
|
|
11064
|
+
p.recursive = true;
|
|
11065
|
+
} else if (a === "-n" || a === "--line-number" || a === "-H" || a === "--with-filename" || a === "--no-heading") {
|
|
11066
|
+
} else if (a === "-e" || a === "--regexp") {
|
|
11067
|
+
const v = args2[++i2];
|
|
11068
|
+
if (v === void 0 || p.pattern !== void 0) return void 0;
|
|
11069
|
+
p.pattern = v;
|
|
11070
|
+
} else if (a.startsWith("--include=")) {
|
|
11071
|
+
p.includes.push(a.slice("--include=".length));
|
|
11072
|
+
} else if (a === "--include" || a === "-g" || a === "--glob") {
|
|
11073
|
+
const v = args2[++i2];
|
|
11074
|
+
if (v === void 0) return void 0;
|
|
11075
|
+
p.includes.push(v);
|
|
11076
|
+
} else if (a.length > 2 && /^-[a-zA-Z]+$/.test(a)) {
|
|
11077
|
+
const expanded = a.slice(1).split("").map((c2) => `-${c2}`);
|
|
11078
|
+
args2.splice(i2, 1, ...expanded);
|
|
11079
|
+
i2--;
|
|
11080
|
+
} else {
|
|
11081
|
+
return void 0;
|
|
11082
|
+
}
|
|
11083
|
+
}
|
|
11084
|
+
if (p.pattern === void 0) {
|
|
11085
|
+
const first = positionals.shift();
|
|
11086
|
+
if (first === void 0 || first === "") return void 0;
|
|
11087
|
+
p.pattern = first;
|
|
11088
|
+
}
|
|
11089
|
+
if (positionals.length > 1) return void 0;
|
|
11090
|
+
p.path = positionals[0];
|
|
11091
|
+
return p;
|
|
11092
|
+
}
|
|
11093
|
+
function rewriteCommand(cmd, bin = "codeindex") {
|
|
11094
|
+
const line = cmd.trim();
|
|
11095
|
+
if (!line || SHELL_METACHARS.test(line)) return void 0;
|
|
11096
|
+
const tokens = tokenize2(line);
|
|
11097
|
+
if (!tokens || tokens.length < 2) return void 0;
|
|
11098
|
+
const [head, ...args2] = tokens;
|
|
11099
|
+
if (head === void 0 || !GREP_BINARIES.has(head)) return void 0;
|
|
11100
|
+
const p = parseSearch(head, args2);
|
|
11101
|
+
if (!p || p.pattern === void 0) return void 0;
|
|
11102
|
+
const pattern = p.pattern;
|
|
11103
|
+
if (!p.recursive) return void 0;
|
|
11104
|
+
const path = p.path;
|
|
11105
|
+
const out2 = [bin, "grep", shellQuote(pattern)];
|
|
11106
|
+
if (path && path !== "." && path !== "./") {
|
|
11107
|
+
out2.push("--scope", shellQuote(path.replace(/\/+$/, "")));
|
|
11108
|
+
}
|
|
11109
|
+
if (p.ignoreCase) out2.push("--ignore-case");
|
|
11110
|
+
for (const g of p.includes) out2.push("--include", shellQuote(g));
|
|
11111
|
+
return out2.join(" ");
|
|
11112
|
+
}
|
|
11113
|
+
var SHELL_METACHARS, GREP_BINARIES;
|
|
11114
|
+
var init_rewrite = __esm({
|
|
11115
|
+
"src/rewrite.ts"() {
|
|
11116
|
+
"use strict";
|
|
11117
|
+
SHELL_METACHARS = /[|&;<>`\n\r$(){}]/;
|
|
11118
|
+
GREP_BINARIES = /* @__PURE__ */ new Set(["grep", "egrep", "rg", "ripgrep"]);
|
|
11119
|
+
}
|
|
11120
|
+
});
|
|
11121
|
+
|
|
10992
11122
|
// src/engine.ts
|
|
10993
11123
|
init_types();
|
|
10994
11124
|
init_walk();
|
|
@@ -11130,8 +11260,8 @@ init_loader();
|
|
|
11130
11260
|
// src/ast/grammars-pull.ts
|
|
11131
11261
|
init_types();
|
|
11132
11262
|
import { createHash as createHash2 } from "crypto";
|
|
11133
|
-
import { mkdirSync, writeFileSync } from "fs";
|
|
11134
|
-
import { dirname as dirname2, resolve, sep as sep2 } from "path";
|
|
11263
|
+
import { existsSync as existsSync2, mkdirSync, mkdtempSync, readFileSync as readFileSync3, renameSync, rmSync, writeFileSync } from "fs";
|
|
11264
|
+
import { dirname as dirname2, join as join3, resolve, sep as sep2 } from "path";
|
|
11135
11265
|
import { gunzipSync } from "zlib";
|
|
11136
11266
|
var DEFAULT_GRAMMARS_URL = `https://github.com/maxgfr/codeindex/releases/download/v${ENGINE_VERSION}/grammars-${ENGINE_VERSION}.tar.gz`;
|
|
11137
11267
|
function resolveGrammarsPullTarget() {
|
|
@@ -11223,6 +11353,109 @@ function extractGrammarsTarball(bytes, destDir) {
|
|
|
11223
11353
|
const raw = b.length >= 2 && b[0] === 31 && b[1] === 139 ? gunzipSync(b) : b;
|
|
11224
11354
|
return extractTarInto(raw, destDir);
|
|
11225
11355
|
}
|
|
11356
|
+
async function pullGrammars(cacheDir, opts = {}) {
|
|
11357
|
+
const note = opts.onNote ?? (() => {
|
|
11358
|
+
});
|
|
11359
|
+
const target = resolveGrammarsPullTarget();
|
|
11360
|
+
let expected;
|
|
11361
|
+
if (target.sha256Url) {
|
|
11362
|
+
try {
|
|
11363
|
+
expected = await fetchExpectedSha256(target.sha256Url);
|
|
11364
|
+
} catch (e) {
|
|
11365
|
+
note(`codeindex: could not fetch checksum (${e instanceof Error ? e.message : String(e)}) \u2014 proceeding unverified
|
|
11366
|
+
`);
|
|
11367
|
+
}
|
|
11368
|
+
}
|
|
11369
|
+
const runtime = join3(cacheDir, "web-tree-sitter.wasm");
|
|
11370
|
+
const markerPath = join3(dirname2(cacheDir), `${ENGINE_VERSION}.sha256`);
|
|
11371
|
+
if (existsSync2(runtime) && expected && existsSync2(markerPath)) {
|
|
11372
|
+
let marker = "";
|
|
11373
|
+
try {
|
|
11374
|
+
marker = readFileSync3(markerPath, "utf8").trim();
|
|
11375
|
+
} catch {
|
|
11376
|
+
}
|
|
11377
|
+
if (marker === expected) {
|
|
11378
|
+
return { ok: true, status: "up-to-date", cacheDir, message: `codeindex: grammars already present at ${cacheDir} (up to date)
|
|
11379
|
+
` };
|
|
11380
|
+
}
|
|
11381
|
+
}
|
|
11382
|
+
note(`codeindex: fetching grammars from ${target.url} \u2192 ${cacheDir}
|
|
11383
|
+
`);
|
|
11384
|
+
let bytes;
|
|
11385
|
+
try {
|
|
11386
|
+
bytes = await fetchGrammarsTarball(target.url, expected);
|
|
11387
|
+
} catch (e) {
|
|
11388
|
+
return {
|
|
11389
|
+
ok: false,
|
|
11390
|
+
status: "failed",
|
|
11391
|
+
cacheDir,
|
|
11392
|
+
message: `codeindex: pull failed \u2014 ${e instanceof Error ? e.message : String(e)} (nothing written)
|
|
11393
|
+
`
|
|
11394
|
+
};
|
|
11395
|
+
}
|
|
11396
|
+
let tmp;
|
|
11397
|
+
try {
|
|
11398
|
+
mkdirSync(dirname2(cacheDir), { recursive: true });
|
|
11399
|
+
tmp = mkdtempSync(join3(dirname2(cacheDir), ".grammars-tmp-"));
|
|
11400
|
+
extractGrammarsTarball(bytes, tmp);
|
|
11401
|
+
if (!existsSync2(join3(tmp, "web-tree-sitter.wasm"))) {
|
|
11402
|
+
throw new Error("archive is missing web-tree-sitter.wasm");
|
|
11403
|
+
}
|
|
11404
|
+
if (existsSync2(cacheDir)) rmSync(cacheDir, { recursive: true, force: true });
|
|
11405
|
+
renameSync(tmp, cacheDir);
|
|
11406
|
+
tmp = void 0;
|
|
11407
|
+
if (expected) writeFileSync(markerPath, expected + "\n");
|
|
11408
|
+
} catch (e) {
|
|
11409
|
+
if (tmp) {
|
|
11410
|
+
try {
|
|
11411
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
11412
|
+
} catch {
|
|
11413
|
+
}
|
|
11414
|
+
}
|
|
11415
|
+
return {
|
|
11416
|
+
ok: false,
|
|
11417
|
+
status: "failed",
|
|
11418
|
+
cacheDir,
|
|
11419
|
+
message: `codeindex: pull failed \u2014 ${e instanceof Error ? e.message : String(e)} (nothing written)
|
|
11420
|
+
`
|
|
11421
|
+
};
|
|
11422
|
+
}
|
|
11423
|
+
return { ok: true, status: "pulled", cacheDir, message: `codeindex: grammars extracted \u2192 ${cacheDir}
|
|
11424
|
+
` };
|
|
11425
|
+
}
|
|
11426
|
+
|
|
11427
|
+
// src/ast/warm.ts
|
|
11428
|
+
init_loader();
|
|
11429
|
+
async function warmGrammars(opts = {}) {
|
|
11430
|
+
const label = opts.label ?? "codeindex";
|
|
11431
|
+
const notes = [];
|
|
11432
|
+
const note = (msg) => {
|
|
11433
|
+
notes.push(msg);
|
|
11434
|
+
if (opts.onNote) opts.onNote(msg);
|
|
11435
|
+
else process.stderr.write(msg);
|
|
11436
|
+
};
|
|
11437
|
+
const noPull = process.env.CODEINDEX_NO_GRAMMARS_PULL;
|
|
11438
|
+
const mayPull = (opts.pull ?? true) && !(noPull && noPull.trim() && noPull !== "0");
|
|
11439
|
+
const keys = [...opts.keys ?? allGrammarKeys()];
|
|
11440
|
+
let pulled = false;
|
|
11441
|
+
if (resolveGrammarsTier().tier === "none" && mayPull) {
|
|
11442
|
+
note(`${label}: tree-sitter grammars not found locally \u2014 pulling them into the shared cache (once per machine)\u2026
|
|
11443
|
+
`);
|
|
11444
|
+
const res = await pullGrammars(sharedGrammarsCacheDir(), { onNote: note });
|
|
11445
|
+
note(res.message);
|
|
11446
|
+
pulled = res.ok && res.status === "pulled";
|
|
11447
|
+
}
|
|
11448
|
+
await ensureGrammars(keys);
|
|
11449
|
+
const tier = resolveGrammarsTier().tier;
|
|
11450
|
+
const ready = keys.some((k) => grammarReady(k));
|
|
11451
|
+
if (!ready) {
|
|
11452
|
+
note(
|
|
11453
|
+
`${label}: no tree-sitter grammars available (offline?) \u2014 extracting with the regex tier, so symbols and call sites are less precise. Run \`codeindex grammars pull\` once online to enable AST precision.
|
|
11454
|
+
`
|
|
11455
|
+
);
|
|
11456
|
+
}
|
|
11457
|
+
return { tier, ready, pulled, notes };
|
|
11458
|
+
}
|
|
11226
11459
|
|
|
11227
11460
|
// src/engine.ts
|
|
11228
11461
|
init_resolve();
|
|
@@ -11498,6 +11731,7 @@ init_deadcode();
|
|
|
11498
11731
|
init_complexity();
|
|
11499
11732
|
init_viz();
|
|
11500
11733
|
init_mcp();
|
|
11734
|
+
init_rewrite();
|
|
11501
11735
|
init_hash();
|
|
11502
11736
|
init_sort();
|
|
11503
11737
|
init_util();
|
|
@@ -11506,8 +11740,8 @@ init_util();
|
|
|
11506
11740
|
init_types();
|
|
11507
11741
|
init_types();
|
|
11508
11742
|
init_loader();
|
|
11509
|
-
import { existsSync as
|
|
11510
|
-
import {
|
|
11743
|
+
import { existsSync as existsSync5, mkdirSync as mkdirSync3, readFileSync as readFileSync8, writeFileSync as writeFileSync4 } from "fs";
|
|
11744
|
+
import { join as join15, resolve as resolve2 } from "path";
|
|
11511
11745
|
init_pipeline();
|
|
11512
11746
|
init_hash();
|
|
11513
11747
|
init_graph_json();
|
|
@@ -11574,8 +11808,27 @@ Commands:
|
|
|
11574
11808
|
repomap Token-budgeted map of the highest-PageRank files (--budget-tokens)
|
|
11575
11809
|
hotspots Churn \xD7 size ranking of the files where work concentrates (JSON)
|
|
11576
11810
|
coupling Change coupling: files that change together (JSON; --since <ref>)
|
|
11577
|
-
|
|
11578
|
-
|
|
11811
|
+
deadcode Dead-code candidates in two labeled tiers: 'unreferenced' (no
|
|
11812
|
+
call site binds AND nothing references the name) and 'uncalled'
|
|
11813
|
+
(referenced \u2014 re-export, type position \u2014 but never called)
|
|
11814
|
+
complexity Cyclomatic-complexity estimates, most-complex first. Pass a file
|
|
11815
|
+
positional for one file; omit for the repo-wide top
|
|
11816
|
+
risk Complexity \xD7 git-churn ranking (JSON; --since <ref> to bound)
|
|
11817
|
+
mermaid Mermaid diagram of the module graph; pass a module positional to
|
|
11818
|
+
focus on one neighborhood
|
|
11819
|
+
rewrite Map an expensive tree-wide search onto its indexed equivalent:
|
|
11820
|
+
cli.mjs rewrite '<command line>'. Prints the replacement command
|
|
11821
|
+
and exits 0, or exits 1 when it has no opinion (run the original).
|
|
11822
|
+
Deliberately conservative \u2014 any shell metacharacter or unknown
|
|
11823
|
+
flag refuses the rewrite
|
|
11824
|
+
mcp Run as an MCP server over stdio (26 tools: scan_summary, graph,
|
|
11825
|
+
symbols, callers, workspaces, churn, symbols_overview,
|
|
11826
|
+
find_symbol, find_references, repo_map, hotspots, coupling,
|
|
11827
|
+
dead_code, complexity, mermaid, grep, search, embed_status,
|
|
11828
|
+
check_rules, the memory quartet and the three symbolic-edit
|
|
11829
|
+
writes). Flags: --repo <dir> pins ONE repository so the per-tool
|
|
11830
|
+
repo argument becomes optional (an explicit per-call repo still
|
|
11831
|
+
wins); --server-name <name> overrides the announced serverInfo
|
|
11579
11832
|
version Print the engine version
|
|
11580
11833
|
|
|
11581
11834
|
Flags:
|
|
@@ -11605,6 +11858,8 @@ Flags:
|
|
|
11605
11858
|
--recall \`callers\`: recall-oriented binding (issue #7) \u2014 relaxes
|
|
11606
11859
|
the JS/TS import gate to unique repo-wide names and labels
|
|
11607
11860
|
each site corroborated|unique-name
|
|
11861
|
+
--ignore-case \`grep\`: case-insensitive matching
|
|
11862
|
+
--max-hits <n> \`grep\`: cap returned hits (default 200)
|
|
11608
11863
|
`;
|
|
11609
11864
|
function parseFlags(args2) {
|
|
11610
11865
|
const flags2 = { repo: process.cwd(), include: [], exclude: [], gitignore: true, ignoreDirs: [], noAst: false, fuzzy: true, semantic: false };
|
|
@@ -11671,6 +11926,26 @@ function scanOptions(flags2, precomputedWalk) {
|
|
|
11671
11926
|
};
|
|
11672
11927
|
}
|
|
11673
11928
|
var SCANLESS_COMMANDS = /* @__PURE__ */ new Set(["grep", "churn", "coupling", "workspaces", "grammars"]);
|
|
11929
|
+
function parseMcpFlags(argv) {
|
|
11930
|
+
let defaultRepo;
|
|
11931
|
+
let name2;
|
|
11932
|
+
for (let i2 = 0; i2 < argv.length; i2++) {
|
|
11933
|
+
const a = argv[i2];
|
|
11934
|
+
if (a === "--repo") {
|
|
11935
|
+
const v = argv[++i2];
|
|
11936
|
+
if (!v) throw new Error("--repo requires a directory");
|
|
11937
|
+
defaultRepo = resolve2(v);
|
|
11938
|
+
} else if (a === "--server-name") {
|
|
11939
|
+
const v = argv[++i2];
|
|
11940
|
+
if (!v) throw new Error("--server-name requires a value");
|
|
11941
|
+
name2 = v;
|
|
11942
|
+
} else {
|
|
11943
|
+
throw new Error(`unknown flag for \`mcp\`: ${a}`);
|
|
11944
|
+
}
|
|
11945
|
+
}
|
|
11946
|
+
if (defaultRepo && !existsSync5(defaultRepo)) throw new Error(`--repo path does not exist: ${defaultRepo}`);
|
|
11947
|
+
return { defaultRepo, serverInfo: name2 ? { name: name2 } : void 0 };
|
|
11948
|
+
}
|
|
11674
11949
|
async function runCli(argv) {
|
|
11675
11950
|
const [cmd, ...rest] = argv;
|
|
11676
11951
|
if (!cmd || cmd === "help" || cmd === "--help" || cmd === "-h") {
|
|
@@ -11681,13 +11956,23 @@ async function runCli(argv) {
|
|
|
11681
11956
|
process.stdout.write(ENGINE_VERSION + "\n");
|
|
11682
11957
|
return;
|
|
11683
11958
|
}
|
|
11959
|
+
if (cmd === "rewrite") {
|
|
11960
|
+
const { rewriteCommand: rewriteCommand2 } = await Promise.resolve().then(() => (init_rewrite(), rewrite_exports));
|
|
11961
|
+
const rewritten = rewriteCommand2(rest.join(" "));
|
|
11962
|
+
if (!rewritten) {
|
|
11963
|
+
process.exitCode = 1;
|
|
11964
|
+
return;
|
|
11965
|
+
}
|
|
11966
|
+
process.stdout.write(rewritten + "\n");
|
|
11967
|
+
return;
|
|
11968
|
+
}
|
|
11684
11969
|
if (cmd === "mcp") {
|
|
11685
11970
|
const { runMcpServer: runMcpServer2 } = await Promise.resolve().then(() => (init_mcp(), mcp_exports));
|
|
11686
|
-
await runMcpServer2();
|
|
11971
|
+
await runMcpServer2(parseMcpFlags(rest));
|
|
11687
11972
|
return;
|
|
11688
11973
|
}
|
|
11689
11974
|
const flags2 = parseFlags(rest);
|
|
11690
|
-
if (!
|
|
11975
|
+
if (!existsSync5(flags2.repo)) throw new Error(`--repo path does not exist: ${flags2.repo}`);
|
|
11691
11976
|
const scans = !SCANLESS_COMMANDS.has(cmd) && !(cmd === "embed" && flags2.positional !== "build");
|
|
11692
11977
|
let precomputedWalk;
|
|
11693
11978
|
if (scans && !flags2.noAst) {
|
|
@@ -11707,7 +11992,7 @@ async function runCli(argv) {
|
|
|
11707
11992
|
let cache;
|
|
11708
11993
|
let meta = {};
|
|
11709
11994
|
try {
|
|
11710
|
-
const parsed = JSON.parse(
|
|
11995
|
+
const parsed = JSON.parse(readFileSync8(cachePath, "utf8"));
|
|
11711
11996
|
if (parsed.schemaVersion === SCHEMA_VERSION && parsed.extractorVersion === EXTRACTOR_VERSION) {
|
|
11712
11997
|
cache = new Map(Object.entries(parsed.files));
|
|
11713
11998
|
meta = {
|
|
@@ -11728,7 +12013,7 @@ async function runCli(argv) {
|
|
|
11728
12013
|
const embedPath = join15(outDir, "embeddings.bin");
|
|
11729
12014
|
const artifactSha = (path) => {
|
|
11730
12015
|
try {
|
|
11731
|
-
return sha1(
|
|
12016
|
+
return sha1(readFileSync8(path));
|
|
11732
12017
|
} catch {
|
|
11733
12018
|
return void 0;
|
|
11734
12019
|
}
|
|
@@ -11948,7 +12233,7 @@ async function runCli(argv) {
|
|
|
11948
12233
|
const cacheDir = sharedGrammarsCacheDir();
|
|
11949
12234
|
if (sub === "status") {
|
|
11950
12235
|
const info2 = resolveGrammarsTier();
|
|
11951
|
-
const runtimePresent = info2.dir ?
|
|
12236
|
+
const runtimePresent = info2.dir ? existsSync5(join15(info2.dir, "web-tree-sitter.wasm")) : false;
|
|
11952
12237
|
const target = resolveGrammarsPullTarget();
|
|
11953
12238
|
const status = {
|
|
11954
12239
|
engineVersion: ENGINE_VERSION,
|
|
@@ -11961,77 +12246,15 @@ async function runCli(argv) {
|
|
|
11961
12246
|
};
|
|
11962
12247
|
emit(JSON.stringify(status, null, 2) + "\n", flags2.out);
|
|
11963
12248
|
} else if (sub === "pull") {
|
|
11964
|
-
const
|
|
11965
|
-
|
|
11966
|
-
if (
|
|
11967
|
-
try {
|
|
11968
|
-
expected = await fetchExpectedSha256(target.sha256Url);
|
|
11969
|
-
} catch (e) {
|
|
11970
|
-
process.stderr.write(
|
|
11971
|
-
`codeindex: could not fetch checksum (${e instanceof Error ? e.message : String(e)}) \u2014 proceeding unverified
|
|
11972
|
-
`
|
|
11973
|
-
);
|
|
11974
|
-
}
|
|
11975
|
-
}
|
|
11976
|
-
const runtime = join15(cacheDir, "web-tree-sitter.wasm");
|
|
11977
|
-
const markerPath = join15(dirname4(cacheDir), `${ENGINE_VERSION}.sha256`);
|
|
11978
|
-
if (existsSync4(runtime) && expected && existsSync4(markerPath)) {
|
|
11979
|
-
let marker = "";
|
|
11980
|
-
try {
|
|
11981
|
-
marker = readFileSync7(markerPath, "utf8").trim();
|
|
11982
|
-
} catch {
|
|
11983
|
-
}
|
|
11984
|
-
if (marker === expected) {
|
|
11985
|
-
process.stderr.write(`codeindex: grammars already present at ${cacheDir} (up to date)
|
|
11986
|
-
`);
|
|
11987
|
-
return;
|
|
11988
|
-
}
|
|
11989
|
-
}
|
|
11990
|
-
process.stderr.write(`codeindex: fetching grammars from ${target.url} \u2192 ${cacheDir}
|
|
11991
|
-
`);
|
|
11992
|
-
let bytes;
|
|
11993
|
-
try {
|
|
11994
|
-
bytes = await fetchGrammarsTarball(target.url, expected);
|
|
11995
|
-
} catch (e) {
|
|
11996
|
-
process.stderr.write(`codeindex: pull failed \u2014 ${e instanceof Error ? e.message : String(e)} (nothing written)
|
|
11997
|
-
`);
|
|
11998
|
-
process.exitCode = 1;
|
|
11999
|
-
return;
|
|
12000
|
-
}
|
|
12001
|
-
let tmp;
|
|
12002
|
-
try {
|
|
12003
|
-
mkdirSync3(dirname4(cacheDir), { recursive: true });
|
|
12004
|
-
tmp = mkdtempSync(join15(dirname4(cacheDir), ".grammars-tmp-"));
|
|
12005
|
-
extractGrammarsTarball(bytes, tmp);
|
|
12006
|
-
if (!existsSync4(join15(tmp, "web-tree-sitter.wasm"))) {
|
|
12007
|
-
throw new Error("archive is missing web-tree-sitter.wasm");
|
|
12008
|
-
}
|
|
12009
|
-
if (existsSync4(cacheDir)) rmSync2(cacheDir, { recursive: true, force: true });
|
|
12010
|
-
renameSync(tmp, cacheDir);
|
|
12011
|
-
tmp = void 0;
|
|
12012
|
-
if (expected) writeFileSync4(markerPath, expected + "\n");
|
|
12013
|
-
} catch (e) {
|
|
12014
|
-
if (tmp) {
|
|
12015
|
-
try {
|
|
12016
|
-
rmSync2(tmp, { recursive: true, force: true });
|
|
12017
|
-
} catch {
|
|
12018
|
-
}
|
|
12019
|
-
}
|
|
12020
|
-
process.stderr.write(
|
|
12021
|
-
`codeindex: pull failed \u2014 ${e instanceof Error ? e.message : String(e)} (nothing written)
|
|
12022
|
-
`
|
|
12023
|
-
);
|
|
12024
|
-
process.exitCode = 1;
|
|
12025
|
-
return;
|
|
12026
|
-
}
|
|
12027
|
-
process.stderr.write(`codeindex: grammars extracted \u2192 ${cacheDir}
|
|
12028
|
-
`);
|
|
12249
|
+
const res = await pullGrammars(cacheDir, { onNote: (m) => process.stderr.write(m) });
|
|
12250
|
+
process.stderr.write(res.message);
|
|
12251
|
+
if (!res.ok) process.exitCode = 1;
|
|
12029
12252
|
} else {
|
|
12030
12253
|
throw new Error("grammars needs a subcommand: status | pull");
|
|
12031
12254
|
}
|
|
12032
12255
|
} else if (cmd === "rules") {
|
|
12033
12256
|
if (!flags2.config) throw new Error("rules needs --config <codeindex.rules.json>");
|
|
12034
|
-
const rules = parseRules(JSON.parse(
|
|
12257
|
+
const rules = parseRules(JSON.parse(readFileSync8(flags2.config, "utf8")));
|
|
12035
12258
|
const { graph } = buildIndexArtifacts(flags2.repo, scanOptions(flags2, precomputedWalk));
|
|
12036
12259
|
const violations = checkRules(graph, rules);
|
|
12037
12260
|
const errors = violations.filter((v) => v.severity === "error").length;
|
|
@@ -12076,7 +12299,8 @@ async function runCli(argv) {
|
|
|
12076
12299
|
emit(renderMermaid(graph, { module: flags2.positional }), flags2.out);
|
|
12077
12300
|
} else if (cmd === "grep") {
|
|
12078
12301
|
if (!flags2.positional) throw new Error("grep needs a pattern: cli.mjs grep <pattern> --repo <dir>");
|
|
12079
|
-
const
|
|
12302
|
+
const scopeGlobs = flags2.scope ? [`${flags2.scope.replace(/\/+$/, "")}/**`] : [];
|
|
12303
|
+
const globs = [...scopeGlobs, ...flags2.include, ...flags2.exclude.map((g) => `!${g}`)];
|
|
12080
12304
|
const hits = grepRepo(flags2.repo, flags2.positional, {
|
|
12081
12305
|
globs: globs.length ? globs : void 0,
|
|
12082
12306
|
ignoreCase: flags2.ignoreCase,
|
|
@@ -12182,6 +12406,7 @@ export {
|
|
|
12182
12406
|
parseGitignore,
|
|
12183
12407
|
parseRules,
|
|
12184
12408
|
probeEndpoint,
|
|
12409
|
+
pullGrammars,
|
|
12185
12410
|
quantize,
|
|
12186
12411
|
rankHotspots,
|
|
12187
12412
|
rankedKeywords,
|
|
@@ -12204,6 +12429,7 @@ export {
|
|
|
12204
12429
|
resolveGrammarsTier,
|
|
12205
12430
|
resolveImport,
|
|
12206
12431
|
resolveUniqueSymbol,
|
|
12432
|
+
rewriteCommand,
|
|
12207
12433
|
riskHotspots,
|
|
12208
12434
|
roundHalfToEven,
|
|
12209
12435
|
rrf,
|
|
@@ -12228,6 +12454,7 @@ export {
|
|
|
12228
12454
|
untestedModules,
|
|
12229
12455
|
untrackedFiles,
|
|
12230
12456
|
walk,
|
|
12457
|
+
warmGrammars,
|
|
12231
12458
|
wordpiece,
|
|
12232
12459
|
writeMemory
|
|
12233
12460
|
};
|