grepmax 0.21.0 → 0.21.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +41 -0
- package/dist/commands/config.js +7 -1
- package/dist/eval-tokens.js +220 -0
- package/dist/lib/core/languages.js +40 -0
- package/dist/lib/daemon/watcher-manager.js +2 -2
- package/dist/lib/graph/graph-builder.js +52 -11
- package/dist/lib/index/batch-processor.js +1 -1
- package/dist/lib/utils/file-utils.js +33 -0
- package/dist/lib/utils/stale-hint.js +7 -2
- package/dist/lib/workers/orchestrator.js +1 -1
- package/package.json +3 -5
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "grepmax",
|
|
11
11
|
"source": "./plugins/grepmax",
|
|
12
12
|
"description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
|
|
13
|
-
"version": "0.21.
|
|
13
|
+
"version": "0.21.2",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Robert Owens",
|
|
16
16
|
"email": "robowens@me.com"
|
package/README.md
CHANGED
|
@@ -289,6 +289,45 @@ All data lives in `~/.gmax/`:
|
|
|
289
289
|
}
|
|
290
290
|
```
|
|
291
291
|
|
|
292
|
+
### Model Tier
|
|
293
|
+
|
|
294
|
+
gmax embeds with IBM's Granite r2 code-embedding models. Two tiers are available:
|
|
295
|
+
|
|
296
|
+
| Tier | Model | Dim | Params |
|
|
297
|
+
| --- | --- | --- | --- |
|
|
298
|
+
| `small` (default) | `granite-embedding-small-english-r2` | 384 | 47M |
|
|
299
|
+
| `standard` | `granite-embedding-english-r2` | 768 | 149M |
|
|
300
|
+
|
|
301
|
+
**384d is the default — and benchmarking says keep it.** On gmax's own 97-case
|
|
302
|
+
retrieval eval (`pnpm bench:recall`), the larger 768d model scored **~10 points
|
|
303
|
+
*worse* on Recall@10** than 384d, consistently on both the MLX GPU and ONNX CPU
|
|
304
|
+
embedding paths, with and without ColBERT rerank:
|
|
305
|
+
|
|
306
|
+
| Model | Recall@10 | MRR@10 |
|
|
307
|
+
| --- | --- | --- |
|
|
308
|
+
| `small` / 384d | **0.72** | **0.51** |
|
|
309
|
+
| `standard` / 768d | 0.62 | 0.44 |
|
|
310
|
+
|
|
311
|
+
<sub>gmax repo, 97 cases, MLX GPU, rerank off (the shipped default). The CPU/q4
|
|
312
|
+
path and the rerank-on path show the same ~0.10 Recall@10 gap, so it isn't a
|
|
313
|
+
quantization artifact — the bigger model just retrieves worse on code here.</sub>
|
|
314
|
+
|
|
315
|
+
Bigger isn't better for this workload: 768d roughly doubles embedding compute,
|
|
316
|
+
worker RAM, and dense-vector storage while *lowering* recall on code search.
|
|
317
|
+
Unless you have a measured reason to switch (e.g. a recall complaint on a very
|
|
318
|
+
large repo — benchmark it first), stay on 384d.
|
|
319
|
+
|
|
320
|
+
To change tiers:
|
|
321
|
+
|
|
322
|
+
```bash
|
|
323
|
+
gmax config --model-tier standard # a dim change needs a full re-embed
|
|
324
|
+
gmax repair --rebuild # drops the shared table, re-embeds every project at the new dim
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
A vector-dimension change can't be applied by a per-project `gmax index --reset`
|
|
328
|
+
— the shared `chunks` table is fixed-width, so `gmax repair --rebuild` is the
|
|
329
|
+
sanctioned path.
|
|
330
|
+
|
|
292
331
|
### Ignoring Files
|
|
293
332
|
|
|
294
333
|
gmax respects `.gitignore` and `.gmaxignore`:
|
|
@@ -339,6 +378,8 @@ GMAX_EVAL_RERANK=1 pnpm bench:oss # toggle ColBERT rerank
|
|
|
339
378
|
|
|
340
379
|
The OSS bench requires the fixture repos to be indexed first — see [`docs/known-limitations.md`](docs/known-limitations.md) for the most recent rerank-on-vs-off comparison across 4 datasets / 131 cases.
|
|
341
380
|
|
|
381
|
+
`pnpm bench:recall` also drives the model-tier comparison behind the 384d default — see [Model Tier](#model-tier) for why the larger 768d model is *not* the default.
|
|
382
|
+
|
|
342
383
|
## Attribution
|
|
343
384
|
|
|
344
385
|
grepmax is built upon the foundation of [mgrep](https://github.com/mixedbread-ai/mgrep) by MixedBread. See the [NOTICE](NOTICE) file for details.
|
package/dist/commands/config.js
CHANGED
|
@@ -84,6 +84,7 @@ Examples:
|
|
|
84
84
|
const newMode = (_c = options.embedMode) !== null && _c !== void 0 ? _c : globalConfig.embedMode;
|
|
85
85
|
const tier = (_d = config_1.MODEL_TIERS[newTier]) !== null && _d !== void 0 ? _d : config_1.MODEL_TIERS.small;
|
|
86
86
|
const tierChanged = newTier !== globalConfig.modelTier;
|
|
87
|
+
const dimChanged = tier.vectorDim !== globalConfig.vectorDim;
|
|
87
88
|
(0, index_config_1.writeGlobalConfig)({
|
|
88
89
|
modelTier: newTier,
|
|
89
90
|
vectorDim: tier.vectorDim,
|
|
@@ -98,7 +99,12 @@ Examples:
|
|
|
98
99
|
});
|
|
99
100
|
console.log(`Updated: embed-mode=${newMode}, model-tier=${newTier} (${tier.vectorDim}d)`);
|
|
100
101
|
if (tierChanged) {
|
|
101
|
-
|
|
102
|
+
// A dimension change can't be fixed by a per-project `gmax index --reset`:
|
|
103
|
+
// the shared `chunks` table is fixed-width, so it must be dropped and
|
|
104
|
+
// rebuilt. A same-dim model swap (future tiers) can use a per-project reset.
|
|
105
|
+
console.log(dimChanged
|
|
106
|
+
? `⚠️ Model tier changed (${globalConfig.vectorDim}d → ${tier.vectorDim}d). The shared vector table is fixed-width — run \`${config_1.REBUILD_COMMAND}\` to drop it and re-embed every project at the new dim.`
|
|
107
|
+
: "⚠️ Model tier changed — run `gmax index --reset` per project to re-embed with the new model.");
|
|
102
108
|
}
|
|
103
109
|
yield (0, exit_1.gracefulExit)();
|
|
104
110
|
}));
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var _a;
|
|
12
|
+
var _b;
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
// Reduce worker pool fan-out during eval to avoid ONNX concurrency issues.
|
|
15
|
+
// Must run before importing modules that read GMAX_WORKER_COUNT (mirrors eval.ts).
|
|
16
|
+
(_a = (_b = process.env).GMAX_WORKER_COUNT) !== null && _a !== void 0 ? _a : (_b.GMAX_WORKER_COUNT = "1");
|
|
17
|
+
// Token-savings benchmark — sibling to eval.ts.
|
|
18
|
+
//
|
|
19
|
+
// eval.ts measures Recall@10/MRR only; it never measures the "fewer tokens"
|
|
20
|
+
// value prop. This harness does, over the SAME 98 `cases` (imported from
|
|
21
|
+
// eval.ts), with an honest, conservative methodology:
|
|
22
|
+
//
|
|
23
|
+
// Baseline (no gmax — what a grep-then-Read agent pays): read the WHOLE first
|
|
24
|
+
// `expectedPath` file → estTokens(content). One file UNDER-counts (after a
|
|
25
|
+
// grep an agent typically reads several), so every reported ratio is a
|
|
26
|
+
// conservative FLOOR, not a best case.
|
|
27
|
+
//
|
|
28
|
+
// gmax pointer: searcher.search(query, 20, {rerank}) rendered through
|
|
29
|
+
// formatMcpPointerSearchResults(..., { query }) — the REAL bytes the
|
|
30
|
+
// `semantic_search detail:pointer` view returns, not an idealized chunk.
|
|
31
|
+
//
|
|
32
|
+
// gmax pointer+symbol: pointer bytes PLUS the top hit's symbol body, obtained
|
|
33
|
+
// by slicing the top result's file on its [startLine..endLine] range. That
|
|
34
|
+
// is what `extract_symbol` fundamentally returns (the symbol's line span),
|
|
35
|
+
// with zero dependency on non-importable MCP handler internals.
|
|
36
|
+
//
|
|
37
|
+
// We deliberately REJECT graphify's "70×/100×" methodology: it compares one
|
|
38
|
+
// query to reading the ENTIRE corpus (corpus_words = nodes*50, benchmark.py:113),
|
|
39
|
+
// which no agent ever does. Whole-file Read is the baseline an agent actually pays.
|
|
40
|
+
//
|
|
41
|
+
// estTokens is a chars/4 proxy (mirrors mcp.ts:2216), NOT a real tokenizer. Both
|
|
42
|
+
// sides use the same estimator, so the RATIO is largely estimator-invariant — no
|
|
43
|
+
// tokenizer dependency is warranted.
|
|
44
|
+
//
|
|
45
|
+
// Preconditions (identical to eval.ts): an indexed store for cwd AND query
|
|
46
|
+
// embeddings available (MLX :8100 or the ONNX CPU fallback).
|
|
47
|
+
//
|
|
48
|
+
// Output: dual-mode like eval.ts — GMAX_EVAL_JSON=1 / --json emits a single JSON
|
|
49
|
+
// object on stdout (human preamble routed to stderr); otherwise human-readable.
|
|
50
|
+
const node_fs_1 = require("node:fs");
|
|
51
|
+
const node_path_1 = require("node:path");
|
|
52
|
+
const mcp_1 = require("./commands/mcp");
|
|
53
|
+
const eval_1 = require("./eval");
|
|
54
|
+
const searcher_1 = require("./lib/search/searcher");
|
|
55
|
+
const vector_db_1 = require("./lib/store/vector-db");
|
|
56
|
+
const exit_1 = require("./lib/utils/exit");
|
|
57
|
+
const project_root_1 = require("./lib/utils/project-root");
|
|
58
|
+
const topK = 20;
|
|
59
|
+
// chars/4 proxy — mirrors the estimator in mcp.ts:2216. Not a tokenizer; fine
|
|
60
|
+
// here because BOTH sides use it, so the ratio is largely estimator-invariant.
|
|
61
|
+
const estTokens = (s) => Math.ceil(s.length / 4);
|
|
62
|
+
// Linear-interpolation quantile over an ascending-sorted array.
|
|
63
|
+
function quantile(sortedAsc, q) {
|
|
64
|
+
if (sortedAsc.length === 0)
|
|
65
|
+
return 0;
|
|
66
|
+
const pos = (sortedAsc.length - 1) * q;
|
|
67
|
+
const base = Math.floor(pos);
|
|
68
|
+
const rest = pos - base;
|
|
69
|
+
const next = sortedAsc[base + 1];
|
|
70
|
+
return next !== undefined
|
|
71
|
+
? sortedAsc[base] + rest * (next - sortedAsc[base])
|
|
72
|
+
: sortedAsc[base];
|
|
73
|
+
}
|
|
74
|
+
function mean(xs) {
|
|
75
|
+
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
76
|
+
}
|
|
77
|
+
function aggregate(ratios) {
|
|
78
|
+
const sorted = [...ratios].sort((a, b) => a - b);
|
|
79
|
+
return {
|
|
80
|
+
medianRatio: Number(quantile(sorted, 0.5).toFixed(2)),
|
|
81
|
+
p25: Number(quantile(sorted, 0.25).toFixed(2)),
|
|
82
|
+
p75: Number(quantile(sorted, 0.75).toFixed(2)),
|
|
83
|
+
meanRatio: Number(mean(sorted).toFixed(2)),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
function run() {
|
|
87
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
88
|
+
var _a, _b, _c;
|
|
89
|
+
const root = process.cwd();
|
|
90
|
+
const projectRoot = (_a = (0, project_root_1.findProjectRoot)(root)) !== null && _a !== void 0 ? _a : root;
|
|
91
|
+
const paths = (0, project_root_1.ensureProjectPaths)(projectRoot);
|
|
92
|
+
const vectorDb = new vector_db_1.VectorDB(paths.lancedbDir);
|
|
93
|
+
const searcher = new searcher_1.Searcher(vectorDb);
|
|
94
|
+
const jsonMode = process.env.GMAX_EVAL_JSON === "1" || process.argv.includes("--json");
|
|
95
|
+
// In JSON mode all human preamble goes to stderr so stdout stays one object.
|
|
96
|
+
const log = jsonMode ? console.error : console.log;
|
|
97
|
+
// Same precondition as eval.ts: bail loudly if the store is empty.
|
|
98
|
+
if (!(yield vectorDb.hasAnyRows())) {
|
|
99
|
+
console.error("❌ Store appears to be empty!");
|
|
100
|
+
console.error(' Run "gmax index" to populate the store with data.');
|
|
101
|
+
process.exit(1);
|
|
102
|
+
}
|
|
103
|
+
// Rerank OFF by default (matches eval.ts) — set GMAX_EVAL_RERANK=1 to measure
|
|
104
|
+
// the full production pipeline. Rendered bytes barely move with rerank; the
|
|
105
|
+
// flag is here so token numbers line up with whichever recall run they sit next to.
|
|
106
|
+
const rerank = process.env.GMAX_EVAL_RERANK === "1";
|
|
107
|
+
log("Starting token-savings benchmark...\n");
|
|
108
|
+
const results = [];
|
|
109
|
+
for (const c of eval_1.cases) {
|
|
110
|
+
// Baseline: a grep-then-Read agent reads the whole first expectedPath file.
|
|
111
|
+
// Pipe-split mirrors evaluateCase's path handling (eval.ts:552-555).
|
|
112
|
+
const firstExpected = c.expectedPath.split("|")[0].trim();
|
|
113
|
+
let baselineTokens = 0;
|
|
114
|
+
let baselineOk = false;
|
|
115
|
+
try {
|
|
116
|
+
const content = (0, node_fs_1.readFileSync)((0, node_path_1.join)(projectRoot, firstExpected), "utf8");
|
|
117
|
+
baselineTokens = estTokens(content);
|
|
118
|
+
baselineOk = true;
|
|
119
|
+
}
|
|
120
|
+
catch (_d) {
|
|
121
|
+
// Fixture path not present in this checkout — excluded from aggregation.
|
|
122
|
+
}
|
|
123
|
+
// gmax pointer: the real rendered bytes of the detail:pointer view.
|
|
124
|
+
const res = yield searcher.search(c.query, topK, { rerank });
|
|
125
|
+
const pointerText = (0, mcp_1.formatMcpPointerSearchResults)(res.data, projectRoot, {
|
|
126
|
+
query: c.query,
|
|
127
|
+
});
|
|
128
|
+
const pointerTokens = estTokens(pointerText);
|
|
129
|
+
// gmax pointer+symbol: pointer bytes plus the top hit's symbol body, sliced
|
|
130
|
+
// by its [startLine..endLine] range (0-based; mcp.ts:482 adds 1 for display).
|
|
131
|
+
let symbolTokens = 0;
|
|
132
|
+
let symbolLabel = "";
|
|
133
|
+
let topHitPath = "";
|
|
134
|
+
const top = res.data[0];
|
|
135
|
+
if (top) {
|
|
136
|
+
const absPath = (0, mcp_1.searchResultPath)(top);
|
|
137
|
+
topHitPath = absPath;
|
|
138
|
+
const start = (0, mcp_1.searchResultStartLine)(top);
|
|
139
|
+
const end = (0, mcp_1.searchResultEndLine)(top, start);
|
|
140
|
+
try {
|
|
141
|
+
const lines = (0, node_fs_1.readFileSync)(absPath, "utf8").split("\n");
|
|
142
|
+
symbolTokens = estTokens(lines.slice(start, end + 1).join("\n"));
|
|
143
|
+
symbolLabel =
|
|
144
|
+
(_c = (_b = top.defined_symbols) === null || _b === void 0 ? void 0 : _b[0]) !== null && _c !== void 0 ? _c : "";
|
|
145
|
+
}
|
|
146
|
+
catch (_e) {
|
|
147
|
+
// Top-hit file unreadable — symbol cost stays 0 (pointer-only).
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
const pointerSymbolTokens = pointerTokens + symbolTokens;
|
|
151
|
+
const skipped = !baselineOk;
|
|
152
|
+
results.push({
|
|
153
|
+
query: c.query,
|
|
154
|
+
expectedPath: c.expectedPath,
|
|
155
|
+
baselineTokens,
|
|
156
|
+
pointerTokens,
|
|
157
|
+
symbolTokens,
|
|
158
|
+
pointerSymbolTokens,
|
|
159
|
+
symbolLabel,
|
|
160
|
+
topHitPath,
|
|
161
|
+
ratioPointer: skipped || pointerTokens === 0 ? 0 : baselineTokens / pointerTokens,
|
|
162
|
+
ratioPointerSymbol: skipped || pointerSymbolTokens === 0
|
|
163
|
+
? 0
|
|
164
|
+
: baselineTokens / pointerSymbolTokens,
|
|
165
|
+
skipped,
|
|
166
|
+
note: skipped ? "baseline file missing" : undefined,
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
const evaluated = results.filter((r) => !r.skipped);
|
|
170
|
+
const pointerAgg = aggregate(evaluated.map((r) => r.ratioPointer));
|
|
171
|
+
const pointerSymbolAgg = aggregate(evaluated.map((r) => r.ratioPointerSymbol));
|
|
172
|
+
const summary = {
|
|
173
|
+
cases: results.length,
|
|
174
|
+
evaluated: evaluated.length,
|
|
175
|
+
skipped: results.length - evaluated.length,
|
|
176
|
+
rerank,
|
|
177
|
+
estimator: "chars/4 proxy (mirrors mcp.ts:2216)",
|
|
178
|
+
baseline: "whole-file Read of first expectedPath — conservative floor (agents read several files after a grep)",
|
|
179
|
+
pointer: pointerAgg,
|
|
180
|
+
pointerSymbol: pointerSymbolAgg,
|
|
181
|
+
storePath: paths.lancedbDir,
|
|
182
|
+
};
|
|
183
|
+
if (jsonMode) {
|
|
184
|
+
process.stdout.write(`${JSON.stringify({ summary, results }, null, 2)}\n`);
|
|
185
|
+
}
|
|
186
|
+
else {
|
|
187
|
+
const fmt = (n) => `${n.toFixed(1)}×`;
|
|
188
|
+
console.log("=".repeat(80));
|
|
189
|
+
console.log(`Token-savings benchmark — store: ${paths.lancedbDir}`);
|
|
190
|
+
console.log("Baseline = whole-file Read of first expectedPath (chars/4, conservative floor)");
|
|
191
|
+
console.log("=".repeat(80));
|
|
192
|
+
for (const r of results) {
|
|
193
|
+
if (r.skipped) {
|
|
194
|
+
console.log(`⏭ ${r.query}`);
|
|
195
|
+
console.log(` => skipped (${r.note}: ${r.expectedPath})`);
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
console.log(r.query);
|
|
199
|
+
console.log(` baseline ${r.baselineTokens} tok | pointer ${r.pointerTokens} tok (${fmt(r.ratioPointer)}) | +symbol ${r.pointerSymbolTokens} tok (${fmt(r.ratioPointerSymbol)})`);
|
|
200
|
+
}
|
|
201
|
+
console.log("=".repeat(80));
|
|
202
|
+
console.log(`gmax pointer median ${fmt(pointerAgg.medianRatio)} (p25 ${fmt(pointerAgg.p25)} · p75 ${fmt(pointerAgg.p75)} · mean ${fmt(pointerAgg.meanRatio)})`);
|
|
203
|
+
console.log(`gmax pointer+symbol median ${fmt(pointerSymbolAgg.medianRatio)} (p25 ${fmt(pointerSymbolAgg.p25)} · p75 ${fmt(pointerSymbolAgg.p75)} · mean ${fmt(pointerSymbolAgg.meanRatio)})`);
|
|
204
|
+
console.log(`Evaluated ${evaluated.length}/${results.length} cases` +
|
|
205
|
+
(summary.skipped ? ` (${summary.skipped} skipped: file missing)` : ""));
|
|
206
|
+
console.log("Estimator: chars/4 proxy — ratio is estimator-invariant. Single-file baseline under-counts → floor.");
|
|
207
|
+
console.log("=".repeat(80));
|
|
208
|
+
}
|
|
209
|
+
yield (0, exit_1.gracefulExit)(0);
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
if (
|
|
213
|
+
// Only auto-run when executed directly (not when imported for experiments/tests)
|
|
214
|
+
require.main === module &&
|
|
215
|
+
process.env.GMAX_EVAL_AUTORUN !== "0") {
|
|
216
|
+
run().catch((err) => {
|
|
217
|
+
console.error("Token benchmark failed:", err);
|
|
218
|
+
(0, exit_1.gracefulExit)(1);
|
|
219
|
+
});
|
|
220
|
+
}
|
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.LANGUAGES = void 0;
|
|
4
4
|
exports.getLanguageByExtension = getLanguageByExtension;
|
|
5
|
+
exports.languageFamily = languageFamily;
|
|
6
|
+
exports.languageFamilyForPath = languageFamilyForPath;
|
|
5
7
|
exports.getGrammarUrl = getGrammarUrl;
|
|
8
|
+
const node_path_1 = require("node:path");
|
|
6
9
|
exports.LANGUAGES = [
|
|
7
10
|
{
|
|
8
11
|
id: "typescript",
|
|
@@ -258,6 +261,43 @@ function getLanguageByExtension(ext) {
|
|
|
258
261
|
const normalized = ext.toLowerCase();
|
|
259
262
|
return exports.LANGUAGES.find((lang) => lang.extensions.includes(normalized));
|
|
260
263
|
}
|
|
264
|
+
// ---------------------------------------------------------------------------
|
|
265
|
+
// Language families — for cross-language phantom-edge suppression
|
|
266
|
+
// ---------------------------------------------------------------------------
|
|
267
|
+
// The shared LanceDB table holds every language's chunks, scoped only by path.
|
|
268
|
+
// A bare-name call-graph match (`array_contains(referenced_symbols, 'render')`)
|
|
269
|
+
// therefore cross-connects a `render` defined in Python to a `render` referenced
|
|
270
|
+
// in TSX. To suppress those phantom edges we collapse languages that genuinely
|
|
271
|
+
// share a call namespace into one "family" and only keep edges within it.
|
|
272
|
+
// Languages absent from this map are their own family (their `id`). The grouping
|
|
273
|
+
// is deliberately conservative: only languages that directly call across one
|
|
274
|
+
// another are merged, so a real cross-file edge is never dropped.
|
|
275
|
+
const LANGUAGE_FAMILIES = {
|
|
276
|
+
// JS/TS ecosystem — these freely import and call across one another.
|
|
277
|
+
typescript: "js_ts",
|
|
278
|
+
tsx: "js_ts",
|
|
279
|
+
javascript: "js_ts",
|
|
280
|
+
// C and C++ share headers and C++ calls into C.
|
|
281
|
+
c: "c_cpp",
|
|
282
|
+
cpp: "c_cpp",
|
|
283
|
+
};
|
|
284
|
+
/** The call-namespace family for a language id (defaults to the id itself). */
|
|
285
|
+
function languageFamily(langId) {
|
|
286
|
+
var _a;
|
|
287
|
+
return (_a = LANGUAGE_FAMILIES[langId]) !== null && _a !== void 0 ? _a : langId;
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* The language family for a file path, or null when the extension is unknown or
|
|
291
|
+
* absent. Null means "unclassifiable, do not filter" — callers must keep edges
|
|
292
|
+
* they can't classify and only drop edges whose family is known *and* differs.
|
|
293
|
+
*/
|
|
294
|
+
function languageFamilyForPath(path) {
|
|
295
|
+
const ext = (0, node_path_1.extname)(path).toLowerCase();
|
|
296
|
+
if (!ext)
|
|
297
|
+
return null;
|
|
298
|
+
const lang = getLanguageByExtension(ext);
|
|
299
|
+
return lang ? languageFamily(lang.id) : null;
|
|
300
|
+
}
|
|
261
301
|
function getGrammarUrl(grammarName) {
|
|
262
302
|
var _a;
|
|
263
303
|
const lang = exports.LANGUAGES.find((l) => { var _a; return ((_a = l.grammar) === null || _a === void 0 ? void 0 : _a.name) === grammarName; });
|
|
@@ -382,9 +382,9 @@ class WatcherManager {
|
|
|
382
382
|
// Fast path: if only mtime changed but size is identical and we have a hash,
|
|
383
383
|
// just verify the hash in-process instead of sending to a worker.
|
|
384
384
|
if ((cached === null || cached === void 0 ? void 0 : cached.hash) && cached.size === stats.size) {
|
|
385
|
-
const {
|
|
385
|
+
const { computeContentHash } = yield Promise.resolve().then(() => __importStar(require("../utils/file-utils")));
|
|
386
386
|
const buf = yield fs.promises.readFile(absPath);
|
|
387
|
-
const hash =
|
|
387
|
+
const hash = computeContentHash(buf, absPath);
|
|
388
388
|
if (hash === cached.hash) {
|
|
389
389
|
// Content unchanged — update mtime in cache and skip worker
|
|
390
390
|
metaCache.put(absPath, Object.assign(Object.assign({}, cached), { mtimeMs: stats.mtimeMs }));
|
|
@@ -10,6 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.GraphBuilder = void 0;
|
|
13
|
+
const languages_1 = require("../core/languages");
|
|
13
14
|
const filter_builder_1 = require("../utils/filter-builder");
|
|
14
15
|
const query_timeout_1 = require("../utils/query-timeout");
|
|
15
16
|
const graph_traversal_1 = require("./graph-traversal");
|
|
@@ -36,8 +37,13 @@ class GraphBuilder {
|
|
|
36
37
|
}
|
|
37
38
|
/**
|
|
38
39
|
* Find all chunks that call the given symbol.
|
|
40
|
+
*
|
|
41
|
+
* `anchorFamily` (the language family of the symbol's definition) enables the
|
|
42
|
+
* cross-language phantom-edge guard: the shared table matches a bare symbol
|
|
43
|
+
* name across every language, so without it a `render` defined in one language
|
|
44
|
+
* picks up callers that merely reference an unrelated `render` in another.
|
|
39
45
|
*/
|
|
40
|
-
getCallers(symbol) {
|
|
46
|
+
getCallers(symbol, anchorFamily) {
|
|
41
47
|
return __awaiter(this, void 0, void 0, function* () {
|
|
42
48
|
const table = yield this.db.ensureTable();
|
|
43
49
|
const escaped = (0, filter_builder_1.escapeSqlString)(symbol);
|
|
@@ -59,7 +65,17 @@ class GraphBuilder {
|
|
|
59
65
|
.where(this.scopeWhere(`(array_contains(referenced_symbols, '${escaped}') OR array_contains(type_referenced_symbols, '${escaped}'))`))
|
|
60
66
|
.limit(100)
|
|
61
67
|
.toArray();
|
|
62
|
-
|
|
68
|
+
// Cross-language phantom-edge guard. When the anchor family is known, drop
|
|
69
|
+
// callers from a *different known* family; callers we can't classify (unknown
|
|
70
|
+
// extension) are kept so a real edge is never lost. No anchor → no filter.
|
|
71
|
+
const guarded = anchorFamily == null
|
|
72
|
+
? rows
|
|
73
|
+
: rows.filter((row) => {
|
|
74
|
+
var _a;
|
|
75
|
+
const fam = (0, languages_1.languageFamilyForPath)(String((_a = row.path) !== null && _a !== void 0 ? _a : ""));
|
|
76
|
+
return fam == null || fam === anchorFamily;
|
|
77
|
+
});
|
|
78
|
+
return guarded.map((row) => this.mapRowToNode(row, symbol, "caller"));
|
|
63
79
|
});
|
|
64
80
|
}
|
|
65
81
|
/**
|
|
@@ -108,8 +124,10 @@ class GraphBuilder {
|
|
|
108
124
|
const center = centerRows.length > 0
|
|
109
125
|
? this.mapRowToNode(centerRows[0], symbol, "center")
|
|
110
126
|
: null;
|
|
111
|
-
// 2. Get Callers
|
|
112
|
-
|
|
127
|
+
// 2. Get Callers — anchored to the center definition's language family so a
|
|
128
|
+
// bare name shared across languages doesn't pull in cross-language callers.
|
|
129
|
+
const anchorFamily = center ? (0, languages_1.languageFamilyForPath)(center.file) : null;
|
|
130
|
+
const callers = yield this.getCallers(symbol, anchorFamily);
|
|
113
131
|
// 3. Get Callees — resolve each to a GraphNode with file:line
|
|
114
132
|
const calleeNames = center ? center.calls.slice(0, 15) : [];
|
|
115
133
|
const calleeNodes = [];
|
|
@@ -204,7 +222,7 @@ class GraphBuilder {
|
|
|
204
222
|
visited.add(caller.symbol);
|
|
205
223
|
let subCallers = [];
|
|
206
224
|
if (remainingDepth > 0) {
|
|
207
|
-
const upstreamCallers = yield this.getCallers(caller.symbol);
|
|
225
|
+
const upstreamCallers = yield this.getCallers(caller.symbol, (0, languages_1.languageFamilyForPath)(caller.file));
|
|
208
226
|
subCallers = yield this.expandCallers(upstreamCallers, remainingDepth - 1, visited);
|
|
209
227
|
}
|
|
210
228
|
trees.push({ node: caller, callers: subCallers });
|
|
@@ -222,7 +240,10 @@ class GraphBuilder {
|
|
|
222
240
|
/** Distinct symbols that reference the given symbol (inbound edges). */
|
|
223
241
|
callersOf(symbol) {
|
|
224
242
|
return __awaiter(this, void 0, void 0, function* () {
|
|
225
|
-
|
|
243
|
+
// Anchor the guard to this symbol's own definition language family so the
|
|
244
|
+
// bare name doesn't pull in callers from an unrelated language.
|
|
245
|
+
const loc = yield this.resolveLocation(symbol);
|
|
246
|
+
const nodes = yield this.getCallers(symbol, loc ? (0, languages_1.languageFamilyForPath)(loc.file) : null);
|
|
226
247
|
const out = [];
|
|
227
248
|
for (const n of nodes) {
|
|
228
249
|
if (n.symbol && n.symbol !== "unknown" && !out.includes(n.symbol)) {
|
|
@@ -246,9 +267,13 @@ class GraphBuilder {
|
|
|
246
267
|
return __awaiter(this, void 0, void 0, function* () {
|
|
247
268
|
var _a, _b;
|
|
248
269
|
const hits = yield (0, graph_traversal_1.bfsNeighbors)(symbol, this.neighborFn(direction), maxHops);
|
|
270
|
+
// Resolve the origin once to anchor every neighbor's location lookup to the
|
|
271
|
+
// same language family, so a shared name doesn't resolve to a foreign file.
|
|
272
|
+
const origin = yield this.resolveLocation(symbol);
|
|
273
|
+
const anchorFamily = origin ? (0, languages_1.languageFamilyForPath)(origin.file) : null;
|
|
249
274
|
const out = [];
|
|
250
275
|
for (const h of hits) {
|
|
251
|
-
const loc = yield this.resolveLocation(h.symbol);
|
|
276
|
+
const loc = yield this.resolveLocation(h.symbol, anchorFamily);
|
|
252
277
|
out.push(Object.assign(Object.assign({}, h), { file: (_a = loc === null || loc === void 0 ? void 0 : loc.file) !== null && _a !== void 0 ? _a : "", line: (_b = loc === null || loc === void 0 ? void 0 : loc.line) !== null && _b !== void 0 ? _b : 0 }));
|
|
253
278
|
}
|
|
254
279
|
return out;
|
|
@@ -260,8 +285,13 @@ class GraphBuilder {
|
|
|
260
285
|
return (0, graph_traversal_1.findPath)(from, to, this.neighborFn(direction), maxHops);
|
|
261
286
|
});
|
|
262
287
|
}
|
|
263
|
-
/**
|
|
264
|
-
|
|
288
|
+
/**
|
|
289
|
+
* Resolve a symbol to a defining chunk's file:line, if indexed. With
|
|
290
|
+
* `anchorFamily` set, prefer the definition in that language family rather than
|
|
291
|
+
* an arbitrary cross-language match (the shared table can hold the same name in
|
|
292
|
+
* several languages); falls back to the first definition when none match.
|
|
293
|
+
*/
|
|
294
|
+
resolveLocation(symbol, anchorFamily) {
|
|
265
295
|
return __awaiter(this, void 0, void 0, function* () {
|
|
266
296
|
const table = yield this.db.ensureTable();
|
|
267
297
|
const escaped = (0, filter_builder_1.escapeSqlString)(symbol);
|
|
@@ -269,11 +299,22 @@ class GraphBuilder {
|
|
|
269
299
|
.query()
|
|
270
300
|
.select(["path", "start_line"])
|
|
271
301
|
.where(this.scopeWhere(`array_contains(defined_symbols, '${escaped}')`))
|
|
272
|
-
.
|
|
302
|
+
// No anchor → keep the cheap single-row fetch. With one, pull a few
|
|
303
|
+
// candidates so we can pick the same-family definition instead of guessing.
|
|
304
|
+
.limit(anchorFamily ? 25 : 1)
|
|
273
305
|
.toArray();
|
|
274
306
|
if (rows.length === 0)
|
|
275
307
|
return null;
|
|
276
|
-
|
|
308
|
+
let r = rows[0];
|
|
309
|
+
if (anchorFamily) {
|
|
310
|
+
const match = rows.find((row) => {
|
|
311
|
+
var _a;
|
|
312
|
+
return (0, languages_1.languageFamilyForPath)(String((_a = row.path) !== null && _a !== void 0 ? _a : "")) ===
|
|
313
|
+
anchorFamily;
|
|
314
|
+
});
|
|
315
|
+
if (match)
|
|
316
|
+
r = match;
|
|
317
|
+
}
|
|
277
318
|
return { file: String(r.path || ""), line: Number(r.start_line || 0) };
|
|
278
319
|
});
|
|
279
320
|
}
|
|
@@ -196,7 +196,7 @@ class ProjectBatchProcessor {
|
|
|
196
196
|
// verify in-process instead of dispatching to a worker (~220ms saved).
|
|
197
197
|
if ((cached === null || cached === void 0 ? void 0 : cached.hash) && cached.size === stats.size) {
|
|
198
198
|
const buf = yield fs.promises.readFile(absPath);
|
|
199
|
-
const hash = (0, file_utils_1.
|
|
199
|
+
const hash = (0, file_utils_1.computeContentHash)(buf, absPath);
|
|
200
200
|
if (hash === cached.hash) {
|
|
201
201
|
metaUpdates.set(absPath, Object.assign(Object.assign({}, cached), { mtimeMs: stats.mtimeMs }));
|
|
202
202
|
continue;
|
|
@@ -43,6 +43,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
43
43
|
};
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.computeBufferHash = computeBufferHash;
|
|
46
|
+
exports.stripMarkdownFrontmatter = stripMarkdownFrontmatter;
|
|
47
|
+
exports.computeContentHash = computeContentHash;
|
|
46
48
|
exports.hasNullByte = hasNullByte;
|
|
47
49
|
exports.readFileSnapshot = readFileSnapshot;
|
|
48
50
|
exports.isIndexableFile = isIndexableFile;
|
|
@@ -58,6 +60,37 @@ const config_1 = require("../../config");
|
|
|
58
60
|
function computeBufferHash(buffer) {
|
|
59
61
|
return (0, node_crypto_1.createHash)("sha256").update(buffer).digest("hex");
|
|
60
62
|
}
|
|
63
|
+
// Files whose leading YAML frontmatter is metadata, not content. A status:/tags:
|
|
64
|
+
// edit bumps mtime and changes the bytes, but the embedding ignores frontmatter —
|
|
65
|
+
// so hashing it would force a needless GPU re-embed on every metadata touch.
|
|
66
|
+
const FRONTMATTER_EXTENSIONS = new Set([".md", ".mdx"]);
|
|
67
|
+
// Leading YAML frontmatter: a whole `---` line at the very top (NOT a bare
|
|
68
|
+
// `startsWith("---")`, which would also match a `---` thematic break or `--- x`),
|
|
69
|
+
// through the next whole `---`/`...` line. Anchored at string start (no `m` flag)
|
|
70
|
+
// with an optional BOM, so only true file-leading frontmatter matches; a doc that
|
|
71
|
+
// opens with a thematic break and no closing fence is left untouched.
|
|
72
|
+
const FRONTMATTER_BLOCK = /^\uFEFF?---[ \t]*\r?\n[\s\S]*?\r?\n(?:---|\.\.\.)[ \t]*(?:\r?\n|$)/;
|
|
73
|
+
/** Strip leading YAML frontmatter from markdown bytes; returns input unchanged otherwise. */
|
|
74
|
+
function stripMarkdownFrontmatter(buffer) {
|
|
75
|
+
const text = buffer.toString("utf8");
|
|
76
|
+
const match = text.match(FRONTMATTER_BLOCK);
|
|
77
|
+
if (!match)
|
|
78
|
+
return buffer;
|
|
79
|
+
return Buffer.from(text.slice(match[0].length), "utf8");
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Hash file content for change detection. For markdown, YAML frontmatter is
|
|
83
|
+
* stripped first so a metadata-only edit doesn't bust `isFileCached` and trigger
|
|
84
|
+
* a re-embed; every other file hashes its exact bytes. Must be used identically
|
|
85
|
+
* across the catchup, watcher, and worker hash paths or they will disagree and
|
|
86
|
+
* re-index in a loop.
|
|
87
|
+
*/
|
|
88
|
+
function computeContentHash(buffer, filePath) {
|
|
89
|
+
if (FRONTMATTER_EXTENSIONS.has((0, node_path_1.extname)(filePath).toLowerCase())) {
|
|
90
|
+
return computeBufferHash(stripMarkdownFrontmatter(buffer));
|
|
91
|
+
}
|
|
92
|
+
return computeBufferHash(buffer);
|
|
93
|
+
}
|
|
61
94
|
function hasNullByte(buffer, sampleLength = 1024) {
|
|
62
95
|
const length = Math.min(buffer.length, sampleLength);
|
|
63
96
|
for (let i = 0; i < length; i++) {
|
|
@@ -104,7 +104,9 @@ function maybeWarnStaleEmbedding(projectRoot, opts) {
|
|
|
104
104
|
`current_dim=${gap.toDim}`,
|
|
105
105
|
`dim_changed=${gap.dimChanged}`,
|
|
106
106
|
`severity=${gap.severity}`,
|
|
107
|
-
|
|
107
|
+
// A dim change needs the global rebuild (shared table is fixed-width); a
|
|
108
|
+
// same-dim model swap can use a per-project reset.
|
|
109
|
+
`fix=${gap.dimChanged ? config_1.REBUILD_COMMAND : "gmax index --reset"}`,
|
|
108
110
|
].join("\t");
|
|
109
111
|
process.stderr.write(`${fields}\n`);
|
|
110
112
|
return;
|
|
@@ -113,7 +115,10 @@ function maybeWarnStaleEmbedding(projectRoot, opts) {
|
|
|
113
115
|
const detail = gap.dimChanged
|
|
114
116
|
? `vector dim ${gap.fromDim}→${gap.toDim} (incompatible — scores invalid until re-embed)`
|
|
115
117
|
: `model '${gap.fromModel}'→'${gap.toModel}' (same dim; results mix models until re-embed)`;
|
|
116
|
-
|
|
118
|
+
const fix = gap.dimChanged
|
|
119
|
+
? `Run '${config_1.REBUILD_COMMAND}'`
|
|
120
|
+
: "Run 'gmax index --reset'";
|
|
121
|
+
process.stderr.write(`${label} gmax: '${name}' indexed with embedding ${detail}. ${fix}. (silence: GMAX_NO_STALE_HINT=1)\n`);
|
|
117
122
|
}
|
|
118
123
|
/**
|
|
119
124
|
* Cross-project guard: when a `--all-projects` / `--projects` search spans
|
|
@@ -244,7 +244,7 @@ class WorkerOrchestrator {
|
|
|
244
244
|
: path.join(PROJECT_ROOT, input.path);
|
|
245
245
|
(0, logger_1.debug)("orch", `processFile start: ${input.path}`);
|
|
246
246
|
const { buffer, mtimeMs, size } = yield (0, file_utils_1.readFileSnapshot)(absolutePath);
|
|
247
|
-
const hash = (0, file_utils_1.
|
|
247
|
+
const hash = (0, file_utils_1.computeContentHash)(buffer, absolutePath);
|
|
248
248
|
if (!(0, file_utils_1.isIndexableFile)(absolutePath, size)) {
|
|
249
249
|
(0, logger_1.debug)("orch", `skip ${input.path} (non-indexable, size=${size})`);
|
|
250
250
|
return { vectors: [], hash, mtimeMs, size, shouldDelete: true };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "grepmax",
|
|
3
|
-
"version": "0.21.
|
|
3
|
+
"version": "0.21.2",
|
|
4
4
|
"author": "Robert Owens <78518764+reowens@users.noreply.github.com>",
|
|
5
5
|
"homepage": "https://github.com/reowens/grepmax",
|
|
6
6
|
"bugs": {
|
|
@@ -25,14 +25,12 @@
|
|
|
25
25
|
"dev": "npx tsc && node dist/index.js",
|
|
26
26
|
"test": "vitest run",
|
|
27
27
|
"test:watch": "vitest",
|
|
28
|
-
"benchmark": "./run-benchmark.sh",
|
|
29
|
-
"benchmark:index": "./run-benchmark.sh $HOME/gmax-benchmarks --index",
|
|
30
|
-
"benchmark:agent": "npx tsx src/bench/benchmark-agent.ts",
|
|
31
|
-
"benchmark:chart": "npx tsx src/bench/generate-benchmark-chart.ts",
|
|
32
28
|
"bench:recall": "npx tsx src/eval.ts",
|
|
33
29
|
"bench:recall:json": "GMAX_EVAL_JSON=1 npx tsx src/eval.ts",
|
|
34
30
|
"bench:oss": "npx tsx src/eval-oss.ts all",
|
|
35
31
|
"bench:oss:json": "GMAX_EVAL_JSON=1 npx tsx src/eval-oss.ts all",
|
|
32
|
+
"bench:tokens": "npx tsx src/eval-tokens.ts",
|
|
33
|
+
"bench:tokens:json": "GMAX_EVAL_JSON=1 npx tsx src/eval-tokens.ts",
|
|
36
34
|
"format": "biome format --write .",
|
|
37
35
|
"format:check": "biome check .",
|
|
38
36
|
"lint": "biome lint .",
|