grepmax 0.21.1 → 0.21.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/dist/config.js +6 -1
- package/dist/eval-tokens.js +220 -0
- package/dist/lib/core/languages.js +40 -0
- package/dist/lib/daemon/watcher-manager.js +2 -2
- package/dist/lib/graph/graph-builder.js +75 -14
- package/dist/lib/index/batch-processor.js +1 -1
- package/dist/lib/index/chunker.js +30 -3
- package/dist/lib/store/vector-db.js +32 -17
- package/dist/lib/utils/file-utils.js +33 -0
- package/dist/lib/workers/orchestrator.js +2 -1
- package/package.json +3 -5
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "grepmax",
|
|
11
11
|
"source": "./plugins/grepmax",
|
|
12
12
|
"description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
|
|
13
|
-
"version": "0.21.
|
|
13
|
+
"version": "0.21.3",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Robert Owens",
|
|
16
16
|
"email": "robowens@me.com"
|
package/dist/config.js
CHANGED
|
@@ -84,7 +84,7 @@ exports.CONFIG = {
|
|
|
84
84
|
// reindex to take effect. Must equal the latest entry's `v` in
|
|
85
85
|
// CHUNKER_VERSION_HISTORY below — see that list for per-version severity and
|
|
86
86
|
// the user-facing note rendered by `gmax doctor` and the staleness hint.
|
|
87
|
-
CHUNKER_VERSION:
|
|
87
|
+
CHUNKER_VERSION: 4,
|
|
88
88
|
};
|
|
89
89
|
/**
|
|
90
90
|
* Per-version record of what changed in the chunker and how much it matters to
|
|
@@ -105,6 +105,11 @@ exports.CHUNKER_VERSION_HISTORY = [
|
|
|
105
105
|
severity: "additive",
|
|
106
106
|
note: "type-position edges; dead/trace miss type-only callers until reindex.",
|
|
107
107
|
},
|
|
108
|
+
{
|
|
109
|
+
v: 4,
|
|
110
|
+
severity: "additive",
|
|
111
|
+
note: "member-call edges recorded separately (obj.method()); substrate for receiver-aware call resolution, no query effect until reindex.",
|
|
112
|
+
},
|
|
108
113
|
];
|
|
109
114
|
/**
|
|
110
115
|
* Describe the gap between an index's stamped chunker version and the current
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var _a;
|
|
12
|
+
var _b;
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
// Reduce worker pool fan-out during eval to avoid ONNX concurrency issues.
|
|
15
|
+
// Must run before importing modules that read GMAX_WORKER_COUNT (mirrors eval.ts).
|
|
16
|
+
(_a = (_b = process.env).GMAX_WORKER_COUNT) !== null && _a !== void 0 ? _a : (_b.GMAX_WORKER_COUNT = "1");
|
|
17
|
+
// Token-savings benchmark — sibling to eval.ts.
|
|
18
|
+
//
|
|
19
|
+
// eval.ts measures Recall@10/MRR only; it never measures the "fewer tokens"
|
|
20
|
+
// value prop. This harness does, over the SAME 98 `cases` (imported from
|
|
21
|
+
// eval.ts), with an honest, conservative methodology:
|
|
22
|
+
//
|
|
23
|
+
// Baseline (no gmax — what a grep-then-Read agent pays): read the WHOLE first
|
|
24
|
+
// `expectedPath` file → estTokens(content). One file UNDER-counts (after a
|
|
25
|
+
// grep an agent typically reads several), so every reported ratio is a
|
|
26
|
+
// conservative FLOOR, not a best case.
|
|
27
|
+
//
|
|
28
|
+
// gmax pointer: searcher.search(query, 20, {rerank}) rendered through
|
|
29
|
+
// formatMcpPointerSearchResults(..., { query }) — the REAL bytes the
|
|
30
|
+
// `semantic_search detail:pointer` view returns, not an idealized chunk.
|
|
31
|
+
//
|
|
32
|
+
// gmax pointer+symbol: pointer bytes PLUS the top hit's symbol body, obtained
|
|
33
|
+
// by slicing the top result's file on its [startLine..endLine] range. That
|
|
34
|
+
// is what `extract_symbol` fundamentally returns (the symbol's line span),
|
|
35
|
+
// with zero dependency on non-importable MCP handler internals.
|
|
36
|
+
//
|
|
37
|
+
// We deliberately REJECT graphify's "70×/100×" methodology: it compares one
|
|
38
|
+
// query to reading the ENTIRE corpus (corpus_words = nodes*50, benchmark.py:113),
|
|
39
|
+
// which no agent ever does. Whole-file Read is the baseline an agent actually pays.
|
|
40
|
+
//
|
|
41
|
+
// estTokens is a chars/4 proxy (mirrors mcp.ts:2216), NOT a real tokenizer. Both
|
|
42
|
+
// sides use the same estimator, so the RATIO is largely estimator-invariant — no
|
|
43
|
+
// tokenizer dependency is warranted.
|
|
44
|
+
//
|
|
45
|
+
// Preconditions (identical to eval.ts): an indexed store for cwd AND query
|
|
46
|
+
// embeddings available (MLX :8100 or the ONNX CPU fallback).
|
|
47
|
+
//
|
|
48
|
+
// Output: dual-mode like eval.ts — GMAX_EVAL_JSON=1 / --json emits a single JSON
|
|
49
|
+
// object on stdout (human preamble routed to stderr); otherwise human-readable.
|
|
50
|
+
const node_fs_1 = require("node:fs");
|
|
51
|
+
const node_path_1 = require("node:path");
|
|
52
|
+
const mcp_1 = require("./commands/mcp");
|
|
53
|
+
const eval_1 = require("./eval");
|
|
54
|
+
const searcher_1 = require("./lib/search/searcher");
|
|
55
|
+
const vector_db_1 = require("./lib/store/vector-db");
|
|
56
|
+
const exit_1 = require("./lib/utils/exit");
|
|
57
|
+
const project_root_1 = require("./lib/utils/project-root");
|
|
58
|
+
const topK = 20;
|
|
59
|
+
// chars/4 proxy — mirrors the estimator in mcp.ts:2216. Not a tokenizer; fine
|
|
60
|
+
// here because BOTH sides use it, so the ratio is largely estimator-invariant.
|
|
61
|
+
const estTokens = (s) => Math.ceil(s.length / 4);
|
|
62
|
+
// Linear-interpolation quantile over an ascending-sorted array.
|
|
63
|
+
function quantile(sortedAsc, q) {
|
|
64
|
+
if (sortedAsc.length === 0)
|
|
65
|
+
return 0;
|
|
66
|
+
const pos = (sortedAsc.length - 1) * q;
|
|
67
|
+
const base = Math.floor(pos);
|
|
68
|
+
const rest = pos - base;
|
|
69
|
+
const next = sortedAsc[base + 1];
|
|
70
|
+
return next !== undefined
|
|
71
|
+
? sortedAsc[base] + rest * (next - sortedAsc[base])
|
|
72
|
+
: sortedAsc[base];
|
|
73
|
+
}
|
|
74
|
+
function mean(xs) {
|
|
75
|
+
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
76
|
+
}
|
|
77
|
+
function aggregate(ratios) {
|
|
78
|
+
const sorted = [...ratios].sort((a, b) => a - b);
|
|
79
|
+
return {
|
|
80
|
+
medianRatio: Number(quantile(sorted, 0.5).toFixed(2)),
|
|
81
|
+
p25: Number(quantile(sorted, 0.25).toFixed(2)),
|
|
82
|
+
p75: Number(quantile(sorted, 0.75).toFixed(2)),
|
|
83
|
+
meanRatio: Number(mean(sorted).toFixed(2)),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
function run() {
|
|
87
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
88
|
+
var _a, _b, _c;
|
|
89
|
+
const root = process.cwd();
|
|
90
|
+
const projectRoot = (_a = (0, project_root_1.findProjectRoot)(root)) !== null && _a !== void 0 ? _a : root;
|
|
91
|
+
const paths = (0, project_root_1.ensureProjectPaths)(projectRoot);
|
|
92
|
+
const vectorDb = new vector_db_1.VectorDB(paths.lancedbDir);
|
|
93
|
+
const searcher = new searcher_1.Searcher(vectorDb);
|
|
94
|
+
const jsonMode = process.env.GMAX_EVAL_JSON === "1" || process.argv.includes("--json");
|
|
95
|
+
// In JSON mode all human preamble goes to stderr so stdout stays one object.
|
|
96
|
+
const log = jsonMode ? console.error : console.log;
|
|
97
|
+
// Same precondition as eval.ts: bail loudly if the store is empty.
|
|
98
|
+
if (!(yield vectorDb.hasAnyRows())) {
|
|
99
|
+
console.error("❌ Store appears to be empty!");
|
|
100
|
+
console.error(' Run "gmax index" to populate the store with data.');
|
|
101
|
+
process.exit(1);
|
|
102
|
+
}
|
|
103
|
+
// Rerank OFF by default (matches eval.ts) — set GMAX_EVAL_RERANK=1 to measure
|
|
104
|
+
// the full production pipeline. Rendered bytes barely move with rerank; the
|
|
105
|
+
// flag is here so token numbers line up with whichever recall run they sit next to.
|
|
106
|
+
const rerank = process.env.GMAX_EVAL_RERANK === "1";
|
|
107
|
+
log("Starting token-savings benchmark...\n");
|
|
108
|
+
const results = [];
|
|
109
|
+
for (const c of eval_1.cases) {
|
|
110
|
+
// Baseline: a grep-then-Read agent reads the whole first expectedPath file.
|
|
111
|
+
// Pipe-split mirrors evaluateCase's path handling (eval.ts:552-555).
|
|
112
|
+
const firstExpected = c.expectedPath.split("|")[0].trim();
|
|
113
|
+
let baselineTokens = 0;
|
|
114
|
+
let baselineOk = false;
|
|
115
|
+
try {
|
|
116
|
+
const content = (0, node_fs_1.readFileSync)((0, node_path_1.join)(projectRoot, firstExpected), "utf8");
|
|
117
|
+
baselineTokens = estTokens(content);
|
|
118
|
+
baselineOk = true;
|
|
119
|
+
}
|
|
120
|
+
catch (_d) {
|
|
121
|
+
// Fixture path not present in this checkout — excluded from aggregation.
|
|
122
|
+
}
|
|
123
|
+
// gmax pointer: the real rendered bytes of the detail:pointer view.
|
|
124
|
+
const res = yield searcher.search(c.query, topK, { rerank });
|
|
125
|
+
const pointerText = (0, mcp_1.formatMcpPointerSearchResults)(res.data, projectRoot, {
|
|
126
|
+
query: c.query,
|
|
127
|
+
});
|
|
128
|
+
const pointerTokens = estTokens(pointerText);
|
|
129
|
+
// gmax pointer+symbol: pointer bytes plus the top hit's symbol body, sliced
|
|
130
|
+
// by its [startLine..endLine] range (0-based; mcp.ts:482 adds 1 for display).
|
|
131
|
+
let symbolTokens = 0;
|
|
132
|
+
let symbolLabel = "";
|
|
133
|
+
let topHitPath = "";
|
|
134
|
+
const top = res.data[0];
|
|
135
|
+
if (top) {
|
|
136
|
+
const absPath = (0, mcp_1.searchResultPath)(top);
|
|
137
|
+
topHitPath = absPath;
|
|
138
|
+
const start = (0, mcp_1.searchResultStartLine)(top);
|
|
139
|
+
const end = (0, mcp_1.searchResultEndLine)(top, start);
|
|
140
|
+
try {
|
|
141
|
+
const lines = (0, node_fs_1.readFileSync)(absPath, "utf8").split("\n");
|
|
142
|
+
symbolTokens = estTokens(lines.slice(start, end + 1).join("\n"));
|
|
143
|
+
symbolLabel =
|
|
144
|
+
(_c = (_b = top.defined_symbols) === null || _b === void 0 ? void 0 : _b[0]) !== null && _c !== void 0 ? _c : "";
|
|
145
|
+
}
|
|
146
|
+
catch (_e) {
|
|
147
|
+
// Top-hit file unreadable — symbol cost stays 0 (pointer-only).
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
const pointerSymbolTokens = pointerTokens + symbolTokens;
|
|
151
|
+
const skipped = !baselineOk;
|
|
152
|
+
results.push({
|
|
153
|
+
query: c.query,
|
|
154
|
+
expectedPath: c.expectedPath,
|
|
155
|
+
baselineTokens,
|
|
156
|
+
pointerTokens,
|
|
157
|
+
symbolTokens,
|
|
158
|
+
pointerSymbolTokens,
|
|
159
|
+
symbolLabel,
|
|
160
|
+
topHitPath,
|
|
161
|
+
ratioPointer: skipped || pointerTokens === 0 ? 0 : baselineTokens / pointerTokens,
|
|
162
|
+
ratioPointerSymbol: skipped || pointerSymbolTokens === 0
|
|
163
|
+
? 0
|
|
164
|
+
: baselineTokens / pointerSymbolTokens,
|
|
165
|
+
skipped,
|
|
166
|
+
note: skipped ? "baseline file missing" : undefined,
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
const evaluated = results.filter((r) => !r.skipped);
|
|
170
|
+
const pointerAgg = aggregate(evaluated.map((r) => r.ratioPointer));
|
|
171
|
+
const pointerSymbolAgg = aggregate(evaluated.map((r) => r.ratioPointerSymbol));
|
|
172
|
+
const summary = {
|
|
173
|
+
cases: results.length,
|
|
174
|
+
evaluated: evaluated.length,
|
|
175
|
+
skipped: results.length - evaluated.length,
|
|
176
|
+
rerank,
|
|
177
|
+
estimator: "chars/4 proxy (mirrors mcp.ts:2216)",
|
|
178
|
+
baseline: "whole-file Read of first expectedPath — conservative floor (agents read several files after a grep)",
|
|
179
|
+
pointer: pointerAgg,
|
|
180
|
+
pointerSymbol: pointerSymbolAgg,
|
|
181
|
+
storePath: paths.lancedbDir,
|
|
182
|
+
};
|
|
183
|
+
if (jsonMode) {
|
|
184
|
+
process.stdout.write(`${JSON.stringify({ summary, results }, null, 2)}\n`);
|
|
185
|
+
}
|
|
186
|
+
else {
|
|
187
|
+
const fmt = (n) => `${n.toFixed(1)}×`;
|
|
188
|
+
console.log("=".repeat(80));
|
|
189
|
+
console.log(`Token-savings benchmark — store: ${paths.lancedbDir}`);
|
|
190
|
+
console.log("Baseline = whole-file Read of first expectedPath (chars/4, conservative floor)");
|
|
191
|
+
console.log("=".repeat(80));
|
|
192
|
+
for (const r of results) {
|
|
193
|
+
if (r.skipped) {
|
|
194
|
+
console.log(`⏭ ${r.query}`);
|
|
195
|
+
console.log(` => skipped (${r.note}: ${r.expectedPath})`);
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
console.log(r.query);
|
|
199
|
+
console.log(` baseline ${r.baselineTokens} tok | pointer ${r.pointerTokens} tok (${fmt(r.ratioPointer)}) | +symbol ${r.pointerSymbolTokens} tok (${fmt(r.ratioPointerSymbol)})`);
|
|
200
|
+
}
|
|
201
|
+
console.log("=".repeat(80));
|
|
202
|
+
console.log(`gmax pointer median ${fmt(pointerAgg.medianRatio)} (p25 ${fmt(pointerAgg.p25)} · p75 ${fmt(pointerAgg.p75)} · mean ${fmt(pointerAgg.meanRatio)})`);
|
|
203
|
+
console.log(`gmax pointer+symbol median ${fmt(pointerSymbolAgg.medianRatio)} (p25 ${fmt(pointerSymbolAgg.p25)} · p75 ${fmt(pointerSymbolAgg.p75)} · mean ${fmt(pointerSymbolAgg.meanRatio)})`);
|
|
204
|
+
console.log(`Evaluated ${evaluated.length}/${results.length} cases` +
|
|
205
|
+
(summary.skipped ? ` (${summary.skipped} skipped: file missing)` : ""));
|
|
206
|
+
console.log("Estimator: chars/4 proxy — ratio is estimator-invariant. Single-file baseline under-counts → floor.");
|
|
207
|
+
console.log("=".repeat(80));
|
|
208
|
+
}
|
|
209
|
+
yield (0, exit_1.gracefulExit)(0);
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
if (
|
|
213
|
+
// Only auto-run when executed directly (not when imported for experiments/tests)
|
|
214
|
+
require.main === module &&
|
|
215
|
+
process.env.GMAX_EVAL_AUTORUN !== "0") {
|
|
216
|
+
run().catch((err) => {
|
|
217
|
+
console.error("Token benchmark failed:", err);
|
|
218
|
+
(0, exit_1.gracefulExit)(1);
|
|
219
|
+
});
|
|
220
|
+
}
|
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.LANGUAGES = void 0;
|
|
4
4
|
exports.getLanguageByExtension = getLanguageByExtension;
|
|
5
|
+
exports.languageFamily = languageFamily;
|
|
6
|
+
exports.languageFamilyForPath = languageFamilyForPath;
|
|
5
7
|
exports.getGrammarUrl = getGrammarUrl;
|
|
8
|
+
const node_path_1 = require("node:path");
|
|
6
9
|
exports.LANGUAGES = [
|
|
7
10
|
{
|
|
8
11
|
id: "typescript",
|
|
@@ -258,6 +261,43 @@ function getLanguageByExtension(ext) {
|
|
|
258
261
|
const normalized = ext.toLowerCase();
|
|
259
262
|
return exports.LANGUAGES.find((lang) => lang.extensions.includes(normalized));
|
|
260
263
|
}
|
|
264
|
+
// ---------------------------------------------------------------------------
|
|
265
|
+
// Language families — for cross-language phantom-edge suppression
|
|
266
|
+
// ---------------------------------------------------------------------------
|
|
267
|
+
// The shared LanceDB table holds every language's chunks, scoped only by path.
|
|
268
|
+
// A bare-name call-graph match (`array_contains(referenced_symbols, 'render')`)
|
|
269
|
+
// therefore cross-connects a `render` defined in Python to a `render` referenced
|
|
270
|
+
// in TSX. To suppress those phantom edges we collapse languages that genuinely
|
|
271
|
+
// share a call namespace into one "family" and only keep edges within it.
|
|
272
|
+
// Languages absent from this map are their own family (their `id`). The grouping
|
|
273
|
+
// is deliberately conservative: only languages that directly call across one
|
|
274
|
+
// another are merged, so a real cross-file edge is never dropped.
|
|
275
|
+
const LANGUAGE_FAMILIES = {
|
|
276
|
+
// JS/TS ecosystem — these freely import and call across one another.
|
|
277
|
+
typescript: "js_ts",
|
|
278
|
+
tsx: "js_ts",
|
|
279
|
+
javascript: "js_ts",
|
|
280
|
+
// C and C++ share headers and C++ calls into C.
|
|
281
|
+
c: "c_cpp",
|
|
282
|
+
cpp: "c_cpp",
|
|
283
|
+
};
|
|
284
|
+
/** The call-namespace family for a language id (defaults to the id itself). */
|
|
285
|
+
function languageFamily(langId) {
|
|
286
|
+
var _a;
|
|
287
|
+
return (_a = LANGUAGE_FAMILIES[langId]) !== null && _a !== void 0 ? _a : langId;
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* The language family for a file path, or null when the extension is unknown or
|
|
291
|
+
* absent. Null means "unclassifiable, do not filter" — callers must keep edges
|
|
292
|
+
* they can't classify and only drop edges whose family is known *and* differs.
|
|
293
|
+
*/
|
|
294
|
+
function languageFamilyForPath(path) {
|
|
295
|
+
const ext = (0, node_path_1.extname)(path).toLowerCase();
|
|
296
|
+
if (!ext)
|
|
297
|
+
return null;
|
|
298
|
+
const lang = getLanguageByExtension(ext);
|
|
299
|
+
return lang ? languageFamily(lang.id) : null;
|
|
300
|
+
}
|
|
261
301
|
function getGrammarUrl(grammarName) {
|
|
262
302
|
var _a;
|
|
263
303
|
const lang = exports.LANGUAGES.find((l) => { var _a; return ((_a = l.grammar) === null || _a === void 0 ? void 0 : _a.name) === grammarName; });
|
|
@@ -382,9 +382,9 @@ class WatcherManager {
|
|
|
382
382
|
// Fast path: if only mtime changed but size is identical and we have a hash,
|
|
383
383
|
// just verify the hash in-process instead of sending to a worker.
|
|
384
384
|
if ((cached === null || cached === void 0 ? void 0 : cached.hash) && cached.size === stats.size) {
|
|
385
|
-
const {
|
|
385
|
+
const { computeContentHash } = yield Promise.resolve().then(() => __importStar(require("../utils/file-utils")));
|
|
386
386
|
const buf = yield fs.promises.readFile(absPath);
|
|
387
|
-
const hash =
|
|
387
|
+
const hash = computeContentHash(buf, absPath);
|
|
388
388
|
if (hash === cached.hash) {
|
|
389
389
|
// Content unchanged — update mtime in cache and skip worker
|
|
390
390
|
metaCache.put(absPath, Object.assign(Object.assign({}, cached), { mtimeMs: stats.mtimeMs }));
|
|
@@ -10,8 +10,10 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.GraphBuilder = void 0;
|
|
13
|
+
const languages_1 = require("../core/languages");
|
|
13
14
|
const filter_builder_1 = require("../utils/filter-builder");
|
|
14
15
|
const query_timeout_1 = require("../utils/query-timeout");
|
|
16
|
+
const callsites_1 = require("./callsites");
|
|
15
17
|
const graph_traversal_1 = require("./graph-traversal");
|
|
16
18
|
class GraphBuilder {
|
|
17
19
|
constructor(db, pathPrefix, excludePrefixes) {
|
|
@@ -36,8 +38,13 @@ class GraphBuilder {
|
|
|
36
38
|
}
|
|
37
39
|
/**
|
|
38
40
|
* Find all chunks that call the given symbol.
|
|
41
|
+
*
|
|
42
|
+
* `anchorFamily` (the language family of the symbol's definition) enables the
|
|
43
|
+
* cross-language phantom-edge guard: the shared table matches a bare symbol
|
|
44
|
+
* name across every language, so without it a `render` defined in one language
|
|
45
|
+
* picks up callers that merely reference an unrelated `render` in another.
|
|
39
46
|
*/
|
|
40
|
-
getCallers(symbol) {
|
|
47
|
+
getCallers(symbol, anchorFamily) {
|
|
41
48
|
return __awaiter(this, void 0, void 0, function* () {
|
|
42
49
|
const table = yield this.db.ensureTable();
|
|
43
50
|
const escaped = (0, filter_builder_1.escapeSqlString)(symbol);
|
|
@@ -59,7 +66,17 @@ class GraphBuilder {
|
|
|
59
66
|
.where(this.scopeWhere(`(array_contains(referenced_symbols, '${escaped}') OR array_contains(type_referenced_symbols, '${escaped}'))`))
|
|
60
67
|
.limit(100)
|
|
61
68
|
.toArray();
|
|
62
|
-
|
|
69
|
+
// Cross-language phantom-edge guard. When the anchor family is known, drop
|
|
70
|
+
// callers from a *different known* family; callers we can't classify (unknown
|
|
71
|
+
// extension) are kept so a real edge is never lost. No anchor → no filter.
|
|
72
|
+
const guarded = anchorFamily == null
|
|
73
|
+
? rows
|
|
74
|
+
: rows.filter((row) => {
|
|
75
|
+
var _a;
|
|
76
|
+
const fam = (0, languages_1.languageFamilyForPath)(String((_a = row.path) !== null && _a !== void 0 ? _a : ""));
|
|
77
|
+
return fam == null || fam === anchorFamily;
|
|
78
|
+
});
|
|
79
|
+
return guarded.map((row) => this.mapRowToNode(row, symbol, "caller"));
|
|
63
80
|
});
|
|
64
81
|
}
|
|
65
82
|
/**
|
|
@@ -108,13 +125,20 @@ class GraphBuilder {
|
|
|
108
125
|
const center = centerRows.length > 0
|
|
109
126
|
? this.mapRowToNode(centerRows[0], symbol, "center")
|
|
110
127
|
: null;
|
|
111
|
-
// 2. Get Callers
|
|
112
|
-
|
|
128
|
+
// 2. Get Callers — anchored to the center definition's language family so a
|
|
129
|
+
// bare name shared across languages doesn't pull in cross-language callers.
|
|
130
|
+
const anchorFamily = center ? (0, languages_1.languageFamilyForPath)(center.file) : null;
|
|
131
|
+
const callers = yield this.getCallers(symbol, anchorFamily);
|
|
113
132
|
// 3. Get Callees — resolve each to a GraphNode with file:line
|
|
114
133
|
const calleeNames = center ? center.calls.slice(0, 15) : [];
|
|
134
|
+
const centerFile = center ? center.file : "";
|
|
115
135
|
const calleeNodes = [];
|
|
116
136
|
for (const name of calleeNames) {
|
|
117
137
|
const esc = (0, filter_builder_1.escapeSqlString)(name);
|
|
138
|
+
// Pull a few candidates (was .limit(1)) so a callee name defined in more
|
|
139
|
+
// than one file can prefer the center's OWN file instead of an arbitrary
|
|
140
|
+
// same-named definition in an unrelated module — the cheapest correct
|
|
141
|
+
// disambiguation, and strictly better than the old first-row guess.
|
|
118
142
|
const rows = yield table
|
|
119
143
|
.query()
|
|
120
144
|
.where(this.scopeWhere(`array_contains(defined_symbols, '${esc}')`))
|
|
@@ -127,12 +151,21 @@ class GraphBuilder {
|
|
|
127
151
|
"parent_symbol",
|
|
128
152
|
"complexity",
|
|
129
153
|
])
|
|
130
|
-
.limit(
|
|
154
|
+
.limit(25)
|
|
131
155
|
.toArray();
|
|
132
156
|
if (rows.length > 0) {
|
|
133
|
-
|
|
157
|
+
// Prefer-self-file: a callee the center also defines locally is almost
|
|
158
|
+
// certainly that local definition. Falls back to the first row when no
|
|
159
|
+
// candidate is in the center's file (unchanged behaviour then).
|
|
160
|
+
const selfRow = rows.find((r) => { var _a; return String((_a = r.path) !== null && _a !== void 0 ? _a : "") === centerFile; });
|
|
161
|
+
calleeNodes.push(this.mapRowToNode((selfRow !== null && selfRow !== void 0 ? selfRow : rows[0]), name, "center"));
|
|
134
162
|
}
|
|
135
|
-
else {
|
|
163
|
+
else if (!(0, callsites_1.isBuiltinCallee)(name)) {
|
|
164
|
+
// Unresolved + a known builtin (.map/.get/forEach) → suppress the
|
|
165
|
+
// phantom callee edge instead of emitting a "(not indexed)" stub.
|
|
166
|
+
// Mirrors the display-layer guard (peek.ts: `c.file || !isBuiltinCallee`):
|
|
167
|
+
// a project symbol that shadows a builtin name still resolves above and
|
|
168
|
+
// is kept, so only genuine builtins are dropped.
|
|
136
169
|
calleeNodes.push({
|
|
137
170
|
symbol: name,
|
|
138
171
|
file: "",
|
|
@@ -204,7 +237,7 @@ class GraphBuilder {
|
|
|
204
237
|
visited.add(caller.symbol);
|
|
205
238
|
let subCallers = [];
|
|
206
239
|
if (remainingDepth > 0) {
|
|
207
|
-
const upstreamCallers = yield this.getCallers(caller.symbol);
|
|
240
|
+
const upstreamCallers = yield this.getCallers(caller.symbol, (0, languages_1.languageFamilyForPath)(caller.file));
|
|
208
241
|
subCallers = yield this.expandCallers(upstreamCallers, remainingDepth - 1, visited);
|
|
209
242
|
}
|
|
210
243
|
trees.push({ node: caller, callers: subCallers });
|
|
@@ -222,7 +255,10 @@ class GraphBuilder {
|
|
|
222
255
|
/** Distinct symbols that reference the given symbol (inbound edges). */
|
|
223
256
|
callersOf(symbol) {
|
|
224
257
|
return __awaiter(this, void 0, void 0, function* () {
|
|
225
|
-
|
|
258
|
+
// Anchor the guard to this symbol's own definition language family so the
|
|
259
|
+
// bare name doesn't pull in callers from an unrelated language.
|
|
260
|
+
const loc = yield this.resolveLocation(symbol);
|
|
261
|
+
const nodes = yield this.getCallers(symbol, loc ? (0, languages_1.languageFamilyForPath)(loc.file) : null);
|
|
226
262
|
const out = [];
|
|
227
263
|
for (const n of nodes) {
|
|
228
264
|
if (n.symbol && n.symbol !== "unknown" && !out.includes(n.symbol)) {
|
|
@@ -246,9 +282,18 @@ class GraphBuilder {
|
|
|
246
282
|
return __awaiter(this, void 0, void 0, function* () {
|
|
247
283
|
var _a, _b;
|
|
248
284
|
const hits = yield (0, graph_traversal_1.bfsNeighbors)(symbol, this.neighborFn(direction), maxHops);
|
|
285
|
+
// Resolve the origin once to anchor every neighbor's location lookup to the
|
|
286
|
+
// same language family, so a shared name doesn't resolve to a foreign file.
|
|
287
|
+
const origin = yield this.resolveLocation(symbol);
|
|
288
|
+
const anchorFamily = origin ? (0, languages_1.languageFamilyForPath)(origin.file) : null;
|
|
249
289
|
const out = [];
|
|
250
290
|
for (const h of hits) {
|
|
251
|
-
const loc = yield this.resolveLocation(h.symbol);
|
|
291
|
+
const loc = yield this.resolveLocation(h.symbol, anchorFamily);
|
|
292
|
+
// Drop unresolved builtins (.map/.get/forEach) reached via callee edges —
|
|
293
|
+
// same resolution-aware rule as buildGraph/peek. A neighbor that resolves
|
|
294
|
+
// to an indexed definition is kept even if it shadows a builtin name.
|
|
295
|
+
if (!loc && (0, callsites_1.isBuiltinCallee)(h.symbol))
|
|
296
|
+
continue;
|
|
252
297
|
out.push(Object.assign(Object.assign({}, h), { file: (_a = loc === null || loc === void 0 ? void 0 : loc.file) !== null && _a !== void 0 ? _a : "", line: (_b = loc === null || loc === void 0 ? void 0 : loc.line) !== null && _b !== void 0 ? _b : 0 }));
|
|
253
298
|
}
|
|
254
299
|
return out;
|
|
@@ -260,8 +305,13 @@ class GraphBuilder {
|
|
|
260
305
|
return (0, graph_traversal_1.findPath)(from, to, this.neighborFn(direction), maxHops);
|
|
261
306
|
});
|
|
262
307
|
}
|
|
263
|
-
/**
|
|
264
|
-
|
|
308
|
+
/**
|
|
309
|
+
* Resolve a symbol to a defining chunk's file:line, if indexed. With
|
|
310
|
+
* `anchorFamily` set, prefer the definition in that language family rather than
|
|
311
|
+
* an arbitrary cross-language match (the shared table can hold the same name in
|
|
312
|
+
* several languages); falls back to the first definition when none match.
|
|
313
|
+
*/
|
|
314
|
+
resolveLocation(symbol, anchorFamily) {
|
|
265
315
|
return __awaiter(this, void 0, void 0, function* () {
|
|
266
316
|
const table = yield this.db.ensureTable();
|
|
267
317
|
const escaped = (0, filter_builder_1.escapeSqlString)(symbol);
|
|
@@ -269,11 +319,22 @@ class GraphBuilder {
|
|
|
269
319
|
.query()
|
|
270
320
|
.select(["path", "start_line"])
|
|
271
321
|
.where(this.scopeWhere(`array_contains(defined_symbols, '${escaped}')`))
|
|
272
|
-
.
|
|
322
|
+
// No anchor → keep the cheap single-row fetch. With one, pull a few
|
|
323
|
+
// candidates so we can pick the same-family definition instead of guessing.
|
|
324
|
+
.limit(anchorFamily ? 25 : 1)
|
|
273
325
|
.toArray();
|
|
274
326
|
if (rows.length === 0)
|
|
275
327
|
return null;
|
|
276
|
-
|
|
328
|
+
let r = rows[0];
|
|
329
|
+
if (anchorFamily) {
|
|
330
|
+
const match = rows.find((row) => {
|
|
331
|
+
var _a;
|
|
332
|
+
return (0, languages_1.languageFamilyForPath)(String((_a = row.path) !== null && _a !== void 0 ? _a : "")) ===
|
|
333
|
+
anchorFamily;
|
|
334
|
+
});
|
|
335
|
+
if (match)
|
|
336
|
+
r = match;
|
|
337
|
+
}
|
|
277
338
|
return { file: String(r.path || ""), line: Number(r.start_line || 0) };
|
|
278
339
|
});
|
|
279
340
|
}
|
|
@@ -196,7 +196,7 @@ class ProjectBatchProcessor {
|
|
|
196
196
|
// verify in-process instead of dispatching to a worker (~220ms saved).
|
|
197
197
|
if ((cached === null || cached === void 0 ? void 0 : cached.hash) && cached.size === stats.size) {
|
|
198
198
|
const buf = yield fs.promises.readFile(absPath);
|
|
199
|
-
const hash = (0, file_utils_1.
|
|
199
|
+
const hash = (0, file_utils_1.computeContentHash)(buf, absPath);
|
|
200
200
|
if (hash === cached.hash) {
|
|
201
201
|
metaUpdates.set(absPath, Object.assign(Object.assign({}, cached), { mtimeMs: stats.mtimeMs }));
|
|
202
202
|
continue;
|
|
@@ -571,6 +571,15 @@ class TreeSitterChunker {
|
|
|
571
571
|
typeReferencedSymbols.push(n);
|
|
572
572
|
}
|
|
573
573
|
};
|
|
574
|
+
// Member-call names (`obj.method()`), recorded additively alongside
|
|
575
|
+
// referencedSymbols (see Chunk.memberReferencedSymbols). Deduped, like
|
|
576
|
+
// type refs — a future resolver queries membership, not occurrence count.
|
|
577
|
+
const memberReferencedSymbols = [];
|
|
578
|
+
const addMemberRef = (n) => {
|
|
579
|
+
if (n && !memberReferencedSymbols.includes(n)) {
|
|
580
|
+
memberReferencedSymbols.push(n);
|
|
581
|
+
}
|
|
582
|
+
};
|
|
574
583
|
// Leaf identifier node types across grammars (a bare name with no
|
|
575
584
|
// named children — `ErrorCodes`, not `a.ErrorCodes`).
|
|
576
585
|
const LEAF_ID_TYPES = new Set([
|
|
@@ -677,24 +686,33 @@ class TreeSitterChunker {
|
|
|
677
686
|
: null;
|
|
678
687
|
if (func) {
|
|
679
688
|
let funcName = func.text;
|
|
689
|
+
let isMemberCall = false;
|
|
680
690
|
// Handle member access (obj.method) to extract just 'method'
|
|
681
691
|
if (func.type === "member_expression") {
|
|
682
692
|
// JS/TS: object.property
|
|
683
693
|
const prop = func.childForFieldName
|
|
684
694
|
? func.childForFieldName("property")
|
|
685
695
|
: null;
|
|
686
|
-
if (prop)
|
|
696
|
+
if (prop) {
|
|
687
697
|
funcName = prop.text;
|
|
698
|
+
isMemberCall = true;
|
|
699
|
+
}
|
|
688
700
|
}
|
|
689
701
|
else if (func.type === "attribute") {
|
|
690
702
|
// Python: object.attribute
|
|
691
703
|
const attr = func.childForFieldName
|
|
692
704
|
? func.childForFieldName("attribute")
|
|
693
705
|
: null;
|
|
694
|
-
if (attr)
|
|
706
|
+
if (attr) {
|
|
695
707
|
funcName = attr.text;
|
|
708
|
+
isMemberCall = true;
|
|
709
|
+
}
|
|
696
710
|
}
|
|
697
711
|
referencedSymbols.push(funcName);
|
|
712
|
+
// Additive: member calls ALSO go here. referencedSymbols is left
|
|
713
|
+
// untouched (no recall loss); this just tags which were members.
|
|
714
|
+
if (isMemberCall)
|
|
715
|
+
addMemberRef(funcName);
|
|
698
716
|
}
|
|
699
717
|
else {
|
|
700
718
|
// Swift/Kotlin: call_expression has no "function" field;
|
|
@@ -702,15 +720,20 @@ class TreeSitterChunker {
|
|
|
702
720
|
const firstChild = ((_a = n.namedChildren) !== null && _a !== void 0 ? _a : [])[0];
|
|
703
721
|
if (firstChild) {
|
|
704
722
|
let funcName = firstChild.text;
|
|
723
|
+
let isMemberCall = false;
|
|
705
724
|
if (firstChild.type === "navigation_expression") {
|
|
706
725
|
const suffix = ((_b = firstChild.namedChildren) !== null && _b !== void 0 ? _b : []).find((c) => c.type === "navigation_suffix");
|
|
707
726
|
const methodId = suffix
|
|
708
727
|
? ((_c = suffix.namedChildren) !== null && _c !== void 0 ? _c : []).find((c) => c.type === "simple_identifier")
|
|
709
728
|
: null;
|
|
710
|
-
if (methodId)
|
|
729
|
+
if (methodId) {
|
|
711
730
|
funcName = methodId.text;
|
|
731
|
+
isMemberCall = true;
|
|
732
|
+
}
|
|
712
733
|
}
|
|
713
734
|
referencedSymbols.push(funcName);
|
|
735
|
+
if (isMemberCall)
|
|
736
|
+
addMemberRef(funcName);
|
|
714
737
|
}
|
|
715
738
|
}
|
|
716
739
|
}
|
|
@@ -905,6 +928,7 @@ class TreeSitterChunker {
|
|
|
905
928
|
definedSymbols,
|
|
906
929
|
referencedSymbols,
|
|
907
930
|
typeReferencedSymbols,
|
|
931
|
+
memberReferencedSymbols,
|
|
908
932
|
role,
|
|
909
933
|
parentSymbol: stack.length > 1
|
|
910
934
|
? stack[stack.length - 1].replace(/^(Class|Method|Function|Interface|Type): /, "")
|
|
@@ -990,6 +1014,9 @@ class TreeSitterChunker {
|
|
|
990
1014
|
if (sub.typeReferencedSymbols) {
|
|
991
1015
|
sub.typeReferencedSymbols = sub.typeReferencedSymbols.filter(occurs);
|
|
992
1016
|
}
|
|
1017
|
+
if (sub.memberReferencedSymbols) {
|
|
1018
|
+
sub.memberReferencedSymbols = sub.memberReferencedSymbols.filter(occurs);
|
|
1019
|
+
}
|
|
993
1020
|
return sub;
|
|
994
1021
|
}
|
|
995
1022
|
splitIfTooBig(chunk) {
|
|
@@ -271,6 +271,7 @@ class VectorDB {
|
|
|
271
271
|
defined_symbols: [],
|
|
272
272
|
referenced_symbols: [],
|
|
273
273
|
type_referenced_symbols: [],
|
|
274
|
+
member_referenced_symbols: [],
|
|
274
275
|
imports: [],
|
|
275
276
|
exports: [],
|
|
276
277
|
role: "",
|
|
@@ -340,6 +341,7 @@ class VectorDB {
|
|
|
340
341
|
new apache_arrow_1.Field("defined_symbols", new apache_arrow_1.List(new apache_arrow_1.Field("item", new apache_arrow_1.Utf8(), true)), true),
|
|
341
342
|
new apache_arrow_1.Field("referenced_symbols", new apache_arrow_1.List(new apache_arrow_1.Field("item", new apache_arrow_1.Utf8(), true)), true),
|
|
342
343
|
new apache_arrow_1.Field("type_referenced_symbols", new apache_arrow_1.List(new apache_arrow_1.Field("item", new apache_arrow_1.Utf8(), true)), true),
|
|
344
|
+
new apache_arrow_1.Field("member_referenced_symbols", new apache_arrow_1.List(new apache_arrow_1.Field("item", new apache_arrow_1.Utf8(), true)), true),
|
|
343
345
|
new apache_arrow_1.Field("imports", new apache_arrow_1.List(new apache_arrow_1.Field("item", new apache_arrow_1.Utf8(), true)), true),
|
|
344
346
|
new apache_arrow_1.Field("exports", new apache_arrow_1.List(new apache_arrow_1.Field("item", new apache_arrow_1.Utf8(), true)), true),
|
|
345
347
|
new apache_arrow_1.Field("role", new apache_arrow_1.Utf8(), true),
|
|
@@ -360,16 +362,27 @@ class VectorDB {
|
|
|
360
362
|
return __awaiter(this, void 0, void 0, function* () {
|
|
361
363
|
const schema = yield table.schema();
|
|
362
364
|
const fields = new Set(schema.fields.map((f) => f.name));
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
365
|
+
// Additive list columns that older tables may predate. Each is checked and
|
|
366
|
+
// added INDEPENDENTLY: a single early-return on the first existing column
|
|
367
|
+
// (as this used to do) would permanently strand every table that already has
|
|
368
|
+
// `type_referenced_symbols` — i.e. all of them — without ever gaining a newer
|
|
369
|
+
// column like `member_referenced_symbols`.
|
|
370
|
+
const additiveListColumns = [
|
|
371
|
+
"type_referenced_symbols",
|
|
372
|
+
"member_referenced_symbols",
|
|
373
|
+
];
|
|
374
|
+
for (const col of additiveListColumns) {
|
|
375
|
+
if (fields.has(col))
|
|
376
|
+
continue;
|
|
377
|
+
try {
|
|
378
|
+
yield table.addColumns(new apache_arrow_1.Field(col, new apache_arrow_1.List(new apache_arrow_1.Field("item", new apache_arrow_1.Utf8(), true)), true));
|
|
379
|
+
(0, logger_1.log)("db", `Added ${col} column to existing table`);
|
|
380
|
+
}
|
|
381
|
+
catch (err) {
|
|
382
|
+
// Lost a race with another writer that already added it, or a transient
|
|
383
|
+
// commit conflict — the next ensureTable() re-checks and no-ops.
|
|
384
|
+
(0, logger_1.debug)("vectordb", `evolveSchema ${col} skipped: ${err.message}`);
|
|
385
|
+
}
|
|
373
386
|
}
|
|
374
387
|
});
|
|
375
388
|
}
|
|
@@ -395,7 +408,7 @@ class VectorDB {
|
|
|
395
408
|
}
|
|
396
409
|
insertBatch(records) {
|
|
397
410
|
return __awaiter(this, void 0, void 0, function* () {
|
|
398
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
|
|
411
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r;
|
|
399
412
|
if (!records.length)
|
|
400
413
|
return;
|
|
401
414
|
this.ensureDiskOk();
|
|
@@ -473,12 +486,14 @@ class VectorDB {
|
|
|
473
486
|
rec.defined_symbols = (_g = rec.defined_symbols) !== null && _g !== void 0 ? _g : [];
|
|
474
487
|
rec.referenced_symbols = (_h = rec.referenced_symbols) !== null && _h !== void 0 ? _h : [];
|
|
475
488
|
rec.type_referenced_symbols = (_j = rec.type_referenced_symbols) !== null && _j !== void 0 ? _j : [];
|
|
476
|
-
rec.
|
|
477
|
-
|
|
478
|
-
rec.
|
|
479
|
-
rec.
|
|
480
|
-
rec.
|
|
481
|
-
rec.
|
|
489
|
+
rec.member_referenced_symbols =
|
|
490
|
+
(_k = rec.member_referenced_symbols) !== null && _k !== void 0 ? _k : [];
|
|
491
|
+
rec.imports = (_l = rec.imports) !== null && _l !== void 0 ? _l : [];
|
|
492
|
+
rec.exports = (_m = rec.exports) !== null && _m !== void 0 ? _m : [];
|
|
493
|
+
rec.role = (_o = rec.role) !== null && _o !== void 0 ? _o : "";
|
|
494
|
+
rec.parent_symbol = (_p = rec.parent_symbol) !== null && _p !== void 0 ? _p : "";
|
|
495
|
+
rec.file_skeleton = (_q = rec.file_skeleton) !== null && _q !== void 0 ? _q : "";
|
|
496
|
+
rec.summary = (_r = rec.summary) !== null && _r !== void 0 ? _r : null;
|
|
482
497
|
}
|
|
483
498
|
try {
|
|
484
499
|
yield this.withWriteGate(() => table.add(records));
|
|
@@ -43,6 +43,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
43
43
|
};
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.computeBufferHash = computeBufferHash;
|
|
46
|
+
exports.stripMarkdownFrontmatter = stripMarkdownFrontmatter;
|
|
47
|
+
exports.computeContentHash = computeContentHash;
|
|
46
48
|
exports.hasNullByte = hasNullByte;
|
|
47
49
|
exports.readFileSnapshot = readFileSnapshot;
|
|
48
50
|
exports.isIndexableFile = isIndexableFile;
|
|
@@ -58,6 +60,37 @@ const config_1 = require("../../config");
|
|
|
58
60
|
function computeBufferHash(buffer) {
|
|
59
61
|
return (0, node_crypto_1.createHash)("sha256").update(buffer).digest("hex");
|
|
60
62
|
}
|
|
63
|
+
// Files whose leading YAML frontmatter is metadata, not content. A status:/tags:
|
|
64
|
+
// edit bumps mtime and changes the bytes, but the embedding ignores frontmatter —
|
|
65
|
+
// so hashing it would force a needless GPU re-embed on every metadata touch.
|
|
66
|
+
const FRONTMATTER_EXTENSIONS = new Set([".md", ".mdx"]);
|
|
67
|
+
// Leading YAML frontmatter: a whole `---` line at the very top (NOT a bare
|
|
68
|
+
// `startsWith("---")`, which would also match a `---` thematic break or `--- x`),
|
|
69
|
+
// through the next whole `---`/`...` line. Anchored at string start (no `m` flag)
|
|
70
|
+
// with an optional BOM, so only true file-leading frontmatter matches; a doc that
|
|
71
|
+
// opens with a thematic break and no closing fence is left untouched.
|
|
72
|
+
const FRONTMATTER_BLOCK = /^\uFEFF?---[ \t]*\r?\n[\s\S]*?\r?\n(?:---|\.\.\.)[ \t]*(?:\r?\n|$)/;
|
|
73
|
+
/** Strip leading YAML frontmatter from markdown bytes; returns input unchanged otherwise. */
|
|
74
|
+
function stripMarkdownFrontmatter(buffer) {
|
|
75
|
+
const text = buffer.toString("utf8");
|
|
76
|
+
const match = text.match(FRONTMATTER_BLOCK);
|
|
77
|
+
if (!match)
|
|
78
|
+
return buffer;
|
|
79
|
+
return Buffer.from(text.slice(match[0].length), "utf8");
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Hash file content for change detection. For markdown, YAML frontmatter is
|
|
83
|
+
* stripped first so a metadata-only edit doesn't bust `isFileCached` and trigger
|
|
84
|
+
* a re-embed; every other file hashes its exact bytes. Must be used identically
|
|
85
|
+
* across the catchup, watcher, and worker hash paths or they will disagree and
|
|
86
|
+
* re-index in a loop.
|
|
87
|
+
*/
|
|
88
|
+
function computeContentHash(buffer, filePath) {
|
|
89
|
+
if (FRONTMATTER_EXTENSIONS.has((0, node_path_1.extname)(filePath).toLowerCase())) {
|
|
90
|
+
return computeBufferHash(stripMarkdownFrontmatter(buffer));
|
|
91
|
+
}
|
|
92
|
+
return computeBufferHash(buffer);
|
|
93
|
+
}
|
|
61
94
|
function hasNullByte(buffer, sampleLength = 1024) {
|
|
62
95
|
const length = Math.min(buffer.length, sampleLength);
|
|
63
96
|
for (let i = 0; i < length; i++) {
|
|
@@ -227,6 +227,7 @@ class WorkerOrchestrator {
|
|
|
227
227
|
defined_symbols: chunk.definedSymbols,
|
|
228
228
|
referenced_symbols: chunk.referencedSymbols,
|
|
229
229
|
type_referenced_symbols: chunk.typeReferencedSymbols,
|
|
230
|
+
member_referenced_symbols: chunk.memberReferencedSymbols,
|
|
230
231
|
role: chunk.role,
|
|
231
232
|
parent_symbol: chunk.parentSymbol,
|
|
232
233
|
file_skeleton: chunk.isAnchor ? skeleton : undefined,
|
|
@@ -244,7 +245,7 @@ class WorkerOrchestrator {
|
|
|
244
245
|
: path.join(PROJECT_ROOT, input.path);
|
|
245
246
|
(0, logger_1.debug)("orch", `processFile start: ${input.path}`);
|
|
246
247
|
const { buffer, mtimeMs, size } = yield (0, file_utils_1.readFileSnapshot)(absolutePath);
|
|
247
|
-
const hash = (0, file_utils_1.
|
|
248
|
+
const hash = (0, file_utils_1.computeContentHash)(buffer, absolutePath);
|
|
248
249
|
if (!(0, file_utils_1.isIndexableFile)(absolutePath, size)) {
|
|
249
250
|
(0, logger_1.debug)("orch", `skip ${input.path} (non-indexable, size=${size})`);
|
|
250
251
|
return { vectors: [], hash, mtimeMs, size, shouldDelete: true };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "grepmax",
|
|
3
|
-
"version": "0.21.
|
|
3
|
+
"version": "0.21.3",
|
|
4
4
|
"author": "Robert Owens <78518764+reowens@users.noreply.github.com>",
|
|
5
5
|
"homepage": "https://github.com/reowens/grepmax",
|
|
6
6
|
"bugs": {
|
|
@@ -25,14 +25,12 @@
|
|
|
25
25
|
"dev": "npx tsc && node dist/index.js",
|
|
26
26
|
"test": "vitest run",
|
|
27
27
|
"test:watch": "vitest",
|
|
28
|
-
"benchmark": "./run-benchmark.sh",
|
|
29
|
-
"benchmark:index": "./run-benchmark.sh $HOME/gmax-benchmarks --index",
|
|
30
|
-
"benchmark:agent": "npx tsx src/bench/benchmark-agent.ts",
|
|
31
|
-
"benchmark:chart": "npx tsx src/bench/generate-benchmark-chart.ts",
|
|
32
28
|
"bench:recall": "npx tsx src/eval.ts",
|
|
33
29
|
"bench:recall:json": "GMAX_EVAL_JSON=1 npx tsx src/eval.ts",
|
|
34
30
|
"bench:oss": "npx tsx src/eval-oss.ts all",
|
|
35
31
|
"bench:oss:json": "GMAX_EVAL_JSON=1 npx tsx src/eval-oss.ts all",
|
|
32
|
+
"bench:tokens": "npx tsx src/eval-tokens.ts",
|
|
33
|
+
"bench:tokens:json": "GMAX_EVAL_JSON=1 npx tsx src/eval-tokens.ts",
|
|
36
34
|
"format": "biome format --write .",
|
|
37
35
|
"format:check": "biome check .",
|
|
38
36
|
"lint": "biome lint .",
|