grepmax 0.26.48 → 0.26.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +77 -40
- package/dist/commands/search-output.js +22 -9
- package/dist/commands/search-run.js +6 -2
- package/dist/commands/search.js +10 -1
- package/dist/commands/serve.js +14 -8
- package/dist/eval-oss.js +2 -1
- package/dist/eval-relevance.js +337 -0
- package/dist/eval.js +4 -3
- package/dist/lib/daemon/ipc-handler.js +12 -0
- package/dist/lib/daemon/search-handler.js +14 -2
- package/dist/lib/eval/relevance-baseline.js +174 -0
- package/dist/lib/search/diagnostics.js +95 -0
- package/dist/lib/search/per-file.js +24 -0
- package/dist/lib/search/searcher.js +72 -13
- package/dist/lib/utils/daemon-search.js +34 -0
- package/package.json +3 -2
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
/** Frozen local multi-corpus baseline, using the existing daemon with no fallback. */
|
|
46
|
+
const node_child_process_1 = require("node:child_process");
|
|
47
|
+
const fs = __importStar(require("node:fs"));
|
|
48
|
+
const path = __importStar(require("node:path"));
|
|
49
|
+
const node_util_1 = require("node:util");
|
|
50
|
+
const lmdb_1 = require("lmdb");
|
|
51
|
+
const config_1 = require("./config");
|
|
52
|
+
const relevance_baseline_1 = require("./lib/eval/relevance-baseline");
|
|
53
|
+
const daemon_client_1 = require("./lib/utils/daemon-client");
|
|
54
|
+
const project_registry_1 = require("./lib/utils/project-registry");
|
|
55
|
+
function git(root) {
|
|
56
|
+
try {
|
|
57
|
+
return {
|
|
58
|
+
head: (0, node_child_process_1.execFileSync)("git", ["rev-parse", "HEAD"], {
|
|
59
|
+
cwd: root,
|
|
60
|
+
encoding: "utf8",
|
|
61
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
62
|
+
}).trim(),
|
|
63
|
+
dirty: (0, node_child_process_1.execFileSync)("git", ["status", "--porcelain"], {
|
|
64
|
+
cwd: root,
|
|
65
|
+
encoding: "utf8",
|
|
66
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
67
|
+
}).trim().length > 0,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
catch (_a) {
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
function run() {
|
|
75
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
76
|
+
var _a, _b, _c, _d;
|
|
77
|
+
var _e, _f;
|
|
78
|
+
const { values } = (0, node_util_1.parseArgs)({
|
|
79
|
+
options: {
|
|
80
|
+
fixture: { type: "string" },
|
|
81
|
+
sha256: { type: "string" },
|
|
82
|
+
output: { type: "string" },
|
|
83
|
+
repeats: { type: "string", default: "2" },
|
|
84
|
+
help: { type: "boolean" },
|
|
85
|
+
diagnostics: { type: "boolean", default: false },
|
|
86
|
+
},
|
|
87
|
+
});
|
|
88
|
+
if (values.help) {
|
|
89
|
+
console.log("pnpm bench:relevance --fixture <json> --sha256 <frozen-sha256> --output <new-json> [--repeats 2] [--diagnostics]");
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
if (!values.fixture || !values.sha256 || !values.output)
|
|
93
|
+
throw new Error("Required: --fixture, --sha256, --output");
|
|
94
|
+
if (fs.existsSync(values.output))
|
|
95
|
+
throw new Error("Output exists; use a new artifact path");
|
|
96
|
+
const checkpoint = `${values.output}.samples.jsonl`;
|
|
97
|
+
if (fs.existsSync(checkpoint))
|
|
98
|
+
throw new Error("Checkpoint exists; use a new artifact path");
|
|
99
|
+
const repeats = Number(values.repeats);
|
|
100
|
+
if (!Number.isInteger(repeats) || repeats < 1 || repeats > 5)
|
|
101
|
+
throw new Error("Repeats must be 1–5");
|
|
102
|
+
const fixturePath = path.resolve(values.fixture);
|
|
103
|
+
const fixture = (0, relevance_baseline_1.parseFrozenFixture)(fs.readFileSync(fixturePath), values.sha256);
|
|
104
|
+
const roots = new Map(fixture.corpora.map((r) => [
|
|
105
|
+
r.id,
|
|
106
|
+
fs.realpathSync(path.resolve(path.dirname(fixturePath), r.root)),
|
|
107
|
+
]));
|
|
108
|
+
const startedAt = new Date().toISOString();
|
|
109
|
+
const ping = yield (0, daemon_client_1.sendDaemonCommand)({ cmd: "ping" }, { timeoutMs: 5000 });
|
|
110
|
+
if (!ping.ok || ping.ready !== true)
|
|
111
|
+
throw new Error("An already-ready daemon is required; no autostart/fallback");
|
|
112
|
+
if (values.diagnostics &&
|
|
113
|
+
((_a = ping.capabilities) === null || _a === void 0 ? void 0 : _a.searchDiagnostics) !== 1)
|
|
114
|
+
throw new Error("Daemon lacks searchDiagnostics v1; install/restart a compatible daemon before measuring diagnostics");
|
|
115
|
+
const before = yield (0, daemon_client_1.sendDaemonCommand)({ cmd: "status" });
|
|
116
|
+
if (!before.ok)
|
|
117
|
+
throw new Error("Cannot capture daemon status");
|
|
118
|
+
const registry = (0, project_registry_1.listProjects)();
|
|
119
|
+
const cache = (0, lmdb_1.open)({
|
|
120
|
+
path: path.join(config_1.PATHS.globalRoot, "cache/meta.lmdb"),
|
|
121
|
+
compression: true,
|
|
122
|
+
readOnly: true,
|
|
123
|
+
});
|
|
124
|
+
const samples = [];
|
|
125
|
+
const preflight = [];
|
|
126
|
+
fs.writeFileSync(checkpoint, "", { flag: "wx" });
|
|
127
|
+
try {
|
|
128
|
+
for (const corpus of fixture.corpora) {
|
|
129
|
+
const root = roots.get(corpus.id);
|
|
130
|
+
const project = registry.find((p) => p.root === root);
|
|
131
|
+
if ((project === null || project === void 0 ? void 0 : project.status) !== "indexed")
|
|
132
|
+
throw new Error(`Corpus is not indexed: ${corpus.id}`);
|
|
133
|
+
const files = [
|
|
134
|
+
...new Set(fixture.cases
|
|
135
|
+
.filter((c) => c.corpus === corpus.id)
|
|
136
|
+
.flatMap((c) => c.expected.map((t) => t.file))),
|
|
137
|
+
];
|
|
138
|
+
const rows = yield (0, daemon_client_1.sendDaemonCommand)({
|
|
139
|
+
cmd: "rows.locate",
|
|
140
|
+
projectRoot: root,
|
|
141
|
+
limit: 500,
|
|
142
|
+
select: ["path", "defined_symbols", "start_line", "end_line"],
|
|
143
|
+
matches: files.map((file) => ({ kind: "path", path: file })),
|
|
144
|
+
});
|
|
145
|
+
if (!rows.ok || !Array.isArray(rows.rows))
|
|
146
|
+
throw new Error(`Target inventory failed: ${corpus.id}`);
|
|
147
|
+
preflight.push({
|
|
148
|
+
corpus: corpus.id,
|
|
149
|
+
root,
|
|
150
|
+
git: git(root),
|
|
151
|
+
project,
|
|
152
|
+
files,
|
|
153
|
+
targetRows: rows.rows,
|
|
154
|
+
cache: files.map((file) => (Object.assign({ file }, cache.get(path.join(root, file))))),
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
// Sequential execution bounds load. Interleave corpora instead of running
|
|
158
|
+
// all of one corpus first; repetitions are reported separately from cases.
|
|
159
|
+
const ordered = [];
|
|
160
|
+
const groups = fixture.corpora.map((r) => fixture.cases.filter((c) => c.corpus === r.id));
|
|
161
|
+
for (let i = 0; i < Math.max(...groups.map((g) => g.length)); i++) {
|
|
162
|
+
for (const group of groups)
|
|
163
|
+
if (group[i])
|
|
164
|
+
ordered.push(group[i]);
|
|
165
|
+
}
|
|
166
|
+
for (let repetition = 1; repetition <= repeats; repetition++) {
|
|
167
|
+
for (const c of ordered) {
|
|
168
|
+
const root = roots.get(c.corpus);
|
|
169
|
+
const exclusions = new Set();
|
|
170
|
+
const inventory = preflight.find((r) => r.corpus === c.corpus);
|
|
171
|
+
for (const t of c.expected) {
|
|
172
|
+
const file = path.join(root, t.file);
|
|
173
|
+
if (fs.realpathSync(file) !== file)
|
|
174
|
+
throw new Error(`Target aliases are not frozen: ${c.id}`);
|
|
175
|
+
for (const reason of (0, relevance_baseline_1.sourceExclusions)(t.sourceSha256, (0, relevance_baseline_1.sha256)(fs.readFileSync(file)), (_b = cache.get(file)) === null || _b === void 0 ? void 0 : _b.hash))
|
|
176
|
+
exclusions.add(reason);
|
|
177
|
+
const indexed = inventory.targetRows
|
|
178
|
+
.flat()
|
|
179
|
+
.some((row) => (0, relevance_baseline_1.matchesTarget)({
|
|
180
|
+
type: "text",
|
|
181
|
+
score: 0,
|
|
182
|
+
metadata: { path: String(row.path), hash: "" },
|
|
183
|
+
defined_symbols: row.defined_symbols,
|
|
184
|
+
generated_metadata: {
|
|
185
|
+
start_line: Number(row.start_line),
|
|
186
|
+
num_lines: Number(row.end_line) - Number(row.start_line) + 1,
|
|
187
|
+
},
|
|
188
|
+
}, root, t));
|
|
189
|
+
if (!indexed)
|
|
190
|
+
exclusions.add("target_not_in_index");
|
|
191
|
+
}
|
|
192
|
+
const start = performance.now();
|
|
193
|
+
const response = exclusions.size === 0
|
|
194
|
+
? yield (0, daemon_client_1.sendDaemonCommand)({
|
|
195
|
+
cmd: "search",
|
|
196
|
+
projectRoot: root,
|
|
197
|
+
query: c.query,
|
|
198
|
+
limit: 20,
|
|
199
|
+
pathPrefix: `${root}${path.sep}`,
|
|
200
|
+
rerank: false,
|
|
201
|
+
explain: true,
|
|
202
|
+
diagnostics: values.diagnostics,
|
|
203
|
+
}, { timeoutMs: 60000 })
|
|
204
|
+
: { ok: false, error: "preflight_excluded" };
|
|
205
|
+
const elapsedMs = performance.now() - start;
|
|
206
|
+
if (values.diagnostics &&
|
|
207
|
+
response.ok &&
|
|
208
|
+
((_c = response.diagnostics) === null || _c === void 0 ? void 0 : _c.schemaVersion) !== 1)
|
|
209
|
+
exclusions.add("diagnostics_missing");
|
|
210
|
+
if (!response.ok)
|
|
211
|
+
exclusions.add(String((_e = response.error) !== null && _e !== void 0 ? _e : "search_failed"));
|
|
212
|
+
const data = Array.isArray(response.data)
|
|
213
|
+
? response.data
|
|
214
|
+
: [];
|
|
215
|
+
const state = response.indexState;
|
|
216
|
+
if (state &&
|
|
217
|
+
(state.indexing ||
|
|
218
|
+
state.verifying ||
|
|
219
|
+
state.degraded ||
|
|
220
|
+
Number(state.pendingFiles) > 0 ||
|
|
221
|
+
Number(state.failedFiles) > 0))
|
|
222
|
+
exclusions.add("index_unsettled");
|
|
223
|
+
const warnings = Array.isArray(response.warnings)
|
|
224
|
+
? response.warnings
|
|
225
|
+
: [];
|
|
226
|
+
if (warnings.length)
|
|
227
|
+
exclusions.add("search_warning");
|
|
228
|
+
for (const t of c.expected) {
|
|
229
|
+
const file = path.join(root, t.file);
|
|
230
|
+
for (const reason of (0, relevance_baseline_1.sourceExclusions)(t.sourceSha256, (0, relevance_baseline_1.sha256)(fs.readFileSync(file)), (_d = cache.get(file)) === null || _d === void 0 ? void 0 : _d.hash))
|
|
231
|
+
exclusions.add(reason);
|
|
232
|
+
if (data.some((d) => { var _a; return ((_a = d.metadata) === null || _a === void 0 ? void 0 : _a.path) === file && d.metadata.hash !== t.sourceSha256; }))
|
|
233
|
+
exclusions.add("returned_row_hash_mismatch");
|
|
234
|
+
}
|
|
235
|
+
const ranked = { data };
|
|
236
|
+
samples.push(Object.assign(Object.assign({}, (0, relevance_baseline_1.scoreRelevance)(ranked, root, c)), { id: c.id, corpus: c.corpus, split: c.split, repetition,
|
|
237
|
+
elapsedMs, exclusions: [...exclusions], query: c.query, intent: c.intent, origin: c.origin, warnings, indexState: state !== null && state !== void 0 ? state : null, error: response.ok ? null : response.error, diagnostics: (_f = response.diagnostics) !== null && _f !== void 0 ? _f : null, results: data.map((d) => { var _a, _b; var _c, _d, _e, _f, _g; return ({
|
|
238
|
+
path: path.relative(root, String((_c = (_a = d.metadata) === null || _a === void 0 ? void 0 : _a.path) !== null && _c !== void 0 ? _c : "")),
|
|
239
|
+
hash: (_d = (_b = d.metadata) === null || _b === void 0 ? void 0 : _b.hash) !== null && _d !== void 0 ? _d : null,
|
|
240
|
+
symbols: (_e = d.defined_symbols) !== null && _e !== void 0 ? _e : [],
|
|
241
|
+
range: (_f = d.generated_metadata) !== null && _f !== void 0 ? _f : null,
|
|
242
|
+
score: d.score,
|
|
243
|
+
scoreBreakdown: (_g = d.scoreBreakdown) !== null && _g !== void 0 ? _g : null,
|
|
244
|
+
}); }) }));
|
|
245
|
+
fs.appendFileSync(checkpoint, `${JSON.stringify(samples[samples.length - 1])}\n`);
|
|
246
|
+
console.error(`${c.id} repeat=${repetition} rank=${samples[samples.length - 1].rank} excluded=${[...exclusions].join(",") || "none"}`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
finally {
|
|
251
|
+
cache.close();
|
|
252
|
+
}
|
|
253
|
+
const afterPing = yield (0, daemon_client_1.sendDaemonCommand)({ cmd: "ping" });
|
|
254
|
+
const after = yield (0, daemon_client_1.sendDaemonCommand)({ cmd: "status" });
|
|
255
|
+
if (!afterPing.ok ||
|
|
256
|
+
after.pid !== before.pid ||
|
|
257
|
+
afterPing.version !== ping.version) {
|
|
258
|
+
for (const sample of samples)
|
|
259
|
+
sample.exclusions.push("daemon_changed");
|
|
260
|
+
}
|
|
261
|
+
if ((0, relevance_baseline_1.sha256)(fs.readFileSync(fixturePath)) !== values.sha256)
|
|
262
|
+
throw new Error("Frozen fixture changed during execution");
|
|
263
|
+
const report = {
|
|
264
|
+
schemaVersion: 1,
|
|
265
|
+
startedAt,
|
|
266
|
+
finishedAt: new Date().toISOString(),
|
|
267
|
+
fixtureSha256: values.sha256,
|
|
268
|
+
fixture,
|
|
269
|
+
source: {
|
|
270
|
+
git: git(process.cwd()),
|
|
271
|
+
node: process.version,
|
|
272
|
+
evaluatorHash: (0, relevance_baseline_1.sha256)(fs.readFileSync(__filename)),
|
|
273
|
+
scoringHash: (0, relevance_baseline_1.sha256)(fs.readFileSync(path.join(__dirname, "lib/eval/relevance-baseline.ts"))),
|
|
274
|
+
searcherHash: (0, relevance_baseline_1.sha256)(fs.readFileSync(path.join(__dirname, "lib/search/searcher.ts"))),
|
|
275
|
+
diagnosticsHash: (0, relevance_baseline_1.sha256)(fs.readFileSync(path.join(__dirname, "lib/search/diagnostics.ts"))),
|
|
276
|
+
},
|
|
277
|
+
execution: {
|
|
278
|
+
transport: "existing-daemon-ipc",
|
|
279
|
+
requestedRerank: false,
|
|
280
|
+
explain: true,
|
|
281
|
+
requestedDiagnostics: values.diagnostics,
|
|
282
|
+
limit: 20,
|
|
283
|
+
repeats,
|
|
284
|
+
ping,
|
|
285
|
+
before,
|
|
286
|
+
afterPing,
|
|
287
|
+
after,
|
|
288
|
+
preflight,
|
|
289
|
+
diagnostics: {
|
|
290
|
+
fusionPool: values.diagnostics
|
|
291
|
+
? "bounded-post-seed-head-200"
|
|
292
|
+
: "unobserved",
|
|
293
|
+
concentrationGate: values.diagnostics
|
|
294
|
+
? "per-sample-diagnostics"
|
|
295
|
+
: "unobserved",
|
|
296
|
+
daemonRankingEnvironment: values.diagnostics
|
|
297
|
+
? "allowlisted-per-sample-diagnostics"
|
|
298
|
+
: "unobserved",
|
|
299
|
+
ftsHealth: values.diagnostics
|
|
300
|
+
? "per-sample-diagnostics"
|
|
301
|
+
: "warnings-and-score-breakdowns-only",
|
|
302
|
+
trueRerankOff: "not-asserted",
|
|
303
|
+
},
|
|
304
|
+
},
|
|
305
|
+
summary: (0, relevance_baseline_1.summarizeRelevance)(samples),
|
|
306
|
+
byCorpus: fixture.corpora.map((r) => (Object.assign({ corpus: r.id }, (0, relevance_baseline_1.summarizeRelevance)(samples.filter((s) => s.corpus === r.id))))),
|
|
307
|
+
bySplit: ["dev", "heldout"].map((split) => (Object.assign({ split }, (0, relevance_baseline_1.summarizeRelevance)(samples.filter((s) => s.split === split))))),
|
|
308
|
+
byIntent: [...new Set(fixture.cases.map((c) => c.intent))].map((intent) => (Object.assign({ intent }, (0, relevance_baseline_1.summarizeRelevance)(samples.filter((s) => s.intent === intent))))),
|
|
309
|
+
samples,
|
|
310
|
+
researchGateReady: false,
|
|
311
|
+
limitations: [
|
|
312
|
+
"Curated source-grounded cases are not observed user-session misses",
|
|
313
|
+
values.diagnostics
|
|
314
|
+
? "Candidate trace is limited to the post-seed fusion head; absence from a truncated trace does not prove retrieval failure"
|
|
315
|
+
: "Pipeline diagnostics were not requested",
|
|
316
|
+
"Client environment cannot change daemon ranking settings",
|
|
317
|
+
"Latencies mix ordinary daemon activity and first/repeated queries",
|
|
318
|
+
],
|
|
319
|
+
};
|
|
320
|
+
fs.writeFileSync(values.output, `${JSON.stringify(report, null, 2)}\n`, {
|
|
321
|
+
flag: "wx",
|
|
322
|
+
});
|
|
323
|
+
console.log(JSON.stringify({
|
|
324
|
+
output: values.output,
|
|
325
|
+
summary: report.summary,
|
|
326
|
+
byCorpus: report.byCorpus,
|
|
327
|
+
bySplit: report.bySplit,
|
|
328
|
+
}, null, 2));
|
|
329
|
+
if (samples.some((s) => s.exclusions.length > 0))
|
|
330
|
+
process.exitCode = 2;
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
if (require.main === module)
|
|
334
|
+
run().catch((error) => {
|
|
335
|
+
console.error(error);
|
|
336
|
+
process.exitCode = 1;
|
|
337
|
+
});
|
package/dist/eval.js
CHANGED
|
@@ -527,9 +527,10 @@ function evaluateCase(response, evalCase, timeMs) {
|
|
|
527
527
|
const avoidRank = response.data.findIndex((chunk) => { var _a, _b, _c; return (_b = (_a = chunk.metadata) === null || _a === void 0 ? void 0 : _a.path) === null || _b === void 0 ? void 0 : _b.toLowerCase().includes(((_c = evalCase.avoidPath) === null || _c === void 0 ? void 0 : _c.toLowerCase()) || "_____"); });
|
|
528
528
|
const hitAvoid = evalCase.avoidPath && avoidRank >= 0 && (rank === -1 || avoidRank < rank);
|
|
529
529
|
const found = rank >= 0 && !hitAvoid;
|
|
530
|
-
const rr = found ? 1 / (rank + 1) : 0;
|
|
531
530
|
const recall = found && rank < 10 ? 1 : 0;
|
|
531
|
+
const rr = recall ? 1 / (rank + 1) : 0;
|
|
532
532
|
return {
|
|
533
|
+
rank: rank + 1,
|
|
533
534
|
rr,
|
|
534
535
|
found,
|
|
535
536
|
recall,
|
|
@@ -619,7 +620,7 @@ function run() {
|
|
|
619
620
|
console.log(`Eval results for store at: ${paths.lancedbDir}`);
|
|
620
621
|
console.log("=".repeat(80));
|
|
621
622
|
results.forEach((r) => {
|
|
622
|
-
const status = r.found ? `rank ${
|
|
623
|
+
const status = r.found ? `rank ${r.rank}` : "❌ missed";
|
|
623
624
|
const emoji = r.found ? (r.rr === 1 ? "🎯" : "✓") : "❌";
|
|
624
625
|
console.log(`${emoji} ${r.query}`);
|
|
625
626
|
console.log(` => ${status} (target: ${r.path}) [${r.timeMs.toFixed(0)}ms]`);
|
|
@@ -628,7 +629,7 @@ function run() {
|
|
|
628
629
|
}
|
|
629
630
|
});
|
|
630
631
|
console.log("=".repeat(80));
|
|
631
|
-
console.log(`MRR: ${mrr.toFixed(3)}`);
|
|
632
|
+
console.log(`MRR@10: ${mrr.toFixed(3)}`);
|
|
632
633
|
console.log(`Recall@10: ${recallAt10.toFixed(3)}`);
|
|
633
634
|
console.log(`Avg query time: ${avgTime.toFixed(0)}ms`);
|
|
634
635
|
console.log(`Total time: ${totalTime.toFixed(0)}ms`);
|
|
@@ -50,6 +50,7 @@ exports.handleCommand = handleCommand;
|
|
|
50
50
|
const fs = __importStar(require("node:fs"));
|
|
51
51
|
const path = __importStar(require("node:path"));
|
|
52
52
|
const config_1 = require("../../config");
|
|
53
|
+
const per_file_1 = require("../search/per-file");
|
|
53
54
|
const logger_1 = require("../utils/logger");
|
|
54
55
|
const operation_coordinator_1 = require("../utils/operation-coordinator");
|
|
55
56
|
const path_containment_1 = require("../utils/path-containment");
|
|
@@ -152,6 +153,8 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
152
153
|
capabilities: {
|
|
153
154
|
exclusiveGenerationRebuild: exports.EXCLUSIVE_GENERATION_REBUILD_PROTOCOL,
|
|
154
155
|
readVerbs: read_verbs_1.READ_VERBS_PROTOCOL,
|
|
156
|
+
searchDiagnostics: 1,
|
|
157
|
+
perFileSearch: 1,
|
|
155
158
|
// `watch`/`unwatch` accept holder/pid/ttlMs. A daemon without this
|
|
156
159
|
// treats `unwatch` as unwatch-for-everyone, so clients releasing a
|
|
157
160
|
// single lease must check it first.
|
|
@@ -231,6 +234,13 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
231
234
|
conn.on("close", onClose);
|
|
232
235
|
try {
|
|
233
236
|
const limitRaw = typeof cmd.limit === "number" ? cmd.limit : 10;
|
|
237
|
+
let maxPerFile;
|
|
238
|
+
try {
|
|
239
|
+
maxPerFile = (0, per_file_1.validateMaxPerFile)(cmd.maxPerFile);
|
|
240
|
+
}
|
|
241
|
+
catch (_m) {
|
|
242
|
+
return { ok: false, error: "invalid maxPerFile" };
|
|
243
|
+
}
|
|
234
244
|
if (!Number.isInteger(limitRaw) || limitRaw < 1 || limitRaw > 1000) {
|
|
235
245
|
return { ok: false, error: "invalid limit" };
|
|
236
246
|
}
|
|
@@ -295,10 +305,12 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
295
305
|
projectRoot,
|
|
296
306
|
query,
|
|
297
307
|
limit: limitRaw,
|
|
308
|
+
maxPerFile,
|
|
298
309
|
filters,
|
|
299
310
|
pathPrefix,
|
|
300
311
|
rerank: cmd.rerank === true,
|
|
301
312
|
explain: cmd.explain === true,
|
|
313
|
+
diagnostics: cmd.diagnostics === true,
|
|
302
314
|
seeds: cmd.seeds &&
|
|
303
315
|
typeof cmd.seeds === "object" &&
|
|
304
316
|
!Array.isArray(cmd.seeds)
|
|
@@ -45,6 +45,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
45
45
|
exports.handleDaemonSearch = handleDaemonSearch;
|
|
46
46
|
const embedding_generation_1 = require("../index/embedding-generation");
|
|
47
47
|
const index_state_footer_1 = require("../output/index-state-footer");
|
|
48
|
+
const per_file_1 = require("../search/per-file");
|
|
48
49
|
const searcher_1 = require("../search/searcher");
|
|
49
50
|
const retriever_1 = require("../skeleton/retriever");
|
|
50
51
|
const project_registry_1 = require("../utils/project-registry");
|
|
@@ -57,6 +58,13 @@ function handleDaemonSearch(deps, payload, signal) {
|
|
|
57
58
|
return __awaiter(this, void 0, void 0, function* () {
|
|
58
59
|
var _a, _b, _c, _d;
|
|
59
60
|
var _e, _f, _g;
|
|
61
|
+
let maxPerFile;
|
|
62
|
+
try {
|
|
63
|
+
maxPerFile = (0, per_file_1.validateMaxPerFile)(payload.maxPerFile);
|
|
64
|
+
}
|
|
65
|
+
catch (_h) {
|
|
66
|
+
return { ok: false, error: "invalid maxPerFile" };
|
|
67
|
+
}
|
|
60
68
|
const { vectorDb, workerPool } = deps;
|
|
61
69
|
if (!vectorDb || !workerPool) {
|
|
62
70
|
return { ok: false, error: "daemon not ready" };
|
|
@@ -103,6 +111,8 @@ function handleDaemonSearch(deps, payload, signal) {
|
|
|
103
111
|
result = yield searcher.search(payload.query, payload.limit, {
|
|
104
112
|
rerank: payload.rerank === true,
|
|
105
113
|
explain: payload.explain === true,
|
|
114
|
+
diagnostics: payload.diagnostics === true,
|
|
115
|
+
maxPerFile,
|
|
106
116
|
seeds: payload.seeds,
|
|
107
117
|
}, payload.filters, payload.pathPrefix, undefined, signal);
|
|
108
118
|
}
|
|
@@ -116,6 +126,8 @@ function handleDaemonSearch(deps, payload, signal) {
|
|
|
116
126
|
const response = { ok: true, data: result.data };
|
|
117
127
|
if ((_c = result.warnings) === null || _c === void 0 ? void 0 : _c.length)
|
|
118
128
|
response.warnings = result.warnings;
|
|
129
|
+
if (payload.diagnostics === true && result.diagnostics)
|
|
130
|
+
response.diagnostics = result.diagnostics;
|
|
119
131
|
// Annotate partial results when the index is still catching up, so an
|
|
120
132
|
// agent can caveat or retry. Settled failures remain visible too.
|
|
121
133
|
const idx = deps.getIndexState(root);
|
|
@@ -152,7 +164,7 @@ function handleDaemonSearch(deps, payload, signal) {
|
|
|
152
164
|
if (sk)
|
|
153
165
|
skeletons[p] = sk;
|
|
154
166
|
}
|
|
155
|
-
catch (
|
|
167
|
+
catch (_j) {
|
|
156
168
|
// best-effort — drop the entry, keep the search result
|
|
157
169
|
}
|
|
158
170
|
}
|
|
@@ -167,7 +179,7 @@ function handleDaemonSearch(deps, payload, signal) {
|
|
|
167
179
|
const builder = new GraphBuilder(vectorDb, root);
|
|
168
180
|
response.graph = yield builder.buildGraphMultiHop(payload.query, 1);
|
|
169
181
|
}
|
|
170
|
-
catch (
|
|
182
|
+
catch (_k) {
|
|
171
183
|
// best-effort — drop graph, keep results
|
|
172
184
|
}
|
|
173
185
|
}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.relevanceFixtureSchema = void 0;
|
|
37
|
+
exports.sha256 = sha256;
|
|
38
|
+
exports.parseFrozenFixture = parseFrozenFixture;
|
|
39
|
+
exports.matchesTarget = matchesTarget;
|
|
40
|
+
exports.scoreRelevance = scoreRelevance;
|
|
41
|
+
exports.summarizeRelevance = summarizeRelevance;
|
|
42
|
+
exports.sourceExclusions = sourceExclusions;
|
|
43
|
+
const node_crypto_1 = require("node:crypto");
|
|
44
|
+
const path = __importStar(require("node:path"));
|
|
45
|
+
const zod_1 = require("zod");
|
|
46
|
+
const digest = zod_1.z.string().regex(/^[a-f0-9]{64}$/);
|
|
47
|
+
const targetSchema = zod_1.z
|
|
48
|
+
.object({
|
|
49
|
+
file: zod_1.z.string().min(1),
|
|
50
|
+
symbol: zod_1.z.string().min(1),
|
|
51
|
+
match: zod_1.z.enum(["symbol-or-range", "range"]).optional(),
|
|
52
|
+
startLine: zod_1.z.number().int().positive(),
|
|
53
|
+
endLine: zod_1.z.number().int().positive(),
|
|
54
|
+
sourceSha256: digest,
|
|
55
|
+
})
|
|
56
|
+
.strict()
|
|
57
|
+
.refine((t) => t.endLine >= t.startLine, "Reversed target range");
|
|
58
|
+
exports.relevanceFixtureSchema = zod_1.z
|
|
59
|
+
.object({
|
|
60
|
+
schemaVersion: zod_1.z.literal(1),
|
|
61
|
+
createdAt: zod_1.z.string().min(1),
|
|
62
|
+
purpose: zod_1.z.string().min(1),
|
|
63
|
+
corpora: zod_1.z
|
|
64
|
+
.array(zod_1.z
|
|
65
|
+
.object({
|
|
66
|
+
id: zod_1.z.string().min(1),
|
|
67
|
+
root: zod_1.z.string().min(1),
|
|
68
|
+
language: zod_1.z.string().min(1),
|
|
69
|
+
})
|
|
70
|
+
.strict())
|
|
71
|
+
.min(1),
|
|
72
|
+
cases: zod_1.z
|
|
73
|
+
.array(zod_1.z
|
|
74
|
+
.object({
|
|
75
|
+
id: zod_1.z.string().min(1),
|
|
76
|
+
corpus: zod_1.z.string().min(1),
|
|
77
|
+
split: zod_1.z.enum(["dev", "heldout"]),
|
|
78
|
+
origin: zod_1.z.enum(["curated-source", "observed-session"]),
|
|
79
|
+
intent: zod_1.z.string().min(1),
|
|
80
|
+
query: zod_1.z.string().min(1),
|
|
81
|
+
expected: zod_1.z.array(targetSchema).min(1),
|
|
82
|
+
})
|
|
83
|
+
.strict())
|
|
84
|
+
.min(1),
|
|
85
|
+
})
|
|
86
|
+
.strict()
|
|
87
|
+
.superRefine((fixture, ctx) => {
|
|
88
|
+
for (const values of [fixture.corpora, fixture.cases]) {
|
|
89
|
+
if (new Set(values.map((v) => v.id)).size !== values.length) {
|
|
90
|
+
ctx.addIssue({ code: "custom", message: "Duplicate fixture IDs" });
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
for (const c of fixture.cases) {
|
|
94
|
+
if (!fixture.corpora.some((r) => r.id === c.corpus)) {
|
|
95
|
+
ctx.addIssue({
|
|
96
|
+
code: "custom",
|
|
97
|
+
message: `Unknown corpus: ${c.corpus}`,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
for (const target of c.expected) {
|
|
101
|
+
if (path.isAbsolute(target.file) ||
|
|
102
|
+
target.file.split(/[\\/]/).includes("..")) {
|
|
103
|
+
ctx.addIssue({
|
|
104
|
+
code: "custom",
|
|
105
|
+
message: "Targets must be contained relative paths",
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
function sha256(bytes) {
|
|
112
|
+
return (0, node_crypto_1.createHash)("sha256").update(bytes).digest("hex");
|
|
113
|
+
}
|
|
114
|
+
function parseFrozenFixture(bytes, expectedSha256) {
|
|
115
|
+
if (!digest.safeParse(expectedSha256).success ||
|
|
116
|
+
sha256(bytes) !== expectedSha256) {
|
|
117
|
+
throw new Error("Fixture checksum mismatch; freeze the reviewed fixture before measuring");
|
|
118
|
+
}
|
|
119
|
+
return exports.relevanceFixtureSchema.parse(JSON.parse(bytes.toString("utf8")));
|
|
120
|
+
}
|
|
121
|
+
/** Exact file plus the frozen matching rule; legacy fixtures allow symbol or range. */
|
|
122
|
+
function matchesTarget(chunk, root, target) {
|
|
123
|
+
var _a, _b, _c, _d;
|
|
124
|
+
if (typeof ((_a = chunk.metadata) === null || _a === void 0 ? void 0 : _a.path) !== "string")
|
|
125
|
+
return false;
|
|
126
|
+
if (path.resolve(chunk.metadata.path) !== path.resolve(root, target.file))
|
|
127
|
+
return false;
|
|
128
|
+
if (target.match !== "range" &&
|
|
129
|
+
((_b = chunk.defined_symbols) === null || _b === void 0 ? void 0 : _b.includes(target.symbol)))
|
|
130
|
+
return true;
|
|
131
|
+
const start = (_c = chunk.generated_metadata) === null || _c === void 0 ? void 0 : _c.start_line;
|
|
132
|
+
const count = (_d = chunk.generated_metadata) === null || _d === void 0 ? void 0 : _d.num_lines;
|
|
133
|
+
if (typeof start !== "number" || typeof count !== "number" || count <= 0)
|
|
134
|
+
return false;
|
|
135
|
+
// SearchResponse ranges start at zero; source fixtures use one-based lines.
|
|
136
|
+
return start + 1 <= target.endLine && start + count >= target.startLine;
|
|
137
|
+
}
|
|
138
|
+
function scoreRelevance(response, root, c) {
|
|
139
|
+
const index = response.data.findIndex((chunk) => c.expected.some((t) => matchesTarget(chunk, root, t)));
|
|
140
|
+
const rank = index + 1;
|
|
141
|
+
return {
|
|
142
|
+
rank,
|
|
143
|
+
rrAt10: rank > 0 && rank <= 10 ? 1 / rank : 0,
|
|
144
|
+
recallAt10: rank > 0 && rank <= 10 ? 1 : 0,
|
|
145
|
+
hitAt1: rank === 1 ? 1 : 0,
|
|
146
|
+
found: rank > 0,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
function summarizeRelevance(samples) {
|
|
150
|
+
const valid = samples.filter((s) => s.exclusions.length === 0);
|
|
151
|
+
const times = valid.map((s) => s.elapsedMs).sort((a, b) => a - b);
|
|
152
|
+
const mean = (key) => valid.length
|
|
153
|
+
? valid.reduce((sum, s) => sum + s[key], 0) / valid.length
|
|
154
|
+
: null;
|
|
155
|
+
return {
|
|
156
|
+
cases: new Set(samples.map((s) => s.id)).size,
|
|
157
|
+
samples: samples.length,
|
|
158
|
+
validSamples: valid.length,
|
|
159
|
+
excludedSamples: samples.length - valid.length,
|
|
160
|
+
mrrAt10: mean("rrAt10"),
|
|
161
|
+
recallAt10: mean("recallAt10"),
|
|
162
|
+
hitsAt1: mean("hitAt1"),
|
|
163
|
+
medianMs: times.length ? times[Math.ceil(times.length / 2) - 1] : null,
|
|
164
|
+
p95Ms: times.length ? times[Math.ceil(times.length * 0.95) - 1] : null,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
function sourceExclusions(frozenHash, sourceHash, cacheHash) {
|
|
168
|
+
const reasons = [];
|
|
169
|
+
if (sourceHash !== frozenHash)
|
|
170
|
+
reasons.push("source_changed");
|
|
171
|
+
if (cacheHash !== frozenHash)
|
|
172
|
+
reasons.push("index_cache_hash_mismatch");
|
|
173
|
+
return reasons;
|
|
174
|
+
}
|