ko-qmd 2.8.3-ko.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1376 -0
- package/LICENSE +21 -0
- package/README.ko.md +86 -0
- package/README.md +1307 -0
- package/bin/qmd +192 -0
- package/dist/ast.d.ts +65 -0
- package/dist/ast.js +334 -0
- package/dist/bench/bench.d.ts +35 -0
- package/dist/bench/bench.js +327 -0
- package/dist/bench/score.d.ts +33 -0
- package/dist/bench/score.js +88 -0
- package/dist/bench/types.d.ts +80 -0
- package/dist/bench/types.js +8 -0
- package/dist/cli/build-info.json +4 -0
- package/dist/cli/embed-lock.d.ts +24 -0
- package/dist/cli/embed-lock.js +94 -0
- package/dist/cli/formatter.d.ts +120 -0
- package/dist/cli/formatter.js +355 -0
- package/dist/cli/mcp-pid.d.ts +25 -0
- package/dist/cli/mcp-pid.js +86 -0
- package/dist/cli/qmd.d.ts +43 -0
- package/dist/cli/qmd.js +4424 -0
- package/dist/cli/version.d.ts +42 -0
- package/dist/cli/version.js +80 -0
- package/dist/collections.d.ts +167 -0
- package/dist/collections.js +410 -0
- package/dist/db.d.ts +63 -0
- package/dist/db.js +130 -0
- package/dist/hangul.d.ts +54 -0
- package/dist/hangul.js +140 -0
- package/dist/index.d.ts +237 -0
- package/dist/index.js +244 -0
- package/dist/llm.d.ts +543 -0
- package/dist/llm.js +1747 -0
- package/dist/maintenance.d.ts +33 -0
- package/dist/maintenance.js +52 -0
- package/dist/mcp/origin-guard.d.ts +67 -0
- package/dist/mcp/origin-guard.js +137 -0
- package/dist/mcp/server.d.ts +116 -0
- package/dist/mcp/server.js +883 -0
- package/dist/paths.d.ts +1 -0
- package/dist/paths.js +4 -0
- package/dist/store.d.ts +1100 -0
- package/dist/store.js +4965 -0
- package/dist/trust.d.ts +152 -0
- package/dist/trust.js +249 -0
- package/package.json +114 -0
- package/scripts/build.mjs +72 -0
- package/scripts/check-package-grammars.mjs +29 -0
- package/scripts/package-smoke.mjs +65 -0
- package/scripts/test-all.mjs +38 -0
- package/skills/qmd/SKILL.md +316 -0
- package/skills/qmd/references/mcp-setup.md +119 -0
- package/skills/release/SKILL.md +141 -0
- package/skills/release/scripts/install-hooks.sh +38 -0
- package/skills/release/scripts/release-context.sh +129 -0
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* QMD Benchmark Harness
|
|
3
|
+
*
|
|
4
|
+
* Runs queries from a fixture file against multiple search backends
|
|
5
|
+
* and measures precision@k, recall, MRR, F1, and latency.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* qmd bench <fixture.json> [--json] [--collection <name>]
|
|
9
|
+
*
|
|
10
|
+
* Backends tested:
|
|
11
|
+
* - bm25: BM25 keyword search (searchLex)
|
|
12
|
+
* - vector: Vector similarity search (searchVector)
|
|
13
|
+
* - hybrid: BM25 + vector RRF fusion without reranking
|
|
14
|
+
* - full: Full hybrid pipeline with LLM reranking
|
|
15
|
+
*/
|
|
16
|
+
import { readFileSync } from "node:fs";
|
|
17
|
+
import { resolve } from "node:path";
|
|
18
|
+
import { createStore, getDefaultDbPath, } from "../index.js";
|
|
19
|
+
import { scoreResults } from "./score.js";
|
|
20
|
+
function parseStructuredQuery(query) {
|
|
21
|
+
const lines = query.split("\n").map((line, idx) => ({
|
|
22
|
+
trimmed: line.trim(),
|
|
23
|
+
number: idx + 1,
|
|
24
|
+
})).filter(line => line.trimmed.length > 0);
|
|
25
|
+
if (lines.length === 0)
|
|
26
|
+
return undefined;
|
|
27
|
+
const prefixRe = /^(lex|vec|hyde):\s*/i;
|
|
28
|
+
const intentRe = /^intent:\s*/i;
|
|
29
|
+
const searches = [];
|
|
30
|
+
let intent;
|
|
31
|
+
for (const line of lines) {
|
|
32
|
+
if (intentRe.test(line.trimmed)) {
|
|
33
|
+
if (intent !== undefined) {
|
|
34
|
+
throw new Error(`Line ${line.number}: only one intent: line is allowed per benchmark query.`);
|
|
35
|
+
}
|
|
36
|
+
intent = line.trimmed.replace(intentRe, "").trim();
|
|
37
|
+
if (!intent) {
|
|
38
|
+
throw new Error(`Line ${line.number}: intent: must include text.`);
|
|
39
|
+
}
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
const match = line.trimmed.match(prefixRe);
|
|
43
|
+
if (match) {
|
|
44
|
+
const type = match[1].toLowerCase();
|
|
45
|
+
const text = line.trimmed.slice(match[0].length).trim();
|
|
46
|
+
if (!text) {
|
|
47
|
+
throw new Error(`Line ${line.number} (${type}:) must include text.`);
|
|
48
|
+
}
|
|
49
|
+
searches.push({ type, query: text, line: line.number });
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
if (lines.length === 1) {
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
throw new Error(`Line ${line.number} is missing a lex:/vec:/hyde:/intent: prefix.`);
|
|
56
|
+
}
|
|
57
|
+
if (intent && searches.length === 0) {
|
|
58
|
+
throw new Error("intent: cannot appear alone. Add at least one lex:, vec:, or hyde: line.");
|
|
59
|
+
}
|
|
60
|
+
return searches.length > 0 ? { searches, intent } : undefined;
|
|
61
|
+
}
|
|
62
|
+
function uniqueFiles(files, limit) {
|
|
63
|
+
const seen = new Set();
|
|
64
|
+
const out = [];
|
|
65
|
+
for (const file of files) {
|
|
66
|
+
if (seen.has(file))
|
|
67
|
+
continue;
|
|
68
|
+
seen.add(file);
|
|
69
|
+
out.push(file);
|
|
70
|
+
if (out.length >= limit)
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
return out;
|
|
74
|
+
}
|
|
75
|
+
const BACKENDS = [
|
|
76
|
+
{
|
|
77
|
+
name: "bm25",
|
|
78
|
+
run: async (store, query, limit, collection) => {
|
|
79
|
+
const structured = parseStructuredQuery(query.query);
|
|
80
|
+
const lexQueries = structured?.searches.filter(q => q.type === "lex");
|
|
81
|
+
if (structured) {
|
|
82
|
+
const files = [];
|
|
83
|
+
for (const lex of lexQueries ?? []) {
|
|
84
|
+
const results = await store.searchLex(lex.query, { limit, collection });
|
|
85
|
+
files.push(...results.map((r) => r.filepath));
|
|
86
|
+
}
|
|
87
|
+
return uniqueFiles(files, limit);
|
|
88
|
+
}
|
|
89
|
+
const results = await store.searchLex(query.query, { limit, collection });
|
|
90
|
+
return results.map((r) => r.filepath);
|
|
91
|
+
},
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
name: "vector",
|
|
95
|
+
run: async (store, query, limit, collection) => {
|
|
96
|
+
const structured = parseStructuredQuery(query.query);
|
|
97
|
+
const vectorQueries = structured?.searches.filter(q => q.type === "vec" || q.type === "hyde");
|
|
98
|
+
if (structured) {
|
|
99
|
+
const files = [];
|
|
100
|
+
for (const vectorQuery of vectorQueries ?? []) {
|
|
101
|
+
const results = await store.searchVector(vectorQuery.query, { limit, collection });
|
|
102
|
+
files.push(...results.map((r) => r.filepath));
|
|
103
|
+
}
|
|
104
|
+
return uniqueFiles(files, limit);
|
|
105
|
+
}
|
|
106
|
+
const results = await store.searchVector(query.query, { limit, collection });
|
|
107
|
+
return results.map((r) => r.filepath);
|
|
108
|
+
},
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
name: "hybrid",
|
|
112
|
+
run: async (store, query, limit, collection) => {
|
|
113
|
+
const structured = parseStructuredQuery(query.query);
|
|
114
|
+
const results = structured
|
|
115
|
+
? await store.search({ queries: structured.searches, intent: structured.intent, limit, collection, rerank: false })
|
|
116
|
+
: await store.search({ query: query.query, limit, collection, rerank: false });
|
|
117
|
+
return results.map((r) => r.file);
|
|
118
|
+
},
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
name: "full",
|
|
122
|
+
run: async (store, query, limit, collection) => {
|
|
123
|
+
const structured = parseStructuredQuery(query.query);
|
|
124
|
+
const results = structured
|
|
125
|
+
? await store.search({ queries: structured.searches, intent: structured.intent, limit, collection, rerank: true })
|
|
126
|
+
: await store.search({ query: query.query, limit, collection, rerank: true });
|
|
127
|
+
return results.map((r) => r.file);
|
|
128
|
+
},
|
|
129
|
+
},
|
|
130
|
+
];
|
|
131
|
+
async function runQuery(store, backend, query, collection) {
|
|
132
|
+
// A fixture entry missing expected_in_top_k used to make limit NaN, and every backend
|
|
133
|
+
// returned zero results with no error — the fixture looked like a retrieval failure.
|
|
134
|
+
const expectedInTopK = query.expected_in_top_k ?? query.expected_files.length;
|
|
135
|
+
const limit = Math.max(Number.isFinite(expectedInTopK) ? expectedInTopK : 0, 10);
|
|
136
|
+
const start = Date.now();
|
|
137
|
+
let resultFiles;
|
|
138
|
+
try {
|
|
139
|
+
resultFiles = await backend.run(store, query, limit, collection);
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
// Backend may not be available (e.g., no embeddings for vector search)
|
|
143
|
+
return {
|
|
144
|
+
precision_at_k: 0,
|
|
145
|
+
recall: 0,
|
|
146
|
+
recall_at_1: 0,
|
|
147
|
+
recall_at_3: 0,
|
|
148
|
+
recall_at_5: 0,
|
|
149
|
+
mrr: 0,
|
|
150
|
+
f1: 0,
|
|
151
|
+
hits_at_k: 0,
|
|
152
|
+
total_expected: query.expected_files.length,
|
|
153
|
+
latency_ms: Date.now() - start,
|
|
154
|
+
top_files: [],
|
|
155
|
+
matched_files: [],
|
|
156
|
+
unmatched_expected_files: query.expected_files,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
const latency_ms = Date.now() - start;
|
|
160
|
+
const scores = scoreResults(resultFiles, query.expected_files, expectedInTopK);
|
|
161
|
+
return {
|
|
162
|
+
...scores,
|
|
163
|
+
total_expected: query.expected_files.length,
|
|
164
|
+
latency_ms,
|
|
165
|
+
top_files: resultFiles.slice(0, 10),
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
function formatTable(results) {
|
|
169
|
+
const lines = [];
|
|
170
|
+
const pad = (s, n) => s.slice(0, n).padEnd(n);
|
|
171
|
+
const num = (n) => n.toFixed(2).padStart(5);
|
|
172
|
+
lines.push(`${pad("Query", 25)} ${pad("Backend", 8)} ${pad("P@k", 6)} ${pad("R@1", 6)} ${pad("R@3", 6)} ${pad("R@5", 6)} ${pad("MRR", 6)} ${pad("F1", 6)} ${pad("ms", 8)}`);
|
|
173
|
+
lines.push("-".repeat(88));
|
|
174
|
+
for (const r of results) {
|
|
175
|
+
for (const [backend, br] of Object.entries(r.backends)) {
|
|
176
|
+
lines.push(`${pad(r.id, 25)} ${pad(backend, 8)} ${num(br.precision_at_k)} ${num(br.recall_at_1)} ${num(br.recall_at_3)} ${num(br.recall_at_5)} ${num(br.mrr)} ${num(br.f1)} ${String(Math.round(br.latency_ms)).padStart(7)}ms`);
|
|
177
|
+
}
|
|
178
|
+
lines.push("");
|
|
179
|
+
}
|
|
180
|
+
return lines.join("\n");
|
|
181
|
+
}
|
|
182
|
+
function computeSummary(results) {
|
|
183
|
+
const summary = {};
|
|
184
|
+
// Collect all backend names
|
|
185
|
+
const backendNames = new Set();
|
|
186
|
+
for (const r of results) {
|
|
187
|
+
for (const name of Object.keys(r.backends)) {
|
|
188
|
+
backendNames.add(name);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
for (const name of Array.from(backendNames)) {
|
|
192
|
+
let totalP = 0, totalR = 0, totalR1 = 0, totalR3 = 0, totalR5 = 0, totalMrr = 0, totalF1 = 0, totalLat = 0, count = 0;
|
|
193
|
+
for (const r of results) {
|
|
194
|
+
const br = r.backends[name];
|
|
195
|
+
if (!br)
|
|
196
|
+
continue;
|
|
197
|
+
totalP += br.precision_at_k;
|
|
198
|
+
totalR += br.recall;
|
|
199
|
+
totalR1 += br.recall_at_1;
|
|
200
|
+
totalR3 += br.recall_at_3;
|
|
201
|
+
totalR5 += br.recall_at_5;
|
|
202
|
+
totalMrr += br.mrr;
|
|
203
|
+
totalF1 += br.f1;
|
|
204
|
+
totalLat += br.latency_ms;
|
|
205
|
+
count++;
|
|
206
|
+
}
|
|
207
|
+
if (count > 0) {
|
|
208
|
+
summary[name] = {
|
|
209
|
+
avg_precision: totalP / count,
|
|
210
|
+
avg_recall: totalR / count,
|
|
211
|
+
avg_recall_at_1: totalR1 / count,
|
|
212
|
+
avg_recall_at_3: totalR3 / count,
|
|
213
|
+
avg_recall_at_5: totalR5 / count,
|
|
214
|
+
avg_mrr: totalMrr / count,
|
|
215
|
+
avg_f1: totalF1 / count,
|
|
216
|
+
avg_latency_ms: totalLat / count,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return summary;
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Fail fast when the fixture's collection is missing or empty so `qmd bench`
|
|
224
|
+
* does not spend minutes printing a wall of 0.00 (#716).
|
|
225
|
+
*/
|
|
226
|
+
export function assertBenchCollectionReady(collections, collection) {
|
|
227
|
+
if (collection) {
|
|
228
|
+
const match = collections.find(c => c.name === collection);
|
|
229
|
+
if (!match) {
|
|
230
|
+
const names = collections.map(c => c.name);
|
|
231
|
+
const hint = names.length > 0
|
|
232
|
+
? `Available: ${names.join(", ")}. Run 'qmd ls' to inspect.`
|
|
233
|
+
: "Run 'qmd ls' to see available collections.";
|
|
234
|
+
throw new Error(`Collection not found: ${collection}\n${hint}`);
|
|
235
|
+
}
|
|
236
|
+
if (match.active_count === 0) {
|
|
237
|
+
throw new Error(`Collection '${collection}' has no indexed documents.\nRun 'qmd update', then 'qmd ls ${collection}' to confirm files are indexed before bench.`);
|
|
238
|
+
}
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
const total = collections.reduce((n, c) => n + c.active_count, 0);
|
|
242
|
+
if (collections.length === 0 || total === 0) {
|
|
243
|
+
throw new Error("No indexed documents found.\nIndex a collection with 'qmd collection add' / 'qmd update' before running bench.");
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
export function benchSummaryAllZero(summary) {
|
|
247
|
+
const entries = Object.values(summary);
|
|
248
|
+
if (entries.length === 0)
|
|
249
|
+
return false;
|
|
250
|
+
return entries.every(s => s.avg_precision === 0 && s.avg_recall === 0 && s.avg_mrr === 0);
|
|
251
|
+
}
|
|
252
|
+
export function allZeroBenchWarning(collection) {
|
|
253
|
+
const lsHint = collection ? `qmd ls ${collection}` : "qmd ls";
|
|
254
|
+
return `All benchmark scores were 0.00 — the collection is likely unindexed or the fixture's expected files are missing. Check with '${lsHint}'.`;
|
|
255
|
+
}
|
|
256
|
+
export async function runBenchmark(fixturePath, options = {}) {
|
|
257
|
+
// Load fixture
|
|
258
|
+
const raw = readFileSync(resolve(fixturePath), "utf-8");
|
|
259
|
+
const fixture = JSON.parse(raw);
|
|
260
|
+
if (!fixture.queries || !Array.isArray(fixture.queries)) {
|
|
261
|
+
throw new Error("Invalid fixture: missing 'queries' array");
|
|
262
|
+
}
|
|
263
|
+
// Open store
|
|
264
|
+
const store = await createStore({
|
|
265
|
+
dbPath: options.dbPath ?? getDefaultDbPath(),
|
|
266
|
+
...(options.configPath ? { configPath: options.configPath } : {}),
|
|
267
|
+
...(options.config ? { config: options.config } : {}),
|
|
268
|
+
});
|
|
269
|
+
// Filter backends if requested
|
|
270
|
+
const activeBackends = options.backends
|
|
271
|
+
? BACKENDS.filter(b => options.backends.includes(b.name))
|
|
272
|
+
: BACKENDS;
|
|
273
|
+
const collection = options.collection ?? fixture.collection;
|
|
274
|
+
const results = [];
|
|
275
|
+
try {
|
|
276
|
+
const collections = await store.listCollections();
|
|
277
|
+
assertBenchCollectionReady(collections, collection);
|
|
278
|
+
// Run queries
|
|
279
|
+
for (const query of fixture.queries) {
|
|
280
|
+
const backends = {};
|
|
281
|
+
for (const backend of activeBackends) {
|
|
282
|
+
if (!options.json) {
|
|
283
|
+
process.stderr.write(` ${query.id} / ${backend.name}...`);
|
|
284
|
+
}
|
|
285
|
+
backends[backend.name] = await runQuery(store, backend, query, collection);
|
|
286
|
+
if (!options.json) {
|
|
287
|
+
process.stderr.write(` ${Math.round(backends[backend.name].latency_ms)}ms\n`);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
results.push({
|
|
291
|
+
id: query.id,
|
|
292
|
+
query: query.query,
|
|
293
|
+
type: query.type,
|
|
294
|
+
backends,
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
finally {
|
|
299
|
+
await store.close();
|
|
300
|
+
}
|
|
301
|
+
const summary = computeSummary(results);
|
|
302
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, "").slice(0, 15);
|
|
303
|
+
const benchResult = {
|
|
304
|
+
timestamp,
|
|
305
|
+
fixture: fixturePath,
|
|
306
|
+
results,
|
|
307
|
+
summary,
|
|
308
|
+
};
|
|
309
|
+
// Output
|
|
310
|
+
if (benchSummaryAllZero(summary)) {
|
|
311
|
+
process.stderr.write(`\n${allZeroBenchWarning(collection)}\n`);
|
|
312
|
+
}
|
|
313
|
+
if (options.json) {
|
|
314
|
+
console.log(JSON.stringify(benchResult, null, 2));
|
|
315
|
+
}
|
|
316
|
+
else {
|
|
317
|
+
console.log("\n" + formatTable(results));
|
|
318
|
+
console.log("Summary:");
|
|
319
|
+
console.log("-".repeat(70));
|
|
320
|
+
const pad = (s, n) => s.slice(0, n).padEnd(n);
|
|
321
|
+
const num = (n) => n.toFixed(3).padStart(6);
|
|
322
|
+
for (const [name, s] of Object.entries(summary)) {
|
|
323
|
+
console.log(` ${pad(name, 8)} P@k=${num(s.avg_precision)} R@1=${num(s.avg_recall_at_1)} R@3=${num(s.avg_recall_at_3)} R@5=${num(s.avg_recall_at_5)} MRR=${num(s.avg_mrr)} F1=${num(s.avg_f1)} Avg=${Math.round(s.avg_latency_ms)}ms`);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
return benchResult;
|
|
327
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scoring functions for the QMD benchmark harness.
|
|
3
|
+
*
|
|
4
|
+
* Computes precision@k, recall, MRR, and F1 for search results
|
|
5
|
+
* against ground-truth expected files.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Normalize a file path for comparison.
|
|
9
|
+
* Strips qmd:// prefix, lowercases, removes leading/trailing slashes.
|
|
10
|
+
*/
|
|
11
|
+
export declare function normalizePath(p: string): string;
|
|
12
|
+
/**
|
|
13
|
+
* Check if two paths refer to the same file.
|
|
14
|
+
* Handles different path formats by comparing normalized suffixes.
|
|
15
|
+
*/
|
|
16
|
+
export declare function pathsMatch(result: string, expected: string): boolean;
|
|
17
|
+
type ScoreMetrics = {
|
|
18
|
+
precision_at_k: number;
|
|
19
|
+
recall: number;
|
|
20
|
+
recall_at_1: number;
|
|
21
|
+
recall_at_3: number;
|
|
22
|
+
recall_at_5: number;
|
|
23
|
+
mrr: number;
|
|
24
|
+
f1: number;
|
|
25
|
+
hits_at_k: number;
|
|
26
|
+
matched_files: string[];
|
|
27
|
+
unmatched_expected_files: string[];
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* Score a set of search results against expected files.
|
|
31
|
+
*/
|
|
32
|
+
export declare function scoreResults(resultFiles: string[], expectedFiles: string[], topK: number): ScoreMetrics;
|
|
33
|
+
export {};
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scoring functions for the QMD benchmark harness.
|
|
3
|
+
*
|
|
4
|
+
* Computes precision@k, recall, MRR, and F1 for search results
|
|
5
|
+
* against ground-truth expected files.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Normalize a file path for comparison.
|
|
9
|
+
* Strips qmd:// prefix, lowercases, removes leading/trailing slashes.
|
|
10
|
+
*/
|
|
11
|
+
export function normalizePath(p) {
|
|
12
|
+
if (p.startsWith("qmd://")) {
|
|
13
|
+
// qmd://collection/docs/readme.md → docs/readme.md
|
|
14
|
+
const withoutScheme = p.slice("qmd://".length);
|
|
15
|
+
const slashIdx = withoutScheme.indexOf("/");
|
|
16
|
+
p = slashIdx >= 0 ? withoutScheme.slice(slashIdx + 1) : withoutScheme;
|
|
17
|
+
}
|
|
18
|
+
return p.toLowerCase().replace(/^\/+|\/+$/g, "");
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Check if two paths refer to the same file.
|
|
22
|
+
* Handles different path formats by comparing normalized suffixes.
|
|
23
|
+
*/
|
|
24
|
+
export function pathsMatch(result, expected) {
|
|
25
|
+
const nr = normalizePath(result);
|
|
26
|
+
const ne = normalizePath(expected);
|
|
27
|
+
if (nr === ne)
|
|
28
|
+
return true;
|
|
29
|
+
if (nr.endsWith(ne) || ne.endsWith(nr))
|
|
30
|
+
return true;
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
function hitsWithin(resultFiles, expectedFiles, k) {
|
|
34
|
+
const topKResults = resultFiles.slice(0, k);
|
|
35
|
+
let hits = 0;
|
|
36
|
+
for (const expected of expectedFiles) {
|
|
37
|
+
if (topKResults.some(r => pathsMatch(r, expected))) {
|
|
38
|
+
hits++;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return hits;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Score a set of search results against expected files.
|
|
45
|
+
*/
|
|
46
|
+
export function scoreResults(resultFiles, expectedFiles, topK) {
|
|
47
|
+
// Count hits in top-k
|
|
48
|
+
const hitsAtK = hitsWithin(resultFiles, expectedFiles, topK);
|
|
49
|
+
const matchedFiles = [];
|
|
50
|
+
const unmatchedExpectedFiles = [];
|
|
51
|
+
for (const expected of expectedFiles) {
|
|
52
|
+
if (resultFiles.some(r => pathsMatch(r, expected))) {
|
|
53
|
+
matchedFiles.push(expected);
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
unmatchedExpectedFiles.push(expected);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
// MRR: reciprocal rank of first relevant result
|
|
60
|
+
let mrr = 0;
|
|
61
|
+
for (let i = 0; i < resultFiles.length; i++) {
|
|
62
|
+
if (expectedFiles.some(e => pathsMatch(resultFiles[i], e))) {
|
|
63
|
+
mrr = 1 / (i + 1);
|
|
64
|
+
break;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
const denominator = Math.min(topK, expectedFiles.length);
|
|
68
|
+
const precision_at_k = denominator > 0 ? hitsAtK / denominator : 0;
|
|
69
|
+
const recall = expectedFiles.length > 0 ? matchedFiles.length / expectedFiles.length : 0;
|
|
70
|
+
const recall_at_1 = expectedFiles.length > 0 ? hitsWithin(resultFiles, expectedFiles, 1) / expectedFiles.length : 0;
|
|
71
|
+
const recall_at_3 = expectedFiles.length > 0 ? hitsWithin(resultFiles, expectedFiles, 3) / expectedFiles.length : 0;
|
|
72
|
+
const recall_at_5 = expectedFiles.length > 0 ? hitsWithin(resultFiles, expectedFiles, 5) / expectedFiles.length : 0;
|
|
73
|
+
const f1 = precision_at_k + recall > 0
|
|
74
|
+
? 2 * (precision_at_k * recall) / (precision_at_k + recall)
|
|
75
|
+
: 0;
|
|
76
|
+
return {
|
|
77
|
+
precision_at_k,
|
|
78
|
+
recall,
|
|
79
|
+
recall_at_1,
|
|
80
|
+
recall_at_3,
|
|
81
|
+
recall_at_5,
|
|
82
|
+
mrr,
|
|
83
|
+
f1,
|
|
84
|
+
hits_at_k: hitsAtK,
|
|
85
|
+
matched_files: matchedFiles,
|
|
86
|
+
unmatched_expected_files: unmatchedExpectedFiles,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for the QMD benchmark harness.
|
|
3
|
+
*
|
|
4
|
+
* A benchmark fixture defines queries with expected results.
|
|
5
|
+
* The harness runs each query through multiple search backends
|
|
6
|
+
* and measures precision, recall, MRR, and latency.
|
|
7
|
+
*/
|
|
8
|
+
export interface BenchmarkQuery {
|
|
9
|
+
/** Unique identifier for the query */
|
|
10
|
+
id: string;
|
|
11
|
+
/** The search query text */
|
|
12
|
+
query: string;
|
|
13
|
+
/** Query difficulty/type for grouping results */
|
|
14
|
+
type: "exact" | "semantic" | "topical" | "cross-domain" | "alias";
|
|
15
|
+
/** Human-readable description of what this tests */
|
|
16
|
+
description: string;
|
|
17
|
+
/** File paths (relative to collection) that should appear in results */
|
|
18
|
+
expected_files: string[];
|
|
19
|
+
/** How many of expected_files should appear in top-k results */
|
|
20
|
+
expected_in_top_k?: number;
|
|
21
|
+
}
|
|
22
|
+
export interface BenchmarkFixture {
|
|
23
|
+
/** Description of the benchmark */
|
|
24
|
+
description: string;
|
|
25
|
+
/** Fixture format version */
|
|
26
|
+
version: number;
|
|
27
|
+
/** Optional collection to search within */
|
|
28
|
+
collection?: string;
|
|
29
|
+
/** The test queries */
|
|
30
|
+
queries: BenchmarkQuery[];
|
|
31
|
+
}
|
|
32
|
+
export interface BackendResult {
|
|
33
|
+
/** Fraction of top-k results that are relevant */
|
|
34
|
+
precision_at_k: number;
|
|
35
|
+
/** Fraction of expected files found anywhere in results */
|
|
36
|
+
recall: number;
|
|
37
|
+
/** Fraction of expected files found in the first result */
|
|
38
|
+
recall_at_1: number;
|
|
39
|
+
/** Fraction of expected files found in the top 3 results */
|
|
40
|
+
recall_at_3: number;
|
|
41
|
+
/** Fraction of expected files found in the top 5 results */
|
|
42
|
+
recall_at_5: number;
|
|
43
|
+
/** Reciprocal rank of first relevant result (1/rank, 0 if not found) */
|
|
44
|
+
mrr: number;
|
|
45
|
+
/** Harmonic mean of precision_at_k and recall */
|
|
46
|
+
f1: number;
|
|
47
|
+
/** Number of expected files found in top-k */
|
|
48
|
+
hits_at_k: number;
|
|
49
|
+
/** Total expected files */
|
|
50
|
+
total_expected: number;
|
|
51
|
+
/** Wall-clock latency in milliseconds */
|
|
52
|
+
latency_ms: number;
|
|
53
|
+
/** Top result file paths (for inspection) */
|
|
54
|
+
top_files: string[];
|
|
55
|
+
/** Expected files that were found anywhere in the returned result set */
|
|
56
|
+
matched_files: string[];
|
|
57
|
+
/** Expected files missing from the returned result set */
|
|
58
|
+
unmatched_expected_files: string[];
|
|
59
|
+
}
|
|
60
|
+
export interface QueryResult {
|
|
61
|
+
id: string;
|
|
62
|
+
query: string;
|
|
63
|
+
type: string;
|
|
64
|
+
backends: Record<string, BackendResult>;
|
|
65
|
+
}
|
|
66
|
+
export interface BenchmarkResult {
|
|
67
|
+
timestamp: string;
|
|
68
|
+
fixture: string;
|
|
69
|
+
results: QueryResult[];
|
|
70
|
+
summary: Record<string, {
|
|
71
|
+
avg_precision: number;
|
|
72
|
+
avg_recall: number;
|
|
73
|
+
avg_recall_at_1: number;
|
|
74
|
+
avg_recall_at_3: number;
|
|
75
|
+
avg_recall_at_5: number;
|
|
76
|
+
avg_mrr: number;
|
|
77
|
+
avg_f1: number;
|
|
78
|
+
avg_latency_ms: number;
|
|
79
|
+
}>;
|
|
80
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Process-level exclusive lock for `qmd embed`.
|
|
3
|
+
*
|
|
4
|
+
* Concurrent embed runs against the same index can race on vectors_vec
|
|
5
|
+
* (UNIQUE constraint on hash_seq). This lockfile keeps a second process from
|
|
6
|
+
* starting while another embed holds the lock. Stale files left by crashed
|
|
7
|
+
* processes are recovered via PID identity checks (same spirit as mcp-pid.ts).
|
|
8
|
+
*/
|
|
9
|
+
export type EmbedLockHandle = {
|
|
10
|
+
lockPath: string;
|
|
11
|
+
release: () => void;
|
|
12
|
+
};
|
|
13
|
+
/** Lockfile path sibling to the index database. */
|
|
14
|
+
export declare function embedLockPathForDb(dbPath: string): string;
|
|
15
|
+
/** True if `pid` still owns a live embed/qmd process (or is this process). */
|
|
16
|
+
export declare function isLiveEmbedLockHolder(pid: number): boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Try to acquire an exclusive embed lock at `lockPath`.
|
|
19
|
+
* Returns a handle with `release()` on success, or `null` if another live
|
|
20
|
+
* qmd process already holds the lock.
|
|
21
|
+
*/
|
|
22
|
+
export declare function tryAcquireEmbedLock(lockPath: string): EmbedLockHandle | null;
|
|
23
|
+
/** User-facing message when a second embed is skipped. */
|
|
24
|
+
export declare const EMBED_LOCK_BUSY_MESSAGE = "Another embed process is already running. Skipping.";
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Process-level exclusive lock for `qmd embed`.
|
|
3
|
+
*
|
|
4
|
+
* Concurrent embed runs against the same index can race on vectors_vec
|
|
5
|
+
* (UNIQUE constraint on hash_seq). This lockfile keeps a second process from
|
|
6
|
+
* starting while another embed holds the lock. Stale files left by crashed
|
|
7
|
+
* processes are recovered via PID identity checks (same spirit as mcp-pid.ts).
|
|
8
|
+
*/
|
|
9
|
+
import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
|
|
10
|
+
import { dirname, join } from "node:path";
|
|
11
|
+
import { isQmdMcpPid } from "./mcp-pid.js";
|
|
12
|
+
/** Lockfile path sibling to the index database. */
|
|
13
|
+
export function embedLockPathForDb(dbPath) {
|
|
14
|
+
return join(dirname(dbPath), ".qmd-embed.lock");
|
|
15
|
+
}
|
|
16
|
+
function readLockPid(lockPath) {
|
|
17
|
+
try {
|
|
18
|
+
const raw = readFileSync(lockPath, "utf-8").trim();
|
|
19
|
+
const pid = parseInt(raw, 10);
|
|
20
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
21
|
+
return null;
|
|
22
|
+
return pid;
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/** True if `pid` still owns a live embed/qmd process (or is this process). */
|
|
29
|
+
export function isLiveEmbedLockHolder(pid) {
|
|
30
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
31
|
+
return false;
|
|
32
|
+
// Same-process re-check: we obviously still hold our own lock.
|
|
33
|
+
if (pid === process.pid)
|
|
34
|
+
return true;
|
|
35
|
+
return isQmdMcpPid(pid);
|
|
36
|
+
}
|
|
37
|
+
function createOwnedLock(lockPath) {
|
|
38
|
+
writeFileSync(lockPath, `${process.pid}\n`, { flag: "wx" });
|
|
39
|
+
let released = false;
|
|
40
|
+
const release = () => {
|
|
41
|
+
if (released)
|
|
42
|
+
return;
|
|
43
|
+
released = true;
|
|
44
|
+
try {
|
|
45
|
+
if (!existsSync(lockPath))
|
|
46
|
+
return;
|
|
47
|
+
const written = readLockPid(lockPath);
|
|
48
|
+
if (written === process.pid)
|
|
49
|
+
unlinkSync(lockPath);
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
// best-effort cleanup
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
return { lockPath, release };
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Try to acquire an exclusive embed lock at `lockPath`.
|
|
59
|
+
* Returns a handle with `release()` on success, or `null` if another live
|
|
60
|
+
* qmd process already holds the lock.
|
|
61
|
+
*/
|
|
62
|
+
export function tryAcquireEmbedLock(lockPath) {
|
|
63
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
64
|
+
try {
|
|
65
|
+
return createOwnedLock(lockPath);
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
const code = typeof err === "object" && err !== null && "code" in err
|
|
69
|
+
? err.code
|
|
70
|
+
: undefined;
|
|
71
|
+
if (code !== "EEXIST")
|
|
72
|
+
throw err;
|
|
73
|
+
const holderPid = readLockPid(lockPath);
|
|
74
|
+
if (holderPid !== null && isLiveEmbedLockHolder(holderPid)) {
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
// Stale / unreadable / recycled PID — remove and retry once.
|
|
78
|
+
try {
|
|
79
|
+
unlinkSync(lockPath);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
// Another process may have claimed it; loop and try wx again.
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
// Final attempt lost the race to a live holder (or repeated EEXIST).
|
|
87
|
+
const holderPid = readLockPid(lockPath);
|
|
88
|
+
if (holderPid !== null && isLiveEmbedLockHolder(holderPid)) {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
/** User-facing message when a second embed is skipped. */
|
|
94
|
+
export const EMBED_LOCK_BUSY_MESSAGE = "Another embed process is already running. Skipping.";
|