@wei840222/qmd 2026.8.23

package/dist/bench/bench.js

@@ -0,0 +1,338 @@
+/**
+ * QMD Benchmark Harness
+ *
+ * Runs queries from a fixture file against multiple search backends
+ * and measures precision@k, recall, MRR, F1, and latency.
+ *
+ * Usage:
+ *   qmd bench <fixture.json> [--json] [--collection <name>]
+ *
+ * Backends tested:
+ *   - bm25: BM25 keyword search (searchLex)
+ *   - vector: Vector similarity search (searchVector)
+ *   - hybrid: BM25 + vector RRF fusion without reranking
+ *   - full: Full hybrid pipeline with LLM reranking
+ */
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { createStore, getDefaultDbPath, } from "../index.js";
+import { resolveFixtureQueries } from "./fixture.js";
+import { scoreResults } from "./score.js";
+function parseStructuredQuery(query) {
+    const lines = query.split("\n").map((line, idx) => ({
+        trimmed: line.trim(),
+        number: idx + 1,
+    })).filter(line => line.trimmed.length > 0);
+    if (lines.length === 0)
+        return undefined;
+    const prefixRe = /^(lex|vec|hyde):\s*/i;
+    const intentRe = /^intent:\s*/i;
+    const searches = [];
+    let intent;
+    for (const line of lines) {
+        if (intentRe.test(line.trimmed)) {
+            if (intent !== undefined) {
+                throw new Error(`Line ${line.number}: only one intent: line is allowed per benchmark query.`);
+            }
+            intent = line.trimmed.replace(intentRe, "").trim();
+            if (!intent) {
+                throw new Error(`Line ${line.number}: intent: must include text.`);
+            }
+            continue;
+        }
+        const match = line.trimmed.match(prefixRe);
+        if (match) {
+            const type = match[1].toLowerCase();
+            const text = line.trimmed.slice(match[0].length).trim();
+            if (!text) {
+                throw new Error(`Line ${line.number} (${type}:) must include text.`);
+            }
+            searches.push({ type, query: text, line: line.number });
+            continue;
+        }
+        if (lines.length === 1) {
+            return undefined;
+        }
+        throw new Error(`Line ${line.number} is missing a lex:/vec:/hyde:/intent: prefix.`);
+    }
+    if (intent && searches.length === 0) {
+        throw new Error("intent: cannot appear alone. Add at least one lex:, vec:, or hyde: line.");
+    }
+    return searches.length > 0 ? { searches, intent } : undefined;
+}
+function uniqueFiles(files, limit) {
+    const seen = new Set();
+    const out = [];
+    for (const file of files) {
+        if (seen.has(file))
+            continue;
+        seen.add(file);
+        out.push(file);
+        if (out.length >= limit)
+            break;
+    }
+    return out;
+}
+const BACKENDS = [
+    {
+        name: "bm25",
+        run: async (store, query, limit, collection) => {
+            const structured = parseStructuredQuery(query.query);
+            const lexQueries = structured?.searches.filter(q => q.type === "lex");
+            if (structured) {
+                const files = [];
+                for (const lex of lexQueries ?? []) {
+                    const results = await store.searchLex(lex.query, { limit, collection });
+                    files.push(...results.map((r) => r.filepath));
+                }
+                return uniqueFiles(files, limit);
+            }
+            const results = await store.searchLex(query.query, { limit, collection });
+            return results.map((r) => r.filepath);
+        },
+    },
+    {
+        name: "vector",
+        run: async (store, query, limit, collection) => {
+            const structured = parseStructuredQuery(query.query);
+            const vectorQueries = structured?.searches.filter(q => q.type === "vec" || q.type === "hyde");
+            if (structured) {
+                const files = [];
+                for (const vectorQuery of vectorQueries ?? []) {
+                    const results = await store.searchVector(vectorQuery.query, { limit, collection });
+                    files.push(...results.map((r) => r.filepath));
+                }
+                return uniqueFiles(files, limit);
+            }
+            const results = await store.searchVector(query.query, { limit, collection });
+            return results.map((r) => r.filepath);
+        },
+    },
+    {
+        name: "hybrid",
+        run: async (store, query, limit, collection) => {
+            const structured = parseStructuredQuery(query.query);
+            const results = structured
+                ? await store.search({ queries: structured.searches, rerankContext: structured.intent, limit, collection, rerank: false })
+                : await store.search({ query: query.query, limit, collection, rerank: false });
+            return results.map((r) => r.file);
+        },
+    },
+    {
+        name: "full",
+        run: async (store, query, limit, collection) => {
+            const structured = parseStructuredQuery(query.query);
+            const results = structured
+                ? await store.search({ queries: structured.searches, rerankContext: structured.intent, limit, collection, rerank: true })
+                : await store.search({ query: query.query, limit, collection, rerank: true });
+            return results.map((r) => r.file);
+        },
+    },
+];
+async function runQuery(store, backend, query, collection) {
+    const limit = Math.max(query.expected_in_top_k, 10);
+    const start = Date.now();
+    let resultFiles;
+    try {
+        resultFiles = await backend.run(store, query, limit, collection);
+    }
+    catch {
+        // Backend may not be available (e.g., no embeddings for vector search)
+        return {
+            precision_at_k: 0,
+            recall: 0,
+            recall_at_1: 0,
+            recall_at_3: 0,
+            recall_at_5: 0,
+            recall_at_10: 0,
+            mrr: 0,
+            mrr_at_10: 0,
+            f1: 0,
+            hits_at_k: 0,
+            false_positive_count: 0,
+            false_positive_files: [],
+            total_expected: query.expected_files.length,
+            latency_ms: Date.now() - start,
+            top_files: [],
+            matched_files: [],
+            unmatched_expected_files: query.expected_files,
+        };
+    }
+    const latency_ms = Date.now() - start;
+    const scores = scoreResults(resultFiles, query.expected_files, query.expected_in_top_k, query.must_not_match_files);
+    return {
+        ...scores,
+        total_expected: query.expected_files.length,
+        latency_ms,
+        top_files: resultFiles.slice(0, 10),
+    };
+}
+function formatTable(results) {
+    const lines = [];
+    const pad = (s, n) => s.slice(0, n).padEnd(n);
+    const num = (n) => n.toFixed(2).padStart(5);
+    lines.push(`${pad("Query", 25)} ${pad("Backend", 8)} ${pad("P@k", 6)} ${pad("R@1", 6)} ${pad("R@3", 6)} ${pad("R@5", 6)} ${pad("R@10", 6)} ${pad("MRR@10", 7)} ${pad("FP", 4)} ${pad("F1", 6)} ${pad("ms", 8)}`);
+    lines.push("-".repeat(108));
+    for (const r of results) {
+        for (const [backend, br] of Object.entries(r.backends)) {
+            lines.push(`${pad(r.id, 25)} ${pad(backend, 8)} ${num(br.precision_at_k)} ${num(br.recall_at_1)} ${num(br.recall_at_3)} ${num(br.recall_at_5)} ${num(br.recall_at_10)} ${num(br.mrr_at_10)} ${String(br.false_positive_count).padStart(3)} ${num(br.f1)} ${String(Math.round(br.latency_ms)).padStart(7)}ms`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+function computeSummary(results) {
+    const summary = {};
+    // Collect all backend names
+    const backendNames = new Set();
+    for (const r of results) {
+        for (const name of Object.keys(r.backends)) {
+            backendNames.add(name);
+        }
+    }
+    for (const name of Array.from(backendNames)) {
+        let totalP = 0, totalR = 0, totalR1 = 0, totalR3 = 0, totalR5 = 0, totalR10 = 0;
+        let totalMrr = 0, totalMrr10 = 0, totalFp = 0, totalF1 = 0, totalLat = 0, count = 0;
+        for (const r of results) {
+            const br = r.backends[name];
+            if (!br)
+                continue;
+            totalP += br.precision_at_k;
+            totalR += br.recall;
+            totalR1 += br.recall_at_1;
+            totalR3 += br.recall_at_3;
+            totalR5 += br.recall_at_5;
+            totalR10 += br.recall_at_10;
+            totalMrr += br.mrr;
+            totalMrr10 += br.mrr_at_10;
+            totalFp += br.false_positive_count;
+            totalF1 += br.f1;
+            totalLat += br.latency_ms;
+            count++;
+        }
+        if (count > 0) {
+            summary[name] = {
+                avg_precision: totalP / count,
+                avg_recall: totalR / count,
+                avg_recall_at_1: totalR1 / count,
+                avg_recall_at_3: totalR3 / count,
+                avg_recall_at_5: totalR5 / count,
+                avg_recall_at_10: totalR10 / count,
+                avg_mrr: totalMrr / count,
+                avg_mrr_at_10: totalMrr10 / count,
+                false_positive_count: totalFp,
+                avg_f1: totalF1 / count,
+                avg_latency_ms: totalLat / count,
+            };
+        }
+    }
+    return summary;
+}
+/**
+ * Fail fast when the fixture's collection is missing or empty so `qmd bench`
+ * does not spend minutes printing a wall of 0.00 (#716).
+ */
+export function assertBenchCollectionReady(collections, collection) {
+    if (collection) {
+        const match = collections.find(c => c.name === collection);
+        if (!match) {
+            const names = collections.map(c => c.name);
+            const hint = names.length > 0
+                ? `Available: ${names.join(", ")}. Run 'qmd ls' to inspect.`
+                : "Run 'qmd ls' to see available collections.";
+            throw new Error(`Collection not found: ${collection}\n${hint}`);
+        }
+        if (match.active_count === 0) {
+            throw new Error(`Collection '${collection}' has no indexed documents.\nRun 'qmd update', then 'qmd ls ${collection}' to confirm files are indexed before bench.`);
+        }
+        return;
+    }
+    const total = collections.reduce((n, c) => n + c.active_count, 0);
+    if (collections.length === 0 || total === 0) {
+        throw new Error("No indexed documents found.\nIndex a collection with 'qmd collection add' / 'qmd update' before running bench.");
+    }
+}
+export function benchSummaryAllZero(summary) {
+    const entries = Object.values(summary);
+    if (entries.length === 0)
+        return false;
+    return entries.every(s => s.avg_precision === 0 && s.avg_recall === 0 && s.avg_mrr === 0);
+}
+export function allZeroBenchWarning(collection) {
+    const lsHint = collection ? `qmd ls ${collection}` : "qmd ls";
+    return `All benchmark scores were 0.00 — the collection is likely unindexed or the fixture's expected files are missing. Check with '${lsHint}'.`;
+}
+export async function runBenchmark(fixturePath, options = {}) {
+    // Load fixture
+    const raw = readFileSync(resolve(fixturePath), "utf-8");
+    const fixture = JSON.parse(raw);
+    if (!fixture.queries || !Array.isArray(fixture.queries)) {
+        throw new Error("Invalid fixture: missing 'queries' array");
+    }
+    const queries = resolveFixtureQueries(fixture);
+    // Open store
+    const store = await createStore({
+        dbPath: options.dbPath ?? getDefaultDbPath(),
+        ...(options.configPath ? { configPath: options.configPath } : {}),
+        ...(options.config ? { config: options.config } : {}),
+    });
+    // Filter backends if requested
+    const activeBackends = options.backends
+        ? BACKENDS.filter(b => options.backends.includes(b.name))
+        : BACKENDS;
+    const collection = options.collection ?? fixture.collection;
+    const results = [];
+    try {
+        const collections = await store.listCollections();
+        assertBenchCollectionReady(collections, collection);
+        // Run queries
+        for (const query of queries) {
+            const backends = {};
+            for (const backend of activeBackends) {
+                if (!options.json) {
+                    process.stderr.write(`  ${query.id} / ${backend.name}...`);
+                }
+                backends[backend.name] = await runQuery(store, backend, query, collection);
+                if (!options.json) {
+                    process.stderr.write(` ${Math.round(backends[backend.name].latency_ms)}ms\n`);
+                }
+            }
+            results.push({
+                id: query.id,
+                query: query.query,
+                type: query.type,
+                ...(query.scenario_tags ? { scenario_tags: query.scenario_tags } : {}),
+                backends,
+            });
+        }
+    }
+    finally {
+        await store.close();
+    }
+    const summary = computeSummary(results);
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "").slice(0, 15);
+    const benchResult = {
+        timestamp,
+        fixture: fixturePath,
+        results,
+        summary,
+    };
+    // Output
+    if (benchSummaryAllZero(summary)) {
+        process.stderr.write(`\n${allZeroBenchWarning(collection)}\n`);
+    }
+    if (options.json) {
+        console.log(JSON.stringify(benchResult, null, 2));
+    }
+    else {
+        console.log("\n" + formatTable(results));
+        console.log("Summary:");
+        console.log("-".repeat(70));
+        const pad = (s, n) => s.slice(0, n).padEnd(n);
+        const num = (n) => n.toFixed(3).padStart(6);
+        for (const [name, s] of Object.entries(summary)) {
+            console.log(`  ${pad(name, 8)} P@k=${num(s.avg_precision)} R@1=${num(s.avg_recall_at_1)} R@3=${num(s.avg_recall_at_3)} R@5=${num(s.avg_recall_at_5)} R@10=${num(s.avg_recall_at_10)} MRR@10=${num(s.avg_mrr_at_10)} FP=${s.false_positive_count} F1=${num(s.avg_f1)} Avg=${Math.round(s.avg_latency_ms)}ms`);
+        }
+    }
+    return benchResult;
+}

package/dist/bench/cjk-baseline.d.ts

@@ -0,0 +1,36 @@
+import type { BenchmarkFixture } from "./types.js";
+export declare const CJK_BASELINE_SCORING_SCHEMA = "qmd-char-bm25-v1";
+export declare const CJK_LEXICAL_SCORING_SCHEMA = "qmd-cjk-lexical-rrf-v1";
+export interface LexicalBaselineStore {
+    searchCharLex(query: string, options: {
+        limit: number;
+        collection?: string;
+    }): Promise<Array<{
+        filepath: string;
+    }>>;
+}
+export interface LexicalBaselineOptions {
+    warmupRuns?: number;
+    measuredRuns?: number;
+    scoringSchema?: string;
+}
+export interface LexicalBaselineObservation {
+    corpus_sha256: string;
+    scoring_schema: string;
+    runs: {
+        warmup: number;
+        measured: number;
+    };
+    metrics: {
+        recall_at_10: number;
+        mrr_at_10: number;
+        shared_character_false_positive_count: number;
+    };
+    latency_ms: {
+        p50: number;
+        p95: number;
+    };
+}
+export declare function hashBenchmarkCorpus(fixture: BenchmarkFixture, documentsDirectory: string): string;
+export declare function hashBaselineImplementation(projectRoot: string): string;
+export declare function observeLexicalBaseline(store: LexicalBaselineStore, fixture: BenchmarkFixture, documentsDirectory: string, options?: LexicalBaselineOptions): Promise<LexicalBaselineObservation>;

package/dist/bench/cjk-baseline.js

@@ -0,0 +1,111 @@
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { resolveFixtureQueries } from "./fixture.js";
+import { percentile, scoreResults } from "./score.js";
+export const CJK_BASELINE_SCORING_SCHEMA = "qmd-char-bm25-v1";
+export const CJK_LEXICAL_SCORING_SCHEMA = "qmd-cjk-lexical-rrf-v1";
+const CJK_BASELINE_IMPLEMENTATION_FILES = [
+    "src/bench/cjk-baseline.ts",
+    "src/bench/fixture.ts",
+    "src/bench/score.ts",
+    "src/db.ts",
+    "src/store.ts",
+];
+function positiveInteger(value, name) {
+    if (!Number.isInteger(value) || value <= 0) {
+        throw new RangeError(`${name} must be a positive integer`);
+    }
+    return value;
+}
+function roundMetric(value) {
+    return Number(value.toFixed(6));
+}
+export function hashBenchmarkCorpus(fixture, documentsDirectory) {
+    if (!fixture.documents?.length) {
+        throw new Error("Benchmark fixture must define documents before its corpus can be hashed");
+    }
+    const hash = createHash("sha256");
+    hash.update(JSON.stringify(fixture));
+    for (const document of [...fixture.documents].sort((a, b) => a.id.localeCompare(b.id))) {
+        hash.update("\0");
+        hash.update(document.id);
+        hash.update("\0");
+        hash.update(readFileSync(join(documentsDirectory, document.file)));
+    }
+    return hash.digest("hex");
+}
+export function hashBaselineImplementation(projectRoot) {
+    const hash = createHash("sha256");
+    for (const file of CJK_BASELINE_IMPLEMENTATION_FILES) {
+        hash.update(file);
+        hash.update("\0");
+        hash.update(readFileSync(join(projectRoot, file)));
+        hash.update("\0");
+    }
+    return hash.digest("hex");
+}
+export async function observeLexicalBaseline(store, fixture, documentsDirectory, options = {}) {
+    const warmupRuns = positiveInteger(options.warmupRuns ?? 5, "warmupRuns");
+    const measuredRuns = positiveInteger(options.measuredRuns ?? 50, "measuredRuns");
+    const queries = resolveFixtureQueries(fixture);
+    if (queries.length === 0)
+        throw new Error("Benchmark fixture must contain at least one query");
+    const runQuery = async (query) => {
+        const startedAt = performance.now();
+        const results = await store.searchCharLex(query.query, {
+            limit: Math.max(query.expected_in_top_k, 10),
+            ...(fixture.collection ? { collection: fixture.collection } : {}),
+        });
+        const latency = performance.now() - startedAt;
+        const scores = scoreResults(results.map(result => result.filepath), query.expected_files, 10, query.must_not_match_files);
+        return {
+            latency,
+            ranking: results.slice(0, 10).map(result => result.filepath),
+            scores,
+        };
+    };
+    for (let run = 0; run < warmupRuns; run++) {
+        for (const query of queries)
+            await runQuery(query);
+    }
+    const latencies = [];
+    let totalRecallAt10 = 0;
+    let totalMrrAt10 = 0;
+    let totalSharedCharacterFalsePositives = 0;
+    const firstRankings = new Map();
+    for (let run = 0; run < measuredRuns; run++) {
+        for (const query of queries) {
+            const { latency, ranking, scores } = await runQuery(query);
+            const firstRanking = firstRankings.get(query.id);
+            if (firstRanking === undefined) {
+                firstRankings.set(query.id, ranking);
+            }
+            else if (firstRanking.length !== ranking.length
+                || firstRanking.some((file, index) => file !== ranking[index])) {
+                throw new Error(`Lexical ranking changed between measured runs for query '${query.id}'`);
+            }
+            latencies.push(latency);
+            totalRecallAt10 += scores.recall_at_10;
+            totalMrrAt10 += scores.mrr_at_10;
+            if (query.scenario_tags?.includes("shared-character")) {
+                totalSharedCharacterFalsePositives += scores.false_positive_count;
+            }
+        }
+    }
+    const measuredQueries = measuredRuns * queries.length;
+    return {
+        corpus_sha256: hashBenchmarkCorpus(fixture, documentsDirectory),
+        scoring_schema: options.scoringSchema ?? CJK_BASELINE_SCORING_SCHEMA,
+        runs: { warmup: warmupRuns, measured: measuredRuns },
+        metrics: {
+            recall_at_10: roundMetric(totalRecallAt10 / measuredQueries),
+            mrr_at_10: roundMetric(totalMrrAt10 / measuredQueries),
+            shared_character_false_positive_count: roundMetric(totalSharedCharacterFalsePositives / measuredRuns),
+        },
+        latency_ms: {
+            p50: percentile(latencies, 0.5),
+            p95: percentile(latencies, 0.95),
+        },
+    };
+}

package/dist/bench/fixture.d.ts

@@ -0,0 +1,2 @@
+import type { BenchmarkFixture, ResolvedBenchmarkQuery } from "./types.js";
+export declare function resolveFixtureQueries(fixture: BenchmarkFixture): ResolvedBenchmarkQuery[];

package/dist/bench/fixture.js

@@ -0,0 +1,84 @@
+function assertUniqueValues(values, label, queryId) {
+    const seen = new Set();
+    for (const value of values) {
+        const scope = queryId ? ` query '${queryId}'` : "";
+        if (value.trim().length === 0) {
+            throw new Error(`Invalid fixture${scope}: empty ${label}`);
+        }
+        if (seen.has(value)) {
+            throw new Error(`Invalid fixture${scope}: duplicate ${label} '${value}'`);
+        }
+        seen.add(value);
+    }
+}
+function buildDocumentMap(fixture) {
+    const documents = new Map();
+    const files = new Set();
+    for (const document of fixture.documents ?? []) {
+        if (document.id.trim().length === 0) {
+            throw new Error("Invalid fixture: empty document id");
+        }
+        if (document.file.trim().length === 0) {
+            throw new Error(`Invalid fixture: empty file for document id '${document.id}'`);
+        }
+        if (documents.has(document.id)) {
+            throw new Error(`Invalid fixture: duplicate document id '${document.id}'`);
+        }
+        if (files.has(document.file)) {
+            throw new Error(`Invalid fixture: duplicate document file '${document.file}'`);
+        }
+        documents.set(document.id, document.file);
+        files.add(document.file);
+    }
+    return documents;
+}
+function resolveDocumentIds(documentIds, documents, queryId) {
+    return documentIds.map((documentId) => {
+        const file = documents.get(documentId);
+        if (!file) {
+            throw new Error(`Invalid fixture query '${queryId}': unknown document id '${documentId}'`);
+        }
+        return file;
+    });
+}
+function resolveQuery(query, documents) {
+    if (query.expected_files && query.relevant_doc_ids) {
+        throw new Error(`Invalid fixture query '${query.id}': use expected_files or relevant_doc_ids, not both`);
+    }
+    const relevantIds = query.relevant_doc_ids ?? [];
+    const mustNotMatchIds = query.must_not_match_doc_ids ?? [];
+    assertUniqueValues(relevantIds, "relevant document id", query.id);
+    assertUniqueValues(mustNotMatchIds, "must-not-match document id", query.id);
+    if (query.expected_files)
+        assertUniqueValues(query.expected_files, "expected file", query.id);
+    for (const documentId of relevantIds) {
+        if (mustNotMatchIds.includes(documentId)) {
+            throw new Error(`Invalid fixture query '${query.id}': document id '${documentId}' cannot be both relevant and must-not-match`);
+        }
+    }
+    const expectedFiles = query.expected_files
+        ? [...query.expected_files]
+        : resolveDocumentIds(relevantIds, documents, query.id);
+    if (expectedFiles.length === 0) {
+        throw new Error(`Invalid fixture query '${query.id}': no relevant documents`);
+    }
+    return {
+        ...query,
+        expected_files: expectedFiles,
+        must_not_match_files: resolveDocumentIds(mustNotMatchIds, documents, query.id),
+    };
+}
+export function resolveFixtureQueries(fixture) {
+    const documents = buildDocumentMap(fixture);
+    const queryIds = new Set();
+    return fixture.queries.map((query) => {
+        if (query.id.trim().length === 0) {
+            throw new Error("Invalid fixture: empty query id");
+        }
+        if (queryIds.has(query.id)) {
+            throw new Error(`Invalid fixture: duplicate query id '${query.id}'`);
+        }
+        queryIds.add(query.id);
+        return resolveQuery(query, documents);
+    });
+}

package/dist/bench/score.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Scoring functions for the QMD benchmark harness.
+ *
+ * Computes precision@k, recall, MRR, and F1 for search results
+ * against ground-truth expected files.
+ */
+/**
+ * Normalize a file path for comparison.
+ * Strips qmd:// prefix, lowercases, removes leading/trailing slashes.
+ */
+export declare function normalizePath(p: string): string;
+/**
+ * Check if two paths refer to the same file.
+ * Handles different path formats by comparing normalized suffixes.
+ */
+export declare function pathsMatch(result: string, expected: string): boolean;
+export declare function percentile(values: number[], quantile: number): number;
+type ScoreMetrics = {
+    precision_at_k: number;
+    recall: number;
+    recall_at_1: number;
+    recall_at_3: number;
+    recall_at_5: number;
+    recall_at_10: number;
+    mrr: number;
+    mrr_at_10: number;
+    f1: number;
+    hits_at_k: number;
+    false_positive_count: number;
+    false_positive_files: string[];
+    matched_files: string[];
+    unmatched_expected_files: string[];
+};
+/**
+ * Score a set of search results against expected files.
+ */
+export declare function scoreResults(resultFiles: string[], expectedFiles: string[], topK: number, mustNotMatchFiles?: string[]): ScoreMetrics;
+export {};