@tobilu/qmd 2.0.1 → 2.1.0

package/dist/bench/bench.js

@@ -0,0 +1,185 @@
+/**
+ * QMD Benchmark Harness
+ *
+ * Runs queries from a fixture file against multiple search backends
+ * and measures precision@k, recall, MRR, F1, and latency.
+ *
+ * Usage:
+ *   qmd bench <fixture.json> [--json] [--collection <name>]
+ *
+ * Backends tested:
+ *   - bm25: BM25 keyword search (searchLex)
+ *   - vector: Vector similarity search (searchVector)
+ *   - hybrid: BM25 + vector RRF fusion without reranking
+ *   - full: Full hybrid pipeline with LLM reranking
+ */
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { createStore, getDefaultDbPath, } from "../index.js";
+import { scoreResults } from "./score.js";
+const BACKENDS = [
+    {
+        name: "bm25",
+        run: async (store, query, limit, collection) => {
+            const results = await store.searchLex(query, { limit, collection });
+            return results.map((r) => r.filepath);
+        },
+    },
+    {
+        name: "vector",
+        run: async (store, query, limit, collection) => {
+            const results = await store.searchVector(query, { limit, collection });
+            return results.map((r) => r.filepath);
+        },
+    },
+    {
+        name: "hybrid",
+        run: async (store, query, limit, collection) => {
+            const results = await store.search({ query, limit, collection, rerank: false });
+            return results.map((r) => r.file);
+        },
+    },
+    {
+        name: "full",
+        run: async (store, query, limit, collection) => {
+            const results = await store.search({ query, limit, collection, rerank: true });
+            return results.map((r) => r.file);
+        },
+    },
+];
+async function runQuery(store, backend, query, collection) {
+    const limit = Math.max(query.expected_in_top_k, 10);
+    const start = Date.now();
+    let resultFiles;
+    try {
+        resultFiles = await backend.run(store, query.query, limit, collection);
+    }
+    catch (err) {
+        // Backend may not be available (e.g., no embeddings for vector search)
+        return {
+            precision_at_k: 0,
+            recall: 0,
+            mrr: 0,
+            f1: 0,
+            hits_at_k: 0,
+            total_expected: query.expected_files.length,
+            latency_ms: Date.now() - start,
+            top_files: [],
+        };
+    }
+    const latency_ms = Date.now() - start;
+    const scores = scoreResults(resultFiles, query.expected_files, query.expected_in_top_k);
+    return {
+        ...scores,
+        total_expected: query.expected_files.length,
+        latency_ms,
+        top_files: resultFiles.slice(0, 10),
+    };
+}
+function formatTable(results) {
+    const lines = [];
+    const pad = (s, n) => s.slice(0, n).padEnd(n);
+    const num = (n) => n.toFixed(2).padStart(5);
+    lines.push(`${pad("Query", 25)} ${pad("Backend", 8)} ${pad("P@k", 6)} ${pad("Recall", 7)} ${pad("MRR", 6)} ${pad("F1", 6)} ${pad("ms", 8)}`);
+    lines.push("-".repeat(70));
+    for (const r of results) {
+        for (const [backend, br] of Object.entries(r.backends)) {
+            lines.push(`${pad(r.id, 25)} ${pad(backend, 8)} ${num(br.precision_at_k)} ${num(br.recall)}  ${num(br.mrr)} ${num(br.f1)} ${String(Math.round(br.latency_ms)).padStart(7)}ms`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+function computeSummary(results) {
+    const summary = {};
+    // Collect all backend names
+    const backendNames = new Set();
+    for (const r of results) {
+        for (const name of Object.keys(r.backends)) {
+            backendNames.add(name);
+        }
+    }
+    for (const name of backendNames) {
+        let totalP = 0, totalR = 0, totalMrr = 0, totalF1 = 0, totalLat = 0, count = 0;
+        for (const r of results) {
+            const br = r.backends[name];
+            if (!br)
+                continue;
+            totalP += br.precision_at_k;
+            totalR += br.recall;
+            totalMrr += br.mrr;
+            totalF1 += br.f1;
+            totalLat += br.latency_ms;
+            count++;
+        }
+        if (count > 0) {
+            summary[name] = {
+                avg_precision: totalP / count,
+                avg_recall: totalR / count,
+                avg_mrr: totalMrr / count,
+                avg_f1: totalF1 / count,
+                avg_latency_ms: totalLat / count,
+            };
+        }
+    }
+    return summary;
+}
+export async function runBenchmark(fixturePath, options = {}) {
+    // Load fixture
+    const raw = readFileSync(resolve(fixturePath), "utf-8");
+    const fixture = JSON.parse(raw);
+    if (!fixture.queries || !Array.isArray(fixture.queries)) {
+        throw new Error("Invalid fixture: missing 'queries' array");
+    }
+    // Open store
+    const store = await createStore({ dbPath: getDefaultDbPath() });
+    // Filter backends if requested
+    const activeBackends = options.backends
+        ? BACKENDS.filter(b => options.backends.includes(b.name))
+        : BACKENDS;
+    const collection = options.collection ?? fixture.collection;
+    // Run queries
+    const results = [];
+    for (const query of fixture.queries) {
+        const backends = {};
+        for (const backend of activeBackends) {
+            if (!options.json) {
+                process.stderr.write(`  ${query.id} / ${backend.name}...`);
+            }
+            backends[backend.name] = await runQuery(store, backend, query, collection);
+            if (!options.json) {
+                process.stderr.write(` ${Math.round(backends[backend.name].latency_ms)}ms\n`);
+            }
+        }
+        results.push({
+            id: query.id,
+            query: query.query,
+            type: query.type,
+            backends,
+        });
+    }
+    await store.close();
+    const summary = computeSummary(results);
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "").slice(0, 15);
+    const benchResult = {
+        timestamp,
+        fixture: fixturePath,
+        results,
+        summary,
+    };
+    // Output
+    if (options.json) {
+        console.log(JSON.stringify(benchResult, null, 2));
+    }
+    else {
+        console.log("\n" + formatTable(results));
+        console.log("Summary:");
+        console.log("-".repeat(70));
+        const pad = (s, n) => s.slice(0, n).padEnd(n);
+        const num = (n) => n.toFixed(3).padStart(6);
+        for (const [name, s] of Object.entries(summary)) {
+            console.log(`  ${pad(name, 8)} P@k=${num(s.avg_precision)} Recall=${num(s.avg_recall)} MRR=${num(s.avg_mrr)} F1=${num(s.avg_f1)} Avg=${Math.round(s.avg_latency_ms)}ms`);
+        }
+    }
+    return benchResult;
+}

package/dist/bench/score.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Scoring functions for the QMD benchmark harness.
+ *
+ * Computes precision@k, recall, MRR, and F1 for search results
+ * against ground-truth expected files.
+ */
+/**
+ * Normalize a file path for comparison.
+ * Strips qmd:// prefix, lowercases, removes leading/trailing slashes.
+ */
+export declare function normalizePath(p: string): string;
+/**
+ * Check if two paths refer to the same file.
+ * Handles different path formats by comparing normalized suffixes.
+ */
+export declare function pathsMatch(result: string, expected: string): boolean;
+/**
+ * Score a set of search results against expected files.
+ */
+export declare function scoreResults(resultFiles: string[], expectedFiles: string[], topK: number): {
+    precision_at_k: number;
+    recall: number;
+    mrr: number;
+    f1: number;
+    hits_at_k: number;
+};

package/dist/bench/score.js

@@ -0,0 +1,67 @@
+/**
+ * Scoring functions for the QMD benchmark harness.
+ *
+ * Computes precision@k, recall, MRR, and F1 for search results
+ * against ground-truth expected files.
+ */
+/**
+ * Normalize a file path for comparison.
+ * Strips qmd:// prefix, lowercases, removes leading/trailing slashes.
+ */
+export function normalizePath(p) {
+    if (p.startsWith("qmd://")) {
+        // qmd://collection/path/to/file → path/to/file
+        const withoutScheme = p.slice("qmd://".length);
+        const slashIdx = withoutScheme.indexOf("/");
+        p = slashIdx >= 0 ? withoutScheme.slice(slashIdx + 1) : withoutScheme;
+    }
+    return p.toLowerCase().replace(/^\/+|\/+$/g, "");
+}
+/**
+ * Check if two paths refer to the same file.
+ * Handles different path formats by comparing normalized suffixes.
+ */
+export function pathsMatch(result, expected) {
+    const nr = normalizePath(result);
+    const ne = normalizePath(expected);
+    if (nr === ne)
+        return true;
+    if (nr.endsWith(ne) || ne.endsWith(nr))
+        return true;
+    return false;
+}
+/**
+ * Score a set of search results against expected files.
+ */
+export function scoreResults(resultFiles, expectedFiles, topK) {
+    // Count hits in top-k
+    const topKResults = resultFiles.slice(0, topK);
+    let hitsAtK = 0;
+    for (const expected of expectedFiles) {
+        if (topKResults.some(r => pathsMatch(r, expected))) {
+            hitsAtK++;
+        }
+    }
+    // Count total hits anywhere
+    let totalHits = 0;
+    for (const expected of expectedFiles) {
+        if (resultFiles.some(r => pathsMatch(r, expected))) {
+            totalHits++;
+        }
+    }
+    // MRR: reciprocal rank of first relevant result
+    let mrr = 0;
+    for (let i = 0; i < resultFiles.length; i++) {
+        if (expectedFiles.some(e => pathsMatch(resultFiles[i], e))) {
+            mrr = 1 / (i + 1);
+            break;
+        }
+    }
+    const denominator = Math.min(topK, expectedFiles.length);
+    const precision_at_k = denominator > 0 ? hitsAtK / denominator : 0;
+    const recall = expectedFiles.length > 0 ? totalHits / expectedFiles.length : 0;
+    const f1 = precision_at_k + recall > 0
+        ? 2 * (precision_at_k * recall) / (precision_at_k + recall)
+        : 0;
+    return { precision_at_k, recall, mrr, f1, hits_at_k: hitsAtK };
+}

package/dist/bench/types.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Types for the QMD benchmark harness.
+ *
+ * A benchmark fixture defines queries with expected results.
+ * The harness runs each query through multiple search backends
+ * and measures precision, recall, MRR, and latency.
+ */
+export interface BenchmarkQuery {
+    /** Unique identifier for the query */
+    id: string;
+    /** The search query text */
+    query: string;
+    /** Query difficulty/type for grouping results */
+    type: "exact" | "semantic" | "topical" | "cross-domain" | "alias";
+    /** Human-readable description of what this tests */
+    description: string;
+    /** File paths (relative to collection) that should appear in results */
+    expected_files: string[];
+    /** How many of expected_files should appear in top-k results */
+    expected_in_top_k: number;
+}
+export interface BenchmarkFixture {
+    /** Description of the benchmark */
+    description: string;
+    /** Fixture format version */
+    version: number;
+    /** Optional collection to search within */
+    collection?: string;
+    /** The test queries */
+    queries: BenchmarkQuery[];
+}
+export interface BackendResult {
+    /** Fraction of top-k results that are relevant */
+    precision_at_k: number;
+    /** Fraction of expected files found anywhere in results */
+    recall: number;
+    /** Reciprocal rank of first relevant result (1/rank, 0 if not found) */
+    mrr: number;
+    /** Harmonic mean of precision_at_k and recall */
+    f1: number;
+    /** Number of expected files found in top-k */
+    hits_at_k: number;
+    /** Total expected files */
+    total_expected: number;
+    /** Wall-clock latency in milliseconds */
+    latency_ms: number;
+    /** Top result file paths (for inspection) */
+    top_files: string[];
+}
+export interface QueryResult {
+    id: string;
+    query: string;
+    type: string;
+    backends: Record<string, BackendResult>;
+}
+export interface BenchmarkResult {
+    timestamp: string;
+    fixture: string;
+    results: QueryResult[];
+    summary: Record<string, {
+        avg_precision: number;
+        avg_recall: number;
+        avg_mrr: number;
+        avg_f1: number;
+        avg_latency_ms: number;
+    }>;
+}

package/dist/bench/types.js

@@ -0,0 +1,8 @@
+/**
+ * Types for the QMD benchmark harness.
+ *
+ * A benchmark fixture defines queries with expected results.
+ * The harness runs each query through multiple search backends
+ * and measures precision, recall, MRR, and latency.
+ */
+export {};

package/dist/cli/formatter.js

@@ -54,8 +54,11 @@ export function searchResultsToJson(results, opts = {}) {
     const query = opts.query || "";
     const output = results.map(row => {
         const bodyStr = row.body || "";
+        const snippetInfo = bodyStr
+            ? extractSnippet(bodyStr, query, 300, row.chunkPos, undefined, opts.intent)
+            : undefined;
         let body = opts.full ? bodyStr : undefined;
-        let snippet = !opts.full ? extractSnippet(bodyStr, query, 300, row.chunkPos, undefined, opts.intent).snippet : undefined;
+        let snippet = !opts.full ? snippetInfo?.snippet : undefined;
         if (opts.lineNumbers) {
             if (body)
                 body = addLineNumbers(body);
@@ -66,6 +69,7 @@ export function searchResultsToJson(results, opts = {}) {
             docid: `#${row.docid}`,
             score: Math.round(row.score * 100) / 100,
             file: row.displayPath,
+            ...(snippetInfo && { line: snippetInfo.line }),
             title: row.title,
             ...(row.context && { context: row.context }),
             ...(body && { body }),

package/dist/cli/qmd.d.ts

@@ -1 +1,2 @@
-export {};
+export declare function buildEditorUri(template: string, absolutePath: string, line: number, col: number): string;
+export declare function termLink(text: string, url: string, isTTY?: boolean): string;