@tobilu/qmd 2.0.1 → 2.5.1

package/dist/bench/score.js

@@ -0,0 +1,88 @@
+/**
+ * Scoring functions for the QMD benchmark harness.
+ *
+ * Computes precision@k, recall, MRR, and F1 for search results
+ * against ground-truth expected files.
+ */
+/**
+ * Normalize a file path for comparison.
+ * Strips qmd:// prefix, lowercases, removes leading/trailing slashes.
+ */
+export function normalizePath(p) {
+    if (p.startsWith("qmd://")) {
+        // qmd://collection/docs/readme.md → docs/readme.md
+        const withoutScheme = p.slice("qmd://".length);
+        const slashIdx = withoutScheme.indexOf("/");
+        p = slashIdx >= 0 ? withoutScheme.slice(slashIdx + 1) : withoutScheme;
+    }
+    return p.toLowerCase().replace(/^\/+|\/+$/g, "");
+}
+/**
+ * Check if two paths refer to the same file.
+ * Handles different path formats by comparing normalized suffixes.
+ */
+export function pathsMatch(result, expected) {
+    const nr = normalizePath(result);
+    const ne = normalizePath(expected);
+    if (nr === ne)
+        return true;
+    if (nr.endsWith(ne) || ne.endsWith(nr))
+        return true;
+    return false;
+}
+function hitsWithin(resultFiles, expectedFiles, k) {
+    const topKResults = resultFiles.slice(0, k);
+    let hits = 0;
+    for (const expected of expectedFiles) {
+        if (topKResults.some(r => pathsMatch(r, expected))) {
+            hits++;
+        }
+    }
+    return hits;
+}
+/**
+ * Score a set of search results against expected files.
+ */
+export function scoreResults(resultFiles, expectedFiles, topK) {
+    // Count hits in top-k
+    const hitsAtK = hitsWithin(resultFiles, expectedFiles, topK);
+    const matchedFiles = [];
+    const unmatchedExpectedFiles = [];
+    for (const expected of expectedFiles) {
+        if (resultFiles.some(r => pathsMatch(r, expected))) {
+            matchedFiles.push(expected);
+        }
+        else {
+            unmatchedExpectedFiles.push(expected);
+        }
+    }
+    // MRR: reciprocal rank of first relevant result
+    let mrr = 0;
+    for (let i = 0; i < resultFiles.length; i++) {
+        if (expectedFiles.some(e => pathsMatch(resultFiles[i], e))) {
+            mrr = 1 / (i + 1);
+            break;
+        }
+    }
+    const denominator = Math.min(topK, expectedFiles.length);
+    const precision_at_k = denominator > 0 ? hitsAtK / denominator : 0;
+    const recall = expectedFiles.length > 0 ? matchedFiles.length / expectedFiles.length : 0;
+    const recall_at_1 = expectedFiles.length > 0 ? hitsWithin(resultFiles, expectedFiles, 1) / expectedFiles.length : 0;
+    const recall_at_3 = expectedFiles.length > 0 ? hitsWithin(resultFiles, expectedFiles, 3) / expectedFiles.length : 0;
+    const recall_at_5 = expectedFiles.length > 0 ? hitsWithin(resultFiles, expectedFiles, 5) / expectedFiles.length : 0;
+    const f1 = precision_at_k + recall > 0
+        ? 2 * (precision_at_k * recall) / (precision_at_k + recall)
+        : 0;
+    return {
+        precision_at_k,
+        recall,
+        recall_at_1,
+        recall_at_3,
+        recall_at_5,
+        mrr,
+        f1,
+        hits_at_k: hitsAtK,
+        matched_files: matchedFiles,
+        unmatched_expected_files: unmatchedExpectedFiles,
+    };
+}

package/dist/bench/types.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Types for the QMD benchmark harness.
+ *
+ * A benchmark fixture defines queries with expected results.
+ * The harness runs each query through multiple search backends
+ * and measures precision, recall, MRR, and latency.
+ */
+export interface BenchmarkQuery {
+    /** Unique identifier for the query */
+    id: string;
+    /** The search query text */
+    query: string;
+    /** Query difficulty/type for grouping results */
+    type: "exact" | "semantic" | "topical" | "cross-domain" | "alias";
+    /** Human-readable description of what this tests */
+    description: string;
+    /** File paths (relative to collection) that should appear in results */
+    expected_files: string[];
+    /** How many of expected_files should appear in top-k results */
+    expected_in_top_k: number;
+}
+export interface BenchmarkFixture {
+    /** Description of the benchmark */
+    description: string;
+    /** Fixture format version */
+    version: number;
+    /** Optional collection to search within */
+    collection?: string;
+    /** The test queries */
+    queries: BenchmarkQuery[];
+}
+export interface BackendResult {
+    /** Fraction of top-k results that are relevant */
+    precision_at_k: number;
+    /** Fraction of expected files found anywhere in results */
+    recall: number;
+    /** Fraction of expected files found in the first result */
+    recall_at_1: number;
+    /** Fraction of expected files found in the top 3 results */
+    recall_at_3: number;
+    /** Fraction of expected files found in the top 5 results */
+    recall_at_5: number;
+    /** Reciprocal rank of first relevant result (1/rank, 0 if not found) */
+    mrr: number;
+    /** Harmonic mean of precision_at_k and recall */
+    f1: number;
+    /** Number of expected files found in top-k */
+    hits_at_k: number;
+    /** Total expected files */
+    total_expected: number;
+    /** Wall-clock latency in milliseconds */
+    latency_ms: number;
+    /** Top result file paths (for inspection) */
+    top_files: string[];
+    /** Expected files that were found anywhere in the returned result set */
+    matched_files: string[];
+    /** Expected files missing from the returned result set */
+    unmatched_expected_files: string[];
+}
+export interface QueryResult {
+    id: string;
+    query: string;
+    type: string;
+    backends: Record<string, BackendResult>;
+}
+export interface BenchmarkResult {
+    timestamp: string;
+    fixture: string;
+    results: QueryResult[];
+    summary: Record<string, {
+        avg_precision: number;
+        avg_recall: number;
+        avg_recall_at_1: number;
+        avg_recall_at_3: number;
+        avg_recall_at_5: number;
+        avg_mrr: number;
+        avg_f1: number;
+        avg_latency_ms: number;
+    }>;
+}

package/dist/bench/types.js

@@ -0,0 +1,8 @@
+/**
+ * Types for the QMD benchmark harness.
+ *
+ * A benchmark fixture defines queries with expected results.
+ * The harness runs each query through multiple search backends
+ * and measures precision, recall, MRR, and latency.
+ */
+export {};

package/dist/cli/formatter.js

@@ -54,8 +54,11 @@ export function searchResultsToJson(results, opts = {}) {
     const query = opts.query || "";
     const output = results.map(row => {
         const bodyStr = row.body || "";
+        const snippetInfo = bodyStr
+            ? extractSnippet(bodyStr, query, 300, row.chunkPos, undefined, opts.intent)
+            : undefined;
         let body = opts.full ? bodyStr : undefined;
-        let snippet = !opts.full ? extractSnippet(bodyStr, query, 300, row.chunkPos, undefined, opts.intent).snippet : undefined;
+        let snippet = !opts.full ? snippetInfo?.snippet : undefined;
         if (opts.lineNumbers) {
             if (body)
                 body = addLineNumbers(body);
@@ -66,6 +69,7 @@ export function searchResultsToJson(results, opts = {}) {
             docid: `#${row.docid}`,
             score: Math.round(row.score * 100) / 100,
             file: row.displayPath,
+            ...(snippetInfo && { line: snippetInfo.line }),
             title: row.title,
             ...(row.context && { context: row.context }),
             ...(body && { body }),

package/dist/cli/qmd.d.ts

@@ -1 +1,28 @@
+import { type OutputFormat } from "./formatter.js";
+type CliLifecycleWritable = {
+    write(chunk: string | Uint8Array, callback?: (error?: Error | null) => void): boolean;
+};
+type FinishSuccessfulCliCommandOptions = {
+    command: string;
+    format?: OutputFormat;
+    cleanup?: () => Promise<void>;
+    exit?: (code: number) => void;
+    immediateExit?: (code: number) => void;
+    stdout?: CliLifecycleWritable;
+    stderr?: CliLifecycleWritable;
+    platform?: NodeJS.Platform;
+};
+/**
+ * Finish a successful CLI command after output has been flushed. On macOS JSON
+ * query runs, skip normal native teardown and use Node/Bun's immediate exit path:
+ * ggml Metal can abort from C++ finalizers after valid JSON has already been
+ * produced (#368). This wrapper is only reached after the command completed, so
+ * real query failures still exit through the normal error path before this runs.
+ */
+export declare function finishSuccessfulCliCommand(options: FinishSuccessfulCliCommandOptions): Promise<void>;
+export declare function resolveEmbedModelForCli(): string;
+export declare function resolveGenerateModelForCli(): string;
+export declare function resolveRerankModelForCli(): string;
+export declare function buildEditorUri(template: string, absolutePath: string, line: number, col: number): string;
+export declare function termLink(text: string, url: string, isTTY?: boolean): string;
 export {};