@joycodetech/qmd-ja 2.5.3-ja.3

package/dist/bench/bench.js

@@ -0,0 +1,280 @@
+/**
+ * QMD Benchmark Harness
+ *
+ * Runs queries from a fixture file against multiple search backends
+ * and measures precision@k, recall, MRR, F1, and latency.
+ *
+ * Usage:
+ *   qmd bench <fixture.json> [--json] [--collection <name>]
+ *
+ * Backends tested:
+ *   - bm25: BM25 keyword search (searchLex)
+ *   - vector: Vector similarity search (searchVector)
+ *   - hybrid: BM25 + vector RRF fusion without reranking
+ *   - full: Full hybrid pipeline with LLM reranking
+ */
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { createStore, getDefaultDbPath, } from "../index.js";
+import { scoreResults } from "./score.js";
+function parseStructuredQuery(query) {
+    const lines = query.split("\n").map((line, idx) => ({
+        trimmed: line.trim(),
+        number: idx + 1,
+    })).filter(line => line.trimmed.length > 0);
+    if (lines.length === 0)
+        return undefined;
+    const prefixRe = /^(lex|vec|hyde):\s*/i;
+    const intentRe = /^intent:\s*/i;
+    const searches = [];
+    let intent;
+    for (const line of lines) {
+        if (intentRe.test(line.trimmed)) {
+            if (intent !== undefined) {
+                throw new Error(`Line ${line.number}: only one intent: line is allowed per benchmark query.`);
+            }
+            intent = line.trimmed.replace(intentRe, "").trim();
+            if (!intent) {
+                throw new Error(`Line ${line.number}: intent: must include text.`);
+            }
+            continue;
+        }
+        const match = line.trimmed.match(prefixRe);
+        if (match) {
+            const type = match[1].toLowerCase();
+            const text = line.trimmed.slice(match[0].length).trim();
+            if (!text) {
+                throw new Error(`Line ${line.number} (${type}:) must include text.`);
+            }
+            searches.push({ type, query: text, line: line.number });
+            continue;
+        }
+        if (lines.length === 1) {
+            return undefined;
+        }
+        throw new Error(`Line ${line.number} is missing a lex:/vec:/hyde:/intent: prefix.`);
+    }
+    if (intent && searches.length === 0) {
+        throw new Error("intent: cannot appear alone. Add at least one lex:, vec:, or hyde: line.");
+    }
+    return searches.length > 0 ? { searches, intent } : undefined;
+}
+function uniqueFiles(files, limit) {
+    const seen = new Set();
+    const out = [];
+    for (const file of files) {
+        if (seen.has(file))
+            continue;
+        seen.add(file);
+        out.push(file);
+        if (out.length >= limit)
+            break;
+    }
+    return out;
+}
+const BACKENDS = [
+    {
+        name: "bm25",
+        run: async (store, query, limit, collection) => {
+            const structured = parseStructuredQuery(query.query);
+            const lexQueries = structured?.searches.filter(q => q.type === "lex");
+            if (structured) {
+                const files = [];
+                for (const lex of lexQueries ?? []) {
+                    const results = await store.searchLex(lex.query, { limit, collection });
+                    files.push(...results.map((r) => r.filepath));
+                }
+                return uniqueFiles(files, limit);
+            }
+            const results = await store.searchLex(query.query, { limit, collection });
+            return results.map((r) => r.filepath);
+        },
+    },
+    {
+        name: "vector",
+        run: async (store, query, limit, collection) => {
+            const structured = parseStructuredQuery(query.query);
+            const vectorQueries = structured?.searches.filter(q => q.type === "vec" || q.type === "hyde");
+            if (structured) {
+                const files = [];
+                for (const vectorQuery of vectorQueries ?? []) {
+                    const results = await store.searchVector(vectorQuery.query, { limit, collection });
+                    files.push(...results.map((r) => r.filepath));
+                }
+                return uniqueFiles(files, limit);
+            }
+            const results = await store.searchVector(query.query, { limit, collection });
+            return results.map((r) => r.filepath);
+        },
+    },
+    {
+        name: "hybrid",
+        run: async (store, query, limit, collection) => {
+            const structured = parseStructuredQuery(query.query);
+            const results = structured
+                ? await store.search({ queries: structured.searches, intent: structured.intent, limit, collection, rerank: false })
+                : await store.search({ query: query.query, limit, collection, rerank: false });
+            return results.map((r) => r.file);
+        },
+    },
+    {
+        name: "full",
+        run: async (store, query, limit, collection) => {
+            const structured = parseStructuredQuery(query.query);
+            const results = structured
+                ? await store.search({ queries: structured.searches, intent: structured.intent, limit, collection, rerank: true })
+                : await store.search({ query: query.query, limit, collection, rerank: true });
+            return results.map((r) => r.file);
+        },
+    },
+];
+async function runQuery(store, backend, query, collection) {
+    const limit = Math.max(query.expected_in_top_k, 10);
+    const start = Date.now();
+    let resultFiles;
+    try {
+        resultFiles = await backend.run(store, query, limit, collection);
+    }
+    catch {
+        // Backend may not be available (e.g., no embeddings for vector search)
+        return {
+            precision_at_k: 0,
+            recall: 0,
+            recall_at_1: 0,
+            recall_at_3: 0,
+            recall_at_5: 0,
+            mrr: 0,
+            f1: 0,
+            hits_at_k: 0,
+            total_expected: query.expected_files.length,
+            latency_ms: Date.now() - start,
+            top_files: [],
+            matched_files: [],
+            unmatched_expected_files: query.expected_files,
+        };
+    }
+    const latency_ms = Date.now() - start;
+    const scores = scoreResults(resultFiles, query.expected_files, query.expected_in_top_k);
+    return {
+        ...scores,
+        total_expected: query.expected_files.length,
+        latency_ms,
+        top_files: resultFiles.slice(0, 10),
+    };
+}
+function formatTable(results) {
+    const lines = [];
+    const pad = (s, n) => s.slice(0, n).padEnd(n);
+    const num = (n) => n.toFixed(2).padStart(5);
+    lines.push(`${pad("Query", 25)} ${pad("Backend", 8)} ${pad("P@k", 6)} ${pad("R@1", 6)} ${pad("R@3", 6)} ${pad("R@5", 6)} ${pad("MRR", 6)} ${pad("F1", 6)} ${pad("ms", 8)}`);
+    lines.push("-".repeat(88));
+    for (const r of results) {
+        for (const [backend, br] of Object.entries(r.backends)) {
+            lines.push(`${pad(r.id, 25)} ${pad(backend, 8)} ${num(br.precision_at_k)} ${num(br.recall_at_1)} ${num(br.recall_at_3)} ${num(br.recall_at_5)} ${num(br.mrr)} ${num(br.f1)} ${String(Math.round(br.latency_ms)).padStart(7)}ms`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+function computeSummary(results) {
+    const summary = {};
+    // Collect all backend names
+    const backendNames = new Set();
+    for (const r of results) {
+        for (const name of Object.keys(r.backends)) {
+            backendNames.add(name);
+        }
+    }
+    for (const name of Array.from(backendNames)) {
+        let totalP = 0, totalR = 0, totalR1 = 0, totalR3 = 0, totalR5 = 0, totalMrr = 0, totalF1 = 0, totalLat = 0, count = 0;
+        for (const r of results) {
+            const br = r.backends[name];
+            if (!br)
+                continue;
+            totalP += br.precision_at_k;
+            totalR += br.recall;
+            totalR1 += br.recall_at_1;
+            totalR3 += br.recall_at_3;
+            totalR5 += br.recall_at_5;
+            totalMrr += br.mrr;
+            totalF1 += br.f1;
+            totalLat += br.latency_ms;
+            count++;
+        }
+        if (count > 0) {
+            summary[name] = {
+                avg_precision: totalP / count,
+                avg_recall: totalR / count,
+                avg_recall_at_1: totalR1 / count,
+                avg_recall_at_3: totalR3 / count,
+                avg_recall_at_5: totalR5 / count,
+                avg_mrr: totalMrr / count,
+                avg_f1: totalF1 / count,
+                avg_latency_ms: totalLat / count,
+            };
+        }
+    }
+    return summary;
+}
+export async function runBenchmark(fixturePath, options = {}) {
+    // Load fixture
+    const raw = readFileSync(resolve(fixturePath), "utf-8");
+    const fixture = JSON.parse(raw);
+    if (!fixture.queries || !Array.isArray(fixture.queries)) {
+        throw new Error("Invalid fixture: missing 'queries' array");
+    }
+    // Open store
+    const store = await createStore({
+        dbPath: options.dbPath ?? getDefaultDbPath(),
+        ...(options.configPath ? { configPath: options.configPath } : {}),
+    });
+    // Filter backends if requested
+    const activeBackends = options.backends
+        ? BACKENDS.filter(b => options.backends.includes(b.name))
+        : BACKENDS;
+    const collection = options.collection ?? fixture.collection;
+    // Run queries
+    const results = [];
+    for (const query of fixture.queries) {
+        const backends = {};
+        for (const backend of activeBackends) {
+            if (!options.json) {
+                process.stderr.write(`  ${query.id} / ${backend.name}...`);
+            }
+            backends[backend.name] = await runQuery(store, backend, query, collection);
+            if (!options.json) {
+                process.stderr.write(` ${Math.round(backends[backend.name].latency_ms)}ms\n`);
+            }
+        }
+        results.push({
+            id: query.id,
+            query: query.query,
+            type: query.type,
+            backends,
+        });
+    }
+    await store.close();
+    const summary = computeSummary(results);
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "").slice(0, 15);
+    const benchResult = {
+        timestamp,
+        fixture: fixturePath,
+        results,
+        summary,
+    };
+    // Output
+    if (options.json) {
+        console.log(JSON.stringify(benchResult, null, 2));
+    }
+    else {
+        console.log("\n" + formatTable(results));
+        console.log("Summary:");
+        console.log("-".repeat(70));
+        const pad = (s, n) => s.slice(0, n).padEnd(n);
+        const num = (n) => n.toFixed(3).padStart(6);
+        for (const [name, s] of Object.entries(summary)) {
+            console.log(`  ${pad(name, 8)} P@k=${num(s.avg_precision)} R@1=${num(s.avg_recall_at_1)} R@3=${num(s.avg_recall_at_3)} R@5=${num(s.avg_recall_at_5)} MRR=${num(s.avg_mrr)} F1=${num(s.avg_f1)} Avg=${Math.round(s.avg_latency_ms)}ms`);
+        }
+    }
+    return benchResult;
+}

package/dist/bench/score.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Scoring functions for the QMD benchmark harness.
+ *
+ * Computes precision@k, recall, MRR, and F1 for search results
+ * against ground-truth expected files.
+ */
+/**
+ * Normalize a file path for comparison.
+ * Strips qmd:// prefix, lowercases, removes leading/trailing slashes.
+ */
+export declare function normalizePath(p: string): string;
+/**
+ * Check if two paths refer to the same file.
+ * Handles different path formats by comparing normalized suffixes.
+ */
+export declare function pathsMatch(result: string, expected: string): boolean;
+type ScoreMetrics = {
+    precision_at_k: number;
+    recall: number;
+    recall_at_1: number;
+    recall_at_3: number;
+    recall_at_5: number;
+    mrr: number;
+    f1: number;
+    hits_at_k: number;
+    matched_files: string[];
+    unmatched_expected_files: string[];
+};
+/**
+ * Score a set of search results against expected files.
+ */
+export declare function scoreResults(resultFiles: string[], expectedFiles: string[], topK: number): ScoreMetrics;
+export {};

package/dist/bench/score.js

@@ -0,0 +1,88 @@
+/**
+ * Scoring functions for the QMD benchmark harness.
+ *
+ * Computes precision@k, recall, MRR, and F1 for search results
+ * against ground-truth expected files.
+ */
+/**
+ * Normalize a file path for comparison.
+ * Strips qmd:// prefix, lowercases, removes leading/trailing slashes.
+ */
+export function normalizePath(p) {
+    if (p.startsWith("qmd://")) {
+        // qmd://collection/docs/readme.md → docs/readme.md
+        const withoutScheme = p.slice("qmd://".length);
+        const slashIdx = withoutScheme.indexOf("/");
+        p = slashIdx >= 0 ? withoutScheme.slice(slashIdx + 1) : withoutScheme;
+    }
+    return p.toLowerCase().replace(/^\/+|\/+$/g, "");
+}
+/**
+ * Check if two paths refer to the same file.
+ * Handles different path formats by comparing normalized suffixes.
+ */
+export function pathsMatch(result, expected) {
+    const nr = normalizePath(result);
+    const ne = normalizePath(expected);
+    if (nr === ne)
+        return true;
+    if (nr.endsWith(ne) || ne.endsWith(nr))
+        return true;
+    return false;
+}
+function hitsWithin(resultFiles, expectedFiles, k) {
+    const topKResults = resultFiles.slice(0, k);
+    let hits = 0;
+    for (const expected of expectedFiles) {
+        if (topKResults.some(r => pathsMatch(r, expected))) {
+            hits++;
+        }
+    }
+    return hits;
+}
+/**
+ * Score a set of search results against expected files.
+ */
+export function scoreResults(resultFiles, expectedFiles, topK) {
+    // Count hits in top-k
+    const hitsAtK = hitsWithin(resultFiles, expectedFiles, topK);
+    const matchedFiles = [];
+    const unmatchedExpectedFiles = [];
+    for (const expected of expectedFiles) {
+        if (resultFiles.some(r => pathsMatch(r, expected))) {
+            matchedFiles.push(expected);
+        }
+        else {
+            unmatchedExpectedFiles.push(expected);
+        }
+    }
+    // MRR: reciprocal rank of first relevant result
+    let mrr = 0;
+    for (let i = 0; i < resultFiles.length; i++) {
+        if (expectedFiles.some(e => pathsMatch(resultFiles[i], e))) {
+            mrr = 1 / (i + 1);
+            break;
+        }
+    }
+    const denominator = Math.min(topK, expectedFiles.length);
+    const precision_at_k = denominator > 0 ? hitsAtK / denominator : 0;
+    const recall = expectedFiles.length > 0 ? matchedFiles.length / expectedFiles.length : 0;
+    const recall_at_1 = expectedFiles.length > 0 ? hitsWithin(resultFiles, expectedFiles, 1) / expectedFiles.length : 0;
+    const recall_at_3 = expectedFiles.length > 0 ? hitsWithin(resultFiles, expectedFiles, 3) / expectedFiles.length : 0;
+    const recall_at_5 = expectedFiles.length > 0 ? hitsWithin(resultFiles, expectedFiles, 5) / expectedFiles.length : 0;
+    const f1 = precision_at_k + recall > 0
+        ? 2 * (precision_at_k * recall) / (precision_at_k + recall)
+        : 0;
+    return {
+        precision_at_k,
+        recall,
+        recall_at_1,
+        recall_at_3,
+        recall_at_5,
+        mrr,
+        f1,
+        hits_at_k: hitsAtK,
+        matched_files: matchedFiles,
+        unmatched_expected_files: unmatchedExpectedFiles,
+    };
+}

package/dist/bench/types.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Types for the QMD benchmark harness.
+ *
+ * A benchmark fixture defines queries with expected results.
+ * The harness runs each query through multiple search backends
+ * and measures precision, recall, MRR, and latency.
+ */
+export interface BenchmarkQuery {
+    /** Unique identifier for the query */
+    id: string;
+    /** The search query text */
+    query: string;
+    /** Query difficulty/type for grouping results */
+    type: "exact" | "semantic" | "topical" | "cross-domain" | "alias";
+    /** Human-readable description of what this tests */
+    description: string;
+    /** File paths (relative to collection) that should appear in results */
+    expected_files: string[];
+    /** How many of expected_files should appear in top-k results */
+    expected_in_top_k: number;
+}
+export interface BenchmarkFixture {
+    /** Description of the benchmark */
+    description: string;
+    /** Fixture format version */
+    version: number;
+    /** Optional collection to search within */
+    collection?: string;
+    /** The test queries */
+    queries: BenchmarkQuery[];
+}
+export interface BackendResult {
+    /** Fraction of top-k results that are relevant */
+    precision_at_k: number;
+    /** Fraction of expected files found anywhere in results */
+    recall: number;
+    /** Fraction of expected files found in the first result */
+    recall_at_1: number;
+    /** Fraction of expected files found in the top 3 results */
+    recall_at_3: number;
+    /** Fraction of expected files found in the top 5 results */
+    recall_at_5: number;
+    /** Reciprocal rank of first relevant result (1/rank, 0 if not found) */
+    mrr: number;
+    /** Harmonic mean of precision_at_k and recall */
+    f1: number;
+    /** Number of expected files found in top-k */
+    hits_at_k: number;
+    /** Total expected files */
+    total_expected: number;
+    /** Wall-clock latency in milliseconds */
+    latency_ms: number;
+    /** Top result file paths (for inspection) */
+    top_files: string[];
+    /** Expected files that were found anywhere in the returned result set */
+    matched_files: string[];
+    /** Expected files missing from the returned result set */
+    unmatched_expected_files: string[];
+}
+export interface QueryResult {
+    id: string;
+    query: string;
+    type: string;
+    backends: Record<string, BackendResult>;
+}
+export interface BenchmarkResult {
+    timestamp: string;
+    fixture: string;
+    results: QueryResult[];
+    summary: Record<string, {
+        avg_precision: number;
+        avg_recall: number;
+        avg_recall_at_1: number;
+        avg_recall_at_3: number;
+        avg_recall_at_5: number;
+        avg_mrr: number;
+        avg_f1: number;
+        avg_latency_ms: number;
+    }>;
+}

package/dist/bench/types.js

@@ -0,0 +1,8 @@
+/**
+ * Types for the QMD benchmark harness.
+ *
+ * A benchmark fixture defines queries with expected results.
+ * The harness runs each query through multiple search backends
+ * and measures precision, recall, MRR, and latency.
+ */
+export {};

package/dist/cli/formatter.d.ts

@@ -0,0 +1,120 @@
+/**
+ * formatter.ts - Output formatting utilities for QMD
+ *
+ * Provides methods to format search results and documents into various output formats:
+ * JSON, CSV, XML, Markdown, files list, and CLI (colored terminal output).
+ */
+import type { SearchResult, MultiGetResult, DocumentResult } from "../store.js";
+export type { SearchResult, MultiGetResult, DocumentResult };
+export type MultiGetFile = {
+    filepath: string;
+    displayPath: string;
+    title: string;
+    body: string;
+    context?: string | null;
+    skipped: false;
+} | {
+    filepath: string;
+    displayPath: string;
+    title: string;
+    body: string;
+    context?: string | null;
+    skipped: true;
+    skipReason: string;
+};
+export type OutputFormat = "cli" | "csv" | "md" | "xml" | "files" | "json";
+export type FormatOptions = {
+    full?: boolean;
+    query?: string;
+    useColor?: boolean;
+    lineNumbers?: boolean;
+    intent?: string;
+};
+/**
+ * Add line numbers to text content.
+ * Each line becomes: "{lineNum}: {content}"
+ * @param text The text to add line numbers to
+ * @param startLine Optional starting line number (default: 1)
+ */
+export declare function addLineNumbers(text: string, startLine?: number): string;
+/**
+ * Extract short docid from a full hash (first 6 characters).
+ */
+export declare function getDocid(hash: string): string;
+export declare function escapeCSV(value: string | null | number): string;
+export declare function escapeXml(str: string): string;
+/**
+ * Format search results as JSON
+ */
+export declare function searchResultsToJson(results: SearchResult[], opts?: FormatOptions): string;
+/**
+ * Format search results as CSV
+ */
+export declare function searchResultsToCsv(results: SearchResult[], opts?: FormatOptions): string;
+/**
+ * Format search results as simple files list (docid,score,filepath,context)
+ */
+export declare function searchResultsToFiles(results: SearchResult[]): string;
+/**
+ * Format search results as Markdown
+ */
+export declare function searchResultsToMarkdown(results: SearchResult[], opts?: FormatOptions): string;
+/**
+ * Format search results as XML
+ */
+export declare function searchResultsToXml(results: SearchResult[], opts?: FormatOptions): string;
+/**
+ * Format search results for MCP (simpler CSV format with pre-extracted snippets)
+ */
+export declare function searchResultsToMcpCsv(results: {
+    docid: string;
+    file: string;
+    title: string;
+    score: number;
+    context: string | null;
+    snippet: string;
+}[]): string;
+/**
+ * Format documents as JSON
+ */
+export declare function documentsToJson(results: MultiGetFile[]): string;
+/**
+ * Format documents as CSV
+ */
+export declare function documentsToCsv(results: MultiGetFile[]): string;
+/**
+ * Format documents as files list
+ */
+export declare function documentsToFiles(results: MultiGetFile[]): string;
+/**
+ * Format documents as Markdown
+ */
+export declare function documentsToMarkdown(results: MultiGetFile[]): string;
+/**
+ * Format documents as XML
+ */
+export declare function documentsToXml(results: MultiGetFile[]): string;
+/**
+ * Format a single DocumentResult as JSON
+ */
+export declare function documentToJson(doc: DocumentResult): string;
+/**
+ * Format a single DocumentResult as Markdown
+ */
+export declare function documentToMarkdown(doc: DocumentResult): string;
+/**
+ * Format a single DocumentResult as XML
+ */
+export declare function documentToXml(doc: DocumentResult): string;
+/**
+ * Format a single document to the specified format
+ */
+export declare function formatDocument(doc: DocumentResult, format: OutputFormat): string;
+/**
+ * Format search results to the specified output format
+ */
+export declare function formatSearchResults(results: SearchResult[], format: OutputFormat, opts?: FormatOptions): string;
+/**
+ * Format documents to the specified output format
+ */
+export declare function formatDocuments(results: MultiGetFile[], format: OutputFormat): string;