@tobilu/qmd 2.0.1 → 2.5.1

package/dist/ast.js

@@ -0,0 +1,334 @@
+/**
+ * AST-aware chunking support via web-tree-sitter.
+ *
+ * Provides language detection, AST break point extraction for supported
+ * code file types, and a stub for future symbol extraction.
+ *
+ * All functions degrade gracefully: parse failures or unsupported languages
+ * return empty arrays, falling back to regex-only chunking.
+ *
+ * ## Dependency Note
+ *
+ * Grammar packages (tree-sitter-typescript, etc.) are listed as
+ * optionalDependencies with pinned versions. They ship native prebuilds
+ * and source files (~72 MB total) but QMD only uses the .wasm files
+ * (~5 MB). If install size becomes a concern, the .wasm files can be
+ * bundled directly in the repo (e.g. assets/grammars/) and resolved
+ * via import.meta.url instead of require.resolve(), eliminating the
+ * grammar packages entirely.
+ */
+import { createRequire } from "node:module";
+import { extname } from "node:path";
+const EXTENSION_MAP = {
+    ".ts": "typescript",
+    ".tsx": "tsx",
+    ".js": "javascript",
+    ".jsx": "tsx",
+    ".mts": "typescript",
+    ".cts": "typescript",
+    ".mjs": "javascript",
+    ".cjs": "javascript",
+    ".py": "python",
+    ".go": "go",
+    ".rs": "rust",
+};
+/**
+ * Detect language from file path extension.
+ * Returns null for unsupported or unknown extensions (including .md).
+ */
+export function detectLanguage(filepath) {
+    const ext = extname(filepath).toLowerCase();
+    return EXTENSION_MAP[ext] ?? null;
+}
+// =============================================================================
+// Grammar Resolution
+// =============================================================================
+/**
+ * Maps language to the npm package and wasm filename for the grammar.
+ */
+const GRAMMAR_MAP = {
+    typescript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm", version: "0.23.2" },
+    tsx: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-tsx.wasm", version: "0.23.2" },
+    javascript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm", version: "0.23.2" },
+    python: { pkg: "tree-sitter-python", wasm: "tree-sitter-python.wasm", version: "0.23.4" },
+    go: { pkg: "tree-sitter-go", wasm: "tree-sitter-go.wasm", version: "0.23.4" },
+    rust: { pkg: "tree-sitter-rust", wasm: "tree-sitter-rust.wasm", version: "0.24.0" },
+};
+export function formatGrammarLoadError(language, err) {
+    const grammar = GRAMMAR_MAP[language];
+    const detail = err instanceof Error ? err.message : String(err);
+    return `${grammar.pkg}/${grammar.wasm} failed to load (${detail}); falling back to regex chunking. ` +
+        `Repair a broken global install with: bun add ${grammar.pkg}@${grammar.version}`;
+}
+// =============================================================================
+// Per-Language Query Definitions
+// =============================================================================
+/**
+ * Tree-sitter S-expression queries for each language.
+ * Each capture name maps to a break point score via SCORE_MAP.
+ *
+ * For TypeScript/JavaScript, we match export_statement wrappers to get the
+ * correct start position (before `export`), plus bare declarations for
+ * non-exported code.
+ */
+const LANGUAGE_QUERIES = {
+    typescript: `
+    (export_statement) @export
+    (class_declaration) @class
+    (function_declaration) @func
+    (method_definition) @method
+    (interface_declaration) @iface
+    (type_alias_declaration) @type
+    (enum_declaration) @enum
+    (import_statement) @import
+    (lexical_declaration (variable_declarator value: (arrow_function))) @func
+    (lexical_declaration (variable_declarator value: (function_expression))) @func
+  `,
+    tsx: `
+    (export_statement) @export
+    (class_declaration) @class
+    (function_declaration) @func
+    (method_definition) @method
+    (interface_declaration) @iface
+    (type_alias_declaration) @type
+    (enum_declaration) @enum
+    (import_statement) @import
+    (lexical_declaration (variable_declarator value: (arrow_function))) @func
+    (lexical_declaration (variable_declarator value: (function_expression))) @func
+  `,
+    javascript: `
+    (export_statement) @export
+    (class_declaration) @class
+    (function_declaration) @func
+    (method_definition) @method
+    (import_statement) @import
+    (lexical_declaration (variable_declarator value: (arrow_function))) @func
+    (lexical_declaration (variable_declarator value: (function_expression))) @func
+  `,
+    python: `
+    (class_definition) @class
+    (function_definition) @func
+    (decorated_definition) @decorated
+    (import_statement) @import
+    (import_from_statement) @import
+  `,
+    go: `
+    (type_declaration) @type
+    (function_declaration) @func
+    (method_declaration) @method
+    (import_declaration) @import
+  `,
+    rust: `
+    (struct_item) @struct
+    (impl_item) @impl
+    (function_item) @func
+    (trait_item) @trait
+    (enum_item) @enum
+    (use_declaration) @import
+    (type_item) @type
+    (mod_item) @mod
+  `,
+};
+/**
+ * Score mapping from capture names to break point scores.
+ * Aligned with the markdown BREAK_PATTERNS scale (h1=100, h2=90, etc.)
+ * so findBestCutoff() decay works unchanged.
+ */
+const SCORE_MAP = {
+    class: 100,
+    iface: 100,
+    struct: 100,
+    trait: 100,
+    impl: 100,
+    mod: 100,
+    export: 90,
+    func: 90,
+    method: 90,
+    decorated: 90,
+    type: 80,
+    enum: 80,
+    import: 60,
+};
+// =============================================================================
+// Parser Caching & Initialization
+// =============================================================================
+let ParserClass = null;
+let LanguageClass = null;
+let QueryClass = null;
+let initPromise = null;
+/** Languages that have already failed to load — warn only once per process. */
+const failedLanguages = new Set();
+/** Last grammar load error by language, for status output. */
+const grammarLoadErrors = new Map();
+/** Cached grammar load promises. */
+const grammarCache = new Map();
+/** Cached compiled queries per language. */
+const queryCache = new Map();
+/**
+ * Initialize web-tree-sitter. Called once and cached.
+ */
+async function ensureInit() {
+    if (!initPromise) {
+        initPromise = (async () => {
+            const mod = await import("web-tree-sitter");
+            ParserClass = mod.Parser;
+            LanguageClass = mod.Language;
+            QueryClass = mod.Query;
+            await ParserClass.init();
+        })();
+    }
+    return initPromise;
+}
+/**
+ * Resolve the filesystem path to a grammar .wasm file.
+ * Uses createRequire to resolve from installed dependency packages.
+ */
+function resolveGrammarPath(language) {
+    const { pkg, wasm } = GRAMMAR_MAP[language];
+    const require = createRequire(import.meta.url);
+    return require.resolve(`${pkg}/${wasm}`);
+}
+/**
+ * Load and cache a grammar for the given language.
+ * Returns null on failure (logs once per language).
+ */
+async function loadGrammar(language) {
+    if (failedLanguages.has(language))
+        return null;
+    const wasmKey = GRAMMAR_MAP[language].wasm;
+    if (!grammarCache.has(wasmKey)) {
+        grammarCache.set(wasmKey, (async () => {
+            const path = resolveGrammarPath(language);
+            return LanguageClass.load(path);
+        })());
+    }
+    try {
+        return await grammarCache.get(wasmKey);
+    }
+    catch (err) {
+        failedLanguages.add(language);
+        grammarCache.delete(wasmKey);
+        const message = formatGrammarLoadError(language, err);
+        grammarLoadErrors.set(language, message);
+        console.warn(`[qmd] AST grammar unavailable for ${language}: ${message}`);
+        return null;
+    }
+}
+/**
+ * Get or create a compiled query for the given language.
+ */
+function getQuery(language, grammar) {
+    if (!queryCache.has(language)) {
+        const source = LANGUAGE_QUERIES[language];
+        const query = new QueryClass(grammar, source);
+        queryCache.set(language, query);
+    }
+    return queryCache.get(language);
+}
+// =============================================================================
+// AST Break Point Extraction
+// =============================================================================
+/**
+ * Parse a source file and return break points at AST node boundaries.
+ *
+ * Returns an empty array for unsupported languages, parse failures,
+ * or grammar loading failures. Never throws.
+ *
+ * @param content - The file content to parse.
+ * @param filepath - The file path (used for language detection).
+ * @returns Array of BreakPoint objects suitable for merging with regex break points.
+ */
+export async function getASTBreakPoints(content, filepath) {
+    const language = detectLanguage(filepath);
+    if (!language)
+        return [];
+    try {
+        await ensureInit();
+        const grammar = await loadGrammar(language);
+        if (!grammar)
+            return [];
+        const parser = new ParserClass();
+        parser.setLanguage(grammar);
+        const tree = parser.parse(content);
+        if (!tree) {
+            parser.delete();
+            return [];
+        }
+        const query = getQuery(language, grammar);
+        const captures = query.captures(tree.rootNode);
+        // Deduplicate: at each byte position, keep the highest-scoring capture.
+        // This handles cases like export_statement wrapping a class_declaration
+        // at different offsets — we want the outermost (earliest) position.
+        const seen = new Map();
+        for (const cap of captures) {
+            const pos = cap.node.startIndex;
+            const score = SCORE_MAP[cap.name] ?? 20;
+            const type = `ast:${cap.name}`;
+            const existing = seen.get(pos);
+            if (!existing || score > existing.score) {
+                seen.set(pos, { pos, score, type });
+            }
+        }
+        tree.delete();
+        parser.delete();
+        return Array.from(seen.values()).sort((a, b) => a.pos - b.pos);
+    }
+    catch (err) {
+        console.warn(`[qmd] AST parse failed for ${filepath}, falling back to regex: ${err instanceof Error ? err.message : err}`);
+        return [];
+    }
+}
+// =============================================================================
+// Health / Status
+// =============================================================================
+/**
+ * Check which tree-sitter grammars are available.
+ * Returns a status object for each supported language.
+ */
+export async function getASTStatus() {
+    const languages = [];
+    try {
+        await ensureInit();
+    }
+    catch (err) {
+        return {
+            available: false,
+            languages: Object.keys(GRAMMAR_MAP).map(lang => ({
+                language: lang,
+                available: false,
+                error: `web-tree-sitter init failed: ${err instanceof Error ? err.message : err}`,
+            })),
+        };
+    }
+    for (const lang of Object.keys(GRAMMAR_MAP)) {
+        try {
+            const grammar = await loadGrammar(lang);
+            if (grammar) {
+                // Also verify the query compiles
+                getQuery(lang, grammar);
+                languages.push({ language: lang, available: true });
+            }
+            else {
+                languages.push({ language: lang, available: false, error: grammarLoadErrors.get(lang) ?? "grammar failed to load" });
+            }
+        }
+        catch (err) {
+            languages.push({
+                language: lang,
+                available: false,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
+    return {
+        available: languages.some(l => l.available),
+        languages,
+    };
+}
+/**
+ * Extract symbol metadata for code within a byte range.
+ * Stubbed for Phase 2 — returns empty array.
+ */
+export function extractSymbols(_content, _language, _startPos, _endPos) {
+    return [];
+}

package/dist/bench/bench.d.ts

@@ -0,0 +1,23 @@
+/**
+ * QMD Benchmark Harness
+ *
+ * Runs queries from a fixture file against multiple search backends
+ * and measures precision@k, recall, MRR, F1, and latency.
+ *
+ * Usage:
+ *   qmd bench <fixture.json> [--json] [--collection <name>]
+ *
+ * Backends tested:
+ *   - bm25: BM25 keyword search (searchLex)
+ *   - vector: Vector similarity search (searchVector)
+ *   - hybrid: BM25 + vector RRF fusion without reranking
+ *   - full: Full hybrid pipeline with LLM reranking
+ */
+import type { BenchmarkResult } from "./types.js";
+export declare function runBenchmark(fixturePath: string, options?: {
+    json?: boolean;
+    collection?: string;
+    backends?: string[];
+    dbPath?: string;
+    configPath?: string;
+}): Promise<BenchmarkResult>;

package/dist/bench/bench.js

@@ -0,0 +1,280 @@
+/**
+ * QMD Benchmark Harness
+ *
+ * Runs queries from a fixture file against multiple search backends
+ * and measures precision@k, recall, MRR, F1, and latency.
+ *
+ * Usage:
+ *   qmd bench <fixture.json> [--json] [--collection <name>]
+ *
+ * Backends tested:
+ *   - bm25: BM25 keyword search (searchLex)
+ *   - vector: Vector similarity search (searchVector)
+ *   - hybrid: BM25 + vector RRF fusion without reranking
+ *   - full: Full hybrid pipeline with LLM reranking
+ */
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { createStore, getDefaultDbPath, } from "../index.js";
+import { scoreResults } from "./score.js";
+function parseStructuredQuery(query) {
+    const lines = query.split("\n").map((line, idx) => ({
+        trimmed: line.trim(),
+        number: idx + 1,
+    })).filter(line => line.trimmed.length > 0);
+    if (lines.length === 0)
+        return undefined;
+    const prefixRe = /^(lex|vec|hyde):\s*/i;
+    const intentRe = /^intent:\s*/i;
+    const searches = [];
+    let intent;
+    for (const line of lines) {
+        if (intentRe.test(line.trimmed)) {
+            if (intent !== undefined) {
+                throw new Error(`Line ${line.number}: only one intent: line is allowed per benchmark query.`);
+            }
+            intent = line.trimmed.replace(intentRe, "").trim();
+            if (!intent) {
+                throw new Error(`Line ${line.number}: intent: must include text.`);
+            }
+            continue;
+        }
+        const match = line.trimmed.match(prefixRe);
+        if (match) {
+            const type = match[1].toLowerCase();
+            const text = line.trimmed.slice(match[0].length).trim();
+            if (!text) {
+                throw new Error(`Line ${line.number} (${type}:) must include text.`);
+            }
+            searches.push({ type, query: text, line: line.number });
+            continue;
+        }
+        if (lines.length === 1) {
+            return undefined;
+        }
+        throw new Error(`Line ${line.number} is missing a lex:/vec:/hyde:/intent: prefix.`);
+    }
+    if (intent && searches.length === 0) {
+        throw new Error("intent: cannot appear alone. Add at least one lex:, vec:, or hyde: line.");
+    }
+    return searches.length > 0 ? { searches, intent } : undefined;
+}
+function uniqueFiles(files, limit) {
+    const seen = new Set();
+    const out = [];
+    for (const file of files) {
+        if (seen.has(file))
+            continue;
+        seen.add(file);
+        out.push(file);
+        if (out.length >= limit)
+            break;
+    }
+    return out;
+}
+const BACKENDS = [
+    {
+        name: "bm25",
+        run: async (store, query, limit, collection) => {
+            const structured = parseStructuredQuery(query.query);
+            const lexQueries = structured?.searches.filter(q => q.type === "lex");
+            if (structured) {
+                const files = [];
+                for (const lex of lexQueries ?? []) {
+                    const results = await store.searchLex(lex.query, { limit, collection });
+                    files.push(...results.map((r) => r.filepath));
+                }
+                return uniqueFiles(files, limit);
+            }
+            const results = await store.searchLex(query.query, { limit, collection });
+            return results.map((r) => r.filepath);
+        },
+    },
+    {
+        name: "vector",
+        run: async (store, query, limit, collection) => {
+            const structured = parseStructuredQuery(query.query);
+            const vectorQueries = structured?.searches.filter(q => q.type === "vec" || q.type === "hyde");
+            if (structured) {
+                const files = [];
+                for (const vectorQuery of vectorQueries ?? []) {
+                    const results = await store.searchVector(vectorQuery.query, { limit, collection });
+                    files.push(...results.map((r) => r.filepath));
+                }
+                return uniqueFiles(files, limit);
+            }
+            const results = await store.searchVector(query.query, { limit, collection });
+            return results.map((r) => r.filepath);
+        },
+    },
+    {
+        name: "hybrid",
+        run: async (store, query, limit, collection) => {
+            const structured = parseStructuredQuery(query.query);
+            const results = structured
+                ? await store.search({ queries: structured.searches, intent: structured.intent, limit, collection, rerank: false })
+                : await store.search({ query: query.query, limit, collection, rerank: false });
+            return results.map((r) => r.file);
+        },
+    },
+    {
+        name: "full",
+        run: async (store, query, limit, collection) => {
+            const structured = parseStructuredQuery(query.query);
+            const results = structured
+                ? await store.search({ queries: structured.searches, intent: structured.intent, limit, collection, rerank: true })
+                : await store.search({ query: query.query, limit, collection, rerank: true });
+            return results.map((r) => r.file);
+        },
+    },
+];
+async function runQuery(store, backend, query, collection) {
+    const limit = Math.max(query.expected_in_top_k, 10);
+    const start = Date.now();
+    let resultFiles;
+    try {
+        resultFiles = await backend.run(store, query, limit, collection);
+    }
+    catch {
+        // Backend may not be available (e.g., no embeddings for vector search)
+        return {
+            precision_at_k: 0,
+            recall: 0,
+            recall_at_1: 0,
+            recall_at_3: 0,
+            recall_at_5: 0,
+            mrr: 0,
+            f1: 0,
+            hits_at_k: 0,
+            total_expected: query.expected_files.length,
+            latency_ms: Date.now() - start,
+            top_files: [],
+            matched_files: [],
+            unmatched_expected_files: query.expected_files,
+        };
+    }
+    const latency_ms = Date.now() - start;
+    const scores = scoreResults(resultFiles, query.expected_files, query.expected_in_top_k);
+    return {
+        ...scores,
+        total_expected: query.expected_files.length,
+        latency_ms,
+        top_files: resultFiles.slice(0, 10),
+    };
+}
+function formatTable(results) {
+    const lines = [];
+    const pad = (s, n) => s.slice(0, n).padEnd(n);
+    const num = (n) => n.toFixed(2).padStart(5);
+    lines.push(`${pad("Query", 25)} ${pad("Backend", 8)} ${pad("P@k", 6)} ${pad("R@1", 6)} ${pad("R@3", 6)} ${pad("R@5", 6)} ${pad("MRR", 6)} ${pad("F1", 6)} ${pad("ms", 8)}`);
+    lines.push("-".repeat(88));
+    for (const r of results) {
+        for (const [backend, br] of Object.entries(r.backends)) {
+            lines.push(`${pad(r.id, 25)} ${pad(backend, 8)} ${num(br.precision_at_k)} ${num(br.recall_at_1)} ${num(br.recall_at_3)} ${num(br.recall_at_5)} ${num(br.mrr)} ${num(br.f1)} ${String(Math.round(br.latency_ms)).padStart(7)}ms`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+function computeSummary(results) {
+    const summary = {};
+    // Collect all backend names
+    const backendNames = new Set();
+    for (const r of results) {
+        for (const name of Object.keys(r.backends)) {
+            backendNames.add(name);
+        }
+    }
+    for (const name of Array.from(backendNames)) {
+        let totalP = 0, totalR = 0, totalR1 = 0, totalR3 = 0, totalR5 = 0, totalMrr = 0, totalF1 = 0, totalLat = 0, count = 0;
+        for (const r of results) {
+            const br = r.backends[name];
+            if (!br)
+                continue;
+            totalP += br.precision_at_k;
+            totalR += br.recall;
+            totalR1 += br.recall_at_1;
+            totalR3 += br.recall_at_3;
+            totalR5 += br.recall_at_5;
+            totalMrr += br.mrr;
+            totalF1 += br.f1;
+            totalLat += br.latency_ms;
+            count++;
+        }
+        if (count > 0) {
+            summary[name] = {
+                avg_precision: totalP / count,
+                avg_recall: totalR / count,
+                avg_recall_at_1: totalR1 / count,
+                avg_recall_at_3: totalR3 / count,
+                avg_recall_at_5: totalR5 / count,
+                avg_mrr: totalMrr / count,
+                avg_f1: totalF1 / count,
+                avg_latency_ms: totalLat / count,
+            };
+        }
+    }
+    return summary;
+}
+export async function runBenchmark(fixturePath, options = {}) {
+    // Load fixture
+    const raw = readFileSync(resolve(fixturePath), "utf-8");
+    const fixture = JSON.parse(raw);
+    if (!fixture.queries || !Array.isArray(fixture.queries)) {
+        throw new Error("Invalid fixture: missing 'queries' array");
+    }
+    // Open store
+    const store = await createStore({
+        dbPath: options.dbPath ?? getDefaultDbPath(),
+        ...(options.configPath ? { configPath: options.configPath } : {}),
+    });
+    // Filter backends if requested
+    const activeBackends = options.backends
+        ? BACKENDS.filter(b => options.backends.includes(b.name))
+        : BACKENDS;
+    const collection = options.collection ?? fixture.collection;
+    // Run queries
+    const results = [];
+    for (const query of fixture.queries) {
+        const backends = {};
+        for (const backend of activeBackends) {
+            if (!options.json) {
+                process.stderr.write(`  ${query.id} / ${backend.name}...`);
+            }
+            backends[backend.name] = await runQuery(store, backend, query, collection);
+            if (!options.json) {
+                process.stderr.write(` ${Math.round(backends[backend.name].latency_ms)}ms\n`);
+            }
+        }
+        results.push({
+            id: query.id,
+            query: query.query,
+            type: query.type,
+            backends,
+        });
+    }
+    await store.close();
+    const summary = computeSummary(results);
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "").slice(0, 15);
+    const benchResult = {
+        timestamp,
+        fixture: fixturePath,
+        results,
+        summary,
+    };
+    // Output
+    if (options.json) {
+        console.log(JSON.stringify(benchResult, null, 2));
+    }
+    else {
+        console.log("\n" + formatTable(results));
+        console.log("Summary:");
+        console.log("-".repeat(70));
+        const pad = (s, n) => s.slice(0, n).padEnd(n);
+        const num = (n) => n.toFixed(3).padStart(6);
+        for (const [name, s] of Object.entries(summary)) {
+            console.log(`  ${pad(name, 8)} P@k=${num(s.avg_precision)} R@1=${num(s.avg_recall_at_1)} R@3=${num(s.avg_recall_at_3)} R@5=${num(s.avg_recall_at_5)} MRR=${num(s.avg_mrr)} F1=${num(s.avg_f1)} Avg=${Math.round(s.avg_latency_ms)}ms`);
+        }
+    }
+    return benchResult;
+}

package/dist/bench/score.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Scoring functions for the QMD benchmark harness.
+ *
+ * Computes precision@k, recall, MRR, and F1 for search results
+ * against ground-truth expected files.
+ */
+/**
+ * Normalize a file path for comparison.
+ * Strips qmd:// prefix, lowercases, removes leading/trailing slashes.
+ */
+export declare function normalizePath(p: string): string;
+/**
+ * Check if two paths refer to the same file.
+ * Handles different path formats by comparing normalized suffixes.
+ */
+export declare function pathsMatch(result: string, expected: string): boolean;
+type ScoreMetrics = {
+    precision_at_k: number;
+    recall: number;
+    recall_at_1: number;
+    recall_at_3: number;
+    recall_at_5: number;
+    mrr: number;
+    f1: number;
+    hits_at_k: number;
+    matched_files: string[];
+    unmatched_expected_files: string[];
+};
+/**
+ * Score a set of search results against expected files.
+ */
+export declare function scoreResults(resultFiles: string[], expectedFiles: string[], topK: number): ScoreMetrics;
+export {};