@joycodetech/qmd-ja 2.5.3

package/bin/qmd

@@ -0,0 +1,162 @@
+#!/usr/bin/env node
+// 2>/dev/null; if command -v node >/dev/null 2>&1; then exec node "$0" "$@"; else exec bun "$0" "$@"; fi
+// Cross-platform launcher for qmd.
+//
+// Previously this was a POSIX shell script with `#!/bin/sh`, which meant npm
+// on Windows generated shims that tried to route through `/bin/sh` — a path
+// that doesn't exist on Windows, so `qmd` failed immediately after a global
+// install. Rewriting the launcher in Node.js lets npm generate native
+// cmd/ps1/sh shims that invoke `node` directly on every platform.
+
+import { spawn, spawnSync } from "node:child_process";
+import { existsSync, realpathSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+// Resolve symlinks so global installs (npm link / npm install -g) can find
+// the actual package directory instead of the global bin directory.
+const self = realpathSync(fileURLToPath(import.meta.url));
+const pkgDir = resolve(dirname(self), "..");
+const jsEntry = resolve(pkgDir, "dist/cli/qmd.js");
+const tsEntry = resolve(pkgDir, "src/cli/qmd.ts");
+
+// MCP stdio reserves stdout exclusively for JSON-RPC frames. node-llama-cpp
+// / llama.cpp / ggml can write native logs directly to stdout before JS-level
+// log handlers are attached, so seed the native quiet env before Node/Bun imports
+// the CLI and its LLM modules. Preserve explicit user values when provided.
+if (process.argv[2] === "mcp") {
+  process.env.LLAMA_LOG_LEVEL = process.env.LLAMA_LOG_LEVEL || "error";
+  process.env.GGML_LOG_LEVEL = process.env.GGML_LOG_LEVEL || "error";
+  process.env.GGML_BACKEND_SILENT = process.env.GGML_BACKEND_SILENT || "1";
+}
+
+// libggml-metal on macOS uses "residency sets" to keep allocated model memory
+// resident across inference requests (180-second keep_alive timer). The
+// process-static device destructor that runs during libc exit() asserts the
+// residency set is empty (ggml-org/llama.cpp#22593); the keep_alive hasn't
+// expired by exit, so the assertion fails and ggml_abort dumps a multi-kB
+// stack trace to stderr even when the user-visible results were already
+// emitted correctly. No JS-side dispose can prevent it because the static
+// destructor runs in __cxa_finalize_ranges, after every JS-reachable cleanup.
+//
+// For QMD's short-lived CLI workflow, residency sets provide no observable
+// performance benefit (subsequent requests don't reuse the warm mapping —
+// measured: identical wall time with and without on M3 Pro), so disable them
+// by default on darwin. The env var must be set BEFORE the native llama.cpp
+// binding loads, which is why it lives here in the launcher rather than in
+// the JS entry point. Opt back in with QMD_METAL_KEEP_RESIDENCY=1 if you
+// run long-lived qmd processes (the MCP daemon may benefit on hot reload)
+// or are triaging an upstream Metal teardown fix.
+if (process.platform === "darwin" && process.env.QMD_METAL_KEEP_RESIDENCY !== "1") {
+  process.env.GGML_METAL_NO_RESIDENCY = process.env.GGML_METAL_NO_RESIDENCY || "1";
+}
+
+function hasBun() {
+  try {
+    const res = spawnSync("bun", ["--version"], { stdio: "ignore", shell: process.platform === "win32" });
+    return res.status === 0;
+  } catch {
+    return false;
+  }
+}
+
+// In published packages, bin/qmd must run dist/. In a git checkout, however,
+// dist/ is often ignored and can be stale after git reset or branch switches.
+// Prefer source mode only for checkouts so ./bin/qmd reflects the checked-out
+// source without changing packaged/runtime behavior.
+//
+// Critical: source-mode detection must NOT trigger when a package manager
+// installed us. `pnpm install -g .` (and `npm install -g .`) copy the entire
+// working tree — including .git/, bun.lock, package-lock.json, src/, and even
+// node_modules/ — into <prefix>/node_modules/@tobilu/qmd/, so .git and a
+// lockfile being present is not a reliable "this is a working tree" signal.
+// What IS reliable: a package-manager install always lands the package
+// directory inside a `node_modules/` segment; a bare working-tree checkout
+// (with `bun link` or a direct path invocation) does not. Gate source mode
+// on that. Allow QMD_SOURCE_MODE=1 / =0 as an explicit override for the
+// rare case where the heuristic disagrees with the user.
+const sourceOverride = process.env.QMD_SOURCE_MODE;
+const looksInstalled = pkgDir.split("/").includes("node_modules");
+const sourceAllowed = sourceOverride === "1"
+  || (sourceOverride !== "0" && !looksInstalled);
+
+let useSourceMode = false;
+let sourceRunner = null;
+let sourceArgs = [];
+
+if (sourceAllowed && existsSync(resolve(pkgDir, ".git")) && existsSync(tsEntry)) {
+  // Lockfile-driven runner selection — mirror the dist-mode logic below so
+  // source mode picks the same runtime the user's deps were installed for.
+  // package-lock.json wins over bun.lock when both are present: pnpm/npm
+  // installs ship the Node-ABI native modules (better-sqlite3, sqlite-vec),
+  // and running Bun against them produces ABI mismatches. This also fixes
+  // pnpm-global installs, which copy the whole working tree — including .git
+  // and bun.lock — into the install dir and used to route through Bun even
+  // when the user installed via npm/pnpm.
+  const hasNpmLock = existsSync(resolve(pkgDir, "package-lock.json"));
+  const hasBunLock = existsSync(resolve(pkgDir, "bun.lock")) || existsSync(resolve(pkgDir, "bun.lockb"));
+  const tsxEntry = resolve(pkgDir, "node_modules/tsx/dist/cli.mjs");
+  const tsxAvailable = existsSync(tsxEntry);
+
+  if (hasNpmLock && tsxAvailable) {
+    useSourceMode = true;
+    sourceRunner = "node";
+    sourceArgs = [tsxEntry, tsEntry, ...process.argv.slice(2)];
+  } else if (hasBunLock && hasBun()) {
+    useSourceMode = true;
+    sourceRunner = "bun";
+    sourceArgs = [tsEntry, ...process.argv.slice(2)];
+  } else if (tsxAvailable) {
+    useSourceMode = true;
+    sourceRunner = "node";
+    sourceArgs = [tsxEntry, tsEntry, ...process.argv.slice(2)];
+  }
+}
+
+if (!useSourceMode && !existsSync(jsEntry)) {
+  console.error(`qmd is not built: missing ${jsEntry}`);
+  console.error("Run: bun install && bun run build");
+  console.error("Or:  npm install && npm run build");
+  console.error("After building, run: qmd doctor");
+  process.exit(1);
+}
+
+// Detect the package manager that installed dependencies by checking lockfiles.
+// $BUN_INSTALL is intentionally NOT checked — it only indicates that bun exists
+// on the system, not that it was used to install this package (see #361).
+//
+// package-lock.json takes priority: if it exists, npm installed the native
+// modules for Node. The repo ships bun.lock, so without this check, source
+// builds that use npm would be incorrectly routed to bun, causing ABI
+// mismatches with better-sqlite3 / sqlite-vec (see #381).
+let runnerName = "node";
+if (existsSync(resolve(pkgDir, "package-lock.json"))) {
+  runnerName = "node";
+} else if (existsSync(resolve(pkgDir, "bun.lock")) || existsSync(resolve(pkgDir, "bun.lockb"))) {
+  runnerName = "bun";
+} else {
+  runnerName = "node";
+}
+
+const runner = useSourceMode ? sourceRunner : (runnerName === "node" ? "node" : "bun");
+const args = useSourceMode ? sourceArgs : [jsEntry, ...process.argv.slice(2)];
+const needsShell = (runner === "bun") && process.platform === "win32";
+
+const child = spawn(runner, args, {
+  stdio: "inherit",
+  shell: needsShell,
+});
+
+child.on("exit", (code, signal) => {
+  if (signal) {
+    process.kill(process.pid, signal);
+  } else {
+    process.exit(code ?? 0);
+  }
+});
+
+child.on("error", (err) => {
+  const name = useSourceMode ? sourceRunner : runnerName;
+  console.error(`qmd: failed to launch ${name}: ${err.message}`);
+  process.exit(1);
+});

package/dist/ast.d.ts

@@ -0,0 +1,65 @@
+/**
+ * AST-aware chunking support via web-tree-sitter.
+ *
+ * Provides language detection, AST break point extraction for supported
+ * code file types, and a stub for future symbol extraction.
+ *
+ * All functions degrade gracefully: parse failures or unsupported languages
+ * return empty arrays, falling back to regex-only chunking.
+ *
+ * ## Dependency Note
+ *
+ * Grammar packages (tree-sitter-typescript, etc.) are listed as
+ * optionalDependencies with pinned versions. They ship native prebuilds
+ * and source files (~72 MB total) but QMD only uses the .wasm files
+ * (~5 MB). If install size becomes a concern, the .wasm files can be
+ * bundled directly in the repo (e.g. assets/grammars/) and resolved
+ * via import.meta.url instead of require.resolve(), eliminating the
+ * grammar packages entirely.
+ */
+import type { BreakPoint } from "./store.js";
+export type SupportedLanguage = "typescript" | "tsx" | "javascript" | "python" | "go" | "rust";
+/**
+ * Detect language from file path extension.
+ * Returns null for unsupported or unknown extensions (including .md).
+ */
+export declare function detectLanguage(filepath: string): SupportedLanguage | null;
+export declare function formatGrammarLoadError(language: SupportedLanguage, err: unknown): string;
+/**
+ * Parse a source file and return break points at AST node boundaries.
+ *
+ * Returns an empty array for unsupported languages, parse failures,
+ * or grammar loading failures. Never throws.
+ *
+ * @param content - The file content to parse.
+ * @param filepath - The file path (used for language detection).
+ * @returns Array of BreakPoint objects suitable for merging with regex break points.
+ */
+export declare function getASTBreakPoints(content: string, filepath: string): Promise<BreakPoint[]>;
+/**
+ * Check which tree-sitter grammars are available.
+ * Returns a status object for each supported language.
+ */
+export declare function getASTStatus(): Promise<{
+    available: boolean;
+    languages: {
+        language: SupportedLanguage;
+        available: boolean;
+        error?: string;
+    }[];
+}>;
+/**
+ * Metadata about a code symbol within a chunk.
+ * Stubbed for Phase 2 — always returns empty array in Phase 1.
+ */
+export interface SymbolInfo {
+    name: string;
+    kind: string;
+    signature?: string;
+    line: number;
+}
+/**
+ * Extract symbol metadata for code within a byte range.
+ * Stubbed for Phase 2 — returns empty array.
+ */
+export declare function extractSymbols(_content: string, _language: string, _startPos: number, _endPos: number): SymbolInfo[];

package/dist/ast.js

@@ -0,0 +1,334 @@
+/**
+ * AST-aware chunking support via web-tree-sitter.
+ *
+ * Provides language detection, AST break point extraction for supported
+ * code file types, and a stub for future symbol extraction.
+ *
+ * All functions degrade gracefully: parse failures or unsupported languages
+ * return empty arrays, falling back to regex-only chunking.
+ *
+ * ## Dependency Note
+ *
+ * Grammar packages (tree-sitter-typescript, etc.) are listed as
+ * optionalDependencies with pinned versions. They ship native prebuilds
+ * and source files (~72 MB total) but QMD only uses the .wasm files
+ * (~5 MB). If install size becomes a concern, the .wasm files can be
+ * bundled directly in the repo (e.g. assets/grammars/) and resolved
+ * via import.meta.url instead of require.resolve(), eliminating the
+ * grammar packages entirely.
+ */
+import { createRequire } from "node:module";
+import { extname } from "node:path";
+const EXTENSION_MAP = {
+    ".ts": "typescript",
+    ".tsx": "tsx",
+    ".js": "javascript",
+    ".jsx": "tsx",
+    ".mts": "typescript",
+    ".cts": "typescript",
+    ".mjs": "javascript",
+    ".cjs": "javascript",
+    ".py": "python",
+    ".go": "go",
+    ".rs": "rust",
+};
+/**
+ * Detect language from file path extension.
+ * Returns null for unsupported or unknown extensions (including .md).
+ */
+export function detectLanguage(filepath) {
+    const ext = extname(filepath).toLowerCase();
+    return EXTENSION_MAP[ext] ?? null;
+}
+// =============================================================================
+// Grammar Resolution
+// =============================================================================
+/**
+ * Maps language to the npm package and wasm filename for the grammar.
+ */
+const GRAMMAR_MAP = {
+    typescript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm", version: "0.23.2" },
+    tsx: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-tsx.wasm", version: "0.23.2" },
+    javascript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm", version: "0.23.2" },
+    python: { pkg: "tree-sitter-python", wasm: "tree-sitter-python.wasm", version: "0.23.4" },
+    go: { pkg: "tree-sitter-go", wasm: "tree-sitter-go.wasm", version: "0.23.4" },
+    rust: { pkg: "tree-sitter-rust", wasm: "tree-sitter-rust.wasm", version: "0.24.0" },
+};
+export function formatGrammarLoadError(language, err) {
+    const grammar = GRAMMAR_MAP[language];
+    const detail = err instanceof Error ? err.message : String(err);
+    return `${grammar.pkg}/${grammar.wasm} failed to load (${detail}); falling back to regex chunking. ` +
+        `Repair a broken global install with: bun add ${grammar.pkg}@${grammar.version}`;
+}
+// =============================================================================
+// Per-Language Query Definitions
+// =============================================================================
+/**
+ * Tree-sitter S-expression queries for each language.
+ * Each capture name maps to a break point score via SCORE_MAP.
+ *
+ * For TypeScript/JavaScript, we match export_statement wrappers to get the
+ * correct start position (before `export`), plus bare declarations for
+ * non-exported code.
+ */
+const LANGUAGE_QUERIES = {
+    typescript: `
+    (export_statement) @export
+    (class_declaration) @class
+    (function_declaration) @func
+    (method_definition) @method
+    (interface_declaration) @iface
+    (type_alias_declaration) @type
+    (enum_declaration) @enum
+    (import_statement) @import
+    (lexical_declaration (variable_declarator value: (arrow_function))) @func
+    (lexical_declaration (variable_declarator value: (function_expression))) @func
+  `,
+    tsx: `
+    (export_statement) @export
+    (class_declaration) @class
+    (function_declaration) @func
+    (method_definition) @method
+    (interface_declaration) @iface
+    (type_alias_declaration) @type
+    (enum_declaration) @enum
+    (import_statement) @import
+    (lexical_declaration (variable_declarator value: (arrow_function))) @func
+    (lexical_declaration (variable_declarator value: (function_expression))) @func
+  `,
+    javascript: `
+    (export_statement) @export
+    (class_declaration) @class
+    (function_declaration) @func
+    (method_definition) @method
+    (import_statement) @import
+    (lexical_declaration (variable_declarator value: (arrow_function))) @func
+    (lexical_declaration (variable_declarator value: (function_expression))) @func
+  `,
+    python: `
+    (class_definition) @class
+    (function_definition) @func
+    (decorated_definition) @decorated
+    (import_statement) @import
+    (import_from_statement) @import
+  `,
+    go: `
+    (type_declaration) @type
+    (function_declaration) @func
+    (method_declaration) @method
+    (import_declaration) @import
+  `,
+    rust: `
+    (struct_item) @struct
+    (impl_item) @impl
+    (function_item) @func
+    (trait_item) @trait
+    (enum_item) @enum
+    (use_declaration) @import
+    (type_item) @type
+    (mod_item) @mod
+  `,
+};
+/**
+ * Score mapping from capture names to break point scores.
+ * Aligned with the markdown BREAK_PATTERNS scale (h1=100, h2=90, etc.)
+ * so findBestCutoff() decay works unchanged.
+ */
+const SCORE_MAP = {
+    class: 100,
+    iface: 100,
+    struct: 100,
+    trait: 100,
+    impl: 100,
+    mod: 100,
+    export: 90,
+    func: 90,
+    method: 90,
+    decorated: 90,
+    type: 80,
+    enum: 80,
+    import: 60,
+};
+// =============================================================================
+// Parser Caching & Initialization
+// =============================================================================
+let ParserClass = null;
+let LanguageClass = null;
+let QueryClass = null;
+let initPromise = null;
+/** Languages that have already failed to load — warn only once per process. */
+const failedLanguages = new Set();
+/** Last grammar load error by language, for status output. */
+const grammarLoadErrors = new Map();
+/** Cached grammar load promises. */
+const grammarCache = new Map();
+/** Cached compiled queries per language. */
+const queryCache = new Map();
+/**
+ * Initialize web-tree-sitter. Called once and cached.
+ */
+async function ensureInit() {
+    if (!initPromise) {
+        initPromise = (async () => {
+            const mod = await import("web-tree-sitter");
+            ParserClass = mod.Parser;
+            LanguageClass = mod.Language;
+            QueryClass = mod.Query;
+            await ParserClass.init();
+        })();
+    }
+    return initPromise;
+}
+/**
+ * Resolve the filesystem path to a grammar .wasm file.
+ * Uses createRequire to resolve from installed dependency packages.
+ */
+function resolveGrammarPath(language) {
+    const { pkg, wasm } = GRAMMAR_MAP[language];
+    const require = createRequire(import.meta.url);
+    return require.resolve(`${pkg}/${wasm}`);
+}
+/**
+ * Load and cache a grammar for the given language.
+ * Returns null on failure (logs once per language).
+ */
+async function loadGrammar(language) {
+    if (failedLanguages.has(language))
+        return null;
+    const wasmKey = GRAMMAR_MAP[language].wasm;
+    if (!grammarCache.has(wasmKey)) {
+        grammarCache.set(wasmKey, (async () => {
+            const path = resolveGrammarPath(language);
+            return LanguageClass.load(path);
+        })());
+    }
+    try {
+        return await grammarCache.get(wasmKey);
+    }
+    catch (err) {
+        failedLanguages.add(language);
+        grammarCache.delete(wasmKey);
+        const message = formatGrammarLoadError(language, err);
+        grammarLoadErrors.set(language, message);
+        console.warn(`[qmd] AST grammar unavailable for ${language}: ${message}`);
+        return null;
+    }
+}
+/**
+ * Get or create a compiled query for the given language.
+ */
+function getQuery(language, grammar) {
+    if (!queryCache.has(language)) {
+        const source = LANGUAGE_QUERIES[language];
+        const query = new QueryClass(grammar, source);
+        queryCache.set(language, query);
+    }
+    return queryCache.get(language);
+}
+// =============================================================================
+// AST Break Point Extraction
+// =============================================================================
+/**
+ * Parse a source file and return break points at AST node boundaries.
+ *
+ * Returns an empty array for unsupported languages, parse failures,
+ * or grammar loading failures. Never throws.
+ *
+ * @param content - The file content to parse.
+ * @param filepath - The file path (used for language detection).
+ * @returns Array of BreakPoint objects suitable for merging with regex break points.
+ */
+export async function getASTBreakPoints(content, filepath) {
+    const language = detectLanguage(filepath);
+    if (!language)
+        return [];
+    try {
+        await ensureInit();
+        const grammar = await loadGrammar(language);
+        if (!grammar)
+            return [];
+        const parser = new ParserClass();
+        parser.setLanguage(grammar);
+        const tree = parser.parse(content);
+        if (!tree) {
+            parser.delete();
+            return [];
+        }
+        const query = getQuery(language, grammar);
+        const captures = query.captures(tree.rootNode);
+        // Deduplicate: at each byte position, keep the highest-scoring capture.
+        // This handles cases like export_statement wrapping a class_declaration
+        // at different offsets — we want the outermost (earliest) position.
+        const seen = new Map();
+        for (const cap of captures) {
+            const pos = cap.node.startIndex;
+            const score = SCORE_MAP[cap.name] ?? 20;
+            const type = `ast:${cap.name}`;
+            const existing = seen.get(pos);
+            if (!existing || score > existing.score) {
+                seen.set(pos, { pos, score, type });
+            }
+        }
+        tree.delete();
+        parser.delete();
+        return Array.from(seen.values()).sort((a, b) => a.pos - b.pos);
+    }
+    catch (err) {
+        console.warn(`[qmd] AST parse failed for ${filepath}, falling back to regex: ${err instanceof Error ? err.message : err}`);
+        return [];
+    }
+}
+// =============================================================================
+// Health / Status
+// =============================================================================
+/**
+ * Check which tree-sitter grammars are available.
+ * Returns a status object for each supported language.
+ */
+export async function getASTStatus() {
+    const languages = [];
+    try {
+        await ensureInit();
+    }
+    catch (err) {
+        return {
+            available: false,
+            languages: Object.keys(GRAMMAR_MAP).map(lang => ({
+                language: lang,
+                available: false,
+                error: `web-tree-sitter init failed: ${err instanceof Error ? err.message : err}`,
+            })),
+        };
+    }
+    for (const lang of Object.keys(GRAMMAR_MAP)) {
+        try {
+            const grammar = await loadGrammar(lang);
+            if (grammar) {
+                // Also verify the query compiles
+                getQuery(lang, grammar);
+                languages.push({ language: lang, available: true });
+            }
+            else {
+                languages.push({ language: lang, available: false, error: grammarLoadErrors.get(lang) ?? "grammar failed to load" });
+            }
+        }
+        catch (err) {
+            languages.push({
+                language: lang,
+                available: false,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
+    return {
+        available: languages.some(l => l.available),
+        languages,
+    };
+}
+/**
+ * Extract symbol metadata for code within a byte range.
+ * Stubbed for Phase 2 — returns empty array.
+ */
+export function extractSymbols(_content, _language, _startPos, _endPos) {
+    return [];
+}

package/dist/bench/bench.d.ts

@@ -0,0 +1,23 @@
+/**
+ * QMD Benchmark Harness
+ *
+ * Runs queries from a fixture file against multiple search backends
+ * and measures precision@k, recall, MRR, F1, and latency.
+ *
+ * Usage:
+ *   qmd bench <fixture.json> [--json] [--collection <name>]
+ *
+ * Backends tested:
+ *   - bm25: BM25 keyword search (searchLex)
+ *   - vector: Vector similarity search (searchVector)
+ *   - hybrid: BM25 + vector RRF fusion without reranking
+ *   - full: Full hybrid pipeline with LLM reranking
+ */
+import type { BenchmarkResult } from "./types.js";
+export declare function runBenchmark(fixturePath: string, options?: {
+    json?: boolean;
+    collection?: string;
+    backends?: string[];
+    dbPath?: string;
+    configPath?: string;
+}): Promise<BenchmarkResult>;