@tobilu/qmd 2.0.1 → 2.5.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tobilu/qmd",
-  "version": "2.0.1",
+  "version": "2.5.1",
   "description": "Query Markup Documents - On-device hybrid search for markdown files with BM25, vector search, and LLM reranking",
   "type": "module",
   "main": "dist/index.js",
@@ -17,13 +17,23 @@
   "files": [
     "bin/",
     "dist/",
+    "skills/",
+    "scripts/build.mjs",
+    "scripts/check-package-grammars.mjs",
+    "scripts/package-smoke.mjs",
+    "scripts/test-all.mjs",
     "LICENSE",
     "CHANGELOG.md"
   ],
   "scripts": {
     "prepare": "[ -d .git ] && ./scripts/install-hooks.sh || true",
-    "build": "tsc -p tsconfig.build.json && printf '#!/usr/bin/env node\n' | cat - dist/cli/qmd.js > dist/cli/qmd.tmp && mv dist/cli/qmd.tmp dist/cli/qmd.js && chmod +x dist/cli/qmd.js",
-    "test": "vitest run --reporter=verbose test/",
+    "build": "node scripts/build.mjs",
+    "test": "node scripts/test-all.mjs",
+    "test:types": "node ./node_modules/typescript/bin/tsc -p tsconfig.build.json --noEmit",
+    "test:node": "node ./node_modules/vitest/vitest.mjs run --reporter=verbose --testTimeout 60000",
+    "test:bun": "bun test --timeout 60000 --preload ./src/test-preload.ts",
+    "test:unit": "CI=true node ./node_modules/vitest/vitest.mjs run --reporter=verbose --testTimeout 60000 test/ && CI=true bun test --timeout 60000 --preload ./src/test-preload.ts test/",
+    "test:package": "node scripts/package-smoke.mjs",
     "qmd": "tsx src/cli/qmd.ts",
     "index": "tsx src/cli/qmd.ts index",
     "vector": "tsx src/cli/qmd.ts vector",
@@ -31,7 +41,8 @@
     "vsearch": "tsx src/cli/qmd.ts vsearch",
     "rerank": "tsx src/cli/qmd.ts rerank",
     "inspector": "npx @modelcontextprotocol/inspector tsx src/cli/qmd.ts mcp",
-    "release": "./scripts/release.sh"
+    "release": "./scripts/release.sh",
+    "smoke:package-grammars": "node scripts/check-package-grammars.mjs"
   },
   "publishConfig": {
     "access": "public"
@@ -45,26 +56,43 @@
     "url": "https://github.com/tobi/qmd/issues"
   },
   "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.25.1",
-    "better-sqlite3": "^12.4.5",
-    "fast-glob": "^3.3.0",
-    "node-llama-cpp": "^3.17.1",
-    "picomatch": "^4.0.0",
-    "sqlite-vec": "^0.1.7-alpha.2",
-    "yaml": "^2.8.2",
-    "zod": "^4.2.1"
+    "@modelcontextprotocol/sdk": "1.29.0",
+    "better-sqlite3": "12.10.0",
+    "fast-glob": "3.3.3",
+    "node-llama-cpp": "3.18.1",
+    "picomatch": "4.0.4",
+    "sqlite-vec": "0.1.9",
+    "tree-sitter-go": "0.25.0",
+    "tree-sitter-python": "0.25.0",
+    "tree-sitter-rust": "0.24.0",
+    "tree-sitter-typescript": "0.23.2",
+    "web-tree-sitter": "0.26.8",
+    "yaml": "2.9.0",
+    "zod": "4.2.1"
   },
   "optionalDependencies": {
-    "sqlite-vec-darwin-arm64": "^0.1.7-alpha.2",
-    "sqlite-vec-darwin-x64": "^0.1.7-alpha.2",
-    "sqlite-vec-linux-arm64": "^0.1.7-alpha.2",
-    "sqlite-vec-linux-x64": "^0.1.7-alpha.2",
-    "sqlite-vec-windows-x64": "^0.1.7-alpha.2"
+    "sqlite-vec-darwin-arm64": "0.1.9",
+    "sqlite-vec-darwin-x64": "0.1.9",
+    "sqlite-vec-linux-arm64": "0.1.9",
+    "sqlite-vec-linux-x64": "0.1.9",
+    "sqlite-vec-windows-x64": "0.1.9"
   },
   "devDependencies": {
-    "@types/better-sqlite3": "^7.6.0",
-    "tsx": "^4.0.0",
-    "vitest": "^3.0.0"
+    "@types/better-sqlite3": "7.6.13",
+    "tsx": "4.21.0",
+    "vitest": "3.2.4"
+  },
+  "pnpm": {
+    "onlyBuiltDependencies": [
+      "better-sqlite3",
+      "esbuild",
+      "node-llama-cpp",
+      "tree-sitter-go",
+      "tree-sitter-javascript",
+      "tree-sitter-python",
+      "tree-sitter-rust",
+      "tree-sitter-typescript"
+    ]
   },
   "peerDependencies": {
     "typescript": "^5.9.3"

package/scripts/build.mjs

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { chmodSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = join(fileURLToPath(new URL("..", import.meta.url)));
+
+function run(command, args, options = {}) {
+  const result = spawnSync(command, args, {
+    cwd: root,
+    stdio: "inherit",
+    shell: process.platform === "win32",
+    ...options,
+  });
+  if (result.status !== 0) {
+    process.exit(result.status ?? 1);
+  }
+}
+
+run(process.execPath, [join(root, "node_modules", "typescript", "bin", "tsc"), "-p", "tsconfig.build.json"]);
+
+const cliPath = join(root, "dist", "cli", "qmd.js");
+const tmpPath = `${cliPath}.tmp`;
+const built = readFileSync(cliPath, "utf8");
+const withoutExistingShebang = built.startsWith("#!") ? built.slice(built.indexOf("\n") + 1) : built;
+writeFileSync(tmpPath, `#!/usr/bin/env node\n${withoutExistingShebang}`);
+renameSync(tmpPath, cliPath);
+chmodSync(cliPath, 0o755);

package/scripts/check-package-grammars.mjs

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+import { createRequire } from "node:module";
+
+const require = createRequire(import.meta.url);
+
+const grammars = [
+  "tree-sitter-typescript/tree-sitter-typescript.wasm",
+  "tree-sitter-typescript/tree-sitter-tsx.wasm",
+  "tree-sitter-python/tree-sitter-python.wasm",
+  "tree-sitter-go/tree-sitter-go.wasm",
+  "tree-sitter-rust/tree-sitter-rust.wasm",
+];
+
+let ok = true;
+for (const grammar of grammars) {
+  try {
+    const resolved = require.resolve(grammar);
+    console.log(`ok ${grammar} -> ${resolved}`);
+  } catch (err) {
+    ok = false;
+    console.error(`missing ${grammar}`);
+    console.error(err instanceof Error ? err.message : String(err));
+  }
+}
+
+if (!ok) {
+  console.error("\nAST grammar package smoke check failed. Run `bun install` locally or repair a broken global install with the matching `bun add tree-sitter-...@<version>` command shown by `qmd status`.");
+  process.exit(1);
+}

package/scripts/package-smoke.mjs

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = fileURLToPath(new URL("..", import.meta.url));
+const pkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8"));
+
+function run(label, command, args, options = {}) {
+  console.log(`==> ${label}`);
+  const { quiet, ...spawnOptions } = options;
+  const result = spawnSync(command, args, {
+    cwd: root,
+    stdio: quiet ? "pipe" : "inherit",
+    shell: process.platform === "win32",
+    ...spawnOptions,
+  });
+  if (result.status !== 0) {
+    console.error(`Package smoke failed: ${label}`);
+    if (quiet) {
+      if (result.stdout) process.stderr.write(result.stdout);
+      if (result.stderr) process.stderr.write(result.stderr);
+    }
+    process.exit(result.status ?? 1);
+  }
+}
+
+function assertPath(path, label = path) {
+  const full = join(root, path);
+  if (!existsSync(full)) {
+    console.error(`Package smoke failed: missing ${label} (${path})`);
+    process.exit(1);
+  }
+  return full;
+}
+
+run("build compiled package", process.execPath, ["scripts/build.mjs"]);
+run("AST grammar runtime packages", process.execPath, ["scripts/check-package-grammars.mjs"]);
+
+for (const entry of pkg.files ?? []) {
+  assertPath(entry.replace(/\/$/, ""), `package.json files[] entry ${entry}`);
+}
+
+for (const [name, binPath] of Object.entries(pkg.bin ?? {})) {
+  const full = assertPath(binPath, `bin ${name}`);
+  const mode = statSync(full).mode;
+  if ((mode & 0o111) === 0) {
+    console.error(`Package smoke failed: bin ${name} is not executable (${binPath})`);
+    process.exit(1);
+  }
+}
+
+assertPath("dist/index.js", "compiled main export");
+assertPath("dist/index.d.ts", "compiled type export");
+assertPath("dist/cli/qmd.js", "compiled CLI");
+
+run("compiled CLI under Node", process.execPath, ["dist/cli/qmd.js", "--help"], { quiet: true });
+run("package wrapper", "sh", ["bin/qmd", "--help"], { quiet: true });
+
+if (process.env.QMD_SKIP_BUN_SMOKE === "1") {
+  console.log("==> compiled CLI under Bun (skipped by QMD_SKIP_BUN_SMOKE=1)");
+} else {
+  run("compiled CLI under Bun", "bun", ["dist/cli/qmd.js", "--help"], { quiet: true });
+}

package/scripts/test-all.mjs

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = fileURLToPath(new URL("..", import.meta.url));
+
+function run(label, command, args, options = {}) {
+  console.log(`==> ${label}`);
+  const { env: extraEnv, ...spawnOptions } = options;
+  const result = spawnSync(command, args, {
+    cwd: root,
+    stdio: "inherit",
+    shell: process.platform === "win32",
+    env: { ...process.env, ...(extraEnv ?? {}) },
+    ...spawnOptions,
+  });
+  if (result.status !== 0) {
+    console.error(`Test task failed: ${label}`);
+    process.exit(result.status ?? 1);
+  }
+}
+
+run("TypeScript build typecheck", process.execPath, [join(root, "node_modules", "typescript", "bin", "tsc"), "-p", "tsconfig.build.json", "--noEmit"]);
+run("Vitest suite under Node", process.execPath, [join(root, "node_modules", "vitest", "vitest.mjs"), "run", "--reporter=verbose", "--testTimeout", "60000", "test/"], { env: { CI: "true" } });
+run("Bun test suite", "bun", ["test", "--timeout", "60000", "--preload", "./src/test-preload.ts", "test/"], { env: { CI: "true" } });
+run("Package smoke", process.execPath, ["scripts/package-smoke.mjs"]);

package/skills/qmd/SKILL.md

@@ -0,0 +1,203 @@
+---
+name: qmd
+description: Search local markdown knowledge bases, notes, docs, and wikis with QMD. Use when users ask to find notes, retrieve documents, inspect a wiki, answer from indexed markdown, or set up QMD access.
+license: MIT
+compatibility: Requires qmd CLI or MCP server. Install via `npm install -g @tobilu/qmd`.
+metadata:
+  author: tobi
+  version: "2.1.0"
+allowed-tools: Bash(qmd:*), mcp__qmd__*
+---
+
+# QMD - Query Markdown Documents
+
+## How search works
+
+QMD searches local markdown collections: notes, docs, wikis, transcripts, and
+project knowledge bases. Use it before web search when the answer may already be
+in indexed local files.
+
+The workflow is always:
+
+1. Search for candidate documents.
+2. Retrieve the full source with `qmd get` or `qmd multi-get`.
+3. Answer from retrieved text, citing paths or docids.
+
+Do not answer from snippets alone when the user needs facts, decisions, quotes,
+or nuance. Snippets are only leads.
+
+Typical loop:
+
+```bash
+qmd search "merchant reality support interviews" -n 5
+# leads: #abc123 concepts/customer-proximity.md; #def432 sources/merchant-call.md
+qmd multi-get "#abc123,#def432" --md
+```
+
+For harder searches, use `qmd query` structured queries with `intent:`, `lex:`,
+`vec:`, and `hyde:` fields.
+
+When reporting what you retrieved, a compact note is enough; do not paste whole
+files unless needed:
+
+```text
+Retrieved:
+- #abc123 concepts/customer-proximity.md
+- #def432 sources/merchant-call.md
+```
+
+## Pick the right search mode
+
+Use **BM25 lexical search** when you know exact words, titles, names, code
+symbols, or rare phrases:
+
+```bash
+qmd search "cockpit OKR Goodhart" -n 10
+qmd search '"AI Before Headcount"' -c concepts -n 5
+```
+
+Use **hybrid semantic search** when the user describes an idea indirectly, uses
+different wording than the source, or needs conceptual recall:
+
+```bash
+qmd query "decision quality depends on surfacing assumptions and context" -n 10
+qmd query --json --explain "metrics as cockpit instruments but not OKRs"
+```
+
+Use **structured queries** for hard searches. They combine exact anchors with
+semantic recall:
+
+```bash
+qmd query $'intent: Find the concept note about metrics as instruments without letting OKRs replace judgment.\nlex: cockpit instruments OKR Goodhart metrics judgment\nvec: data informed not metric driven product judgment\nhyde: A concept note says metrics are useful like cockpit instruments, but leaders should remain data-informed rather than metric-driven because OKRs and dashboards can Goodhart product judgment.'
+```
+
+Structured query fields:
+
+- `intent:` states what you are trying to find and what to avoid.
+- `lex:` uses exact terms, aliases, titles, and rare words.
+- `vec:` paraphrases the idea in natural language.
+- `hyde:` describes the document or answer that would satisfy the request.
+
+If `qmd query` is slow or model/GPU setup fails, fall back to `qmd search` with
+better lexical terms.
+
+## Retrieve sources
+
+Search results include docids like `#abc123` and `qmd://...` paths. Fetch them:
+
+```bash
+qmd get "#abc123"
+qmd get qmd://concepts/ai-before-headcount.md --full
+qmd multi-get "#abc123,#def432" --md
+qmd multi-get 'concepts/{ai-before-headcount.md,data-informed-not-metric-driven.md}' --md
+qmd multi-get 'sources/podcast-2025-*.md' -l 80
+```
+
+Use `multi-get` when comparing several hits or gathering context across pages.
+Use `--full` when the exact source matters.
+
+## Discover what is indexed
+
+```bash
+qmd collection list
+qmd ls
+qmd status
+```
+
+Add collection filters when broad searches drift into the wrong corpus:
+
+```bash
+qmd search "headcount autonomous agents" -c concepts -n 10
+qmd query "merchant support product reality" -c concepts -c sources -n 10
+```
+
+Omit `-c` to search everything.
+
+## MCP Tool: `query`
+
+When using the MCP server, prefer structured searches:
+
+```json
+{
+  "searches": [
+    { "type": "lex", "query": "cockpit OKR Goodhart" },
+    { "type": "vec", "query": "data informed not metric driven product judgment" },
+    { "type": "hyde", "query": "A concept note explains that metrics are useful as instruments, but leaders should not let OKRs or dashboards replace judgment." }
+  ],
+  "intent": "Find the concept note about using metrics as instruments without becoming metric-driven.",
+  "collections": ["concepts"],
+  "limit": 10
+}
+```
+
+Query types:
+
+- `lex` — BM25 keyword search. Best for exact terms, names, titles, and code.
+- `vec` — vector semantic search. Best for natural-language concepts.
+- `hyde` — vector search using a hypothetical answer/document passage.
+
+## Query craft
+
+Good QMD searches mix three things:
+
+1. **Title/alias anchors:** exact page titles, named entities, phrases.
+2. **Semantic paraphrase:** how a human would describe the idea.
+3. **Negative space:** enough intent to avoid nearby-but-wrong concepts.
+
+Examples:
+
+```bash
+# Exact-ish title lookup
+qmd search '"arm the rebels" merchants tools big companies' -c concepts
+
+# Semantic concept lookup
+qmd query $'intent: Find the customer proximity concept, not generic customer delight.\nlex: support pseudonymous merchant customer interviews\nvec: founder stays close to merchant reality through support and product use'
+
+# Source lookup
+qmd search "six-week cadence WhatsApp merchant relationships Shawn Ryan" -c sources -n 10
+```
+
+## Setup and maintenance
+
+Only mutate indexes when the user asked for setup or maintenance. Searching and
+retrieving are safe; collection/index mutation is not a casual first step.
+
+```bash
+npm install -g @tobilu/qmd
+qmd collection add ~/notes --name notes
+qmd update
+qmd embed
+```
+
+Health and diagnostics:
+
+```bash
+qmd doctor
+qmd status
+qmd pull
+```
+
+`qmd doctor` checks config, model cache, device/GPU setup, vector fingerprints,
+and common environment overrides. If a model-backed command fails, run it before
+changing configuration.
+
+## MCP setup
+
+See `references/mcp-setup.md` for Claude Code, Claude Desktop, OpenClaw, and HTTP
+server configuration.
+
+## Pitfalls
+
+- **Do not stop at snippets.** Fetch documents before making claims.
+- **Do not overuse semantic search.** If you know exact titles or terms, BM25 is
+  faster and often better.
+- **Do not mutate indexes casually.** `qmd collection add`, `qmd update`, and
+  `qmd embed` change local state and can be expensive.
+- **Model-backed commands can be environment-sensitive.** If `qmd query`,
+  `qmd vsearch`, or reranking fails because local models/GPU are unavailable,
+  use `qmd search` and stronger lexical/structured terms.
+- **Ambiguous user wording needs intent.** Add `intent:` rather than hoping query
+  expansion guesses the right domain.
+- **Collection names matter.** Search `concepts` for synthesized wiki pages,
+  `sources` for transcripts/raw source pages, and docs collections for code or
+  project documentation.

package/skills/qmd/references/mcp-setup.md

@@ -0,0 +1,102 @@
+# QMD MCP Server Setup
+
+## Install
+
+```bash
+npm install -g @tobilu/qmd
+qmd collection add ~/path/to/markdown --name myknowledge
+qmd embed
+```
+
+## Configure MCP Client
+
+**Claude Code** (`~/.claude/settings.json`):
+```json
+{
+  "mcpServers": {
+    "qmd": { "command": "qmd", "args": ["mcp"] }
+  }
+}
+```
+
+**Claude Desktop** (`~/Library/Application Support/Claude/claude_desktop_config.json`):
+```json
+{
+  "mcpServers": {
+    "qmd": { "command": "qmd", "args": ["mcp"] }
+  }
+}
+```
+
+**OpenClaw** (`~/.openclaw/openclaw.json`):
+```json
+{
+  "mcp": {
+    "servers": {
+      "qmd": { "command": "qmd", "args": ["mcp"] }
+    }
+  }
+}
+```
+
+## HTTP Mode
+
+```bash
+qmd mcp --http              # Port 8181
+qmd mcp --http --daemon     # Background
+qmd mcp stop                # Stop daemon
+```
+
+## Tools
+
+### structured_search
+
+Search with pre-expanded queries.
+
+```json
+{
+  "searches": [
+    { "type": "lex", "query": "keyword phrases" },
+    { "type": "vec", "query": "natural language question" },
+    { "type": "hyde", "query": "hypothetical answer passage..." }
+  ],
+  "limit": 10,
+  "collection": "optional",
+  "minScore": 0.0
+}
+```
+
+| Type | Method | Input |
+|------|--------|-------|
+| `lex` | BM25 | Keywords (2-5 terms) |
+| `vec` | Vector | Question |
+| `hyde` | Vector | Answer passage (50-100 words) |
+
+### get
+
+Retrieve document by path or `#docid`.
+
+| Param | Type | Description |
+|-------|------|-------------|
+| `path` | string | File path or `#docid` |
+| `full` | bool? | Return full content |
+| `lineNumbers` | bool? | Add line numbers |
+
+### multi_get
+
+Retrieve multiple documents.
+
+| Param | Type | Description |
+|-------|------|-------------|
+| `pattern` | string | Glob or comma-separated list |
+| `maxBytes` | number? | Skip large files (default 10KB) |
+
+### status
+
+Index health and collections. No params.
+
+## Troubleshooting
+
+- **Not starting**: `which qmd`, `qmd mcp` manually
+- **No results**: `qmd collection list`, `qmd embed`
+- **Slow first search**: Normal, models loading (~3GB)