@joycodetech/qmd-ja 2.5.3-ja.3

package/package.json

@@ -0,0 +1,130 @@
+{
+  "name": "@joycodetech/qmd-ja",
+  "version": "2.5.3-ja.3",
+  "description": "Japanese-enhanced fork of qmd — On-device hybrid search with Vaporetto WASM morphological tokenizer for accurate Japanese BM25 full-text search",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "bin": {
+    "qmd-ja": "bin/qmd-ja"
+  },
+  "files": [
+    "bin/",
+    "dist/",
+    "vendor/vaporetto-node-wasm/vaporetto_node_wasm.js",
+    "vendor/vaporetto-node-wasm/vaporetto_node_wasm_bg.wasm",
+    "vendor/vaporetto-node-wasm/vaporetto_node_wasm.d.ts",
+    "vendor/vaporetto-node-wasm/vaporetto_node_wasm_bg.wasm.d.ts",
+    "vendor/vaporetto-node-wasm/package.json",
+    "models/vaporetto-bccwj.model",
+    "skills/",
+    "scripts/build.mjs",
+    "scripts/check-package-grammars.mjs",
+    "scripts/package-smoke.mjs",
+    "scripts/test-all.mjs",
+    "LICENSE",
+    "CHANGELOG.md"
+  ],
+  "scripts": {
+    "prepare": "[ -d .git ] && ./scripts/install-hooks.sh || true",
+    "build": "node scripts/build.mjs",
+    "test": "node scripts/test-all.mjs",
+    "test:types": "node ./node_modules/typescript/bin/tsc -p tsconfig.build.json --noEmit",
+    "test:node": "node ./node_modules/vitest/vitest.mjs run --reporter=verbose --testTimeout 60000",
+    "test:bun": "bun test --timeout 60000 --preload ./src/test-preload.ts",
+    "test:unit": "CI=true node ./node_modules/vitest/vitest.mjs run --reporter=verbose --testTimeout 60000 test/ && CI=true bun test --timeout 60000 --preload ./src/test-preload.ts test/",
+    "test:package": "node scripts/package-smoke.mjs",
+    "qmd": "tsx src/cli/qmd.ts",
+    "index": "tsx src/cli/qmd.ts index",
+    "vector": "tsx src/cli/qmd.ts vector",
+    "search": "tsx src/cli/qmd.ts search",
+    "vsearch": "tsx src/cli/qmd.ts vsearch",
+    "rerank": "tsx src/cli/qmd.ts rerank",
+    "inspector": "npx @modelcontextprotocol/inspector tsx src/cli/qmd.ts mcp",
+    "release": "./scripts/release.sh",
+    "smoke:package-grammars": "node scripts/check-package-grammars.mjs"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/joycodetech/qmd-ja.git"
+  },
+  "homepage": "https://github.com/joycodetech/qmd-ja#readme",
+  "bugs": {
+    "url": "https://github.com/joycodetech/qmd-ja/issues"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "1.29.0",
+    "@types/kuromoji": "^0.1.3",
+    "better-sqlite3": "12.10.0",
+    "fast-glob": "3.3.3",
+    "kuromoji": "^0.1.2",
+    "node-llama-cpp": "3.18.1",
+    "picomatch": "4.0.4",
+    "sqlite-vec": "0.1.9",
+    "tree-sitter-go": "0.25.0",
+    "tree-sitter-python": "0.25.0",
+    "tree-sitter-rust": "0.24.0",
+    "tree-sitter-typescript": "0.23.2",
+    "web-tree-sitter": "0.26.8",
+    "yaml": "2.9.0",
+    "zod": "4.2.1"
+  },
+  "optionalDependencies": {
+    "sqlite-vec-darwin-arm64": "0.1.9",
+    "sqlite-vec-darwin-x64": "0.1.9",
+    "sqlite-vec-linux-arm64": "0.1.9",
+    "sqlite-vec-linux-x64": "0.1.9",
+    "sqlite-vec-windows-x64": "0.1.9"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "7.6.13",
+    "tsx": "4.21.0",
+    "vitest": "3.2.4"
+  },
+  "pnpm": {
+    "onlyBuiltDependencies": [
+      "better-sqlite3",
+      "esbuild",
+      "node-llama-cpp",
+      "tree-sitter-go",
+      "tree-sitter-javascript",
+      "tree-sitter-python",
+      "tree-sitter-rust",
+      "tree-sitter-typescript"
+    ]
+  },
+  "peerDependencies": {
+    "typescript": "^5.9.3"
+  },
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "keywords": [
+    "markdown",
+    "search",
+    "fts",
+    "full-text-search",
+    "vector",
+    "semantic-search",
+    "sqlite",
+    "bm25",
+    "embeddings",
+    "rag",
+    "mcp",
+    "reranking",
+    "knowledge-base",
+    "local-ai",
+    "llm"
+  ],
+  "author": "Koz Oda <oss@joycodetech.jp>",
+  "license": "MIT"
+}

package/scripts/build.mjs

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { chmodSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = join(fileURLToPath(new URL("..", import.meta.url)));
+
+function run(command, args, options = {}) {
+  const result = spawnSync(command, args, {
+    cwd: root,
+    stdio: "inherit",
+    shell: process.platform === "win32",
+    ...options,
+  });
+  if (result.status !== 0) {
+    process.exit(result.status ?? 1);
+  }
+}
+
+run(process.execPath, [join(root, "node_modules", "typescript", "bin", "tsc"), "-p", "tsconfig.build.json"]);
+
+
+const cliPath = join(root, "dist", "cli", "qmd.js");
+const tmpPath = `${cliPath}.tmp`;
+const built = readFileSync(cliPath, "utf8");
+const withoutExistingShebang = built.startsWith("#!") ? built.slice(built.indexOf("\n") + 1) : built;
+writeFileSync(tmpPath, `#!/usr/bin/env node\n${withoutExistingShebang}`);
+renameSync(tmpPath, cliPath);
+chmodSync(cliPath, 0o755);

package/scripts/check-package-grammars.mjs

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+import { createRequire } from "node:module";
+
+const require = createRequire(import.meta.url);
+
+const grammars = [
+  "tree-sitter-typescript/tree-sitter-typescript.wasm",
+  "tree-sitter-typescript/tree-sitter-tsx.wasm",
+  "tree-sitter-python/tree-sitter-python.wasm",
+  "tree-sitter-go/tree-sitter-go.wasm",
+  "tree-sitter-rust/tree-sitter-rust.wasm",
+];
+
+let ok = true;
+for (const grammar of grammars) {
+  try {
+    const resolved = require.resolve(grammar);
+    console.log(`ok ${grammar} -> ${resolved}`);
+  } catch (err) {
+    ok = false;
+    console.error(`missing ${grammar}`);
+    console.error(err instanceof Error ? err.message : String(err));
+  }
+}
+
+if (!ok) {
+  console.error("\nAST grammar package smoke check failed. Run `bun install` locally or repair a broken global install with the matching `bun add tree-sitter-...@<version>` command shown by `qmd status`.");
+  process.exit(1);
+}

package/scripts/package-smoke.mjs

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = fileURLToPath(new URL("..", import.meta.url));
+const pkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8"));
+
+function run(label, command, args, options = {}) {
+  console.log(`==> ${label}`);
+  const { quiet, ...spawnOptions } = options;
+  const result = spawnSync(command, args, {
+    cwd: root,
+    stdio: quiet ? "pipe" : "inherit",
+    shell: process.platform === "win32",
+    ...spawnOptions,
+  });
+  if (result.status !== 0) {
+    console.error(`Package smoke failed: ${label}`);
+    if (quiet) {
+      if (result.stdout) process.stderr.write(result.stdout);
+      if (result.stderr) process.stderr.write(result.stderr);
+    }
+    process.exit(result.status ?? 1);
+  }
+}
+
+function assertPath(path, label = path) {
+  const full = join(root, path);
+  if (!existsSync(full)) {
+    console.error(`Package smoke failed: missing ${label} (${path})`);
+    process.exit(1);
+  }
+  return full;
+}
+
+run("build compiled package", process.execPath, ["scripts/build.mjs"]);
+run("AST grammar runtime packages", process.execPath, ["scripts/check-package-grammars.mjs"]);
+
+for (const entry of pkg.files ?? []) {
+  assertPath(entry.replace(/\/$/, ""), `package.json files[] entry ${entry}`);
+}
+
+for (const [name, binPath] of Object.entries(pkg.bin ?? {})) {
+  const full = assertPath(binPath, `bin ${name}`);
+  const mode = statSync(full).mode;
+  if ((mode & 0o111) === 0) {
+    console.error(`Package smoke failed: bin ${name} is not executable (${binPath})`);
+    process.exit(1);
+  }
+}
+
+assertPath("dist/index.js", "compiled main export");
+assertPath("dist/index.d.ts", "compiled type export");
+assertPath("dist/cli/qmd.js", "compiled CLI");
+
+run("compiled CLI under Node", process.execPath, ["dist/cli/qmd.js", "--help"], { quiet: true });
+run("package wrapper", "sh", ["bin/qmd", "--help"], { quiet: true });
+
+if (process.env.QMD_SKIP_BUN_SMOKE === "1") {
+  console.log("==> compiled CLI under Bun (skipped by QMD_SKIP_BUN_SMOKE=1)");
+} else {
+  run("compiled CLI under Bun", "bun", ["dist/cli/qmd.js", "--help"], { quiet: true });
+}

package/scripts/test-all.mjs

@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = fileURLToPath(new URL("..", import.meta.url));
+
+// Mirror bin/qmd's darwin Metal residency mitigation for test subprocesses.
+// libggml-metal asserts on a non-empty residency set during its static
+// destructor (ggml-org/llama.cpp#22593, fix open as #22595) and dumps a
+// multi-kB backtrace at process exit even when tests pass. The env var must
+// be set BEFORE the subprocess starts because libggml-metal reads it via
+// libc getenv at module-load time. Opt out with QMD_METAL_KEEP_RESIDENCY=1.
+const darwinMetalEnv =
+  process.platform === "darwin" && process.env.QMD_METAL_KEEP_RESIDENCY !== "1"
+    ? { GGML_METAL_NO_RESIDENCY: "1" }
+    : {};
+
+function run(label, command, args, options = {}) {
+  console.log(`==> ${label}`);
+  const { env: extraEnv, ...spawnOptions } = options;
+  const result = spawnSync(command, args, {
+    cwd: root,
+    stdio: "inherit",
+    shell: process.platform === "win32",
+    env: { ...process.env, ...darwinMetalEnv, ...(extraEnv ?? {}) },
+    ...spawnOptions,
+  });
+  if (result.status !== 0) {
+    console.error(`Test task failed: ${label}`);
+    process.exit(result.status ?? 1);
+  }
+}
+
+run("TypeScript build typecheck", process.execPath, [join(root, "node_modules", "typescript", "bin", "tsc"), "-p", "tsconfig.build.json", "--noEmit"]);
+run("Vitest suite under Node", process.execPath, [join(root, "node_modules", "vitest", "vitest.mjs"), "run", "--reporter=verbose", "--testTimeout", "60000", "test/"], { env: { CI: "true" } });
+run("Bun test suite", "bun", ["test", "--timeout", "60000", "--preload", "./src/test-preload.ts", "test/"], { env: { CI: "true" } });
+run("Package smoke", process.execPath, ["scripts/package-smoke.mjs"]);

package/skills/qmd/SKILL.md

@@ -0,0 +1,295 @@
+---
+name: qmd
+description: Search local markdown knowledge bases, notes, docs, and wikis with QMD. Use when users ask to find notes, retrieve documents, inspect a wiki, answer from indexed markdown, or set up QMD access.
+license: MIT
+compatibility: Requires qmd CLI or MCP server. Install via `npm install -g @tobilu/qmd`.
+metadata:
+  author: tobi
+  version: "2.2.0"
+allowed-tools: Bash(qmd:*), mcp__qmd__*
+---
+
+# QMD - Query Markdown Documents
+
+## How search works
+
+QMD searches local markdown collections: notes, docs, wikis, transcripts, and
+project knowledge bases. Use it before web search when the answer may already be
+in indexed local files.
+
+The workflow is always:
+
+1. Search for candidate documents.
+2. Retrieve the full source with `qmd get` or `qmd multi-get`.
+3. Answer from retrieved text, citing paths or docids.
+
+Do not answer from snippets alone when the user needs facts, decisions, quotes,
+or nuance. Snippets are only leads.
+
+Typical loop:
+
+```bash
+qmd search "merchant reality support interviews" -n 5
+# leads: #abc123 concepts/customer-proximity.md; #def432 sources/merchant-call.md
+qmd multi-get "#abc123,#def432" --format md
+```
+
+**Default to structured `qmd query` with `intent:`, `lex:`, `vec:`, and `hyde:`
+fields that you write yourself.** You are a better query expander than the
+built-in model: you know the user's actual goal, the domain vocabulary, and the
+nearby-but-wrong concepts to avoid. Do not just paste the user's words into
+`qmd query "..."` and hope the expansion model guesses right — supply the
+`intent:` and craft the lexical and semantic terms deliberately (see
+[Pick the right search mode](#pick-the-right-search-mode)).
+
+When reporting what you retrieved, a compact note is enough; do not paste whole
+files unless needed:
+
+```text
+Retrieved:
+- #abc123 concepts/customer-proximity.md
+- #def432 sources/merchant-call.md
+```
+
+## Pick the right search mode
+
+Use **BM25 lexical search** when you know exact words, titles, names, code
+symbols, or rare phrases:
+
+```bash
+qmd search "cockpit OKR Goodhart" -n 10
+qmd search '"AI Before Headcount"' -c concepts -n 5
+```
+
+Use **`qmd query` with structured fields** when the user describes an idea
+indirectly, uses different wording than the source, or needs conceptual recall.
+**This is the default mode — write the fields yourself rather than leaning on
+query expansion.** Combine exact anchors with semantic recall:
+
+```bash
+qmd query $'intent: Find the concept note about metrics as instruments without letting OKRs replace judgment.\nlex: cockpit instruments OKR Goodhart metrics judgment\nvec: data informed not metric driven product judgment\nhyde: A concept note says metrics are useful like cockpit instruments, but leaders should remain data-informed rather than metric-driven because OKRs and dashboards can Goodhart product judgment.'
+```
+
+Structured query fields (you author each one — do not delegate this to the
+expansion model):
+
+- `intent:` states what you are trying to find **and what to avoid**. Always
+  supply this. It steers ranking away from nearby-but-wrong concepts.
+- `lex:` exact terms, aliases, titles, code symbols, and rare words you expect
+  in the source. This is your own keyword expansion.
+- `vec:` paraphrases the idea in natural language, in source-like wording.
+- `hyde:` describes the document or answer that would satisfy the request.
+
+You do not need all four every time, but you should almost always write at least
+`intent:` plus one of `lex:`/`vec:`. A bare `qmd query "the user's sentence"`
+throws away the context only you have and relies on the built-in expander to
+reconstruct it — prefer the structured form.
+
+If you genuinely have nothing to expand (a single rare token, a verbatim phrase),
+that is a job for `qmd search`, not bare `qmd query`:
+
+```bash
+qmd query --format json --explain $'intent: ...\nlex: ...\nvec: ...'  # inspect ranking
+```
+
+If `qmd query` is slow or model/GPU setup fails, fall back to `qmd search` with
+better lexical terms.
+
+## Retrieve sources
+
+Search results include docids like `#abc123` and `qmd://...` paths. Fetch them:
+
+```bash
+qmd get "#abc123"
+qmd get qmd://concepts/ai-before-headcount.md
+qmd multi-get "#abc123,#def432" --format md
+qmd multi-get 'concepts/{ai-before-headcount.md,data-informed-not-metric-driven.md}' --format md
+qmd multi-get 'sources/podcast-2025-*.md' -l 80
+```
+
+Use `multi-get` when comparing several hits or gathering context across pages.
+
+### Output is line-numbered and carries the docid — cite both
+
+`get` and `multi-get` are **line-numbered by default** and always print the
+document's `#docid` and `qmd://` path. So `get` output looks like:
+
+```text
+qmd://concepts/note.md  #abc123
+---
+
+1: # Metrics as instruments
+2:
+3: Treat dashboards like cockpit instruments...
+```
+
+Cite the docid and exact line numbers in your answer, and use the numbers to ask
+for the next slice. Pass `--no-line-numbers` only when you need raw content to
+copy verbatim (e.g. reproducing a code block).
+
+When you need to open or edit the underlying file (e.g. hand a path to `Read`,
+`Edit`, or an editor), add `--full-path`. It replaces the `qmd://` URL + docid
+header with the document's on-disk path, falling back to the canonical header if
+the file no longer exists on disk:
+
+```text
+$ qmd get "#abc123" --full-path
+/Users/you/notes/concepts/note.md
+---
+
+1: # Metrics as instruments
+```
+
+`--full-path` works the same way on `qmd search` and `qmd query`: result paths
+become the file's on-disk path — `./`-prefixed relative path when the file is
+inside `$PWD`, absolute realpath otherwise — and the per-result `#docid` is
+dropped because the path is the identifier. The leading `./` is intentional so
+the output is unambiguously a filesystem path and cannot be mistaken for a bare
+collection-relative string. Default search/query output still uses `qmd://`
+URIs; only opt into `--full-path` when you specifically need a path you can hand
+to a non-QMD tool.
+
+### Read line ranges with the `:from:count` suffix — never pipe through `sed`/`head`/`tail`
+
+`qmd get` slices files itself. Use the suffix or flags; do **not** shell out to
+`sed -n`, `head`, `tail`, or `awk` to pull a line range. Piping defeats docid
+resolution, virtual-path lookups, line numbering, and the header, and it is
+slower and more error-prone.
+
+The most compact form is a `:from:count` suffix right on the path or docid —
+prefer it:
+
+```bash
+qmd get "#abc123:120:40"                  # 40 lines starting at line 120
+qmd get qmd://concepts/note.md:200:60     # lines 200–259
+qmd get "#abc123:120"                      # from line 120 to end of file
+qmd get "#abc123" --from 120 -l 40         # equivalent, using flags
+```
+
+Suffix and flags:
+
+- `<path>:<from>:<count>` — start at line `<from>`, read `<count>` lines. **Best
+  for reading around a search hit.**
+- `<path>:<from>` — start at `<from>`, read to end of file.
+- `--from <line>` / `-l <lines>` — flag equivalents. Explicit flags override the
+  suffix, so `... :5:2 -l 1` reads 1 line.
+- `--no-line-numbers` — drop the `N:` prefixes (line numbers are on by default).
+
+Wrong: `qmd get "#abc123" | sed -n '120,160p'`
+Right: `qmd get "#abc123:120:40"`
+
+Search results include a `:line` anchor on each hit — feed it straight into
+`qmd get path:line:<n>` to read a window around the match (line numbers in the
+output will start at `line`).
+
+## Discover what is indexed
+
+```bash
+qmd collection list
+qmd ls
+qmd status
+```
+
+Add collection filters when broad searches drift into the wrong corpus:
+
+```bash
+qmd search "headcount autonomous agents" -c concepts -n 10
+qmd query "merchant support product reality" -c concepts -c sources -n 10
+```
+
+Omit `-c` to search everything.
+
+## MCP Tool: `query`
+
+When using the MCP server, prefer structured searches:
+
+```json
+{
+  "searches": [
+    { "type": "lex", "query": "cockpit OKR Goodhart" },
+    { "type": "vec", "query": "data informed not metric driven product judgment" },
+    { "type": "hyde", "query": "A concept note explains that metrics are useful as instruments, but leaders should not let OKRs or dashboards replace judgment." }
+  ],
+  "intent": "Find the concept note about using metrics as instruments without becoming metric-driven.",
+  "collections": ["concepts"],
+  "limit": 10
+}
+```
+
+Query types:
+
+- `lex` — BM25 keyword search. Best for exact terms, names, titles, and code.
+- `vec` — vector semantic search. Best for natural-language concepts.
+- `hyde` — vector search using a hypothetical answer/document passage.
+
+## Query craft
+
+Good QMD searches mix three things:
+
+1. **Title/alias anchors:** exact page titles, named entities, phrases.
+2. **Semantic paraphrase:** how a human would describe the idea.
+3. **Negative space:** enough intent to avoid nearby-but-wrong concepts.
+
+Examples:
+
+```bash
+# Exact-ish title lookup
+qmd search '"arm the rebels" merchants tools big companies' -c concepts
+
+# Semantic concept lookup
+qmd query $'intent: Find the customer proximity concept, not generic customer delight.\nlex: support pseudonymous merchant customer interviews\nvec: founder stays close to merchant reality through support and product use'
+
+# Source lookup
+qmd search "six-week cadence WhatsApp merchant relationships Shawn Ryan" -c sources -n 10
+```
+
+## Setup and maintenance
+
+Only mutate indexes when the user asked for setup or maintenance. Searching and
+retrieving are safe; collection/index mutation is not a casual first step.
+
+```bash
+npm install -g @tobilu/qmd
+qmd collection add ~/notes --name notes
+qmd update
+qmd embed
+```
+
+Health and diagnostics:
+
+```bash
+qmd doctor
+qmd status
+qmd pull
+```
+
+`qmd doctor` checks config, model cache, device/GPU setup, vector fingerprints,
+and common environment overrides. If a model-backed command fails, run it before
+changing configuration.
+
+## MCP setup
+
+See `references/mcp-setup.md` for Claude Code, Claude Desktop, OpenClaw, and HTTP
+server configuration.
+
+## Pitfalls
+
+- **Do not stop at snippets.** Fetch documents before making claims.
+- **Do not slice files with `sed`/`head`/`tail`.** Use the `path:from:count`
+  suffix (e.g. `qmd get "#abc123:120:40"`) or `--from`/`-l`. Output is already
+  line-numbered; piping breaks docid resolution, the header, and virtual paths.
+- **Do not lean on query expansion.** Write `intent:`/`lex:`/`vec:`/`hyde:`
+  yourself. A bare `qmd query "user sentence"` discards the context only you
+  have. You expand the query; the model just ranks.
+- **Do not overuse semantic search.** If you know exact titles or terms, BM25 is
+  faster and often better.
+- **Do not mutate indexes casually.** `qmd collection add`, `qmd update`, and
+  `qmd embed` change local state and can be expensive.
+- **Model-backed commands can be environment-sensitive.** If `qmd query`,
+  `qmd vsearch`, or reranking fails because local models/GPU are unavailable,
+  use `qmd search` and stronger lexical/structured terms.
+- **Ambiguous user wording needs intent.** Add `intent:` rather than hoping query
+  expansion guesses the right domain.
+- **Collection names matter.** Search `concepts` for synthesized wiki pages,
+  `sources` for transcripts/raw source pages, and docs collections for code or
+  project documentation.