@wei840222/qmd 2026.8.23 → 2026.8.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -21
- package/LICENSE +0 -23
- package/README.md +11 -30
- package/THIRD_PARTY_NOTICES.md +2 -2
- package/bin/qmd +16 -116
- package/dist/ast.js +1 -1
- package/dist/cli/build-info.json +2 -2
- package/dist/cli/qmd.d.ts +1 -1
- package/dist/cli/qmd.js +9 -7
- package/dist/db.d.ts +14 -40
- package/dist/db.js +21 -74
- package/dist/hybrid-llm.d.ts +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +5 -1
- package/dist/llm.d.ts +3 -0
- package/dist/llm.js +20 -6
- package/dist/mcp/server.js +3 -1
- package/dist/remote-llm.d.ts +1 -0
- package/dist/remote-llm.js +20 -8
- package/dist/search/zh-dict.txt +13 -7
- package/dist/store.d.ts +12 -2
- package/dist/store.js +31 -8
- package/package.json +10 -12
- package/scripts/check-package-grammars.mjs +1 -1
- package/scripts/package-smoke.mjs +3 -17
- package/scripts/test-all.mjs +0 -1
- package/skills/qmd/SKILL.md +20 -10
- package/skills/qmd/references/mcp-setup.md +4 -20
- package/skills/qmd/references/query-syntax.md +165 -0
package/CHANGELOG.md
CHANGED
|
@@ -4,30 +4,70 @@
|
|
|
4
4
|
|
|
5
5
|
### Added
|
|
6
6
|
|
|
7
|
-
- Added
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
and
|
|
16
|
-
- Added
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
7
|
+
- **Disable HyDE Expansion Control**: Added `--no-hyde` CLI option for `qmd query` and `qmd vsearch`, `includeHyde` parameter to SDK (`store.search`, `store.expandQuery`) and MCP `query` tool, allowing users to disable generating hypothetical document embeddings during query expansion.
|
|
8
|
+
|
|
9
|
+
## [2026.8.23-1] - 2026-08-23
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- **Pure Node.js + pnpm Standardized Toolchain**: Completely standardized the project on
|
|
14
|
+
Node.js (>=22) and pnpm (`pnpm@11.15.1`). Removed all Bun-specific runtime abstractions,
|
|
15
|
+
lockfile heuristics, and dual-runtime test preloads.
|
|
16
|
+
- **Oxlint & Code Quality Fences**: Added Oxlint static analysis rules (`oxlint`) and anti-slop
|
|
17
|
+
guardrails across the codebase.
|
|
18
|
+
- **CJK 3-Way Lexical Retrieval**: Added CJK-aware lexical retrieval with independent character,
|
|
19
|
+
Jieba word, and bigram FTS5 shadow channels, versioned reciprocal rank fusion (RRF), and
|
|
20
|
+
explain traces.
|
|
21
|
+
- **Bundled Traditional Chinese Jieba Dictionary**: Added a bundled Traditional Chinese dictionary
|
|
22
|
+
deterministically combining `@node-rs/jieba` (`2.0.2`), APCLab's Taiwan dictionary, and
|
|
23
|
+
`sysprog21/zhtw-mcp` technical terms. Pinned sources and hashes are recorded in
|
|
24
|
+
`src/search/zh-dict.sources.json` and `THIRD_PARTY_NOTICES.md`.
|
|
25
|
+
- **Custom User Dictionary Support**: Added `user_dict` configuration supporting custom
|
|
26
|
+
user dictionaries with automatic analyzer fingerprint updates.
|
|
27
|
+
- **Typed Remote OpenAI-Compatible Embedding Provider**: Added typed `OpenAIEmbeddingProvider`,
|
|
28
|
+
provider-qualified vector identities, resumable chunk persistence, build leases, remote
|
|
29
|
+
preflight, chunk byte budgeting, and safe OpenAI error handling.
|
|
30
|
+
- **Remote LLM Query Expansion & Chat Reranking**: Supported OpenAI-compatible
|
|
31
|
+
`/v1/chat/completions` API endpoints for remote query expansion and reranking with XML-structured
|
|
32
|
+
prompts and current timestamp injection.
|
|
33
|
+
- **Vector Table Collection Column**: Added `collection` column to vector indexes with automatic
|
|
34
|
+
schema migration and collection-scoped filtering.
|
|
35
|
+
- **Single-Item Batch Embedding Fallback**: Added single-item fallback to gracefully handle and
|
|
36
|
+
recover from oversized batch chunks during embedding and vector repair.
|
|
37
|
+
- **Agent Skill Query Syntax Reference**: Added `skills/qmd/references/query-syntax.md` detailing
|
|
38
|
+
formal EBNF grammar, search operators, and structured MCP JSON payloads, linked directly from
|
|
39
|
+
`skills/qmd/SKILL.md`.
|
|
40
|
+
- **Automated Jules PR Code Reviews**: Integrated Google Jules PR review GitHub Actions workflow
|
|
41
|
+
(`.github/workflows/jules-pr-review.yml`) with automatic `git diff` fallback for large diffs (>20k lines).
|
|
42
|
+
- **Trivy Vulnerability Scanner & Fast CI Checks**: Integrated Trivy filesystem and dependency security
|
|
43
|
+
scanner, TypeScript type checking (`pnpm run test:types`), and Tree-sitter WASM grammars smoke
|
|
44
|
+
(`pnpm run smoke:package-grammars`) in `.github/workflows/ci.yml`.
|
|
24
45
|
|
|
25
46
|
### Changed
|
|
26
47
|
|
|
27
|
-
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
48
|
+
- **Runtime Standardization & Bun Deprecation**: Standardized database layer on `better-sqlite3`
|
|
49
|
+
and `sqlite-vec`. Streamlined `bin/qmd` launcher to a pure Node.js / `tsx` trampoline.
|
|
50
|
+
Replaced all `bun` / `bunx` scripts with `node`, `pnpm`, `tsx`, and `npx`.
|
|
51
|
+
- **Unified `models` Configuration Block**: Replaced disjointed embedding and expand configurations
|
|
52
|
+
with a unified `models:` configuration block supporting `embed`, `generate`, and `rerank`
|
|
53
|
+
endpoints/models.
|
|
54
|
+
- **Generic MCP Client Documentation**: Generalized MCP server configuration instructions across
|
|
55
|
+
`README.md` and `skills/qmd/references/mcp-setup.md` to support all MCP-compatible clients
|
|
56
|
+
(Cursor, Claude Desktop, Zed, OpenClaw, etc.).
|
|
57
|
+
- **Standardized Multi-Holder MIT License**: Updated `LICENSE` to the standard multi-holder copyright
|
|
58
|
+
formatting (Tobi Lutke and Wan, Jiun Wei) for full compatibility with automated license scanners.
|
|
59
|
+
- **Test Suite Semver Suffix Support**: Updated CLI `--version` test in `test/esm-ambiguous-module.test.ts`
|
|
60
|
+
to support Semver prerelease/build suffixes (e.g. `2026.8.23-1`).
|
|
61
|
+
- **Remote LLM Prompt Architecture & Hardening**: Remote query expansion and chat-completions
|
|
62
|
+
reranking now use consistent XML prompt structures, escape untrusted prompt data, preserve
|
|
63
|
+
cross-language technical terms, and discard low-confidence chat rerank scores below `0.1`.
|
|
64
|
+
- **Direct Remote Authorization**: Streamlined remote embedding and execution by removing
|
|
65
|
+
legacy interactive remote consent prompts while enforcing strict identity fingerprinting.
|
|
66
|
+
- **Database Performance Optimizations**: Skipped redundant database write transactions when
|
|
67
|
+
configuration state remains unchanged.
|
|
68
|
+
- **Dependency Upgrades**: Upgraded `@node-rs/jieba` to `2.0.2` and updated package scope to
|
|
69
|
+
`@wei840222/qmd`.
|
|
70
|
+
|
|
31
71
|
- Remote query expansion and chat-completions reranking now use a consistent
|
|
32
72
|
Gemini-style XML prompt structure in both system and user messages, separating
|
|
33
73
|
role, instructions, constraints, output format, context, task, and final
|
|
@@ -102,6 +142,18 @@
|
|
|
102
142
|
reopen, keep CLI update/include settings synchronized, and use atomic durable
|
|
103
143
|
YAML replacement.
|
|
104
144
|
|
|
145
|
+
### Removed
|
|
146
|
+
|
|
147
|
+
- Removed Nix flake configurations (`flake.nix`, `flake.lock`) and related CI jobs.
|
|
148
|
+
- Removed `bun.lock`, `src/test-preload.ts`, and `test/launcher-detection.test.sh`.
|
|
149
|
+
- Removed legacy interactive remote embedding consent workflows in favor of direct configuration-based authorization.
|
|
150
|
+
|
|
151
|
+
### Security
|
|
152
|
+
|
|
153
|
+
- Added Trivy filesystem and dependency security scanning in CI workflows.
|
|
154
|
+
- Bound MCP HTTP server explicitly to `127.0.0.1`.
|
|
155
|
+
- Sanitized remote LLM error handling to prevent leaking credentials or headers.
|
|
156
|
+
|
|
105
157
|
### Compatibility
|
|
106
158
|
|
|
107
159
|
- `vectors_vec` still supports one embedding dimension at a time. Switching
|
package/LICENSE
CHANGED
|
@@ -1,29 +1,6 @@
|
|
|
1
1
|
MIT License
|
|
2
2
|
|
|
3
3
|
Copyright (c) 2024-2026 Tobi Lutke
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE.
|
|
22
|
-
|
|
23
|
-
---
|
|
24
|
-
|
|
25
|
-
MIT License
|
|
26
|
-
|
|
27
4
|
Copyright (c) 2026 Wan, Jiun Wei
|
|
28
5
|
|
|
29
6
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
package/README.md
CHANGED
|
@@ -28,14 +28,13 @@ You can read more about QMD's progress in the [CHANGELOG](CHANGELOG.md).
|
|
|
28
28
|
## Quick Start (Local Embedding Default)
|
|
29
29
|
|
|
30
30
|
```sh
|
|
31
|
-
# Install globally
|
|
31
|
+
# Install globally
|
|
32
32
|
npm install -g @wei840222/qmd
|
|
33
33
|
# or
|
|
34
|
-
|
|
34
|
+
pnpm add -g @wei840222/qmd
|
|
35
35
|
|
|
36
36
|
# Or run directly
|
|
37
37
|
npx @wei840222/qmd ...
|
|
38
|
-
bunx @wei840222/qmd ...
|
|
39
38
|
|
|
40
39
|
# Create collections for your notes, docs, and meeting transcripts
|
|
41
40
|
qmd collection add ~/notes --name notes
|
|
@@ -96,27 +95,9 @@ Although the tool works perfectly fine when you just tell your agent to use it o
|
|
|
96
95
|
- `multi_get` — Batch retrieve by glob pattern, comma-separated list, or docids
|
|
97
96
|
- `status` — Index health and collection info
|
|
98
97
|
|
|
99
|
-
|
|
98
|
+
#### Stdio Transport (Default)
|
|
100
99
|
|
|
101
|
-
|
|
102
|
-
{
|
|
103
|
-
"mcpServers": {
|
|
104
|
-
"qmd": {
|
|
105
|
-
"command": "qmd",
|
|
106
|
-
"args": ["mcp"]
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
```
|
|
111
|
-
|
|
112
|
-
**Claude Code** — Install the plugin (recommended):
|
|
113
|
-
|
|
114
|
-
```bash
|
|
115
|
-
claude plugin marketplace add tobi/qmd
|
|
116
|
-
claude plugin install qmd@qmd
|
|
117
|
-
```
|
|
118
|
-
|
|
119
|
-
Or configure MCP manually in `~/.claude/settings.json`:
|
|
100
|
+
Add QMD to your MCP client configuration (e.g. Cursor, Claude Desktop, Zed, or other MCP-compatible clients):
|
|
120
101
|
|
|
121
102
|
```json
|
|
122
103
|
{
|
|
@@ -211,7 +192,7 @@ results seem unscoped. The HTTP `/query` and `/search` endpoints return
|
|
|
211
192
|
|
|
212
193
|
### SDK / Library Usage
|
|
213
194
|
|
|
214
|
-
Use QMD as a library in your own Node.js
|
|
195
|
+
Use QMD as a library in your own Node.js applications.
|
|
215
196
|
|
|
216
197
|
#### Installation
|
|
217
198
|
|
|
@@ -552,7 +533,6 @@ The `query` command uses **Reciprocal Rank Fusion (RRF)** with position-aware bl
|
|
|
552
533
|
### System Requirements
|
|
553
534
|
|
|
554
535
|
- **Node.js** >= 22
|
|
555
|
-
- **Bun** >= 1.0.0
|
|
556
536
|
- **macOS**: Homebrew SQLite (for extension support)
|
|
557
537
|
```sh
|
|
558
538
|
brew install sqlite
|
|
@@ -729,16 +709,16 @@ Review the resulting pin and dictionary diff before committing an update.
|
|
|
729
709
|
```sh
|
|
730
710
|
npm install -g @wei840222/qmd
|
|
731
711
|
# or
|
|
732
|
-
|
|
712
|
+
pnpm add -g @wei840222/qmd
|
|
733
713
|
```
|
|
734
714
|
|
|
735
715
|
### Development
|
|
736
716
|
|
|
737
717
|
```sh
|
|
738
|
-
git clone https://github.com/
|
|
718
|
+
git clone https://github.com/wei840222/qmd
|
|
739
719
|
cd qmd
|
|
740
|
-
|
|
741
|
-
|
|
720
|
+
pnpm install
|
|
721
|
+
npm link
|
|
742
722
|
```
|
|
743
723
|
|
|
744
724
|
## Usage
|
|
@@ -811,7 +791,7 @@ opt in. Run `qmd status` to verify which grammars are available.
|
|
|
811
791
|
|
|
812
792
|
> **Note:** Tree-sitter grammars are optional dependencies. If they are not
|
|
813
793
|
> installed, `--chunk-strategy auto` falls back to regex-only chunking
|
|
814
|
-
> automatically.
|
|
794
|
+
> automatically.
|
|
815
795
|
|
|
816
796
|
### Context Management
|
|
817
797
|
|
|
@@ -1002,6 +982,7 @@ and `deep-search` (→ `query`).
|
|
|
1002
982
|
--index <name> # Use named index
|
|
1003
983
|
--intent "<text>" # Legacy CLI alias for rerank context (e.g. "web page load times")
|
|
1004
984
|
--no-rerank # Skip LLM reranking (RRF scores only; faster on CPU)
|
|
985
|
+
--no-hyde # Disable HyDE in query expansion (only lex and vec expansions)
|
|
1005
986
|
-C, --candidate-limit <n> # Max candidates to rerank (default: 40)
|
|
1006
987
|
--full-path # Emit on-disk filesystem paths instead of qmd:// URIs
|
|
1007
988
|
# (a result whose file has moved or been deleted since
|
package/THIRD_PARTY_NOTICES.md
CHANGED
|
@@ -4,9 +4,9 @@
|
|
|
4
4
|
|
|
5
5
|
The bundled Traditional Chinese Jieba dictionary combines the following MIT-licensed sources:
|
|
6
6
|
|
|
7
|
-
- `@node-rs/jieba` 2.0.
|
|
7
|
+
- `@node-rs/jieba` 2.0.2 (`dict.txt`), Copyright (c) LongYinan.
|
|
8
8
|
- `APCLab/jieba-tw` (`jieba/dict.txt`) at commit `2bd4c0913cdadc865879b21a00c78e28ef1ba2c5`, Copyright (c) 2013 Sun Junyi.
|
|
9
|
-
- `sysprog21/zhtw-mcp` (`assets/ruleset.json`) at commit `
|
|
9
|
+
- `sysprog21/zhtw-mcp` (`assets/ruleset.json`) at commit `40e669a769bfcafc5f21f9ca6e2f5152b54c8831`, Copyright (c) 2026 National Cheng Kung University, Taiwan.
|
|
10
10
|
|
|
11
11
|
The source pins and content fingerprints are recorded in `src/search/zh-dict.sources.json` in the qmd source distribution. qmd has no runtime source dependency on these upstream repositories.
|
|
12
12
|
|
package/bin/qmd
CHANGED
|
@@ -1,19 +1,16 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// 2>/dev/null;
|
|
2
|
+
// 2>/dev/null; exec node "$0" "$@"
|
|
3
3
|
// Cross-platform launcher for qmd.
|
|
4
4
|
//
|
|
5
|
-
//
|
|
6
|
-
// on Windows generated shims that tried to route through `/bin/sh` — a path
|
|
7
|
-
// that doesn't exist on Windows, so `qmd` failed immediately after a global
|
|
8
|
-
// install. Rewriting the launcher in Node.js lets npm generate native
|
|
5
|
+
// Rewriting the launcher in Node.js lets npm/pnpm generate native
|
|
9
6
|
// cmd/ps1/sh shims that invoke `node` directly on every platform.
|
|
10
7
|
|
|
11
|
-
import { spawn
|
|
8
|
+
import { spawn } from "node:child_process";
|
|
12
9
|
import { existsSync, realpathSync } from "node:fs";
|
|
13
|
-
import { dirname, resolve
|
|
10
|
+
import { dirname, resolve } from "node:path";
|
|
14
11
|
import { fileURLToPath } from "node:url";
|
|
15
12
|
|
|
16
|
-
// Resolve symlinks so global installs (npm link / npm install -g) can find
|
|
13
|
+
// Resolve symlinks so global installs (npm link / npm install -g / pnpm add -g) can find
|
|
17
14
|
// the actual package directory instead of the global bin directory.
|
|
18
15
|
const self = realpathSync(fileURLToPath(import.meta.url));
|
|
19
16
|
const pkgDir = resolve(dirname(self), "..");
|
|
@@ -22,7 +19,7 @@ const tsEntry = resolve(pkgDir, "src/cli/qmd.ts");
|
|
|
22
19
|
|
|
23
20
|
// MCP stdio reserves stdout exclusively for JSON-RPC frames. node-llama-cpp
|
|
24
21
|
// / llama.cpp / ggml can write native logs directly to stdout before JS-level
|
|
25
|
-
// log handlers are attached, so seed the native quiet env before Node
|
|
22
|
+
// log handlers are attached, so seed the native quiet env before Node imports
|
|
26
23
|
// the CLI and its LLM modules. Preserve explicit user values when provided.
|
|
27
24
|
if (process.argv[2] === "mcp") {
|
|
28
25
|
process.env.LLAMA_LOG_LEVEL = process.env.LLAMA_LOG_LEVEL || "error";
|
|
@@ -36,145 +33,49 @@ if (process.argv[2] === "mcp") {
|
|
|
36
33
|
// residency set is empty (ggml-org/llama.cpp#22593); the keep_alive hasn't
|
|
37
34
|
// expired by exit, so the assertion fails and ggml_abort dumps a multi-kB
|
|
38
35
|
// stack trace to stderr even when the user-visible results were already
|
|
39
|
-
// emitted correctly.
|
|
40
|
-
// destructor runs in __cxa_finalize_ranges, after every JS-reachable cleanup.
|
|
36
|
+
// emitted correctly.
|
|
41
37
|
//
|
|
42
38
|
// For QMD's short-lived CLI workflow, residency sets provide no observable
|
|
43
|
-
// performance benefit
|
|
44
|
-
// measured: identical wall time with and without on M3 Pro), so disable them
|
|
45
|
-
// by default on darwin. The env var must be set BEFORE the native llama.cpp
|
|
46
|
-
// binding loads, which is why it lives here in the launcher rather than in
|
|
47
|
-
// the JS entry point. Opt back in with QMD_METAL_KEEP_RESIDENCY=1 if you
|
|
48
|
-
// run long-lived qmd processes (the MCP daemon may benefit on hot reload)
|
|
49
|
-
// or are triaging an upstream Metal teardown fix.
|
|
39
|
+
// performance benefit, so disable them by default on darwin.
|
|
50
40
|
if (process.platform === "darwin" && process.env.QMD_METAL_KEEP_RESIDENCY !== "1") {
|
|
51
41
|
process.env.GGML_METAL_NO_RESIDENCY = process.env.GGML_METAL_NO_RESIDENCY || "1";
|
|
52
42
|
}
|
|
53
43
|
|
|
54
|
-
function hasBun() {
|
|
55
|
-
try {
|
|
56
|
-
const res = spawnSync("bun", ["--version"], { stdio: "ignore", shell: process.platform === "win32" });
|
|
57
|
-
return res.status === 0;
|
|
58
|
-
} catch {
|
|
59
|
-
return false;
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
|
|
63
44
|
// In published packages, bin/qmd must run dist/. In a git checkout, however,
|
|
64
45
|
// dist/ is often ignored and can be stale after git reset or branch switches.
|
|
65
46
|
// Prefer source mode only for checkouts so ./bin/qmd reflects the checked-out
|
|
66
47
|
// source without changing packaged/runtime behavior.
|
|
67
|
-
//
|
|
68
|
-
// Critical: source-mode detection must NOT trigger when a package manager
|
|
69
|
-
// installed us. `pnpm install -g .` (and `npm install -g .`) copy the entire
|
|
70
|
-
// working tree — including .git/, bun.lock, package-lock.json, src/, and even
|
|
71
|
-
// node_modules/ — into <prefix>/node_modules/@tobilu/qmd/, so .git and a
|
|
72
|
-
// lockfile being present is not a reliable "this is a working tree" signal.
|
|
73
|
-
// What IS reliable: a package-manager install always lands the package
|
|
74
|
-
// directory inside a `node_modules/` segment; a bare working-tree checkout
|
|
75
|
-
// (with `bun link` or a direct path invocation) does not. Gate source mode
|
|
76
|
-
// on that. Allow QMD_SOURCE_MODE=1 / =0 as an explicit override for the
|
|
77
|
-
// rare case where the heuristic disagrees with the user.
|
|
78
48
|
const sourceOverride = process.env.QMD_SOURCE_MODE;
|
|
79
49
|
const looksInstalled = pkgDir.split("/").includes("node_modules");
|
|
80
50
|
const sourceAllowed = sourceOverride === "1"
|
|
81
51
|
|| (sourceOverride !== "0" && !looksInstalled);
|
|
82
52
|
|
|
83
53
|
let useSourceMode = false;
|
|
84
|
-
let
|
|
85
|
-
let sourceArgs = [];
|
|
54
|
+
let runnerArgs = [];
|
|
86
55
|
|
|
87
56
|
if (sourceAllowed && existsSync(resolve(pkgDir, ".git")) && existsSync(tsEntry)) {
|
|
88
|
-
// Lockfile-driven runner selection — mirror the dist-mode logic below so
|
|
89
|
-
// source mode picks the same runtime the user's deps were installed for.
|
|
90
|
-
// package-lock.json wins over bun.lock when both are present: pnpm/npm
|
|
91
|
-
// installs ship the Node-ABI native modules (better-sqlite3, sqlite-vec),
|
|
92
|
-
// and running Bun against them produces ABI mismatches. This also fixes
|
|
93
|
-
// pnpm-global installs, which copy the whole working tree — including .git
|
|
94
|
-
// and bun.lock — into the install dir and used to route through Bun even
|
|
95
|
-
// when the user installed via npm/pnpm.
|
|
96
|
-
const hasNpmLock = existsSync(resolve(pkgDir, "package-lock.json"));
|
|
97
|
-
const hasBunLock = existsSync(resolve(pkgDir, "bun.lock")) || existsSync(resolve(pkgDir, "bun.lockb"));
|
|
98
57
|
const tsxEntry = resolve(pkgDir, "node_modules/tsx/dist/cli.mjs");
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if (hasNpmLock && tsxAvailable) {
|
|
102
|
-
useSourceMode = true;
|
|
103
|
-
sourceRunner = "node";
|
|
104
|
-
sourceArgs = [tsxEntry, tsEntry, ...process.argv.slice(2)];
|
|
105
|
-
} else if (hasBunLock && hasBun()) {
|
|
58
|
+
if (existsSync(tsxEntry)) {
|
|
106
59
|
useSourceMode = true;
|
|
107
|
-
|
|
108
|
-
sourceArgs = [tsEntry, ...process.argv.slice(2)];
|
|
109
|
-
} else if (tsxAvailable) {
|
|
110
|
-
useSourceMode = true;
|
|
111
|
-
sourceRunner = "node";
|
|
112
|
-
sourceArgs = [tsxEntry, tsEntry, ...process.argv.slice(2)];
|
|
60
|
+
runnerArgs = [tsxEntry, tsEntry, ...process.argv.slice(2)];
|
|
113
61
|
}
|
|
114
62
|
}
|
|
115
63
|
|
|
116
64
|
if (!useSourceMode && !existsSync(jsEntry)) {
|
|
117
65
|
console.error(`qmd is not built: missing ${jsEntry}`);
|
|
118
|
-
console.error("Run:
|
|
66
|
+
console.error("Run: pnpm install && pnpm run build");
|
|
119
67
|
console.error("Or: npm install && npm run build");
|
|
120
68
|
console.error("After building, run: qmd doctor");
|
|
121
69
|
process.exit(1);
|
|
122
70
|
}
|
|
123
71
|
|
|
124
|
-
// Detect the package manager that installed dependencies by checking lockfiles.
|
|
125
|
-
// $BUN_INSTALL is intentionally NOT checked — it only indicates that bun exists
|
|
126
|
-
// on the system, not that it was used to install this package (see #361).
|
|
127
|
-
//
|
|
128
|
-
// package-lock.json takes priority: if it exists, npm installed the native
|
|
129
|
-
// modules for Node. The repo ships bun.lock, so without this check, source
|
|
130
|
-
// builds that use npm would be incorrectly routed to bun, causing ABI
|
|
131
|
-
// mismatches with better-sqlite3 / sqlite-vec (see #381).
|
|
132
|
-
//
|
|
133
|
-
// Package-manager installs keep their lockfile at the install root, above
|
|
134
|
-
// node_modules/ — never inside the package directory itself. A bun global
|
|
135
|
-
// install writes $BUN_INSTALL/install/global/bun.lock while the package
|
|
136
|
-
// lands in .../install/global/node_modules/@tobilu/qmd/, so the package-dir
|
|
137
|
-
// checks below can't match and qmd would run under whatever node is first
|
|
138
|
-
// on PATH. The native modules were built by the installing runtime; after
|
|
139
|
-
// the next Node major upgrade they fail to load (ERR_DLOPEN_FAILED,
|
|
140
|
-
// NODE_MODULE_VERSION mismatch). Apply the same npm-priority rule at the
|
|
141
|
-
// install root when the package directory itself has no lockfile.
|
|
142
|
-
const pathParts = pkgDir.split(sep);
|
|
143
|
-
const nodeModulesIndex = pathParts.lastIndexOf("node_modules");
|
|
144
|
-
const installRoot = nodeModulesIndex === -1 ? null : pathParts.slice(0, nodeModulesIndex).join(sep);
|
|
145
|
-
const hasRootNpmLock = installRoot !== null && existsSync(resolve(installRoot, "package-lock.json"));
|
|
146
|
-
const hasRootBunLock = installRoot !== null
|
|
147
|
-
&& (existsSync(resolve(installRoot, "bun.lock")) || existsSync(resolve(installRoot, "bun.lockb")));
|
|
148
|
-
|
|
149
|
-
let runnerName = "node";
|
|
150
|
-
if (existsSync(resolve(pkgDir, "package-lock.json"))) {
|
|
151
|
-
runnerName = "node";
|
|
152
|
-
} else if (existsSync(resolve(pkgDir, "bun.lock")) || existsSync(resolve(pkgDir, "bun.lockb"))) {
|
|
153
|
-
runnerName = "bun";
|
|
154
|
-
} else if (hasRootNpmLock) {
|
|
155
|
-
runnerName = "node";
|
|
156
|
-
} else if (hasRootBunLock) {
|
|
157
|
-
runnerName = "bun";
|
|
158
|
-
} else {
|
|
159
|
-
runnerName = "node";
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
const selected = useSourceMode ? sourceRunner : (runnerName === "node" ? "node" : "bun");
|
|
163
72
|
// Pin Node to the binary that launched this trampoline. Native addons
|
|
164
|
-
// (better-sqlite3) are compiled for that install's ABI
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
// If the trampoline itself is running under bun (`bun bin/qmd`), keep
|
|
168
|
-
// PATH `node` so we don't load Node-ABI addons into bun.
|
|
169
|
-
const runner = selected === "node" && typeof process.versions.bun !== "string"
|
|
170
|
-
? process.execPath
|
|
171
|
-
: selected;
|
|
172
|
-
const args = useSourceMode ? sourceArgs : [jsEntry, ...process.argv.slice(2)];
|
|
173
|
-
const needsShell = (runner === "bun") && process.platform === "win32";
|
|
73
|
+
// (better-sqlite3) are compiled for that install's ABI.
|
|
74
|
+
const runner = process.execPath;
|
|
75
|
+
const args = useSourceMode ? runnerArgs : [jsEntry, ...process.argv.slice(2)];
|
|
174
76
|
|
|
175
77
|
const child = spawn(runner, args, {
|
|
176
78
|
stdio: "inherit",
|
|
177
|
-
shell: needsShell,
|
|
178
79
|
});
|
|
179
80
|
|
|
180
81
|
child.on("exit", (code, signal) => {
|
|
@@ -186,7 +87,6 @@ child.on("exit", (code, signal) => {
|
|
|
186
87
|
});
|
|
187
88
|
|
|
188
89
|
child.on("error", (err) => {
|
|
189
|
-
|
|
190
|
-
console.error(`qmd: failed to launch ${name}: ${err.message}`);
|
|
90
|
+
console.error(`qmd: failed to launch node: ${err.message}`);
|
|
191
91
|
process.exit(1);
|
|
192
92
|
});
|
package/dist/ast.js
CHANGED
|
@@ -58,7 +58,7 @@ export function formatGrammarLoadError(language, err) {
|
|
|
58
58
|
const grammar = GRAMMAR_MAP[language];
|
|
59
59
|
const detail = err instanceof Error ? err.message : String(err);
|
|
60
60
|
return `${grammar.pkg}/${grammar.wasm} failed to load (${detail}); falling back to regex chunking. ` +
|
|
61
|
-
`Repair a broken global install with:
|
|
61
|
+
`Repair a broken global install with: pnpm add ${grammar.pkg}@${grammar.version}`;
|
|
62
62
|
}
|
|
63
63
|
// =============================================================================
|
|
64
64
|
// Per-Language Query Definitions
|
package/dist/cli/build-info.json
CHANGED
package/dist/cli/qmd.d.ts
CHANGED
|
@@ -37,7 +37,7 @@ type FinishSuccessfulCliCommandOptions = {
|
|
|
37
37
|
* order, and the process exits cleanly. The `GGML_METAL_NO_RESIDENCY=1` env
|
|
38
38
|
* var that `bin/qmd` exports is a defense-in-depth safety net for paths
|
|
39
39
|
* that still call `process.exit()` after loading the native binding
|
|
40
|
-
* (signal handlers, error paths,
|
|
40
|
+
* (signal handlers, error paths, test runners).
|
|
41
41
|
*
|
|
42
42
|
* If the caller passes an explicit `exit` for testability, we honor it —
|
|
43
43
|
* the lifecycle tests verify the legacy flush → cleanup → exit ordering.
|
package/dist/cli/qmd.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
2
|
+
import { openDatabase, loadSqliteVec } from "../db.js";
|
|
3
3
|
import fastGlob from "fast-glob";
|
|
4
4
|
import { spawn as nodeSpawn } from "child_process";
|
|
5
5
|
import { isQmdMcpPid, mcpDaemonStateFiles } from "./mcp-pid.js";
|
|
@@ -270,7 +270,7 @@ async function flushWritable(stream) {
|
|
|
270
270
|
* order, and the process exits cleanly. The `GGML_METAL_NO_RESIDENCY=1` env
|
|
271
271
|
* var that `bin/qmd` exports is a defense-in-depth safety net for paths
|
|
272
272
|
* that still call `process.exit()` after loading the native binding
|
|
273
|
-
* (signal handlers, error paths,
|
|
273
|
+
* (signal handlers, error paths, test runners).
|
|
274
274
|
*
|
|
275
275
|
* If the caller passes an explicit `exit` for testability, we honor it —
|
|
276
276
|
* the lifecycle tests verify the legacy flush → cleanup → exit ordering.
|
|
@@ -2674,6 +2674,7 @@ async function vectorSearch(query, opts, _model = DEFAULT_EMBED_MODEL) {
|
|
|
2674
2674
|
limit: opts.all ? 500 : (opts.limit || 10),
|
|
2675
2675
|
minScore: opts.minScore || 0.3,
|
|
2676
2676
|
expansionContext: opts.intent,
|
|
2677
|
+
includeHyde: opts.includeHyde,
|
|
2677
2678
|
hooks: {
|
|
2678
2679
|
onExpand: (original, expanded) => {
|
|
2679
2680
|
logExpansionTree(original, expanded);
|
|
@@ -2766,6 +2767,7 @@ async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rera
|
|
|
2766
2767
|
explain: !!opts.explain,
|
|
2767
2768
|
rerankContext: intent,
|
|
2768
2769
|
expansion: opts.expansion,
|
|
2770
|
+
includeHyde: opts.includeHyde,
|
|
2769
2771
|
chunkStrategy: opts.chunkStrategy,
|
|
2770
2772
|
hooks: {
|
|
2771
2773
|
onExpansionDecision: (decision) => {
|
|
@@ -2884,6 +2886,7 @@ function parseCLI() {
|
|
|
2884
2886
|
// Query options
|
|
2885
2887
|
"candidate-limit": { type: "string", short: "C" },
|
|
2886
2888
|
"no-rerank": { type: "boolean", default: false },
|
|
2889
|
+
"no-hyde": { type: "boolean", default: false },
|
|
2887
2890
|
expand: { type: "boolean", default: false },
|
|
2888
2891
|
"no-gpu": { type: "boolean", default: false },
|
|
2889
2892
|
intent: { type: "string" },
|
|
@@ -2960,6 +2963,7 @@ function parseCLI() {
|
|
|
2960
2963
|
lineNumbers: !!values["line-numbers"],
|
|
2961
2964
|
candidateLimit: values["candidate-limit"] ? parseInt(String(values["candidate-limit"]), 10) : undefined,
|
|
2962
2965
|
skipRerank: !!values["no-rerank"],
|
|
2966
|
+
includeHyde: !values["no-hyde"],
|
|
2963
2967
|
explain: !!values.explain,
|
|
2964
2968
|
intent: values.intent,
|
|
2965
2969
|
expansion: values.expand ? "force" : "auto",
|
|
@@ -3459,6 +3463,7 @@ function showHelp() {
|
|
|
3459
3463
|
console.log(" --chunk-strategy <auto|regex> - Chunking mode (default: regex; auto uses AST for code files)");
|
|
3460
3464
|
console.log(" --timeout <minutes> - Embed session cap in minutes (0 = no limit; default 30)");
|
|
3461
3465
|
console.log(" --expand - Force query expansion (auto is the default; lex: skips)");
|
|
3466
|
+
console.log(" --no-hyde - Disable HyDE (hypothetical document) in query expansion");
|
|
3462
3467
|
console.log("");
|
|
3463
3468
|
console.log("Embedding providers & disclosure:");
|
|
3464
3469
|
console.log(" - Local embedding is the default. OpenAI requires explicit provider configuration and OPENAI_API_KEY.");
|
|
@@ -3933,7 +3938,7 @@ async function showDoctor() {
|
|
|
3933
3938
|
const nextSteps = [];
|
|
3934
3939
|
console.log(`${c.bold}QMD Doctor${c.reset}\n`);
|
|
3935
3940
|
console.log(`Index: ${getDbPath()}`);
|
|
3936
|
-
console.log(`Runtime:
|
|
3941
|
+
console.log(`Runtime: better-sqlite3`);
|
|
3937
3942
|
try {
|
|
3938
3943
|
const row = db.prepare(`SELECT sqlite_version() AS version`).get();
|
|
3939
3944
|
doctorCheck("SQLite runtime", true, row.version);
|
|
@@ -4632,11 +4637,8 @@ if (isMain) {
|
|
|
4632
4637
|
const selfPath = fileURLToPath(import.meta.url);
|
|
4633
4638
|
const indexArgs = cli.values.index ? ["--index", String(cli.values.index)] : [];
|
|
4634
4639
|
const hostArgs = host ? ["--host", host] : [];
|
|
4635
|
-
const isBunRuntime = typeof process.versions.bun === "string";
|
|
4636
4640
|
const spawnArgs = selfPath.endsWith(".ts")
|
|
4637
|
-
?
|
|
4638
|
-
? [selfPath, ...indexArgs, "mcp", "--http", "--port", String(port), ...hostArgs]
|
|
4639
|
-
: ["--import", pathJoin(dirname(selfPath), "..", "..", "node_modules", "tsx", "dist", "esm", "index.mjs"), selfPath, ...indexArgs, "mcp", "--http", "--port", String(port), ...hostArgs]
|
|
4641
|
+
? ["--import", pathJoin(dirname(selfPath), "..", "..", "node_modules", "tsx", "dist", "esm", "index.mjs"), selfPath, ...indexArgs, "mcp", "--http", "--port", String(port), ...hostArgs]
|
|
4640
4642
|
: [selfPath, ...indexArgs, "mcp", "--http", "--port", String(port), ...hostArgs];
|
|
4641
4643
|
const child = nodeSpawn(process.execPath, spawnArgs, {
|
|
4642
4644
|
stdio: ["ignore", logFd, logFd],
|
package/dist/db.d.ts
CHANGED
|
@@ -1,27 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* db.ts -
|
|
2
|
+
* db.ts - SQLite database connection and extension management
|
|
3
3
|
*
|
|
4
|
-
* Provides
|
|
5
|
-
* and
|
|
6
|
-
* difference is the import path.
|
|
7
|
-
*
|
|
8
|
-
* On macOS, Apple's system SQLite is compiled with SQLITE_OMIT_LOAD_EXTENSION,
|
|
9
|
-
* which prevents loading native extensions like sqlite-vec. When running under
|
|
10
|
-
* Bun we call Database.setCustomSQLite() to swap in Homebrew's full-featured
|
|
11
|
-
* SQLite build before creating any database instances.
|
|
4
|
+
* Provides Database export and connection management using better-sqlite3
|
|
5
|
+
* and sqlite-vec.
|
|
12
6
|
*/
|
|
13
|
-
|
|
7
|
+
import BetterSqlite3 from "better-sqlite3";
|
|
14
8
|
export type SQLiteValue = string | number | bigint | Buffer | Uint8Array | Float32Array | null;
|
|
15
9
|
export type SQLiteParams = readonly SQLiteValue[];
|
|
16
10
|
/**
|
|
17
|
-
* Open a SQLite database
|
|
11
|
+
* Open a SQLite database using better-sqlite3.
|
|
18
12
|
*
|
|
19
|
-
* `
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* `query` racing a long `embed`, or a first-open schema migration racing any
|
|
24
|
-
* routine command) queue at batch boundaries instead of failing on contact.
|
|
13
|
+
* `better-sqlite3` defaults `busy_timeout` to 0, so concurrent writers throw
|
|
14
|
+
* `SQLITE_BUSY` instead of waiting. WAL improves read-while-write concurrency
|
|
15
|
+
* but does not serialise writers. Setting the timeout at connection open makes
|
|
16
|
+
* parallel processes queue at batch boundaries instead of failing on contact.
|
|
25
17
|
*
|
|
26
18
|
* WAL is enabled here too (with a bounded retry) so connection-level pragmas
|
|
27
19
|
* live in one place and the cold-database journal migration survives concurrent
|
|
@@ -29,37 +21,19 @@ export type SQLiteParams = readonly SQLiteValue[];
|
|
|
29
21
|
*
|
|
30
22
|
* Default 120_000 ms outlasts the worst-case batch commit on a multi-GB
|
|
31
23
|
* index. Override with `QMD_SQLITE_BUSY_TIMEOUT` (value in milliseconds; `0`
|
|
32
|
-
* restores the upstream fail-fast behaviour).
|
|
33
|
-
* https://bun.sh/docs/api/sqlite#busy-timeout.
|
|
24
|
+
* restores the upstream fail-fast behaviour).
|
|
34
25
|
*/
|
|
35
26
|
export declare function openDatabase(path: string): Database;
|
|
36
27
|
/** Open an existing database without changing journal mode, schema, or user data. */
|
|
37
28
|
export declare function openReadOnlyDatabase(path: string): Database;
|
|
38
29
|
/**
|
|
39
|
-
*
|
|
30
|
+
* Database and Statement types used throughout QMD.
|
|
40
31
|
*/
|
|
41
|
-
export
|
|
42
|
-
|
|
43
|
-
prepare(sql: string): Statement;
|
|
44
|
-
loadExtension(path: string): void;
|
|
45
|
-
transaction<T extends (...args: SQLiteValue[]) => unknown>(fn: T): T & {
|
|
46
|
-
immediate: T;
|
|
47
|
-
};
|
|
48
|
-
close(): void;
|
|
49
|
-
}
|
|
50
|
-
export interface Statement {
|
|
51
|
-
run(...params: SQLiteValue[]): {
|
|
52
|
-
changes: number;
|
|
53
|
-
lastInsertRowid: number | bigint;
|
|
54
|
-
};
|
|
55
|
-
get<T = unknown>(...params: SQLiteValue[]): T | undefined;
|
|
56
|
-
all<T = unknown>(...params: SQLiteValue[]): T[];
|
|
57
|
-
iterate<T = unknown>(...params: SQLiteValue[]): IterableIterator<T>;
|
|
58
|
-
}
|
|
32
|
+
export type Database = BetterSqlite3.Database;
|
|
33
|
+
export type Statement<T extends SQLiteParams = SQLiteParams> = BetterSqlite3.Statement<T>;
|
|
59
34
|
/**
|
|
60
35
|
* Load the sqlite-vec extension into a database.
|
|
61
36
|
*
|
|
62
|
-
* Throws with
|
|
63
|
-
* unavailable.
|
|
37
|
+
* Throws with fix instructions when the extension is unavailable.
|
|
64
38
|
*/
|
|
65
39
|
export declare function loadSqliteVec(db: Database): void;
|