@wei840222/qmd 2026.8.23 → 2026.8.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -21
- package/LICENSE +0 -23
- package/README.md +11 -30
- package/THIRD_PARTY_NOTICES.md +2 -2
- package/bin/qmd +16 -116
- package/dist/ast.js +1 -1
- package/dist/cli/build-info.json +2 -2
- package/dist/cli/qmd.d.ts +1 -1
- package/dist/cli/qmd.js +9 -7
- package/dist/db.d.ts +14 -40
- package/dist/db.js +21 -74
- package/dist/hybrid-llm.d.ts +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +5 -1
- package/dist/llm.d.ts +3 -0
- package/dist/llm.js +20 -6
- package/dist/mcp/server.js +3 -1
- package/dist/remote-llm.d.ts +1 -0
- package/dist/remote-llm.js +20 -8
- package/dist/search/zh-dict.txt +13 -7
- package/dist/store.d.ts +12 -2
- package/dist/store.js +31 -8
- package/package.json +10 -12
- package/scripts/check-package-grammars.mjs +1 -1
- package/scripts/package-smoke.mjs +3 -17
- package/scripts/test-all.mjs +0 -1
- package/skills/qmd/SKILL.md +20 -10
- package/skills/qmd/references/mcp-setup.md +4 -20
- package/skills/qmd/references/query-syntax.md +165 -0
package/dist/db.js
CHANGED
|
@@ -1,57 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* db.ts -
|
|
2
|
+
* db.ts - SQLite database connection and extension management
|
|
3
3
|
*
|
|
4
|
-
* Provides
|
|
5
|
-
* and
|
|
6
|
-
* difference is the import path.
|
|
7
|
-
*
|
|
8
|
-
* On macOS, Apple's system SQLite is compiled with SQLITE_OMIT_LOAD_EXTENSION,
|
|
9
|
-
* which prevents loading native extensions like sqlite-vec. When running under
|
|
10
|
-
* Bun we call Database.setCustomSQLite() to swap in Homebrew's full-featured
|
|
11
|
-
* SQLite build before creating any database instances.
|
|
4
|
+
* Provides Database export and connection management using better-sqlite3
|
|
5
|
+
* and sqlite-vec.
|
|
12
6
|
*/
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
let _sqliteVecLoad;
|
|
16
|
-
if (isBun) {
|
|
17
|
-
// Dynamic string prevents tsc from resolving bun:sqlite on Node.js builds
|
|
18
|
-
const bunSqlite = "bun:" + "sqlite";
|
|
19
|
-
const BunDatabase = (await import(/* @vite-ignore */ bunSqlite)).Database;
|
|
20
|
-
// See: https://bun.com/docs/runtime/sqlite#setcustomsqlite
|
|
21
|
-
if (process.platform === "darwin") {
|
|
22
|
-
const homebrewPaths = [
|
|
23
|
-
"/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib", // Apple Silicon
|
|
24
|
-
"/usr/local/opt/sqlite/lib/libsqlite3.dylib", // Intel
|
|
25
|
-
];
|
|
26
|
-
for (const p of homebrewPaths) {
|
|
27
|
-
try {
|
|
28
|
-
BunDatabase.setCustomSQLite(p);
|
|
29
|
-
break;
|
|
30
|
-
}
|
|
31
|
-
catch { }
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
_Database = BunDatabase;
|
|
35
|
-
// setCustomSQLite may have silently failed — test that extensions actually work.
|
|
36
|
-
try {
|
|
37
|
-
const { getLoadablePath } = await import("sqlite-vec");
|
|
38
|
-
const vecPath = getLoadablePath();
|
|
39
|
-
const testDb = new BunDatabase(":memory:");
|
|
40
|
-
testDb.loadExtension(vecPath);
|
|
41
|
-
testDb.close();
|
|
42
|
-
_sqliteVecLoad = (db) => db.loadExtension(vecPath);
|
|
43
|
-
}
|
|
44
|
-
catch {
|
|
45
|
-
// Vector search won't work, but BM25 and other operations are unaffected.
|
|
46
|
-
_sqliteVecLoad = null;
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
else {
|
|
50
|
-
// Dual-runtime: better-sqlite3 matches Database at runtime; published types do not share an interface with bun:sqlite.
|
|
51
|
-
_Database = (await import("better-sqlite3")).default;
|
|
52
|
-
const sqliteVec = await import("sqlite-vec");
|
|
53
|
-
_sqliteVecLoad = (db) => sqliteVec.load(db);
|
|
54
|
-
}
|
|
7
|
+
import BetterSqlite3 from "better-sqlite3";
|
|
8
|
+
import * as sqliteVec from "sqlite-vec";
|
|
55
9
|
function isBusyError(err) {
|
|
56
10
|
if (typeof err !== "object" || err === null)
|
|
57
11
|
return false;
|
|
@@ -86,14 +40,12 @@ function enableWal(db, budgetMs) {
|
|
|
86
40
|
}
|
|
87
41
|
}
|
|
88
42
|
/**
|
|
89
|
-
* Open a SQLite database
|
|
43
|
+
* Open a SQLite database using better-sqlite3.
|
|
90
44
|
*
|
|
91
|
-
* `
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
* `query` racing a long `embed`, or a first-open schema migration racing any
|
|
96
|
-
* routine command) queue at batch boundaries instead of failing on contact.
|
|
45
|
+
* `better-sqlite3` defaults `busy_timeout` to 0, so concurrent writers throw
|
|
46
|
+
* `SQLITE_BUSY` instead of waiting. WAL improves read-while-write concurrency
|
|
47
|
+
* but does not serialise writers. Setting the timeout at connection open makes
|
|
48
|
+
* parallel processes queue at batch boundaries instead of failing on contact.
|
|
97
49
|
*
|
|
98
50
|
* WAL is enabled here too (with a bounded retry) so connection-level pragmas
|
|
99
51
|
* live in one place and the cold-database journal migration survives concurrent
|
|
@@ -101,11 +53,10 @@ function enableWal(db, budgetMs) {
|
|
|
101
53
|
*
|
|
102
54
|
* Default 120_000 ms outlasts the worst-case batch commit on a multi-GB
|
|
103
55
|
* index. Override with `QMD_SQLITE_BUSY_TIMEOUT` (value in milliseconds; `0`
|
|
104
|
-
* restores the upstream fail-fast behaviour).
|
|
105
|
-
* https://bun.sh/docs/api/sqlite#busy-timeout.
|
|
56
|
+
* restores the upstream fail-fast behaviour).
|
|
106
57
|
*/
|
|
107
58
|
export function openDatabase(path) {
|
|
108
|
-
const db = new
|
|
59
|
+
const db = new BetterSqlite3(path);
|
|
109
60
|
const raw = process.env.QMD_SQLITE_BUSY_TIMEOUT;
|
|
110
61
|
const parsed = raw !== undefined && raw !== "" ? Number(raw) : Number.NaN;
|
|
111
62
|
const busyTimeoutMs = Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : 120_000;
|
|
@@ -115,10 +66,8 @@ export function openDatabase(path) {
|
|
|
115
66
|
}
|
|
116
67
|
/** Open an existing database without changing journal mode, schema, or user data. */
|
|
117
68
|
export function openReadOnlyDatabase(path) {
|
|
118
|
-
const options =
|
|
119
|
-
|
|
120
|
-
: { readonly: true, fileMustExist: true };
|
|
121
|
-
const db = new _Database(path, options);
|
|
69
|
+
const options = { readonly: true, fileMustExist: true };
|
|
70
|
+
const db = new BetterSqlite3(path, options);
|
|
122
71
|
const raw = process.env.QMD_SQLITE_BUSY_TIMEOUT;
|
|
123
72
|
const parsed = raw !== undefined && raw !== "" ? Number(raw) : Number.NaN;
|
|
124
73
|
const busyTimeoutMs = Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : 120_000;
|
|
@@ -128,16 +77,14 @@ export function openReadOnlyDatabase(path) {
|
|
|
128
77
|
/**
|
|
129
78
|
* Load the sqlite-vec extension into a database.
|
|
130
79
|
*
|
|
131
|
-
* Throws with
|
|
132
|
-
* unavailable.
|
|
80
|
+
* Throws with fix instructions when the extension is unavailable.
|
|
133
81
|
*/
|
|
134
82
|
export function loadSqliteVec(db) {
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
throw new Error(`sqlite-vec extension is unavailable. ${
|
|
83
|
+
try {
|
|
84
|
+
sqliteVec.load(db);
|
|
85
|
+
}
|
|
86
|
+
catch (err) {
|
|
87
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
88
|
+
throw new Error(`sqlite-vec extension is unavailable. Ensure the sqlite-vec native module is installed correctly: ${message}`);
|
|
141
89
|
}
|
|
142
|
-
_sqliteVecLoad(db);
|
|
143
90
|
}
|
package/dist/hybrid-llm.d.ts
CHANGED
|
@@ -12,6 +12,7 @@ export declare class HybridLLM implements LLM {
|
|
|
12
12
|
expandQuery(query: string, options?: {
|
|
13
13
|
context?: string;
|
|
14
14
|
includeLexical?: boolean;
|
|
15
|
+
includeHyde?: boolean;
|
|
15
16
|
}): Promise<Queryable[]>;
|
|
16
17
|
rerank(query: string, documents: RerankDocument[], options?: RerankOptions): Promise<RerankResult>;
|
|
17
18
|
dispose(): Promise<void>;
|
package/dist/index.d.ts
CHANGED
|
@@ -75,6 +75,8 @@ export interface SearchOptions {
|
|
|
75
75
|
explain?: boolean;
|
|
76
76
|
/** Query expansion policy (default: auto) */
|
|
77
77
|
expansion?: ExpansionMode;
|
|
78
|
+
/** Whether to include HyDE (hypothetical document) in query expansion (default: true) */
|
|
79
|
+
includeHyde?: boolean;
|
|
78
80
|
/** Optional progress/decision hooks for search orchestration */
|
|
79
81
|
hooks?: SearchHooks;
|
|
80
82
|
/** Chunk strategy: "auto" (default, uses AST for code files) or "regex" (legacy) */
|
|
@@ -100,6 +102,10 @@ export interface VectorSearchOptions {
|
|
|
100
102
|
export interface ExpandQueryOptions {
|
|
101
103
|
/** Additional context used only while generating query expansions. */
|
|
102
104
|
expansionContext?: string;
|
|
105
|
+
/** Whether to include lexical (BM25) sub-queries (default: true) */
|
|
106
|
+
includeLexical?: boolean;
|
|
107
|
+
/** Whether to include HyDE (hypothetical document) sub-queries (default: true) */
|
|
108
|
+
includeHyde?: boolean;
|
|
103
109
|
}
|
|
104
110
|
/**
|
|
105
111
|
* Options for creating a QMD store.
|
package/dist/index.js
CHANGED
|
@@ -231,6 +231,7 @@ export async function createStore(options) {
|
|
|
231
231
|
expansionContext: opts.expansionContext,
|
|
232
232
|
rerankContext: opts.rerankContext,
|
|
233
233
|
expansion: opts.expansion,
|
|
234
|
+
includeHyde: opts.includeHyde,
|
|
234
235
|
hooks: opts.hooks,
|
|
235
236
|
candidateLimit: opts.candidateLimit,
|
|
236
237
|
skipRerank,
|
|
@@ -242,7 +243,10 @@ export async function createStore(options) {
|
|
|
242
243
|
const provider = internal.embeddingProvider;
|
|
243
244
|
return internal.searchVec(q, provider?.model ?? internal.llm?.embedModelName ?? DEFAULT_EMBED_MODEL_URI, opts?.limit, opts?.collection);
|
|
244
245
|
},
|
|
245
|
-
expandQuery: async (q, opts) => internal.expandQuery(q, undefined, opts?.expansionContext
|
|
246
|
+
expandQuery: async (q, opts) => internal.expandQuery(q, undefined, opts?.expansionContext, {
|
|
247
|
+
includeLexical: opts?.includeLexical,
|
|
248
|
+
includeHyde: opts?.includeHyde,
|
|
249
|
+
}),
|
|
246
250
|
get: async (pathOrDocid, opts) => internal.findDocument(pathOrDocid, opts),
|
|
247
251
|
getDocumentBody: async (pathOrDocid, opts) => {
|
|
248
252
|
const result = internal.findDocument(pathOrDocid, { includeBody: false });
|
package/dist/llm.d.ts
CHANGED
|
@@ -137,6 +137,7 @@ export interface ILLMSession {
|
|
|
137
137
|
expandQuery(query: string, options?: {
|
|
138
138
|
context?: string;
|
|
139
139
|
includeLexical?: boolean;
|
|
140
|
+
includeHyde?: boolean;
|
|
140
141
|
}): Promise<Queryable[]>;
|
|
141
142
|
rerank(query: string, documents: RerankDocument[], options?: RerankOptions): Promise<RerankResult>;
|
|
142
143
|
/** Whether this session is still valid (not released or aborted) */
|
|
@@ -236,6 +237,7 @@ export interface LLM {
|
|
|
236
237
|
expandQuery(query: string, options?: {
|
|
237
238
|
context?: string;
|
|
238
239
|
includeLexical?: boolean;
|
|
240
|
+
includeHyde?: boolean;
|
|
239
241
|
}): Promise<Queryable[]>;
|
|
240
242
|
/**
|
|
241
243
|
* Rerank documents by relevance to a query
|
|
@@ -465,6 +467,7 @@ export declare class LlamaCpp implements LLM {
|
|
|
465
467
|
expandQuery(query: string, options?: {
|
|
466
468
|
context?: string;
|
|
467
469
|
includeLexical?: boolean;
|
|
470
|
+
includeHyde?: boolean;
|
|
468
471
|
}): Promise<Queryable[]>;
|
|
469
472
|
private static readonly RERANK_TEMPLATE_OVERHEAD;
|
|
470
473
|
private static readonly RERANK_TARGET_DOCS_PER_CONTEXT;
|
package/dist/llm.js
CHANGED
|
@@ -1245,6 +1245,7 @@ export class LlamaCpp {
|
|
|
1245
1245
|
const llama = await this.ensureLlama();
|
|
1246
1246
|
await this.ensureGenerateModel();
|
|
1247
1247
|
const includeLexical = options.includeLexical ?? true;
|
|
1248
|
+
const includeHyde = options.includeHyde ?? true;
|
|
1248
1249
|
const context = options.context;
|
|
1249
1250
|
// Keep the caller-provided expansion context separate from the query. It
|
|
1250
1251
|
// may clarify ambiguous terms, but it is untrusted data rather than an
|
|
@@ -1259,11 +1260,18 @@ export class LlamaCpp {
|
|
|
1259
1260
|
let genContext;
|
|
1260
1261
|
let sequence;
|
|
1261
1262
|
try {
|
|
1263
|
+
const allowedTypes = [];
|
|
1264
|
+
if (includeLexical)
|
|
1265
|
+
allowedTypes.push('"lex"');
|
|
1266
|
+
allowedTypes.push('"vec"');
|
|
1267
|
+
if (includeHyde)
|
|
1268
|
+
allowedTypes.push('"hyde"');
|
|
1269
|
+
const typeRule = allowedTypes.join(' | ');
|
|
1262
1270
|
const grammar = await llama.createGrammar({
|
|
1263
1271
|
grammar: `
|
|
1264
1272
|
root ::= line+
|
|
1265
1273
|
line ::= type ": " content "\\n"
|
|
1266
|
-
type ::=
|
|
1274
|
+
type ::= ${typeRule}
|
|
1267
1275
|
content ::= [^\\n]+
|
|
1268
1276
|
`
|
|
1269
1277
|
});
|
|
@@ -1304,21 +1312,27 @@ export class LlamaCpp {
|
|
|
1304
1312
|
const type = line.slice(0, colonIdx).trim();
|
|
1305
1313
|
if (type !== 'lex' && type !== 'vec' && type !== 'hyde')
|
|
1306
1314
|
return null;
|
|
1315
|
+
if (type === 'lex' && !includeLexical)
|
|
1316
|
+
return null;
|
|
1317
|
+
if (type === 'hyde' && !includeHyde)
|
|
1318
|
+
return null;
|
|
1307
1319
|
const text = line.slice(colonIdx + 1).trim();
|
|
1308
1320
|
if (!hasQueryTerm(text))
|
|
1309
1321
|
return null;
|
|
1310
1322
|
return { type: type, text };
|
|
1311
1323
|
}).filter((q) => q !== null);
|
|
1312
|
-
// Filter out
|
|
1313
|
-
const filtered =
|
|
1324
|
+
// Filter out unwanted types if any slipped through
|
|
1325
|
+
const filtered = queryables
|
|
1326
|
+
.filter(q => (includeLexical || q.type !== 'lex'))
|
|
1327
|
+
.filter(q => (includeHyde || q.type !== 'hyde'));
|
|
1314
1328
|
if (filtered.length > 0)
|
|
1315
1329
|
return filtered;
|
|
1316
1330
|
const fallback = [
|
|
1317
|
-
{ type: 'hyde', text: `Information about ${query}` },
|
|
1318
|
-
{ type: 'lex', text: query },
|
|
1331
|
+
...(includeHyde ? [{ type: 'hyde', text: `Information about ${query}` }] : []),
|
|
1332
|
+
...(includeLexical ? [{ type: 'lex', text: query }] : []),
|
|
1319
1333
|
{ type: 'vec', text: query },
|
|
1320
1334
|
];
|
|
1321
|
-
return
|
|
1335
|
+
return fallback;
|
|
1322
1336
|
}
|
|
1323
1337
|
catch (error) {
|
|
1324
1338
|
console.error("Structured query expansion failed:", error);
|
package/dist/mcp/server.js
CHANGED
|
@@ -255,8 +255,9 @@ Context-aware lex (C++ performance, not sports):
|
|
|
255
255
|
rerankContext: z.string().optional().describe("Additional context used only to rerank results and select snippets/chunks."),
|
|
256
256
|
rerank: z.boolean().optional().default(true).describe("Rerank results using LLM (default: true). Set to false for faster results on CPU-only machines."),
|
|
257
257
|
explain: z.boolean().optional().default(false).describe("Include retrieval traces and the shared query-expansion decision or typed expansion error"),
|
|
258
|
+
includeHyde: z.boolean().optional().default(true).describe("Whether to include HyDE (hypothetical document) in query expansion (default: true)"),
|
|
258
259
|
}),
|
|
259
|
-
}, track(async ({ query, searches, expansion, limit, minScore, candidateLimit, collections, expansionContext, rerankContext, rerank, explain }) => {
|
|
260
|
+
}, track(async ({ query, searches, expansion, includeHyde, limit, minScore, candidateLimit, collections, expansionContext, rerankContext, rerank, explain }) => {
|
|
260
261
|
// Require exactly one of `query` (plain text with an expansion policy) or `searches` (typed sub-queries).
|
|
261
262
|
if (!query && (!searches || searches.length === 0)) {
|
|
262
263
|
return {
|
|
@@ -292,6 +293,7 @@ Context-aware lex (C++ performance, not sports):
|
|
|
292
293
|
rerankContext,
|
|
293
294
|
explain,
|
|
294
295
|
expansion: query ? expansion : undefined,
|
|
296
|
+
includeHyde,
|
|
295
297
|
hooks: explain && query ? {
|
|
296
298
|
onExpansionDecision: decision => { expansionDecision = decision; },
|
|
297
299
|
onExpansionError: event => { expansionError = event; },
|
package/dist/remote-llm.d.ts
CHANGED
|
@@ -38,6 +38,7 @@ export declare class RemoteLLM implements LLM {
|
|
|
38
38
|
expandQuery(query: string, options?: {
|
|
39
39
|
context?: string;
|
|
40
40
|
includeLexical?: boolean;
|
|
41
|
+
includeHyde?: boolean;
|
|
41
42
|
timeZone?: string;
|
|
42
43
|
}): Promise<Queryable[]>;
|
|
43
44
|
rerank(query: string, documents: RerankDocument[], options?: RerankOptions | string | (RerankOptions & {
|
package/dist/remote-llm.js
CHANGED
|
@@ -127,18 +127,29 @@ export class RemoteLLM {
|
|
|
127
127
|
throw new Error("Remote expansion is not configured or circuit is broken.");
|
|
128
128
|
}
|
|
129
129
|
const includeLexical = options?.includeLexical !== false;
|
|
130
|
+
const includeHyde = options?.includeHyde !== false;
|
|
130
131
|
const lexicalOutput = includeLexical ? "lex: keyword-focused search phrase\n" : "";
|
|
131
132
|
const lexicalRule = includeLexical
|
|
132
133
|
? "- lex: preserve precise terms and add only useful synonyms or related keywords; do not write a complete question.\n"
|
|
133
134
|
: "";
|
|
134
135
|
const lexicalExample = includeLexical ? "lex: database connection pool timeout exhaustion\n" : "";
|
|
136
|
+
const hydeOutput = includeHyde ? "hyde: concise hypothetical answer-style passage\n" : "";
|
|
137
|
+
const hydeRule = includeHyde
|
|
138
|
+
? "- hyde: write a concise hypothetical passage describing plausible answer content, describing general concepts without inventing specific fake facts.\n"
|
|
139
|
+
: "";
|
|
140
|
+
const hydeExample = includeHyde ? "hyde: Database connection pool timeout troubleshooting may examine pool limits, active connections, query latency, and connection handling.\n" : "";
|
|
141
|
+
const requestedBackends = [
|
|
142
|
+
includeLexical ? "lex" : null,
|
|
143
|
+
"vec",
|
|
144
|
+
includeHyde ? "hyde" : null,
|
|
145
|
+
].filter(Boolean).join(", ");
|
|
135
146
|
const systemPrompt = `<role>
|
|
136
147
|
You are a specialized assistant for hybrid document-search query expansion.
|
|
137
148
|
You expand search queries to enhance retrieval recall with analytical precision while preserving user intent and constraints.
|
|
138
149
|
</role>
|
|
139
150
|
|
|
140
151
|
<instructions>
|
|
141
|
-
1. Proactively generate one high-quality variation for each requested backend (
|
|
152
|
+
1. Proactively generate one high-quality variation for each requested backend (${requestedBackends}) whenever the query has clear intent.
|
|
142
153
|
2. Preserve query constraints and avoid inventing unmentioned facts.
|
|
143
154
|
3. Return only the requested prefix lines.
|
|
144
155
|
</instructions>
|
|
@@ -150,15 +161,13 @@ You expand search queries to enhance retrieval recall with analytical precision
|
|
|
150
161
|
- Keep the query's primary language and script, while preserving exact identifiers, product names, API names, abbreviations, and established domain terms from the query or context.
|
|
151
162
|
${lexicalRule}- vec: state the search intent as a clear natural-language phrase or question.
|
|
152
163
|
- For space-separated or keyword-list queries, synthesize the scattered terms into a coherent, natural-language phrase or question for vec.
|
|
153
|
-
-
|
|
154
|
-
- For very short or identifier-only queries, retain exact terms without inventing unprovided constraints.
|
|
164
|
+
${hydeRule}- For very short or identifier-only queries, retain exact terms without inventing unprovided constraints.
|
|
155
165
|
</constraints>
|
|
156
166
|
|
|
157
167
|
<output_format>
|
|
158
168
|
Output only prefix lines. Do not include preambles, explanations, markdown, or code fences.
|
|
159
169
|
${lexicalOutput}vec: natural-language semantic search phrase or question
|
|
160
|
-
|
|
161
|
-
Generate at most one line of each listed type.
|
|
170
|
+
${hydeOutput}Generate at most one line of each listed type.
|
|
162
171
|
</output_format>
|
|
163
172
|
|
|
164
173
|
<example>
|
|
@@ -173,8 +182,7 @@ database pool timeout
|
|
|
173
182
|
</task>
|
|
174
183
|
|
|
175
184
|
${lexicalExample}vec: Why is the database connection pool timing out under load?
|
|
176
|
-
|
|
177
|
-
</example>`;
|
|
185
|
+
${hydeExample}</example>`;
|
|
178
186
|
const currentTime = getFormattedLocalTime(new Date(), options?.timeZone ?? this.timeZone);
|
|
179
187
|
const additionalContext = options?.context
|
|
180
188
|
? `Additional context:\n${escapePromptXml(options.context)}`
|
|
@@ -224,7 +232,11 @@ Return only the prefix lines specified in the output format.
|
|
|
224
232
|
const match = /^(lex|vec|hyde)\s*:\s*(.+)$/i.exec(line.trim());
|
|
225
233
|
if (match && match[1] && match[2]) {
|
|
226
234
|
const type = match[1].toLowerCase();
|
|
227
|
-
if (
|
|
235
|
+
if (type === "lex" && !includeLexical)
|
|
236
|
+
continue;
|
|
237
|
+
if (type === "hyde" && !includeHyde)
|
|
238
|
+
continue;
|
|
239
|
+
if (!seenTypes.has(type)) {
|
|
228
240
|
seenTypes.add(type);
|
|
229
241
|
results.push({ type, text: match[2].trim() });
|
|
230
242
|
}
|
package/dist/search/zh-dict.txt
CHANGED
|
@@ -46617,6 +46617,7 @@ c++ 3 nz
|
|
|
46617
46617
|
來歷不明 5 Vi
|
|
46618
46618
|
來歸 4 Vi
|
|
46619
46619
|
來源 1909 N
|
|
46620
|
+
來源位置 1000000 nz
|
|
46620
46621
|
來源國 18 N
|
|
46621
46622
|
來源地 3 N
|
|
46622
46623
|
來源場 4 N
|
|
@@ -53444,7 +53445,7 @@ c++ 3 nz
|
|
|
53444
53445
|
停養 5 Vt
|
|
53445
53446
|
停養區 1 N
|
|
53446
53447
|
停餐 18 N
|
|
53447
|
-
停駐
|
|
53448
|
+
停駐 1000000 nz
|
|
53448
53449
|
停駕 6 Vi
|
|
53449
53450
|
停駛 271 Vi
|
|
53450
53451
|
停驗 2 Vt
|
|
@@ -151253,6 +151254,7 @@ c++ 3 nz
|
|
|
151253
151254
|
多鹽 1 N
|
|
151254
151255
|
多麗 1 N
|
|
151255
151256
|
多麼 338 ADV
|
|
151257
|
+
多點傳送 1000000 nz
|
|
151256
151258
|
多黨 19 N
|
|
151257
151259
|
多黨制 1 N
|
|
151258
151260
|
夛 198 zg
|
|
@@ -181443,6 +181445,7 @@ c++ 3 nz
|
|
|
181443
181445
|
封前朝 2 nr
|
|
181444
181446
|
封包 1000000 nz
|
|
181445
181447
|
封包交換 1000000 nz
|
|
181448
|
+
封包遺失 1000000 nz
|
|
181446
181449
|
封华歆 2 nr
|
|
181447
181450
|
封博望 3 nr
|
|
181448
181451
|
封博陵 2 nr
|
|
@@ -223286,6 +223289,7 @@ c++ 3 nz
|
|
|
223286
223289
|
快反 22 v
|
|
223287
223290
|
快取 1000000 nz
|
|
223288
223291
|
快取記憶體 1000000 nz
|
|
223292
|
+
快取項目 1000000 nz
|
|
223289
223293
|
快咬 3 v
|
|
223290
223294
|
快嘴 11 N
|
|
223291
223295
|
快嘴利舌 3 i
|
|
@@ -327168,7 +327172,7 @@ c++ 3 nz
|
|
|
327168
327172
|
比容 7 n
|
|
327169
327173
|
比对 88 d
|
|
327170
327174
|
比对法 3 n
|
|
327171
|
-
比對
|
|
327175
|
+
比對 1000000 nz
|
|
327172
327176
|
比對市 1 N
|
|
327173
327177
|
比對戰 1 N
|
|
327174
327178
|
比小孔 2 nr
|
|
@@ -345178,7 +345182,7 @@ c++ 3 nz
|
|
|
345178
345182
|
流离颠疐 3 i
|
|
345179
345183
|
流离颠顿 3 i
|
|
345180
345184
|
流移失所 3 i
|
|
345181
|
-
流程
|
|
345185
|
+
流程 1000000 nz
|
|
345182
345186
|
流程再造 1000000 nz
|
|
345183
345187
|
流程化 4 n
|
|
345184
345188
|
流程图 18 n
|
|
@@ -386962,6 +386966,7 @@ c++ 3 nz
|
|
|
386962
386966
|
生命觀 2 N
|
|
386963
386967
|
生命诚可贵 3 i
|
|
386964
386968
|
生命财产 3 l
|
|
386969
|
+
生命週期 1000000 nz
|
|
386965
386970
|
生命體 10 N
|
|
386966
386971
|
生員 3 N
|
|
386967
386972
|
生唐 2 t
|
|
@@ -402286,7 +402291,7 @@ c++ 3 nz
|
|
|
402286
402291
|
相究 1 Vt
|
|
402287
402292
|
相空间 3 n
|
|
402288
402293
|
相竞 16 v
|
|
402289
|
-
相符
|
|
402294
|
+
相符 1000000 nz
|
|
402290
402295
|
相符合 3 nr
|
|
402291
402296
|
相等 18 Vi
|
|
402292
402297
|
相等于 3 l
|
|
@@ -448252,6 +448257,7 @@ c++ 3 nz
|
|
|
448252
448257
|
群憤 1 Vi
|
|
448253
448258
|
群戲 1 N
|
|
448254
448259
|
群批 1 N
|
|
448260
|
+
群播 1000000 nz
|
|
448255
448261
|
群擋球 1 N
|
|
448256
448262
|
群攀 1 N
|
|
448257
448263
|
群攻 1 Vt
|
|
@@ -456405,7 +456411,7 @@ c++ 3 nz
|
|
|
456405
456411
|
背日 3 n
|
|
456406
456412
|
背日性 3 n
|
|
456407
456413
|
背旮旯儿 3 z
|
|
456408
|
-
背景
|
|
456414
|
+
背景 1000000 nz
|
|
456409
456415
|
背景值 15 N
|
|
456410
456416
|
背景光 1 N
|
|
456411
456417
|
背景噪声 10 n
|
|
@@ -501378,7 +501384,7 @@ c++ 3 nz
|
|
|
501378
501384
|
註消 1 N
|
|
501379
501385
|
註生娘娘 44 N
|
|
501380
501386
|
註腳 10 N
|
|
501381
|
-
註解
|
|
501387
|
+
註解 1000000 nz
|
|
501382
501388
|
註記 184 N
|
|
501383
501389
|
註語 1 N
|
|
501384
501390
|
註載 1 N
|
|
@@ -542725,7 +542731,7 @@ c++ 3 nz
|
|
|
542725
542731
|
遮空蔽日 3 i
|
|
542726
542732
|
遮簷 2 N
|
|
542727
542733
|
遮簾 1 N
|
|
542728
|
-
遮罩
|
|
542734
|
+
遮罩 1000000 nz
|
|
542729
542735
|
遮羞 14 Vi
|
|
542730
542736
|
遮羞布 37 N
|
|
542731
542737
|
遮羞費 18 N
|
package/dist/store.d.ts
CHANGED
|
@@ -132,6 +132,8 @@ export type ExpandedQuery = {
|
|
|
132
132
|
};
|
|
133
133
|
export type QueryExpansionOptions = {
|
|
134
134
|
requireResult?: boolean;
|
|
135
|
+
includeLexical?: boolean;
|
|
136
|
+
includeHyde?: boolean;
|
|
135
137
|
};
|
|
136
138
|
export declare function homedir(): string;
|
|
137
139
|
/**
|
|
@@ -313,7 +315,7 @@ export type Store = {
|
|
|
313
315
|
searchVec: (query: string, model: string, limit?: number, collectionFilter?: CollectionFilter, session?: ILLMSession, precomputedEmbedding?: number[]) => Promise<SearchResult[]>;
|
|
314
316
|
expandQuery: (query: string, model?: string, expansionContext?: string, options?: QueryExpansionOptions) => Promise<ExpandedQuery[]>;
|
|
315
317
|
/** Drop the cached expansion for a query so the next call regenerates. */
|
|
316
|
-
invalidateExpansionCache: (query: string, expansionContext?: string) => void;
|
|
318
|
+
invalidateExpansionCache: (query: string, expansionContext?: string, options?: QueryExpansionOptions) => void;
|
|
317
319
|
rerank: (query: string, documents: {
|
|
318
320
|
file: string;
|
|
319
321
|
text: string;
|
|
@@ -634,6 +636,8 @@ export type CacheKeyBody = {
|
|
|
634
636
|
chunk?: string;
|
|
635
637
|
file?: string;
|
|
636
638
|
expansionContext?: string;
|
|
639
|
+
noHyde?: boolean;
|
|
640
|
+
noLex?: boolean;
|
|
637
641
|
};
|
|
638
642
|
export declare function getCacheKey(url: string, body: CacheKeyBody): string;
|
|
639
643
|
export declare function getCachedResult(db: Database, cacheKey: string): string | null;
|
|
@@ -968,7 +972,10 @@ export declare function expandQuery(query: string, model: string | undefined, db
|
|
|
968
972
|
* expansion's sub-queries all came back empty — left in place, the dud entry
|
|
969
973
|
* would replay the same misses on every warm repeat of the query.
|
|
970
974
|
*/
|
|
971
|
-
export declare function deleteExpansionCacheEntry(db: Database, query: string, model?: string, expansionContext?: string
|
|
975
|
+
export declare function deleteExpansionCacheEntry(db: Database, query: string, model?: string, expansionContext?: string, options?: {
|
|
976
|
+
includeLexical?: boolean;
|
|
977
|
+
includeHyde?: boolean;
|
|
978
|
+
}): void;
|
|
972
979
|
export declare function rerank(query: string, documents: {
|
|
973
980
|
file: string;
|
|
974
981
|
text: string;
|
|
@@ -1106,6 +1113,7 @@ export interface HybridQueryOptions {
|
|
|
1106
1113
|
/** Additional context used for reranking and snippet/chunk selection. */
|
|
1107
1114
|
rerankContext?: string;
|
|
1108
1115
|
expansion?: ExpansionMode;
|
|
1116
|
+
includeHyde?: boolean;
|
|
1109
1117
|
skipRerank?: boolean;
|
|
1110
1118
|
chunkStrategy?: ChunkStrategy;
|
|
1111
1119
|
hooks?: SearchHooks;
|
|
@@ -1159,6 +1167,8 @@ export interface VectorSearchOptions {
|
|
|
1159
1167
|
minScore?: number;
|
|
1160
1168
|
/** Additional context used only while generating query expansions. */
|
|
1161
1169
|
expansionContext?: string;
|
|
1170
|
+
/** Whether to include HyDE (hypothetical document) in query expansion (default: true) */
|
|
1171
|
+
includeHyde?: boolean;
|
|
1162
1172
|
hooks?: Pick<SearchHooks, 'onExpand'>;
|
|
1163
1173
|
}
|
|
1164
1174
|
export interface VectorSearchResult {
|
package/dist/store.js
CHANGED
|
@@ -2690,7 +2690,7 @@ export function createStore(dbPath, options = {}) {
|
|
|
2690
2690
|
searchVec: (query, model, limit, collectionFilter, session, precomputedEmbedding) => searchVec(db, query, model, limit, collectionFilter, session, precomputedEmbedding, store.embeddingProvider, store.authorizeRemoteRequest, store.llm),
|
|
2691
2691
|
// Query expansion & reranking
|
|
2692
2692
|
expandQuery: (query, model, expansionContext, options) => expandQuery(query, model ?? store.localLlm?.generateModelName ?? store.llm?.generateModelName ?? DEFAULT_QUERY_MODEL, db, expansionContext, store.llm, options),
|
|
2693
|
-
invalidateExpansionCache: (query, expansionContext) => deleteExpansionCacheEntry(db, query, store.localLlm?.generateModelName ?? store.llm?.generateModelName ?? DEFAULT_QUERY_MODEL, expansionContext),
|
|
2693
|
+
invalidateExpansionCache: (query, expansionContext, options) => deleteExpansionCacheEntry(db, query, store.localLlm?.generateModelName ?? store.llm?.generateModelName ?? DEFAULT_QUERY_MODEL, expansionContext, options),
|
|
2694
2694
|
rerank: (query, documents, model, rerankContext) => {
|
|
2695
2695
|
const llm = getLlm(store);
|
|
2696
2696
|
return rerank(query, documents, model ?? store.localLlm?.rerankModelName ?? llm?.rerankModelName ?? DEFAULT_RERANK_MODEL, db, rerankContext, store.llm ?? llm);
|
|
@@ -3018,7 +3018,7 @@ export function countOrphanedVectors(db) {
|
|
|
3018
3018
|
* Returns the number of orphaned embedding chunks deleted.
|
|
3019
3019
|
*/
|
|
3020
3020
|
export function cleanupOrphanedVectors(db) {
|
|
3021
|
-
// sqlite-vec may not be loaded
|
|
3021
|
+
// sqlite-vec may not be loaded if extension failed to initialize.
|
|
3022
3022
|
// The vectors_vec virtual table can appear in sqlite_master from a prior
|
|
3023
3023
|
// session, but querying it without the vec0 module loaded will crash (#380).
|
|
3024
3024
|
if (!isSqliteVecAvailable()) {
|
|
@@ -4670,8 +4670,16 @@ export function insertEmbedding(db, hash, seq, pos, embedding, model, embeddedAt
|
|
|
4670
4670
|
// Query expansion
|
|
4671
4671
|
// =============================================================================
|
|
4672
4672
|
export async function expandQuery(query, model = DEFAULT_QUERY_MODEL, db, expansionContext, llmOverride, options) {
|
|
4673
|
+
const includeLexical = options?.includeLexical ?? true;
|
|
4674
|
+
const includeHyde = options?.includeHyde ?? true;
|
|
4673
4675
|
// Check cache first — stored as JSON preserving types
|
|
4674
|
-
const cacheKey = getCacheKey("expandQuery", {
|
|
4676
|
+
const cacheKey = getCacheKey("expandQuery", {
|
|
4677
|
+
query,
|
|
4678
|
+
model,
|
|
4679
|
+
...(expansionContext && { expansionContext }),
|
|
4680
|
+
...(!includeHyde && { noHyde: true }),
|
|
4681
|
+
...(!includeLexical && { noLex: true }),
|
|
4682
|
+
});
|
|
4675
4683
|
const cached = getCachedResult(db, cacheKey);
|
|
4676
4684
|
if (cached) {
|
|
4677
4685
|
try {
|
|
@@ -4693,7 +4701,11 @@ export async function expandQuery(query, model = DEFAULT_QUERY_MODEL, db, expans
|
|
|
4693
4701
|
}
|
|
4694
4702
|
const llm = llmOverride ?? getDefaultLlamaCpp();
|
|
4695
4703
|
// Note: LlamaCpp uses hardcoded model, model parameter is ignored
|
|
4696
|
-
const results = await llm.expandQuery(query, {
|
|
4704
|
+
const results = await llm.expandQuery(query, {
|
|
4705
|
+
context: expansionContext,
|
|
4706
|
+
includeLexical,
|
|
4707
|
+
includeHyde,
|
|
4708
|
+
});
|
|
4697
4709
|
// Map Queryable[] → ExpandedQuery[] (same shape, decoupled from llm.ts internals).
|
|
4698
4710
|
// Filter out entries that duplicate the original query text.
|
|
4699
4711
|
const expanded = results
|
|
@@ -4712,8 +4724,14 @@ export async function expandQuery(query, model = DEFAULT_QUERY_MODEL, db, expans
|
|
|
4712
4724
|
* expansion's sub-queries all came back empty — left in place, the dud entry
|
|
4713
4725
|
* would replay the same misses on every warm repeat of the query.
|
|
4714
4726
|
*/
|
|
4715
|
-
export function deleteExpansionCacheEntry(db, query, model = DEFAULT_QUERY_MODEL, expansionContext) {
|
|
4716
|
-
const cacheKey = getCacheKey("expandQuery", {
|
|
4727
|
+
export function deleteExpansionCacheEntry(db, query, model = DEFAULT_QUERY_MODEL, expansionContext, options) {
|
|
4728
|
+
const cacheKey = getCacheKey("expandQuery", {
|
|
4729
|
+
query,
|
|
4730
|
+
model,
|
|
4731
|
+
...(expansionContext && { expansionContext }),
|
|
4732
|
+
...(options?.includeHyde === false && { noHyde: true }),
|
|
4733
|
+
...(options?.includeLexical === false && { noLex: true }),
|
|
4734
|
+
});
|
|
4717
4735
|
db.prepare(`DELETE FROM llm_cache WHERE hash = ?`).run(cacheKey);
|
|
4718
4736
|
}
|
|
4719
4737
|
// =============================================================================
|
|
@@ -5491,6 +5509,7 @@ export async function hybridQuery(store, query, options) {
|
|
|
5491
5509
|
if (hasStrongSignal)
|
|
5492
5510
|
hooks?.onStrongSignal?.(topScore);
|
|
5493
5511
|
// Step 2: Expand query (or skip if strong signal)
|
|
5512
|
+
const includeHyde = options?.includeHyde ?? true;
|
|
5494
5513
|
if (expansionDecision.action === "expand")
|
|
5495
5514
|
hooks?.onExpandStart?.();
|
|
5496
5515
|
const expandStart = Date.now();
|
|
@@ -5500,6 +5519,7 @@ export async function hybridQuery(store, query, options) {
|
|
|
5500
5519
|
? []
|
|
5501
5520
|
: await store.expandQuery(query, undefined, expansionContext, {
|
|
5502
5521
|
requireResult: expansionDecision.reason === "explicit-force",
|
|
5522
|
+
includeHyde,
|
|
5503
5523
|
});
|
|
5504
5524
|
}
|
|
5505
5525
|
catch (error) {
|
|
@@ -5590,7 +5610,7 @@ export async function hybridQuery(store, query, options) {
|
|
|
5590
5610
|
const runnable = expanded.filter(q => q.type === "lex" || hasVectors);
|
|
5591
5611
|
const expansionContributed = rankedListMeta.some(m => m.queryType !== "original");
|
|
5592
5612
|
if (runnable.length > 0 && !expansionContributed) {
|
|
5593
|
-
store.invalidateExpansionCache(query, expansionContext);
|
|
5613
|
+
store.invalidateExpansionCache(query, expansionContext, { includeHyde });
|
|
5594
5614
|
}
|
|
5595
5615
|
}
|
|
5596
5616
|
// Step 4: RRF fusion — original-query FTS and vector lists get 2x weight;
|
|
@@ -5767,11 +5787,14 @@ export async function vectorSearchQuery(store, query, options) {
|
|
|
5767
5787
|
const minScore = options?.minScore ?? 0.3;
|
|
5768
5788
|
const collection = options?.collection;
|
|
5769
5789
|
const expansionContext = options?.expansionContext;
|
|
5790
|
+
const includeHyde = options?.includeHyde ?? true;
|
|
5770
5791
|
if (!hasSearchableVectorIndex(store))
|
|
5771
5792
|
return [];
|
|
5772
5793
|
// Expand query — filter to vec/hyde only (lex queries target FTS, not vector)
|
|
5773
5794
|
const expandStart = Date.now();
|
|
5774
|
-
const allExpanded = await store.expandQuery(query, undefined, expansionContext
|
|
5795
|
+
const allExpanded = await store.expandQuery(query, undefined, expansionContext, {
|
|
5796
|
+
includeHyde,
|
|
5797
|
+
});
|
|
5775
5798
|
const vecExpanded = allExpanded.filter(q => q.type !== 'lex');
|
|
5776
5799
|
options?.hooks?.onExpand?.(query, vecExpanded, Date.now() - expandStart);
|
|
5777
5800
|
const embedModel = store.embeddingProvider?.model ?? getLlm(store).embedModelName;
|