@wei840222/qmd 2026.8.28 → 2026.9.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +84 -2
- package/dist/cli/build-info.json +2 -2
- package/dist/cli/qmd.js +96 -18
- package/dist/collections.js +9 -4
- package/dist/db.d.ts +16 -29
- package/dist/db.js +63 -40
- package/dist/index.d.ts +10 -0
- package/dist/index.js +14 -2
- package/dist/llm.d.ts +7 -1
- package/dist/llm.js +23 -4
- package/dist/mcp/server.js +70 -6
- package/dist/metadata-filter.d.ts +74 -0
- package/dist/metadata-filter.js +279 -0
- package/dist/metadata-store.d.ts +45 -0
- package/dist/metadata-store.js +173 -0
- package/dist/metadata.d.ts +61 -0
- package/dist/metadata.js +215 -0
- package/dist/search/zh-dict.txt +3 -0
- package/dist/store.d.ts +23 -16
- package/dist/store.js +354 -166
- package/package.json +3 -5
- package/scripts/sync-zh-dict.mjs +4 -1
- package/skills/qmd/SKILL.md +11 -2
- package/skills/qmd/references/query-syntax.md +1 -8
- package/skills/release/SKILL.md +0 -141
- package/skills/release/scripts/install-hooks.sh +0 -38
- package/skills/release/scripts/release-context.sh +0 -129
package/dist/db.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* db.ts - SQLite database connection and extension management
|
|
3
3
|
*
|
|
4
|
-
* Provides
|
|
5
|
-
* and sqlite-vec.
|
|
4
|
+
* Provides a synchronous node:sqlite connection with QMD's transaction helper
|
|
5
|
+
* and sqlite-vec extension loading.
|
|
6
6
|
*/
|
|
7
|
-
import
|
|
7
|
+
import { DatabaseSync } from "node:sqlite";
|
|
8
8
|
import * as sqliteVec from "sqlite-vec";
|
|
9
|
+
let savepointSequence = 0;
|
|
9
10
|
function isBusyError(err) {
|
|
10
11
|
if (typeof err !== "object" || err === null)
|
|
11
12
|
return false;
|
|
@@ -18,12 +19,57 @@ function isBusyError(err) {
|
|
|
18
19
|
function sleepSync(ms) {
|
|
19
20
|
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
20
21
|
}
|
|
22
|
+
/** Synchronous SQLite connection with QMD-compatible transactions. */
|
|
23
|
+
export class Database extends DatabaseSync {
|
|
24
|
+
transaction(operation) {
|
|
25
|
+
const execute = (mode, args) => {
|
|
26
|
+
if (!this.isTransaction) {
|
|
27
|
+
this.exec(`BEGIN ${mode}`);
|
|
28
|
+
try {
|
|
29
|
+
const result = operation(...args);
|
|
30
|
+
this.exec("COMMIT");
|
|
31
|
+
return result;
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
try {
|
|
35
|
+
this.exec("ROLLBACK");
|
|
36
|
+
}
|
|
37
|
+
catch { }
|
|
38
|
+
throw error;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const savepoint = `qmd_${++savepointSequence}`;
|
|
42
|
+
this.exec(`SAVEPOINT ${savepoint}`);
|
|
43
|
+
try {
|
|
44
|
+
const result = operation(...args);
|
|
45
|
+
this.exec(`RELEASE ${savepoint}`);
|
|
46
|
+
return result;
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
try {
|
|
50
|
+
this.exec(`ROLLBACK TO ${savepoint}`);
|
|
51
|
+
this.exec(`RELEASE ${savepoint}`);
|
|
52
|
+
}
|
|
53
|
+
catch { }
|
|
54
|
+
throw error;
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
const transaction = ((...args) => execute("DEFERRED", args));
|
|
58
|
+
transaction.deferred = (...args) => execute("DEFERRED", args);
|
|
59
|
+
transaction.immediate = (...args) => execute("IMMEDIATE", args);
|
|
60
|
+
transaction.exclusive = (...args) => execute("EXCLUSIVE", args);
|
|
61
|
+
return transaction;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
function resolveBusyTimeout() {
|
|
65
|
+
const raw = process.env.QMD_SQLITE_BUSY_TIMEOUT;
|
|
66
|
+
const parsed = raw !== undefined && raw !== "" ? Number(raw) : Number.NaN;
|
|
67
|
+
return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : 120_000;
|
|
68
|
+
}
|
|
21
69
|
/**
|
|
22
70
|
* Switch a connection to WAL, retrying on `SQLITE_BUSY` within the busy-timeout
|
|
23
|
-
* budget.
|
|
24
|
-
*
|
|
25
|
-
* cold database throw "database is locked" even with `busy_timeout` set. Once the
|
|
26
|
-
* database is already WAL the pragma is a cheap no-op that does not contend.
|
|
71
|
+
* budget. Migrating the journal needs a brief exclusive lock and does not invoke
|
|
72
|
+
* SQLite's busy handler on every supported runtime.
|
|
27
73
|
*/
|
|
28
74
|
function enableWal(db, budgetMs) {
|
|
29
75
|
const deadline = Date.now() + Math.max(budgetMs, 0);
|
|
@@ -39,46 +85,23 @@ function enableWal(db, budgetMs) {
|
|
|
39
85
|
}
|
|
40
86
|
}
|
|
41
87
|
}
|
|
42
|
-
/**
|
|
43
|
-
* Open a SQLite database using better-sqlite3.
|
|
44
|
-
*
|
|
45
|
-
* `better-sqlite3` defaults `busy_timeout` to 0, so concurrent writers throw
|
|
46
|
-
* `SQLITE_BUSY` instead of waiting. WAL improves read-while-write concurrency
|
|
47
|
-
* but does not serialise writers. Setting the timeout at connection open makes
|
|
48
|
-
* parallel processes queue at batch boundaries instead of failing on contact.
|
|
49
|
-
*
|
|
50
|
-
* WAL is enabled here too (with a bounded retry) so connection-level pragmas
|
|
51
|
-
* live in one place and the cold-database journal migration survives concurrent
|
|
52
|
-
* opens.
|
|
53
|
-
*
|
|
54
|
-
* Default 120_000 ms outlasts the worst-case batch commit on a multi-GB
|
|
55
|
-
* index. Override with `QMD_SQLITE_BUSY_TIMEOUT` (value in milliseconds; `0`
|
|
56
|
-
* restores the upstream fail-fast behaviour).
|
|
57
|
-
*/
|
|
88
|
+
/** Open a writable QMD database using Node's built-in SQLite runtime. */
|
|
58
89
|
export function openDatabase(path) {
|
|
59
|
-
const
|
|
60
|
-
const
|
|
61
|
-
const parsed = raw !== undefined && raw !== "" ? Number(raw) : Number.NaN;
|
|
62
|
-
const busyTimeoutMs = Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : 120_000;
|
|
63
|
-
db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}`);
|
|
90
|
+
const busyTimeoutMs = resolveBusyTimeout();
|
|
91
|
+
const db = new Database(path, { allowExtension: true, timeout: busyTimeoutMs });
|
|
64
92
|
enableWal(db, busyTimeoutMs);
|
|
65
93
|
return db;
|
|
66
94
|
}
|
|
67
95
|
/** Open an existing database without changing journal mode, schema, or user data. */
|
|
68
96
|
export function openReadOnlyDatabase(path) {
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
return db;
|
|
97
|
+
const busyTimeoutMs = resolveBusyTimeout();
|
|
98
|
+
return new Database(path, {
|
|
99
|
+
readOnly: true,
|
|
100
|
+
allowExtension: true,
|
|
101
|
+
timeout: busyTimeoutMs,
|
|
102
|
+
});
|
|
76
103
|
}
|
|
77
|
-
/**
|
|
78
|
-
* Load the sqlite-vec extension into a database.
|
|
79
|
-
*
|
|
80
|
-
* Throws with fix instructions when the extension is unavailable.
|
|
81
|
-
*/
|
|
104
|
+
/** Load the sqlite-vec extension into a database. */
|
|
82
105
|
export function loadSqliteVec(db) {
|
|
83
106
|
try {
|
|
84
107
|
sqliteVec.load(db);
|
package/dist/index.d.ts
CHANGED
|
@@ -17,9 +17,13 @@
|
|
|
17
17
|
* await store.close()
|
|
18
18
|
*/
|
|
19
19
|
import { extractSnippet, addLineNumbers, DEFAULT_MULTI_GET_MAX_BYTES, type Store as InternalStore, type DocumentResult, type DocumentNotFound, type DocumentExcludedByIgnore, type DocumentLookupError, type SearchResult, type HybridQueryResult, type HybridQueryOptions, type HybridQueryExplain, type ExpandedQuery, type StructuredSearchOptions, type MultiGetResult, type IndexStatus, type IndexHealthInfo, type SearchHooks, type ReindexProgress, type ReindexResult, type EmbedProgress, type EmbedResult, type ChunkStrategy } from "./store.js";
|
|
20
|
+
import type { DocumentMetadata, MetadataScalar, MetadataScalarArray, MetadataValue } from "./metadata.js";
|
|
21
|
+
import { parseMetadataFilter, MetadataFilterError, type MetadataFilter, type MetadataFilterGroup, type MetadataFilterNegation, type MetadataCondition } from "./metadata-filter.js";
|
|
20
22
|
import type { ExpansionMode } from "./search/query-expansion.js";
|
|
21
23
|
import { type Collection, type CollectionConfig, type NamedCollection, type ContextMap } from "./collections.js";
|
|
22
24
|
export type { DocumentResult, DocumentNotFound, DocumentExcludedByIgnore, DocumentLookupError, SearchResult, HybridQueryResult, HybridQueryOptions, HybridQueryExplain, ExpandedQuery, StructuredSearchOptions, MultiGetResult, IndexStatus, IndexHealthInfo, SearchHooks, ReindexProgress, ReindexResult, EmbedProgress, EmbedResult, Collection, CollectionConfig, NamedCollection, ContextMap, };
|
|
25
|
+
export type { DocumentMetadata, MetadataScalar, MetadataScalarArray, MetadataValue, MetadataFilter, MetadataFilterGroup, MetadataFilterNegation, MetadataCondition, };
|
|
26
|
+
export { parseMetadataFilter, MetadataFilterError };
|
|
23
27
|
export type { InternalStore };
|
|
24
28
|
export type { ExpansionMode } from "./search/query-expansion.js";
|
|
25
29
|
export { extractSnippet, addLineNumbers, DEFAULT_MULTI_GET_MAX_BYTES };
|
|
@@ -65,6 +69,8 @@ export interface SearchOptions {
|
|
|
65
69
|
collection?: string;
|
|
66
70
|
/** Filter to specific collections */
|
|
67
71
|
collections?: string[];
|
|
72
|
+
/** Metadata filter — every returned result satisfies it */
|
|
73
|
+
filter?: MetadataFilter;
|
|
68
74
|
/** Max results (default: 10) */
|
|
69
75
|
limit?: number;
|
|
70
76
|
/** Max candidates to rerank (default: 40) */
|
|
@@ -88,6 +94,8 @@ export interface SearchOptions {
|
|
|
88
94
|
export interface LexSearchOptions {
|
|
89
95
|
limit?: number;
|
|
90
96
|
collection?: string | string[];
|
|
97
|
+
/** Metadata filter — every returned result satisfies it */
|
|
98
|
+
filter?: MetadataFilter;
|
|
91
99
|
}
|
|
92
100
|
/**
|
|
93
101
|
* Options for searchVector() — vector similarity search.
|
|
@@ -95,6 +103,8 @@ export interface LexSearchOptions {
|
|
|
95
103
|
export interface VectorSearchOptions {
|
|
96
104
|
limit?: number;
|
|
97
105
|
collection?: string | string[];
|
|
106
|
+
/** Metadata filter — every returned result satisfies it */
|
|
107
|
+
filter?: MetadataFilter;
|
|
98
108
|
}
|
|
99
109
|
/**
|
|
100
110
|
* Options for expandQuery() — manual query expansion.
|
package/dist/index.js
CHANGED
|
@@ -20,6 +20,7 @@ import { existsSync } from "node:fs";
|
|
|
20
20
|
import { createStore as createStoreInternal, hybridQuery, structuredSearch, extractSnippet, addLineNumbers, DEFAULT_MULTI_GET_MAX_BYTES, reindexCollection, generateEmbeddings, listCollections as storeListCollections, syncConfigToDb, getStoreCollections, getStoreCollection, getStoreGlobalContext, getStoreContexts, upsertStoreCollection, removeCollection as removeCollectionWithDocuments, renameCollection as renameCollectionWithDocuments, updateStoreContext, removeStoreContext, setStoreGlobalContext, vacuumDatabase, cleanupOrphanedContent, cleanupOrphanedVectors, deleteLLMCache, deleteInactiveDocuments, clearAllEmbeddings, getPendingEmbeddingDocsReadOnly, getIndexHealthReadOnly, getStatusReadOnly, } from "./store.js";
|
|
21
21
|
import { DEFAULT_EMBED_MODEL_URI, LlamaCpp, waitForLLMSessionsToDrain, } from "./llm.js";
|
|
22
22
|
import { LocalEmbeddingProviderOwner } from "./embedding/local.js";
|
|
23
|
+
import { parseMetadataFilter, MetadataFilterError, } from "./metadata-filter.js";
|
|
23
24
|
import { OpenAIEmbeddingProvider, UnavailableOpenAIEmbeddingProvider, } from "./embedding/openai.js";
|
|
24
25
|
import { authorizeRemoteEmbeddingRequest, remoteEmbeddingIdentity, } from "./embedding/remote-embedding.js";
|
|
25
26
|
import { readStoredEmbeddingIdentity } from "./embedding/identity.js";
|
|
@@ -29,6 +30,7 @@ import { rebuildCjkLexicalIndex } from "./search/cjk-index.js";
|
|
|
29
30
|
import { RemoteLLM } from "./remote-llm.js";
|
|
30
31
|
import { HybridLLM } from "./hybrid-llm.js";
|
|
31
32
|
import { createCollectionConfigSource, loadConfig, addCollection as collectionsAddCollection, removeCollection as collectionsRemoveCollection, renameCollection as collectionsRenameCollection, addContext as collectionsAddContext, removeContext as collectionsRemoveContext, setGlobalContext as collectionsSetGlobalContext, } from "./collections.js";
|
|
33
|
+
export { parseMetadataFilter, MetadataFilterError };
|
|
32
34
|
// Re-export utility functions and types used by frontends
|
|
33
35
|
export { extractSnippet, addLineNumbers, DEFAULT_MULTI_GET_MAX_BYTES };
|
|
34
36
|
// Re-export getDefaultDbPath for CLI/MCP that need the default database location
|
|
@@ -208,10 +210,15 @@ export async function createStore(options) {
|
|
|
208
210
|
...(opts.collections ?? []),
|
|
209
211
|
];
|
|
210
212
|
const skipRerank = opts.rerank === false;
|
|
213
|
+
// The SDK is also a JavaScript boundary: TypeScript declarations do not
|
|
214
|
+
// protect plain-JS callers or deserialized input. Apply the same bounded,
|
|
215
|
+
// strict validation used by CLI, MCP, and HTTP before compiling SQL.
|
|
216
|
+
const filter = opts.filter === undefined ? undefined : parseMetadataFilter(opts.filter);
|
|
211
217
|
if (opts.queries) {
|
|
212
218
|
// Pre-expanded queries — use structuredSearch
|
|
213
219
|
return structuredSearch(internal, opts.queries, {
|
|
214
220
|
collections: collections.length > 0 ? collections : undefined,
|
|
221
|
+
filter,
|
|
215
222
|
limit: opts.limit,
|
|
216
223
|
minScore: opts.minScore,
|
|
217
224
|
explain: opts.explain,
|
|
@@ -225,6 +232,7 @@ export async function createStore(options) {
|
|
|
225
232
|
return hybridQuery(internal, opts.query, {
|
|
226
233
|
collections: collections.length > 0 ? collections : undefined,
|
|
227
234
|
collection: collections.length === 1 ? collections[0] : (collections.length > 0 ? collections : undefined),
|
|
235
|
+
filter,
|
|
228
236
|
limit: opts.limit,
|
|
229
237
|
minScore: opts.minScore,
|
|
230
238
|
explain: opts.explain,
|
|
@@ -238,10 +246,14 @@ export async function createStore(options) {
|
|
|
238
246
|
chunkStrategy: opts.chunkStrategy,
|
|
239
247
|
});
|
|
240
248
|
},
|
|
241
|
-
searchLex: async (q, opts) =>
|
|
249
|
+
searchLex: async (q, opts) => {
|
|
250
|
+
const filter = opts?.filter === undefined ? undefined : parseMetadataFilter(opts.filter);
|
|
251
|
+
return internal.searchFTS(q, opts?.limit, opts?.collection, filter);
|
|
252
|
+
},
|
|
242
253
|
searchVector: async (q, opts) => {
|
|
254
|
+
const filter = opts?.filter === undefined ? undefined : parseMetadataFilter(opts.filter);
|
|
243
255
|
const provider = internal.embeddingProvider;
|
|
244
|
-
return internal.searchVec(q, provider?.model ?? internal.llm?.embedModelName ?? DEFAULT_EMBED_MODEL_URI, opts?.limit, opts?.collection);
|
|
256
|
+
return internal.searchVec(q, provider?.model ?? internal.llm?.embedModelName ?? DEFAULT_EMBED_MODEL_URI, opts?.limit, opts?.collection, undefined, undefined, filter);
|
|
245
257
|
},
|
|
246
258
|
expandQuery: async (q, opts) => internal.expandQuery(q, undefined, opts?.expansionContext, {
|
|
247
259
|
includeLexical: opts?.includeLexical,
|
package/dist/llm.d.ts
CHANGED
|
@@ -199,7 +199,13 @@ export type PullResult = {
|
|
|
199
199
|
export type GgufFileInspection = {
|
|
200
200
|
exists: boolean;
|
|
201
201
|
valid: boolean;
|
|
202
|
-
|
|
202
|
+
/**
|
|
203
|
+
* "html" and "invalid" mean the header was read and is confirmed bad.
|
|
204
|
+
* "unreadable" means the read itself failed, so nothing is known about the
|
|
205
|
+
* content. The two stay distinct because only the first justifies deleting
|
|
206
|
+
* the file.
|
|
207
|
+
*/
|
|
208
|
+
kind: "missing" | "gguf" | "html" | "invalid" | "unreadable";
|
|
203
209
|
sizeBytes?: number;
|
|
204
210
|
magic?: string;
|
|
205
211
|
details: string;
|
package/dist/llm.js
CHANGED
|
@@ -235,10 +235,13 @@ export function inspectGgufFile(filePath) {
|
|
|
235
235
|
};
|
|
236
236
|
}
|
|
237
237
|
catch (error) {
|
|
238
|
+
// stat/open/read threw, so the header was never seen. Reporting this as
|
|
239
|
+
// "invalid" would claim a verdict no read supports, and `magic` stays
|
|
240
|
+
// undefined for the same reason.
|
|
238
241
|
return {
|
|
239
242
|
exists: true,
|
|
240
243
|
valid: false,
|
|
241
|
-
kind: "
|
|
244
|
+
kind: "unreadable",
|
|
242
245
|
sizeBytes,
|
|
243
246
|
details: `cannot read model file: ${error instanceof Error ? error.message : String(error)}`,
|
|
244
247
|
};
|
|
@@ -253,11 +256,25 @@ function validateGgufFile(filePath, modelUri) {
|
|
|
253
256
|
const inspection = inspectGgufFile(filePath);
|
|
254
257
|
if (!inspection.exists || inspection.valid)
|
|
255
258
|
return; // let downstream handle missing files
|
|
256
|
-
//
|
|
259
|
+
// A read that failed says nothing about the bytes on disk. Deleting here
|
|
260
|
+
// throws away a file that is usually fine (fd exhaustion, a concurrent
|
|
261
|
+
// loader, a volume that briefly went away) and re-downloading it can cost
|
|
262
|
+
// gigabytes, so surface the real error and leave the file alone.
|
|
263
|
+
if (inspection.kind === "unreadable") {
|
|
264
|
+
throw new Error(`Model file could not be read, so it could not be validated (${inspection.details}).\n` +
|
|
265
|
+
`Model: ${modelUri}\n` +
|
|
266
|
+
`Path: ${filePath}\n\n` +
|
|
267
|
+
`The file has been left in place. If this repeats, check open file limits, ` +
|
|
268
|
+
`permissions, and whether the volume holding the model cache is still mounted.`);
|
|
269
|
+
}
|
|
270
|
+
// Confirmed bad content: remove it so the next attempt re-downloads.
|
|
271
|
+
let removed = true;
|
|
257
272
|
try {
|
|
258
273
|
unlinkSync(filePath);
|
|
259
274
|
}
|
|
260
|
-
catch {
|
|
275
|
+
catch {
|
|
276
|
+
removed = false;
|
|
277
|
+
}
|
|
261
278
|
if (inspection.kind === "html") {
|
|
262
279
|
throw new Error(`Downloaded model file is an HTML page, not a GGUF model (${formatModelFileSize(inspection.sizeBytes ?? 0)}).\n` +
|
|
263
280
|
`Something is intercepting the download from huggingface.co (a proxy, firewall, or captive portal).\n\n` +
|
|
@@ -272,7 +289,9 @@ function validateGgufFile(filePath, modelUri) {
|
|
|
272
289
|
throw new Error(`Model file is not valid GGUF (expected magic "GGUF", got "${inspection.magic ?? "unknown"}", file is ${formatModelFileSize(inspection.sizeBytes ?? 0)}).\n` +
|
|
273
290
|
`Model: ${modelUri}\n` +
|
|
274
291
|
`Path: ${filePath}\n\n` +
|
|
275
|
-
|
|
292
|
+
(removed
|
|
293
|
+
? `The file has been removed. Run the command again to re-download.`
|
|
294
|
+
: `The file could NOT be removed. Delete it manually, then run the command again.`));
|
|
276
295
|
}
|
|
277
296
|
/**
|
|
278
297
|
* node-llama-cpp prints a multi-line download progress bar when the second
|
package/dist/mcp/server.js
CHANGED
|
@@ -15,10 +15,24 @@ import { createMcpHandler, McpServer, ResourceTemplate } from "@modelcontextprot
|
|
|
15
15
|
import { serveStdio } from "@modelcontextprotocol/server/stdio";
|
|
16
16
|
import { z } from "zod";
|
|
17
17
|
import { existsSync } from "fs";
|
|
18
|
-
import { createStore, extractSnippet, addLineNumbers, getDefaultDbPath, DEFAULT_MULTI_GET_MAX_BYTES, } from "../index.js";
|
|
18
|
+
import { createStore, extractSnippet, addLineNumbers, getDefaultDbPath, DEFAULT_MULTI_GET_MAX_BYTES, parseMetadataFilter, } from "../index.js";
|
|
19
19
|
import { getConfigPath } from "../collections.js";
|
|
20
20
|
import { enableProductionMode } from "../store.js";
|
|
21
21
|
import { checkRequestOrigin, resolveOriginGuard } from "./origin-guard.js";
|
|
22
|
+
/**
|
|
23
|
+
* Validate an untrusted `filter` argument through the shared runtime
|
|
24
|
+
* validator. Returns the parse error message when invalid.
|
|
25
|
+
*/
|
|
26
|
+
function validateFilterArgument(filter) {
|
|
27
|
+
if (filter === undefined)
|
|
28
|
+
return {};
|
|
29
|
+
try {
|
|
30
|
+
return { filter: parseMetadataFilter(filter) };
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
return { error: err instanceof Error ? err.message : String(err) };
|
|
34
|
+
}
|
|
35
|
+
}
|
|
22
36
|
// =============================================================================
|
|
23
37
|
// Helper functions
|
|
24
38
|
// =============================================================================
|
|
@@ -251,14 +265,21 @@ Context-aware lex (C++ performance, not sports):
|
|
|
251
265
|
minScore: z.number().optional().default(0).describe("Min relevance 0-1 (default: 0)"),
|
|
252
266
|
candidateLimit: z.number().optional().describe("Maximum candidates to rerank (default: 40, lower = faster but may miss results)"),
|
|
253
267
|
collections: z.array(z.string()).optional().describe("Filter to collections (OR match)"),
|
|
268
|
+
filter: z.record(z.string(), z.unknown()).optional().describe("Metadata filter (recursive JSON AST). Every returned result satisfies it. " +
|
|
269
|
+
"Nodes are operator-discriminated: logical groups {operator:'and'|'or', operands:[...]}, " +
|
|
270
|
+
"negation {operator:'not', operand:{...}}, and conditions {key, operator, value} with " +
|
|
271
|
+
"operators eq/ne/gt/gte/lt/lte (comparison), in/nin/all (membership), exists (presence). " +
|
|
272
|
+
"Values are typed exactly (no coercion); missing keys do not match ne/nin. " +
|
|
273
|
+
"Example: {\"operator\":\"and\",\"operands\":[{\"key\":\"topics\",\"operator\":\"all\",\"value\":[\"typescript\"]}," +
|
|
274
|
+
"{\"key\":\"status\",\"operator\":\"ne\",\"value\":\"draft\"}]}"),
|
|
254
275
|
expansionContext: z.string().optional().describe("Additional context used only to generate lex, vec, and hyde query expansions."),
|
|
255
276
|
rerankContext: z.string().optional().describe("Additional context used only to rerank results and select snippets/chunks."),
|
|
277
|
+
intent: z.string().optional().describe("Background context to disambiguate the query. Example: query='performance', intent='web page load times and Core Web Vitals'. Does not search on its own."),
|
|
256
278
|
rerank: z.boolean().optional().default(true).describe("Rerank results using LLM (default: true). Set to false for faster results on CPU-only machines."),
|
|
257
279
|
explain: z.boolean().optional().default(false).describe("Include retrieval traces and the shared query-expansion decision or typed expansion error"),
|
|
258
280
|
includeHyde: z.boolean().optional().default(true).describe("Whether to include HyDE (hypothetical document) in query expansion (default: true)"),
|
|
259
281
|
}),
|
|
260
|
-
}, track(async ({ query, searches, expansion, includeHyde, limit, minScore, candidateLimit, collections, expansionContext, rerankContext, rerank, explain }) => {
|
|
261
|
-
// Require exactly one of `query` (plain text with an expansion policy) or `searches` (typed sub-queries).
|
|
282
|
+
}, track(async ({ query, searches, expansion, includeHyde, limit, minScore, candidateLimit, collections, filter, expansionContext, rerankContext, intent, rerank, explain }) => {
|
|
262
283
|
if (!query && (!searches || searches.length === 0)) {
|
|
263
284
|
return {
|
|
264
285
|
content: [{ type: "text", text: "Error: provide either 'query' (plain text) or 'searches' (typed sub-queries)" }],
|
|
@@ -271,6 +292,13 @@ Context-aware lex (C++ performance, not sports):
|
|
|
271
292
|
isError: true,
|
|
272
293
|
};
|
|
273
294
|
}
|
|
295
|
+
const filterValidation = validateFilterArgument(filter);
|
|
296
|
+
if (filterValidation.error) {
|
|
297
|
+
return {
|
|
298
|
+
content: [{ type: "text", text: `Error: ${filterValidation.error}` }],
|
|
299
|
+
isError: true,
|
|
300
|
+
};
|
|
301
|
+
}
|
|
274
302
|
// Use default collections if none specified
|
|
275
303
|
const effectiveCollections = collections ?? defaultCollectionNames;
|
|
276
304
|
// Plain `query` follows the requested SDK expansion policy before fusion and reranking;
|
|
@@ -278,6 +306,8 @@ Context-aware lex (C++ performance, not sports):
|
|
|
278
306
|
const searchOptions = query
|
|
279
307
|
? { query }
|
|
280
308
|
: { queries: (searches ?? []).map(s => ({ type: s.type, query: s.query })) };
|
|
309
|
+
const effectiveExpansionContext = expansionContext ?? intent;
|
|
310
|
+
const effectiveRerankContext = rerankContext ?? intent;
|
|
281
311
|
let expansionDecision;
|
|
282
312
|
let expansionError;
|
|
283
313
|
let results;
|
|
@@ -285,12 +315,13 @@ Context-aware lex (C++ performance, not sports):
|
|
|
285
315
|
results = await store.search({
|
|
286
316
|
...searchOptions,
|
|
287
317
|
collections: effectiveCollections.length > 0 ? effectiveCollections : undefined,
|
|
318
|
+
filter: filterValidation.filter,
|
|
288
319
|
limit,
|
|
289
320
|
minScore,
|
|
290
321
|
candidateLimit,
|
|
291
322
|
rerank,
|
|
292
|
-
expansionContext,
|
|
293
|
-
rerankContext,
|
|
323
|
+
expansionContext: effectiveExpansionContext,
|
|
324
|
+
rerankContext: effectiveRerankContext,
|
|
294
325
|
explain,
|
|
295
326
|
expansion: query ? expansion : undefined,
|
|
296
327
|
includeHyde,
|
|
@@ -323,6 +354,7 @@ Context-aware lex (C++ performance, not sports):
|
|
|
323
354
|
title: r.title,
|
|
324
355
|
score: Math.round(r.score * 100) / 100,
|
|
325
356
|
context: r.context,
|
|
357
|
+
...(Object.keys(r.metadata).length > 0 ? { metadata: r.metadata } : {}),
|
|
326
358
|
line,
|
|
327
359
|
snippet: addLineNumbers(snippet, line),
|
|
328
360
|
...(explain && r.explain ? { explain: r.explain } : {}),
|
|
@@ -798,7 +830,21 @@ export async function startMcpHttpServer(port, options = {}) {
|
|
|
798
830
|
// REST endpoint: POST /query (alias: /search) — structured search without MCP protocol
|
|
799
831
|
if ((pathname === "/query" || pathname === "/search") && nodeReq.method === "POST") {
|
|
800
832
|
const rawBody = await collectBody(nodeReq);
|
|
801
|
-
|
|
833
|
+
let parsedParams;
|
|
834
|
+
try {
|
|
835
|
+
parsedParams = JSON.parse(rawBody);
|
|
836
|
+
}
|
|
837
|
+
catch {
|
|
838
|
+
nodeRes.writeHead(400, { "Content-Type": "application/json" });
|
|
839
|
+
nodeRes.end(JSON.stringify({ error: "Invalid JSON body" }));
|
|
840
|
+
return;
|
|
841
|
+
}
|
|
842
|
+
if (typeof parsedParams !== "object" || parsedParams === null || Array.isArray(parsedParams)) {
|
|
843
|
+
nodeRes.writeHead(400, { "Content-Type": "application/json" });
|
|
844
|
+
nodeRes.end(JSON.stringify({ error: "JSON body must be an object" }));
|
|
845
|
+
return;
|
|
846
|
+
}
|
|
847
|
+
const params = parsedParams;
|
|
802
848
|
// Validate required fields
|
|
803
849
|
if (!params.searches || !Array.isArray(params.searches)) {
|
|
804
850
|
nodeRes.writeHead(400, { "Content-Type": "application/json" });
|
|
@@ -811,11 +857,28 @@ export async function startMcpHttpServer(port, options = {}) {
|
|
|
811
857
|
type: s.type,
|
|
812
858
|
query: String(s.query || ""),
|
|
813
859
|
}));
|
|
860
|
+
// Optional metadata filter — must be an object and a valid filter AST
|
|
861
|
+
let restFilter;
|
|
862
|
+
if (params.filter !== undefined) {
|
|
863
|
+
if (typeof params.filter !== "object" || params.filter === null || Array.isArray(params.filter)) {
|
|
864
|
+
nodeRes.writeHead(400, { "Content-Type": "application/json" });
|
|
865
|
+
nodeRes.end(JSON.stringify({ error: "Invalid field: filter (must be an object)" }));
|
|
866
|
+
return;
|
|
867
|
+
}
|
|
868
|
+
const filterValidation = validateFilterArgument(params.filter);
|
|
869
|
+
if (filterValidation.error) {
|
|
870
|
+
nodeRes.writeHead(400, { "Content-Type": "application/json" });
|
|
871
|
+
nodeRes.end(JSON.stringify({ error: filterValidation.error }));
|
|
872
|
+
return;
|
|
873
|
+
}
|
|
874
|
+
restFilter = filterValidation.filter;
|
|
875
|
+
}
|
|
814
876
|
// Use default collections if none specified
|
|
815
877
|
const effectiveCollections = Array.isArray(params.collections) ? params.collections.map(String) : defaultCollectionNames;
|
|
816
878
|
const results = await store.search({
|
|
817
879
|
queries,
|
|
818
880
|
collections: effectiveCollections.length > 0 ? effectiveCollections : undefined,
|
|
881
|
+
filter: restFilter,
|
|
819
882
|
limit: typeof params.limit === "number" ? params.limit : 10,
|
|
820
883
|
minScore: typeof params.minScore === "number" ? params.minScore : 0,
|
|
821
884
|
candidateLimit: typeof params.candidateLimit === "number" ? params.candidateLimit : undefined,
|
|
@@ -835,6 +898,7 @@ export async function startMcpHttpServer(port, options = {}) {
|
|
|
835
898
|
title: r.title,
|
|
836
899
|
score: Math.round(r.score * 100) / 100,
|
|
837
900
|
context: r.context,
|
|
901
|
+
...(Object.keys(r.metadata).length > 0 ? { metadata: r.metadata } : {}),
|
|
838
902
|
line,
|
|
839
903
|
snippet: addLineNumbers(snippet, line),
|
|
840
904
|
};
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* QMD Metadata Filter - Recursive filter AST, strict runtime validation, and
|
|
3
|
+
* parameterized SQL compilation.
|
|
4
|
+
*
|
|
5
|
+
* The filter has one canonical, `operator`-discriminated recursive shape shared
|
|
6
|
+
* by every public search surface (CLI, SDK, MCP, HTTP):
|
|
7
|
+
*
|
|
8
|
+
* { "operator": "and", "operands": [ ... ] }
|
|
9
|
+
* { "operator": "not", "operand": { ... } }
|
|
10
|
+
* { "key": "status", "operator": "eq", "value": "published" }
|
|
11
|
+
*
|
|
12
|
+
* Compilation emits correlated EXISTS/NOT EXISTS subqueries over
|
|
13
|
+
* `document_metadata_values` with every user value bound as a parameter —
|
|
14
|
+
* metadata keys and values are data, never SQL.
|
|
15
|
+
*/
|
|
16
|
+
import type { MetadataScalar, MetadataScalarArray } from "./metadata.js";
|
|
17
|
+
export type MetadataFilter = MetadataFilterGroup | MetadataFilterNegation | MetadataCondition;
|
|
18
|
+
export interface MetadataFilterGroup {
|
|
19
|
+
operator: "and" | "or";
|
|
20
|
+
operands: readonly MetadataFilter[];
|
|
21
|
+
}
|
|
22
|
+
export interface MetadataFilterNegation {
|
|
23
|
+
operator: "not";
|
|
24
|
+
operand: MetadataFilter;
|
|
25
|
+
}
|
|
26
|
+
export type MetadataCondition = {
|
|
27
|
+
key: string;
|
|
28
|
+
operator: "eq" | "ne";
|
|
29
|
+
value: MetadataScalar;
|
|
30
|
+
} | {
|
|
31
|
+
key: string;
|
|
32
|
+
operator: "gt" | "gte" | "lt" | "lte";
|
|
33
|
+
value: string | number;
|
|
34
|
+
} | {
|
|
35
|
+
key: string;
|
|
36
|
+
operator: "in" | "nin" | "all";
|
|
37
|
+
value: MetadataScalarArray;
|
|
38
|
+
} | {
|
|
39
|
+
key: string;
|
|
40
|
+
operator: "exists";
|
|
41
|
+
value: boolean;
|
|
42
|
+
};
|
|
43
|
+
export interface CompiledMetadataFilter {
|
|
44
|
+
sql: string;
|
|
45
|
+
params: (string | number)[];
|
|
46
|
+
}
|
|
47
|
+
/** Raised by parseMetadataFilter with the JSON path of the failing node. */
|
|
48
|
+
export declare class MetadataFilterError extends Error {
|
|
49
|
+
readonly path: string;
|
|
50
|
+
constructor(path: string, message: string);
|
|
51
|
+
}
|
|
52
|
+
/** Defensive limits for recursive filters from untrusted callers. */
|
|
53
|
+
export declare const METADATA_FILTER_LIMITS: {
|
|
54
|
+
readonly maxDepth: 16;
|
|
55
|
+
readonly maxNodes: 256;
|
|
56
|
+
readonly maxGroupOperands: 32;
|
|
57
|
+
readonly maxMembershipValues: 64;
|
|
58
|
+
readonly maxKeyBytes: 128;
|
|
59
|
+
readonly maxStringLength: 1024;
|
|
60
|
+
};
|
|
61
|
+
/**
|
|
62
|
+
* Strictly validate an untrusted value as a MetadataFilter.
|
|
63
|
+
* Rejects unknown operators, unknown properties, operator-incompatible values,
|
|
64
|
+
* and inputs exceeding METADATA_FILTER_LIMITS. Canonicalizes membership value
|
|
65
|
+
* arrays by de-duplicating while preserving order.
|
|
66
|
+
*/
|
|
67
|
+
export declare function parseMetadataFilter(input: unknown): MetadataFilter;
|
|
68
|
+
/**
|
|
69
|
+
* Compile a validated filter into one parameterized SQL predicate correlated
|
|
70
|
+
* against a documents-table alias (e.g. `d`). All keys and values are bound
|
|
71
|
+
* parameters. The caller is responsible for restricting the surrounding query
|
|
72
|
+
* to active documents with current, error-free metadata extraction.
|
|
73
|
+
*/
|
|
74
|
+
export declare function compileMetadataFilter(filter: MetadataFilter, documentsAlias: string): CompiledMetadataFilter;
|