@wei840222/qmd 2026.8.28 → 2026.9.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +84 -2
- package/dist/cli/build-info.json +2 -2
- package/dist/cli/qmd.js +96 -18
- package/dist/collections.js +9 -4
- package/dist/db.d.ts +16 -29
- package/dist/db.js +63 -40
- package/dist/index.d.ts +10 -0
- package/dist/index.js +14 -2
- package/dist/llm.d.ts +7 -1
- package/dist/llm.js +23 -4
- package/dist/mcp/server.js +70 -6
- package/dist/metadata-filter.d.ts +74 -0
- package/dist/metadata-filter.js +279 -0
- package/dist/metadata-store.d.ts +45 -0
- package/dist/metadata-store.js +173 -0
- package/dist/metadata.d.ts +61 -0
- package/dist/metadata.js +215 -0
- package/dist/search/zh-dict.txt +3 -0
- package/dist/store.d.ts +23 -16
- package/dist/store.js +354 -166
- package/package.json +3 -5
- package/scripts/sync-zh-dict.mjs +4 -1
- package/skills/qmd/SKILL.md +11 -2
- package/skills/qmd/references/query-syntax.md +1 -8
- package/skills/release/SKILL.md +0 -141
- package/skills/release/scripts/install-hooks.sh +0 -38
- package/skills/release/scripts/release-context.sh +0 -129
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* QMD Metadata Filter - Recursive filter AST, strict runtime validation, and
|
|
3
|
+
* parameterized SQL compilation.
|
|
4
|
+
*
|
|
5
|
+
* The filter has one canonical, `operator`-discriminated recursive shape shared
|
|
6
|
+
* by every public search surface (CLI, SDK, MCP, HTTP):
|
|
7
|
+
*
|
|
8
|
+
* { "operator": "and", "operands": [ ... ] }
|
|
9
|
+
* { "operator": "not", "operand": { ... } }
|
|
10
|
+
* { "key": "status", "operator": "eq", "value": "published" }
|
|
11
|
+
*
|
|
12
|
+
* Compilation emits correlated EXISTS/NOT EXISTS subqueries over
|
|
13
|
+
* `document_metadata_values` with every user value bound as a parameter —
|
|
14
|
+
* metadata keys and values are data, never SQL.
|
|
15
|
+
*/
|
|
16
|
+
import { METADATA_LIMITS } from "./metadata.js";
|
|
17
|
+
/** Raised by parseMetadataFilter with the JSON path of the failing node. */
|
|
18
|
+
export class MetadataFilterError extends Error {
|
|
19
|
+
path;
|
|
20
|
+
constructor(path, message) {
|
|
21
|
+
super(`Invalid metadata filter at ${path}: ${message}`);
|
|
22
|
+
this.name = "MetadataFilterError";
|
|
23
|
+
this.path = path;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
// =============================================================================
|
|
27
|
+
// Limits
|
|
28
|
+
// =============================================================================
|
|
29
|
+
/** Defensive limits for recursive filters from untrusted callers. */
|
|
30
|
+
export const METADATA_FILTER_LIMITS = {
|
|
31
|
+
maxDepth: 16,
|
|
32
|
+
maxNodes: 256,
|
|
33
|
+
maxGroupOperands: 32,
|
|
34
|
+
maxMembershipValues: 64,
|
|
35
|
+
maxKeyBytes: METADATA_LIMITS.maxKeyBytes,
|
|
36
|
+
maxStringLength: METADATA_LIMITS.maxStringLength,
|
|
37
|
+
};
|
|
38
|
+
const GROUP_OPERATORS = new Set(["and", "or"]);
|
|
39
|
+
const COMPARISON_OPERATORS = new Set(["eq", "ne", "gt", "gte", "lt", "lte"]);
|
|
40
|
+
const ORDERED_OPERATORS = new Set(["gt", "gte", "lt", "lte"]);
|
|
41
|
+
const MEMBERSHIP_OPERATORS = new Set(["in", "nin", "all"]);
|
|
42
|
+
const CONDITION_OPERATORS = new Set([...COMPARISON_OPERATORS, ...MEMBERSHIP_OPERATORS, "exists"]);
|
|
43
|
+
const ALL_OPERATORS = [...GROUP_OPERATORS, "not", ...CONDITION_OPERATORS];
|
|
44
|
+
/**
|
|
45
|
+
* Strictly validate an untrusted value as a MetadataFilter.
|
|
46
|
+
* Rejects unknown operators, unknown properties, operator-incompatible values,
|
|
47
|
+
* and inputs exceeding METADATA_FILTER_LIMITS. Canonicalizes membership value
|
|
48
|
+
* arrays by de-duplicating while preserving order.
|
|
49
|
+
*/
|
|
50
|
+
export function parseMetadataFilter(input) {
|
|
51
|
+
const state = { nodes: 0 };
|
|
52
|
+
return parseFilterNode(input, "$", 1, state);
|
|
53
|
+
}
|
|
54
|
+
function parseFilterNode(input, path, depth, state) {
|
|
55
|
+
if (depth > METADATA_FILTER_LIMITS.maxDepth) {
|
|
56
|
+
throw new MetadataFilterError(path, `exceeds maximum nesting depth of ${METADATA_FILTER_LIMITS.maxDepth}`);
|
|
57
|
+
}
|
|
58
|
+
state.nodes += 1;
|
|
59
|
+
if (state.nodes > METADATA_FILTER_LIMITS.maxNodes) {
|
|
60
|
+
throw new MetadataFilterError(path, `exceeds maximum of ${METADATA_FILTER_LIMITS.maxNodes} nodes`);
|
|
61
|
+
}
|
|
62
|
+
if (typeof input !== "object" || input === null || Array.isArray(input)) {
|
|
63
|
+
throw new MetadataFilterError(path, "each filter node must be an object");
|
|
64
|
+
}
|
|
65
|
+
const node = input;
|
|
66
|
+
const operator = node["operator"];
|
|
67
|
+
if (typeof operator !== "string") {
|
|
68
|
+
throw new MetadataFilterError(path, "missing 'operator' property");
|
|
69
|
+
}
|
|
70
|
+
if (GROUP_OPERATORS.has(operator)) {
|
|
71
|
+
return parseFilterGroup(node, operator, path, depth, state);
|
|
72
|
+
}
|
|
73
|
+
if (operator === "not") {
|
|
74
|
+
return parseFilterNegation(node, path, depth, state);
|
|
75
|
+
}
|
|
76
|
+
if (CONDITION_OPERATORS.has(operator)) {
|
|
77
|
+
return parseFilterCondition(node, operator, path);
|
|
78
|
+
}
|
|
79
|
+
throw new MetadataFilterError(path, `unknown operator '${operator}' — expected one of: ${ALL_OPERATORS.join(", ")}`);
|
|
80
|
+
}
|
|
81
|
+
function parseFilterGroup(node, operator, path, depth, state) {
|
|
82
|
+
rejectUnknownProperties(node, ["operator", "operands"], path);
|
|
83
|
+
const operands = node["operands"];
|
|
84
|
+
if (!Array.isArray(operands)) {
|
|
85
|
+
throw new MetadataFilterError(path, `'${operator}' requires an 'operands' array`);
|
|
86
|
+
}
|
|
87
|
+
if (operands.length === 0) {
|
|
88
|
+
throw new MetadataFilterError(path, `'${operator}' requires a non-empty 'operands' array`);
|
|
89
|
+
}
|
|
90
|
+
if (operands.length > METADATA_FILTER_LIMITS.maxGroupOperands) {
|
|
91
|
+
throw new MetadataFilterError(path, `'${operator}' exceeds maximum of ${METADATA_FILTER_LIMITS.maxGroupOperands} operands`);
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
operator,
|
|
95
|
+
operands: operands.map((operand, index) => parseFilterNode(operand, `${path}.operands[${index}]`, depth + 1, state)),
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
function parseFilterNegation(node, path, depth, state) {
|
|
99
|
+
rejectUnknownProperties(node, ["operator", "operand"], path);
|
|
100
|
+
if (!("operand" in node)) {
|
|
101
|
+
throw new MetadataFilterError(path, "'not' requires exactly one 'operand'");
|
|
102
|
+
}
|
|
103
|
+
return {
|
|
104
|
+
operator: "not",
|
|
105
|
+
operand: parseFilterNode(node["operand"], `${path}.operand`, depth + 1, state),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
function parseFilterCondition(node, operator, path) {
|
|
109
|
+
rejectUnknownProperties(node, ["key", "operator", "value"], path);
|
|
110
|
+
const key = node["key"];
|
|
111
|
+
if (typeof key !== "string" || key.length === 0) {
|
|
112
|
+
throw new MetadataFilterError(path, `'${operator}' requires a non-empty string 'key'`);
|
|
113
|
+
}
|
|
114
|
+
if (Buffer.byteLength(key, "utf-8") > METADATA_FILTER_LIMITS.maxKeyBytes) {
|
|
115
|
+
throw new MetadataFilterError(path, `'key' exceeds ${METADATA_FILTER_LIMITS.maxKeyBytes} bytes`);
|
|
116
|
+
}
|
|
117
|
+
if (!("value" in node)) {
|
|
118
|
+
throw new MetadataFilterError(path, `'${operator}' requires a 'value'`);
|
|
119
|
+
}
|
|
120
|
+
const value = node["value"];
|
|
121
|
+
if (operator === "exists") {
|
|
122
|
+
if (typeof value !== "boolean") {
|
|
123
|
+
throw new MetadataFilterError(`${path}.value`, "'exists' requires a boolean value");
|
|
124
|
+
}
|
|
125
|
+
return { key, operator, value };
|
|
126
|
+
}
|
|
127
|
+
if (MEMBERSHIP_OPERATORS.has(operator)) {
|
|
128
|
+
return {
|
|
129
|
+
key,
|
|
130
|
+
operator: operator,
|
|
131
|
+
value: parseMembershipValues(value, operator, path),
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
// Comparison operators: eq, ne, gt, gte, lt, lte.
|
|
135
|
+
const scalar = parseScalarValue(value, `${path}.value`);
|
|
136
|
+
if (ORDERED_OPERATORS.has(operator) && typeof scalar === "boolean") {
|
|
137
|
+
throw new MetadataFilterError(`${path}.value`, `'${operator}' requires a string or number value`);
|
|
138
|
+
}
|
|
139
|
+
return { key, operator, value: scalar };
|
|
140
|
+
}
|
|
141
|
+
function parseMembershipValues(value, operator, path) {
|
|
142
|
+
if (!Array.isArray(value)) {
|
|
143
|
+
throw new MetadataFilterError(`${path}.value`, `'${operator}' requires an array value`);
|
|
144
|
+
}
|
|
145
|
+
if (value.length === 0) {
|
|
146
|
+
throw new MetadataFilterError(`${path}.value`, `'${operator}' requires a non-empty array value`);
|
|
147
|
+
}
|
|
148
|
+
if (value.length > METADATA_FILTER_LIMITS.maxMembershipValues) {
|
|
149
|
+
throw new MetadataFilterError(`${path}.value`, `'${operator}' exceeds maximum of ${METADATA_FILTER_LIMITS.maxMembershipValues} values`);
|
|
150
|
+
}
|
|
151
|
+
const scalars = value.map((element, index) => parseScalarValue(element, `${path}.value[${index}]`));
|
|
152
|
+
// Narrow each homogeneous case explicitly so the public array union remains
|
|
153
|
+
// precise without discarding type evidence through chained assertions.
|
|
154
|
+
if (scalars.every((scalar) => typeof scalar === "string")) {
|
|
155
|
+
return Array.from(new Set(scalars));
|
|
156
|
+
}
|
|
157
|
+
if (scalars.every((scalar) => typeof scalar === "number")) {
|
|
158
|
+
return Array.from(new Set(scalars));
|
|
159
|
+
}
|
|
160
|
+
if (scalars.every((scalar) => typeof scalar === "boolean")) {
|
|
161
|
+
return Array.from(new Set(scalars));
|
|
162
|
+
}
|
|
163
|
+
throw new MetadataFilterError(`${path}.value`, `'${operator}' requires a homogeneous array of one scalar type`);
|
|
164
|
+
}
|
|
165
|
+
function parseScalarValue(value, path) {
|
|
166
|
+
if (typeof value === "string") {
|
|
167
|
+
if (value.length > METADATA_FILTER_LIMITS.maxStringLength) {
|
|
168
|
+
throw new MetadataFilterError(path, `string exceeds ${METADATA_FILTER_LIMITS.maxStringLength} characters`);
|
|
169
|
+
}
|
|
170
|
+
return value;
|
|
171
|
+
}
|
|
172
|
+
if (typeof value === "number") {
|
|
173
|
+
if (!Number.isFinite(value)) {
|
|
174
|
+
throw new MetadataFilterError(path, "numbers must be finite");
|
|
175
|
+
}
|
|
176
|
+
return value;
|
|
177
|
+
}
|
|
178
|
+
if (typeof value === "boolean")
|
|
179
|
+
return value;
|
|
180
|
+
throw new MetadataFilterError(path, "expected a string, number, or boolean");
|
|
181
|
+
}
|
|
182
|
+
function rejectUnknownProperties(node, allowed, path) {
|
|
183
|
+
for (const property of Object.keys(node)) {
|
|
184
|
+
if (!allowed.includes(property)) {
|
|
185
|
+
throw new MetadataFilterError(path, `unknown property '${property}' — allowed: ${allowed.join(", ")}`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
// =============================================================================
|
|
190
|
+
// SQL compilation
|
|
191
|
+
// =============================================================================
|
|
192
|
+
/**
|
|
193
|
+
* Compile a validated filter into one parameterized SQL predicate correlated
|
|
194
|
+
* against a documents-table alias (e.g. `d`). All keys and values are bound
|
|
195
|
+
* parameters. The caller is responsible for restricting the surrounding query
|
|
196
|
+
* to active documents with current, error-free metadata extraction.
|
|
197
|
+
*/
|
|
198
|
+
export function compileMetadataFilter(filter, documentsAlias) {
|
|
199
|
+
const params = [];
|
|
200
|
+
const sql = compileFilterNode(filter, documentsAlias, params);
|
|
201
|
+
return { sql, params };
|
|
202
|
+
}
|
|
203
|
+
function compileFilterNode(filter, alias, params) {
|
|
204
|
+
switch (filter.operator) {
|
|
205
|
+
case "and":
|
|
206
|
+
case "or": {
|
|
207
|
+
const joiner = filter.operator === "and" ? " AND " : " OR ";
|
|
208
|
+
return `(${filter.operands.map(operand => compileFilterNode(operand, alias, params)).join(joiner)})`;
|
|
209
|
+
}
|
|
210
|
+
case "not":
|
|
211
|
+
return `NOT ${compileFilterNode(filter.operand, alias, params)}`;
|
|
212
|
+
case "exists":
|
|
213
|
+
params.push(filter.key);
|
|
214
|
+
return filter.value
|
|
215
|
+
? buildValueExistsSql(alias, "mv.key = ?")
|
|
216
|
+
: `NOT ${buildValueExistsSql(alias, "mv.key = ?")}`;
|
|
217
|
+
case "eq":
|
|
218
|
+
case "gt":
|
|
219
|
+
case "gte":
|
|
220
|
+
case "lt":
|
|
221
|
+
case "lte": {
|
|
222
|
+
const sqlOperator = { eq: "=", gt: ">", gte: ">=", lt: "<", lte: "<=" }[filter.operator];
|
|
223
|
+
params.push(filter.key, bindScalar(filter.value));
|
|
224
|
+
return buildValueExistsSql(alias, `mv.key = ? AND mv.value_type = '${valueTypeOf(filter.value)}' AND mv.${valueColumnOf(filter.value)} ${sqlOperator} ?`);
|
|
225
|
+
}
|
|
226
|
+
case "ne": {
|
|
227
|
+
// Key must have at least one same-type value, and no same-type value
|
|
228
|
+
// may equal the operand. Missing keys and type mismatches do not match.
|
|
229
|
+
const valueType = valueTypeOf(filter.value);
|
|
230
|
+
params.push(filter.key);
|
|
231
|
+
const presentSql = buildValueExistsSql(alias, `mv.key = ? AND mv.value_type = '${valueType}'`);
|
|
232
|
+
params.push(filter.key, bindScalar(filter.value));
|
|
233
|
+
const equalSql = buildValueExistsSql(alias, `mv.key = ? AND mv.value_type = '${valueType}' AND mv.${valueColumnOf(filter.value)} = ?`);
|
|
234
|
+
return `(${presentSql} AND NOT ${equalSql})`;
|
|
235
|
+
}
|
|
236
|
+
case "in":
|
|
237
|
+
case "nin": {
|
|
238
|
+
const valueType = valueTypeOf(filter.value[0]);
|
|
239
|
+
const column = valueColumnOf(filter.value[0]);
|
|
240
|
+
const placeholders = filter.value.map(() => "?").join(", ");
|
|
241
|
+
if (filter.operator === "in") {
|
|
242
|
+
params.push(filter.key, ...filter.value.map(bindScalar));
|
|
243
|
+
return buildValueExistsSql(alias, `mv.key = ? AND mv.value_type = '${valueType}' AND mv.${column} IN (${placeholders})`);
|
|
244
|
+
}
|
|
245
|
+
params.push(filter.key);
|
|
246
|
+
const presentSql = buildValueExistsSql(alias, `mv.key = ? AND mv.value_type = '${valueType}'`);
|
|
247
|
+
params.push(filter.key, ...filter.value.map(bindScalar));
|
|
248
|
+
const memberSql = buildValueExistsSql(alias, `mv.key = ? AND mv.value_type = '${valueType}' AND mv.${column} IN (${placeholders})`);
|
|
249
|
+
return `(${presentSql} AND NOT ${memberSql})`;
|
|
250
|
+
}
|
|
251
|
+
case "all": {
|
|
252
|
+
const valueType = valueTypeOf(filter.value[0]);
|
|
253
|
+
const column = valueColumnOf(filter.value[0]);
|
|
254
|
+
const memberSqls = filter.value.map(element => {
|
|
255
|
+
params.push(filter.key, bindScalar(element));
|
|
256
|
+
return buildValueExistsSql(alias, `mv.key = ? AND mv.value_type = '${valueType}' AND mv.${column} = ?`);
|
|
257
|
+
});
|
|
258
|
+
return `(${memberSqls.join(" AND ")})`;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
function buildValueExistsSql(alias, conditionSql) {
|
|
263
|
+
return `EXISTS (SELECT 1 FROM document_metadata_values mv WHERE mv.document_id = ${alias}.id AND ${conditionSql})`;
|
|
264
|
+
}
|
|
265
|
+
function valueTypeOf(scalar) {
|
|
266
|
+
return typeof scalar;
|
|
267
|
+
}
|
|
268
|
+
function valueColumnOf(scalar) {
|
|
269
|
+
if (typeof scalar === "string")
|
|
270
|
+
return "text_value";
|
|
271
|
+
if (typeof scalar === "number")
|
|
272
|
+
return "number_value";
|
|
273
|
+
return "boolean_value";
|
|
274
|
+
}
|
|
275
|
+
function bindScalar(scalar) {
|
|
276
|
+
if (typeof scalar === "boolean")
|
|
277
|
+
return scalar ? 1 : 0;
|
|
278
|
+
return scalar;
|
|
279
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* QMD Metadata Store - Schema, persistence, and batch loading for document
|
|
3
|
+
* metadata.
|
|
4
|
+
*
|
|
5
|
+
* Metadata attaches to document identity (`documents.id`), not content
|
|
6
|
+
* identity: two paths can share one content hash while carrying different
|
|
7
|
+
* metadata. SQLite stays a derived index — metadata is rebuilt from source
|
|
8
|
+
* documents on `qmd update`, never mutated in place.
|
|
9
|
+
*
|
|
10
|
+
* `document_metadata` records extraction state per document (including
|
|
11
|
+
* successful-but-empty extraction), so filtered search can distinguish
|
|
12
|
+
* "extracted with no metadata" from "not yet extracted" and "extraction
|
|
13
|
+
* failed". `document_metadata_values` holds one indexed row per scalar value
|
|
14
|
+
* for filtering.
|
|
15
|
+
*/
|
|
16
|
+
import type { Database } from "./db.js";
|
|
17
|
+
import { type DocumentMetadata, type MetadataExtractionResult } from "./metadata.js";
|
|
18
|
+
export declare function initializeMetadataSchema(db: Database): void;
|
|
19
|
+
/**
|
|
20
|
+
* Extract and persist metadata for one document, replacing any prior rows.
|
|
21
|
+
*
|
|
22
|
+
* With `onlyIfStale`, extraction is skipped when the document already has a
|
|
23
|
+
* current-version extraction row — the cheap path for unchanged documents
|
|
24
|
+
* during re-index. Returns the extraction result, or null when skipped.
|
|
25
|
+
*/
|
|
26
|
+
export declare function syncDocumentMetadata(db: Database, documentId: number, content: string, path: string, options?: {
|
|
27
|
+
onlyIfStale?: boolean;
|
|
28
|
+
}): MetadataExtractionResult | null;
|
|
29
|
+
/**
|
|
30
|
+
* Replace a document's metadata rows atomically. A failed extraction persists
|
|
31
|
+
* empty metadata plus the error, so stale metadata never survives a bad edit.
|
|
32
|
+
*/
|
|
33
|
+
export declare function replaceDocumentMetadata(db: Database, documentId: number, extraction: MetadataExtractionResult): void;
|
|
34
|
+
/**
|
|
35
|
+
* Count active documents without a current, error-free metadata extraction.
|
|
36
|
+
* These documents are excluded from filtered search until `qmd update` runs.
|
|
37
|
+
*/
|
|
38
|
+
export declare function countDocumentsPendingMetadata(db: Database): number;
|
|
39
|
+
/**
|
|
40
|
+
* Batch-load canonical metadata for a set of result filepaths
|
|
41
|
+
* (`qmd://collection/path`). One query — never per-result lookups.
|
|
42
|
+
*/
|
|
43
|
+
export declare function getMetadataByFilepath(db: Database, filepaths: readonly string[]): Map<string, DocumentMetadata>;
|
|
44
|
+
/** Parse a stored `metadata_json` column value, tolerating absent rows. */
|
|
45
|
+
export declare function parseMetadataJson(metadataJson: string | null | undefined): DocumentMetadata;
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* QMD Metadata Store - Schema, persistence, and batch loading for document
|
|
3
|
+
* metadata.
|
|
4
|
+
*
|
|
5
|
+
* Metadata attaches to document identity (`documents.id`), not content
|
|
6
|
+
* identity: two paths can share one content hash while carrying different
|
|
7
|
+
* metadata. SQLite stays a derived index — metadata is rebuilt from source
|
|
8
|
+
* documents on `qmd update`, never mutated in place.
|
|
9
|
+
*
|
|
10
|
+
* `document_metadata` records extraction state per document (including
|
|
11
|
+
* successful-but-empty extraction), so filtered search can distinguish
|
|
12
|
+
* "extracted with no metadata" from "not yet extracted" and "extraction
|
|
13
|
+
* failed". `document_metadata_values` holds one indexed row per scalar value
|
|
14
|
+
* for filtering.
|
|
15
|
+
*/
|
|
16
|
+
import { extractDocumentMetadata, METADATA_EXTRACTION_VERSION, } from "./metadata.js";
|
|
17
|
+
// =============================================================================
|
|
18
|
+
// Schema
|
|
19
|
+
// =============================================================================
|
|
20
|
+
export function initializeMetadataSchema(db) {
|
|
21
|
+
db.exec(`
|
|
22
|
+
CREATE TABLE IF NOT EXISTS document_metadata (
|
|
23
|
+
document_id INTEGER PRIMARY KEY,
|
|
24
|
+
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
25
|
+
extraction_version INTEGER NOT NULL,
|
|
26
|
+
extraction_error TEXT,
|
|
27
|
+
extracted_at TEXT NOT NULL,
|
|
28
|
+
FOREIGN KEY (document_id) REFERENCES documents(id) ON DELETE CASCADE
|
|
29
|
+
)
|
|
30
|
+
`);
|
|
31
|
+
db.exec(`
|
|
32
|
+
CREATE TABLE IF NOT EXISTS document_metadata_values (
|
|
33
|
+
document_id INTEGER NOT NULL,
|
|
34
|
+
key TEXT NOT NULL,
|
|
35
|
+
ordinal INTEGER NOT NULL,
|
|
36
|
+
value_type TEXT NOT NULL,
|
|
37
|
+
text_value TEXT,
|
|
38
|
+
number_value REAL,
|
|
39
|
+
boolean_value INTEGER,
|
|
40
|
+
PRIMARY KEY (document_id, key, ordinal),
|
|
41
|
+
FOREIGN KEY (document_id)
|
|
42
|
+
REFERENCES document_metadata(document_id)
|
|
43
|
+
ON DELETE CASCADE,
|
|
44
|
+
CHECK (value_type IN ('string', 'number', 'boolean')),
|
|
45
|
+
CHECK (
|
|
46
|
+
(value_type = 'string' AND text_value IS NOT NULL AND number_value IS NULL AND boolean_value IS NULL)
|
|
47
|
+
OR (value_type = 'number' AND number_value IS NOT NULL AND text_value IS NULL AND boolean_value IS NULL)
|
|
48
|
+
OR (value_type = 'boolean' AND boolean_value IN (0, 1) AND text_value IS NULL AND number_value IS NULL)
|
|
49
|
+
)
|
|
50
|
+
)
|
|
51
|
+
`);
|
|
52
|
+
db.exec(`
|
|
53
|
+
CREATE INDEX IF NOT EXISTS idx_metadata_text_lookup
|
|
54
|
+
ON document_metadata_values(key, text_value, document_id)
|
|
55
|
+
WHERE value_type = 'string'
|
|
56
|
+
`);
|
|
57
|
+
db.exec(`
|
|
58
|
+
CREATE INDEX IF NOT EXISTS idx_metadata_number_lookup
|
|
59
|
+
ON document_metadata_values(key, number_value, document_id)
|
|
60
|
+
WHERE value_type = 'number'
|
|
61
|
+
`);
|
|
62
|
+
db.exec(`
|
|
63
|
+
CREATE INDEX IF NOT EXISTS idx_metadata_boolean_lookup
|
|
64
|
+
ON document_metadata_values(key, boolean_value, document_id)
|
|
65
|
+
WHERE value_type = 'boolean'
|
|
66
|
+
`);
|
|
67
|
+
}
|
|
68
|
+
// =============================================================================
|
|
69
|
+
// Persistence
|
|
70
|
+
// =============================================================================
|
|
71
|
+
/**
|
|
72
|
+
* Extract and persist metadata for one document, replacing any prior rows.
|
|
73
|
+
*
|
|
74
|
+
* With `onlyIfStale`, extraction is skipped when the document already has a
|
|
75
|
+
* current-version extraction row — the cheap path for unchanged documents
|
|
76
|
+
* during re-index. Returns the extraction result, or null when skipped.
|
|
77
|
+
*/
|
|
78
|
+
export function syncDocumentMetadata(db, documentId, content, path, options) {
|
|
79
|
+
if (options?.onlyIfStale && isDocumentMetadataCurrent(db, documentId))
|
|
80
|
+
return null;
|
|
81
|
+
const extraction = extractDocumentMetadata(content, path);
|
|
82
|
+
replaceDocumentMetadata(db, documentId, extraction);
|
|
83
|
+
return extraction;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Replace a document's metadata rows atomically. A failed extraction persists
|
|
87
|
+
* empty metadata plus the error, so stale metadata never survives a bad edit.
|
|
88
|
+
*/
|
|
89
|
+
export function replaceDocumentMetadata(db, documentId, extraction) {
|
|
90
|
+
const replace = db.transaction(() => {
|
|
91
|
+
db.prepare(`
|
|
92
|
+
INSERT INTO document_metadata (document_id, metadata_json, extraction_version, extraction_error, extracted_at)
|
|
93
|
+
VALUES (?, ?, ?, ?, ?)
|
|
94
|
+
ON CONFLICT(document_id) DO UPDATE SET
|
|
95
|
+
metadata_json = excluded.metadata_json,
|
|
96
|
+
extraction_version = excluded.extraction_version,
|
|
97
|
+
extraction_error = excluded.extraction_error,
|
|
98
|
+
extracted_at = excluded.extracted_at
|
|
99
|
+
`).run(documentId, JSON.stringify(extraction.metadata), extraction.extractionVersion, extraction.error ?? null, new Date().toISOString());
|
|
100
|
+
db.prepare(`DELETE FROM document_metadata_values WHERE document_id = ?`).run(documentId);
|
|
101
|
+
const insertValue = db.prepare(`
|
|
102
|
+
INSERT INTO document_metadata_values (document_id, key, ordinal, value_type, text_value, number_value, boolean_value)
|
|
103
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
104
|
+
`);
|
|
105
|
+
for (const [key, value] of Object.entries(extraction.metadata)) {
|
|
106
|
+
const scalars = Array.isArray(value) ? value : [value];
|
|
107
|
+
scalars.forEach((scalar, ordinal) => {
|
|
108
|
+
insertValue.run(documentId, key, ordinal, typeof scalar, typeof scalar === "string" ? scalar : null, typeof scalar === "number" ? scalar : null, typeof scalar === "boolean" ? (scalar ? 1 : 0) : null);
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
replace();
|
|
113
|
+
}
|
|
114
|
+
function isDocumentMetadataCurrent(db, documentId) {
|
|
115
|
+
const row = db.prepare(`SELECT extraction_version FROM document_metadata WHERE document_id = ?`)
|
|
116
|
+
.get(documentId);
|
|
117
|
+
return row?.extraction_version === METADATA_EXTRACTION_VERSION;
|
|
118
|
+
}
|
|
119
|
+
// =============================================================================
|
|
120
|
+
// Queries
|
|
121
|
+
// =============================================================================
|
|
122
|
+
/**
|
|
123
|
+
* Count active documents without a current, error-free metadata extraction.
|
|
124
|
+
* These documents are excluded from filtered search until `qmd update` runs.
|
|
125
|
+
*/
|
|
126
|
+
export function countDocumentsPendingMetadata(db) {
|
|
127
|
+
const row = db.prepare(`
|
|
128
|
+
SELECT COUNT(*) as c FROM documents d
|
|
129
|
+
WHERE d.active = 1
|
|
130
|
+
AND NOT EXISTS (
|
|
131
|
+
SELECT 1 FROM document_metadata dm
|
|
132
|
+
WHERE dm.document_id = d.id
|
|
133
|
+
AND dm.extraction_version = ?
|
|
134
|
+
AND dm.extraction_error IS NULL
|
|
135
|
+
)
|
|
136
|
+
`).get(METADATA_EXTRACTION_VERSION);
|
|
137
|
+
return row.c;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Batch-load canonical metadata for a set of result filepaths
|
|
141
|
+
* (`qmd://collection/path`). One query — never per-result lookups.
|
|
142
|
+
*/
|
|
143
|
+
export function getMetadataByFilepath(db, filepaths) {
|
|
144
|
+
const metadataByFilepath = new Map();
|
|
145
|
+
if (filepaths.length === 0)
|
|
146
|
+
return metadataByFilepath;
|
|
147
|
+
const placeholders = filepaths.map(() => "?").join(", ");
|
|
148
|
+
const stmt = db.prepare(`
|
|
149
|
+
SELECT 'qmd://' || d.collection || '/' || d.path AS filepath, dm.metadata_json
|
|
150
|
+
FROM documents d
|
|
151
|
+
JOIN document_metadata dm ON dm.document_id = d.id
|
|
152
|
+
WHERE d.active = 1
|
|
153
|
+
AND 'qmd://' || d.collection || '/' || d.path IN (${placeholders})
|
|
154
|
+
`);
|
|
155
|
+
const rows = typeof stmt?.all === "function"
|
|
156
|
+
? stmt.all(...filepaths)
|
|
157
|
+
: [];
|
|
158
|
+
for (const row of rows) {
|
|
159
|
+
metadataByFilepath.set(row.filepath, parseMetadataJson(row.metadata_json));
|
|
160
|
+
}
|
|
161
|
+
return metadataByFilepath;
|
|
162
|
+
}
|
|
163
|
+
/** Parse a stored `metadata_json` column value, tolerating absent rows. */
|
|
164
|
+
export function parseMetadataJson(metadataJson) {
|
|
165
|
+
if (!metadataJson)
|
|
166
|
+
return {};
|
|
167
|
+
try {
|
|
168
|
+
return JSON.parse(metadataJson);
|
|
169
|
+
}
|
|
170
|
+
catch {
|
|
171
|
+
return {};
|
|
172
|
+
}
|
|
173
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* QMD Metadata - Public metadata types and frontmatter extraction.
|
|
3
|
+
*
|
|
4
|
+
* Documents opt into metadata through a namespaced Markdown frontmatter block:
|
|
5
|
+
*
|
|
6
|
+
* ---
|
|
7
|
+
* qmd:
|
|
8
|
+
* metadata:
|
|
9
|
+
* topics:
|
|
10
|
+
* - typescript
|
|
11
|
+
* - programming
|
|
12
|
+
* status: published
|
|
13
|
+
* ---
|
|
14
|
+
*
|
|
15
|
+
* Extraction is source-agnostic at the persistence boundary: this module
|
|
16
|
+
* produces a canonical `MetadataExtractionResult`, and future non-frontmatter
|
|
17
|
+
* sources can produce the same shape without touching storage or filtering.
|
|
18
|
+
*
|
|
19
|
+
* The raw document is never modified — frontmatter stays part of the stored,
|
|
20
|
+
* indexed, chunked, and embedded content.
|
|
21
|
+
*/
|
|
22
|
+
export type MetadataScalar = string | number | boolean;
|
|
23
|
+
export type MetadataScalarArray = readonly string[] | readonly number[] | readonly boolean[];
|
|
24
|
+
export type MetadataValue = MetadataScalar | MetadataScalarArray;
|
|
25
|
+
export type DocumentMetadata = Record<string, MetadataValue>;
|
|
26
|
+
/**
|
|
27
|
+
* Result of extracting metadata from one document.
|
|
28
|
+
*
|
|
29
|
+
* `error` is set when the document opted into `qmd.metadata` but the value was
|
|
30
|
+
* invalid — the document still indexes normally, but it is excluded from
|
|
31
|
+
* filtered search until the metadata is corrected and re-indexed.
|
|
32
|
+
*/
|
|
33
|
+
export interface MetadataExtractionResult {
|
|
34
|
+
metadata: DocumentMetadata;
|
|
35
|
+
error?: string;
|
|
36
|
+
extractionVersion: number;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Bump when extraction or normalization semantics change so existing rows are
|
|
40
|
+
* re-extracted on the next `qmd update`.
|
|
41
|
+
*/
|
|
42
|
+
export declare const METADATA_EXTRACTION_VERSION = 1;
|
|
43
|
+
/** Defensive limits for metadata from untrusted repositories. */
|
|
44
|
+
export declare const METADATA_LIMITS: {
|
|
45
|
+
readonly maxFrontmatterBytes: number;
|
|
46
|
+
readonly maxKeys: 64;
|
|
47
|
+
readonly maxKeyBytes: 128;
|
|
48
|
+
readonly maxStringLength: 1024;
|
|
49
|
+
readonly maxArrayLength: 128;
|
|
50
|
+
readonly maxYamlAliasCount: 100;
|
|
51
|
+
readonly maxErrorLength: 200;
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* Extract `qmd.metadata` from a document's leading YAML frontmatter.
|
|
55
|
+
*
|
|
56
|
+
* Never throws. A document without frontmatter, without a `qmd` namespace, or
|
|
57
|
+
* with a non-frontmatter extension yields empty metadata with no error.
|
|
58
|
+
* Invalid frontmatter or invalid metadata yields empty metadata plus a bounded
|
|
59
|
+
* extraction error.
|
|
60
|
+
*/
|
|
61
|
+
export declare function extractDocumentMetadata(content: string, path: string): MetadataExtractionResult;
|