@gmickel/gno 2.3.2 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -4
- package/assets/skill/SKILL.md +30 -0
- package/assets/skill/cli-reference.md +12 -0
- package/assets/skill/examples.md +15 -0
- package/assets/skill/mcp-reference.md +16 -0
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.3.2.zip → gno-browser-clipper-v2.4.0.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.4.0.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +30 -0
- package/spec/db/schema.sql +2 -0
- package/spec/evals.md +48 -0
- package/spec/mcp.md +12 -0
- package/spec/output-schemas/ask.schema.json +12 -0
- package/spec/output-schemas/context-capsule-v1.schema.json +205 -32
- package/spec/output-schemas/context-capsule-verification.schema.json +32 -9
- package/spec/output-schemas/evidence-fixtures.schema.json +176 -0
- package/spec/output-schemas/evidence-report.schema.json +322 -0
- package/spec/output-schemas/evidence-run.schema.json +277 -0
- package/spec/output-schemas/metadata-predicate.schema.json +460 -0
- package/spec/output-schemas/retrieval-trace-filters.schema.json +13 -4
- package/spec/output-schemas/search-results.schema.json +12 -0
- package/spec/output-schemas/status.schema.json +37 -6
- package/src/app/context-runtime-contract.ts +33 -2
- package/src/app/context-runtime-input.ts +17 -0
- package/src/app/context-runtime-types.ts +2 -0
- package/src/app/context-runtime.ts +27 -3
- package/src/app/context-surface.ts +2 -0
- package/src/app/verified-ask.ts +1 -0
- package/src/cli/commands/ask.ts +1 -0
- package/src/cli/commands/context-build.ts +1 -0
- package/src/cli/commands/search.ts +1 -0
- package/src/cli/commands/status.ts +10 -0
- package/src/cli/options.ts +19 -0
- package/src/cli/program.ts +12 -0
- package/src/converters/types.ts +2 -0
- package/src/core/context-capsule-retrieval-schema.ts +8 -0
- package/src/core/context-capsule-schema.ts +47 -3
- package/src/core/context-capsule-validation.ts +14 -2
- package/src/core/context-capsule-verification.ts +1 -0
- package/src/core/context-capsule.ts +16 -2
- package/src/core/context-compiler.ts +3 -0
- package/src/core/record-metadata.ts +1 -1
- package/src/core/retrieval-replay-candidate.ts +5 -0
- package/src/core/retrieval-trace-filter-normalization.ts +4 -0
- package/src/core/retrieval-trace-request.ts +1 -0
- package/src/core/retrieval-trace-session.ts +5 -1
- package/src/core/retrieval-trace.ts +2 -0
- package/src/core/typed-metadata.ts +224 -0
- package/src/ingestion/record-adapter-canonical.ts +13 -2
- package/src/ingestion/record-container.ts +9 -0
- package/src/ingestion/sync.ts +11 -1
- package/src/ingestion/typed-metadata.ts +55 -0
- package/src/mcp/tools/ask.ts +10 -2
- package/src/mcp/tools/context.ts +1 -0
- package/src/mcp/tools/index.ts +12 -1
- package/src/mcp/tools/query.ts +13 -0
- package/src/mcp/tools/search.ts +9 -0
- package/src/mcp/tools/status.ts +4 -0
- package/src/mcp/tools/vsearch.ts +9 -0
- package/src/pipeline/diagnose.ts +6 -0
- package/src/pipeline/filters.ts +65 -0
- package/src/pipeline/graph-retrieval.ts +9 -1
- package/src/pipeline/hybrid.ts +22 -2
- package/src/pipeline/search.ts +11 -0
- package/src/pipeline/types.ts +7 -1
- package/src/pipeline/vsearch.ts +21 -1
- package/src/sdk/client.ts +29 -1
- package/src/sdk/index.ts +4 -0
- package/src/sdk/types.ts +7 -0
- package/src/serve/context-capsule.ts +1 -0
- package/src/serve/public/app.tsx +1 -1
- package/src/serve/public/components/MetadataFilter.tsx +186 -0
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/lib/metadata-filter.ts +28 -0
- package/src/serve/public/lib/retrieval-filters.ts +3 -0
- package/src/serve/public/pages/Ask.tsx +48 -3
- package/src/serve/public/pages/DocView.tsx +33 -0
- package/src/serve/public/pages/Search.tsx +44 -1
- package/src/serve/routes/api.ts +50 -0
- package/src/store/migrations/030-typed-metadata.ts +13 -0
- package/src/store/migrations/index.ts +2 -0
- package/src/store/sqlite/adapter.ts +55 -4
- package/src/store/sqlite/eligibility.ts +12 -0
- package/src/store/sqlite/metadata-predicate.ts +71 -0
- package/src/store/types.ts +11 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.3.2.zip.sha256 +0 -1
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
/** Bounded custom metadata and the shared, type-strict retrieval predicate. */
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
|
|
4
|
+
export type MetadataScalar = string | number | boolean;
|
|
5
|
+
export type TypedValue = MetadataScalar | MetadataScalar[];
|
|
6
|
+
export type TypedMetadata = Record<string, TypedValue>;
|
|
7
|
+
export type MetadataPredicate =
|
|
8
|
+
| { op: "and" | "or"; predicates: MetadataPredicate[] }
|
|
9
|
+
| { op: "not"; predicate: MetadataPredicate }
|
|
10
|
+
| { op: "eq" | "ne"; key: string; value: MetadataScalar }
|
|
11
|
+
| { op: "gt" | "gte" | "lt" | "lte"; key: string; value: number }
|
|
12
|
+
| { op: "in" | "nin" | "all"; key: string; values: MetadataScalar[] }
|
|
13
|
+
| { op: "exists"; key: string; value: boolean };
|
|
14
|
+
|
|
15
|
+
export const TYPED_METADATA_INGEST_VERSION = 7;
|
|
16
|
+
export const METADATA_LIMITS = {
|
|
17
|
+
keys: 64,
|
|
18
|
+
keyLength: 128,
|
|
19
|
+
stringLength: 4096,
|
|
20
|
+
members: 128,
|
|
21
|
+
bytes: 65_536,
|
|
22
|
+
nodes: 128,
|
|
23
|
+
depth: 8,
|
|
24
|
+
} as const;
|
|
25
|
+
const reserved = new Set(["__proto__", "prototype", "constructor"]);
|
|
26
|
+
const keySchema = z
|
|
27
|
+
.string()
|
|
28
|
+
.min(1)
|
|
29
|
+
.max(METADATA_LIMITS.keyLength)
|
|
30
|
+
.refine((key) => !reserved.has(key), "Reserved metadata key");
|
|
31
|
+
const scalarSchema = z.union([
|
|
32
|
+
z.string().max(METADATA_LIMITS.stringLength),
|
|
33
|
+
z.number().finite(),
|
|
34
|
+
z.boolean(),
|
|
35
|
+
]);
|
|
36
|
+
const membersSchema = z
|
|
37
|
+
.array(scalarSchema)
|
|
38
|
+
.max(METADATA_LIMITS.members)
|
|
39
|
+
.refine(
|
|
40
|
+
(values) => values.every((value) => typeof value === typeof values[0]),
|
|
41
|
+
"Array members must have one scalar type"
|
|
42
|
+
);
|
|
43
|
+
export const typedMetadataSchema = z
|
|
44
|
+
.unknown()
|
|
45
|
+
.superRefine((value, ctx) => {
|
|
46
|
+
if (
|
|
47
|
+
value &&
|
|
48
|
+
typeof value === "object" &&
|
|
49
|
+
Object.keys(value).some((key) => reserved.has(key))
|
|
50
|
+
) {
|
|
51
|
+
ctx.addIssue({ code: "custom", message: "Reserved metadata key" });
|
|
52
|
+
}
|
|
53
|
+
})
|
|
54
|
+
.pipe(
|
|
55
|
+
z
|
|
56
|
+
.record(keySchema, z.union([scalarSchema, membersSchema]))
|
|
57
|
+
.refine(
|
|
58
|
+
(value) => Object.keys(value).length <= METADATA_LIMITS.keys,
|
|
59
|
+
"Too many metadata keys"
|
|
60
|
+
)
|
|
61
|
+
.refine(
|
|
62
|
+
(value) =>
|
|
63
|
+
new TextEncoder().encode(JSON.stringify(value)).length <=
|
|
64
|
+
METADATA_LIMITS.bytes,
|
|
65
|
+
"Metadata exceeds 64 KiB"
|
|
66
|
+
)
|
|
67
|
+
);
|
|
68
|
+
|
|
69
|
+
const leafSchemas = [
|
|
70
|
+
z
|
|
71
|
+
.object({ op: z.enum(["eq", "ne"]), key: keySchema, value: scalarSchema })
|
|
72
|
+
.strict(),
|
|
73
|
+
z
|
|
74
|
+
.object({
|
|
75
|
+
op: z.enum(["gt", "gte", "lt", "lte"]),
|
|
76
|
+
key: keySchema,
|
|
77
|
+
value: z.number().finite(),
|
|
78
|
+
})
|
|
79
|
+
.strict(),
|
|
80
|
+
z
|
|
81
|
+
.object({
|
|
82
|
+
op: z.enum(["in", "nin", "all"]),
|
|
83
|
+
key: keySchema,
|
|
84
|
+
values: membersSchema.refine(
|
|
85
|
+
(v) => v.length > 0,
|
|
86
|
+
"Membership requires values"
|
|
87
|
+
),
|
|
88
|
+
})
|
|
89
|
+
.strict(),
|
|
90
|
+
z
|
|
91
|
+
.object({ op: z.literal("exists"), key: keySchema, value: z.boolean() })
|
|
92
|
+
.strict(),
|
|
93
|
+
] as const;
|
|
94
|
+
|
|
95
|
+
/** Finite schema recursion rejects deep/cyclic SDK values before descending.
|
|
96
|
+
* Each level shares its child schema; MCP can export the real input contract. */
|
|
97
|
+
function predicateAtDepth(depth: number): z.ZodType<MetadataPredicate> {
|
|
98
|
+
if (depth === METADATA_LIMITS.depth)
|
|
99
|
+
return z
|
|
100
|
+
.discriminatedUnion("op", leafSchemas)
|
|
101
|
+
.meta({ id: "gnoMetadataPredicateDepth8" });
|
|
102
|
+
const child = predicateAtDepth(depth + 1);
|
|
103
|
+
return z
|
|
104
|
+
.discriminatedUnion("op", [
|
|
105
|
+
...leafSchemas,
|
|
106
|
+
z
|
|
107
|
+
.object({
|
|
108
|
+
op: z.enum(["and", "or"]),
|
|
109
|
+
predicates: z.array(child).min(1).max(128),
|
|
110
|
+
})
|
|
111
|
+
.strict(),
|
|
112
|
+
z.object({ op: z.literal("not"), predicate: child }).strict(),
|
|
113
|
+
])
|
|
114
|
+
.meta({ id: `gnoMetadataPredicateDepth${depth}` });
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export const metadataPredicateSchema = predicateAtDepth(1).superRefine(
|
|
118
|
+
(input, ctx) => {
|
|
119
|
+
const queue: MetadataPredicate[] = [input];
|
|
120
|
+
let nodes = 0;
|
|
121
|
+
while (queue.length) {
|
|
122
|
+
const node = queue.pop();
|
|
123
|
+
if (!node) break;
|
|
124
|
+
if (++nodes > METADATA_LIMITS.nodes) {
|
|
125
|
+
ctx.addIssue({ code: "custom", message: "Filter exceeds 128 nodes" });
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
if ("predicates" in node) queue.push(...node.predicates);
|
|
129
|
+
if ("predicate" in node) queue.push(node.predicate);
|
|
130
|
+
}
|
|
131
|
+
if (
|
|
132
|
+
new TextEncoder().encode(JSON.stringify(input)).length >
|
|
133
|
+
METADATA_LIMITS.bytes
|
|
134
|
+
) {
|
|
135
|
+
ctx.addIssue({ code: "custom", message: "Filter exceeds 64 KiB" });
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
);
|
|
139
|
+
|
|
140
|
+
export function normalizeMetadataPredicate(input: unknown): MetadataPredicate {
|
|
141
|
+
const predicate = metadataPredicateSchema.parse(input);
|
|
142
|
+
const normalize = (node: MetadataPredicate): MetadataPredicate => {
|
|
143
|
+
if ("predicates" in node) {
|
|
144
|
+
const unique = new Map(
|
|
145
|
+
node.predicates.map((child) => {
|
|
146
|
+
const value = normalize(child);
|
|
147
|
+
return [JSON.stringify(value), value];
|
|
148
|
+
})
|
|
149
|
+
);
|
|
150
|
+
return {
|
|
151
|
+
op: node.op,
|
|
152
|
+
predicates: [...unique.entries()]
|
|
153
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
154
|
+
.map(([, value]) => value),
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
if (node.op === "not")
|
|
158
|
+
return { op: "not", predicate: normalize(node.predicate) };
|
|
159
|
+
if ("values" in node)
|
|
160
|
+
return {
|
|
161
|
+
op: node.op,
|
|
162
|
+
key: node.key,
|
|
163
|
+
values: [...new Set(node.values)].sort((a, b) =>
|
|
164
|
+
JSON.stringify(a) < JSON.stringify(b)
|
|
165
|
+
? -1
|
|
166
|
+
: JSON.stringify(a) > JSON.stringify(b)
|
|
167
|
+
? 1
|
|
168
|
+
: 0
|
|
169
|
+
),
|
|
170
|
+
};
|
|
171
|
+
return node;
|
|
172
|
+
};
|
|
173
|
+
return normalize(predicate);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export function matchesMetadataPredicate(
|
|
177
|
+
metadata: TypedMetadata,
|
|
178
|
+
predicate: MetadataPredicate
|
|
179
|
+
): boolean {
|
|
180
|
+
if (predicate.op === "and")
|
|
181
|
+
return predicate.predicates.every((p) =>
|
|
182
|
+
matchesMetadataPredicate(metadata, p)
|
|
183
|
+
);
|
|
184
|
+
if (predicate.op === "or")
|
|
185
|
+
return predicate.predicates.some((p) =>
|
|
186
|
+
matchesMetadataPredicate(metadata, p)
|
|
187
|
+
);
|
|
188
|
+
if (predicate.op === "not")
|
|
189
|
+
return !matchesMetadataPredicate(metadata, predicate.predicate);
|
|
190
|
+
if (!("key" in predicate)) return false;
|
|
191
|
+
const present = Object.hasOwn(metadata, predicate.key);
|
|
192
|
+
if (predicate.op === "exists") return present === predicate.value;
|
|
193
|
+
if (!present) return false;
|
|
194
|
+
const field = metadata[predicate.key];
|
|
195
|
+
if ("values" in predicate) {
|
|
196
|
+
const values = Array.isArray(field) ? field : [field];
|
|
197
|
+
if (predicate.op === "all")
|
|
198
|
+
return (
|
|
199
|
+
Array.isArray(field) && predicate.values.every((v) => field.includes(v))
|
|
200
|
+
);
|
|
201
|
+
const match = values.some((v) =>
|
|
202
|
+
predicate.values.some((wanted) => wanted === v)
|
|
203
|
+
);
|
|
204
|
+
return predicate.op === "in" ? match : !match;
|
|
205
|
+
}
|
|
206
|
+
if (Array.isArray(field)) return false;
|
|
207
|
+
if (predicate.op === "eq") return field === predicate.value;
|
|
208
|
+
if (predicate.op === "ne")
|
|
209
|
+
return typeof field === typeof predicate.value && field !== predicate.value;
|
|
210
|
+
if (typeof field !== "number" || typeof predicate.value !== "number")
|
|
211
|
+
return false;
|
|
212
|
+
switch (predicate.op) {
|
|
213
|
+
case "gt":
|
|
214
|
+
return field > predicate.value;
|
|
215
|
+
case "gte":
|
|
216
|
+
return field >= predicate.value;
|
|
217
|
+
case "lt":
|
|
218
|
+
return field < predicate.value;
|
|
219
|
+
case "lte":
|
|
220
|
+
return field <= predicate.value;
|
|
221
|
+
default:
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
@@ -7,9 +7,11 @@ import type {
|
|
|
7
7
|
RecordAnchor,
|
|
8
8
|
RecordMetadata,
|
|
9
9
|
} from "../converters/types";
|
|
10
|
+
import type { TypedMetadata } from "../core/typed-metadata";
|
|
10
11
|
|
|
11
12
|
import { canonicalize, mirrorHash } from "../converters/canonicalize";
|
|
12
13
|
import { RECORD_METADATA_LIMITS } from "../converters/types";
|
|
14
|
+
import { validateRecordMetadata } from "./typed-metadata";
|
|
13
15
|
|
|
14
16
|
const HASH_PATTERN = /^[a-f\d]{64}$/;
|
|
15
17
|
const CONTROL_PATTERN = new RegExp(
|
|
@@ -33,6 +35,8 @@ const RECORD_ATTACHMENT_DISPOSITIONS = new Set<
|
|
|
33
35
|
>(["inline", "attachment"]);
|
|
34
36
|
|
|
35
37
|
export interface CanonicalRecord {
|
|
38
|
+
typedMetadata?: TypedMetadata;
|
|
39
|
+
metadataError?: string;
|
|
36
40
|
recordKey: string;
|
|
37
41
|
stableId: string;
|
|
38
42
|
sourceLocator: string;
|
|
@@ -44,7 +48,7 @@ export interface CanonicalRecord {
|
|
|
44
48
|
adapterFingerprint: string;
|
|
45
49
|
title?: string;
|
|
46
50
|
languageHint?: string;
|
|
47
|
-
metadata?: RecordMetadata
|
|
51
|
+
metadata?: Omit<RecordMetadata, "custom">;
|
|
48
52
|
anchors?: RecordAnchor[];
|
|
49
53
|
}
|
|
50
54
|
|
|
@@ -187,7 +191,7 @@ const normalizeFailureReference = (
|
|
|
187
191
|
|
|
188
192
|
const normalizeMetadata = (
|
|
189
193
|
metadata: RecordMetadata | undefined
|
|
190
|
-
): RecordMetadata | undefined => {
|
|
194
|
+
): Omit<RecordMetadata, "custom"> | undefined => {
|
|
191
195
|
if (!metadata) return undefined;
|
|
192
196
|
const normalizeList = (
|
|
193
197
|
values: string[] | undefined,
|
|
@@ -348,6 +352,10 @@ export const canonicalRecord = (
|
|
|
348
352
|
const sourceLocator = normalizeLocator(record.sourceLocator);
|
|
349
353
|
const markdown = canonicalize(record.markdown);
|
|
350
354
|
const metadata = normalizeMetadata(record.metadata);
|
|
355
|
+
const custom =
|
|
356
|
+
record.metadata && Object.hasOwn(record.metadata, "custom")
|
|
357
|
+
? validateRecordMetadata(record.metadata.custom, "metadata.custom")
|
|
358
|
+
: undefined;
|
|
351
359
|
const anchors = normalizeAnchors(record.anchors);
|
|
352
360
|
const title = boundedText(record.title, RECORD_METADATA_LIMITS.maxTitleChars);
|
|
353
361
|
const languageHint = boundedText(
|
|
@@ -360,6 +368,7 @@ export const canonicalRecord = (
|
|
|
360
368
|
languageHint,
|
|
361
369
|
markdown,
|
|
362
370
|
metadata,
|
|
371
|
+
...custom,
|
|
363
372
|
sourceLocator,
|
|
364
373
|
stableId,
|
|
365
374
|
title,
|
|
@@ -370,6 +379,7 @@ export const canonicalRecord = (
|
|
|
370
379
|
anchors,
|
|
371
380
|
languageHint,
|
|
372
381
|
metadata,
|
|
382
|
+
...custom,
|
|
373
383
|
sourceLocator,
|
|
374
384
|
stableId,
|
|
375
385
|
title,
|
|
@@ -395,6 +405,7 @@ export const canonicalRecord = (
|
|
|
395
405
|
title,
|
|
396
406
|
languageHint,
|
|
397
407
|
metadata,
|
|
408
|
+
...custom,
|
|
398
409
|
anchors,
|
|
399
410
|
accountingChars: derivedSource.length,
|
|
400
411
|
metadataChars,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { NormalizedContentTypeRule } from "../config";
|
|
2
2
|
import type { Collection } from "../config/types";
|
|
3
3
|
import type { RecordAdapter, RecordMetadata } from "../converters/types";
|
|
4
|
+
import type { TypedMetadata } from "../core/typed-metadata";
|
|
4
5
|
import type {
|
|
5
6
|
ChunkInput,
|
|
6
7
|
DocumentRow,
|
|
@@ -36,6 +37,8 @@ interface RecordDocumentMetadata {
|
|
|
36
37
|
author?: string;
|
|
37
38
|
frontmatterDate?: string;
|
|
38
39
|
dateFields?: Record<string, string>;
|
|
40
|
+
typedMetadata?: TypedMetadata;
|
|
41
|
+
metadataError?: string;
|
|
39
42
|
}
|
|
40
43
|
|
|
41
44
|
interface RecordContainerInput {
|
|
@@ -376,6 +379,12 @@ const persistRecord = async (
|
|
|
376
379
|
author: record.metadata?.author ?? inferred.author,
|
|
377
380
|
frontmatterDate: primaryDate(dateFields) ?? inferred.frontmatterDate,
|
|
378
381
|
dateFields,
|
|
382
|
+
typedMetadata: record.metadataError
|
|
383
|
+
? undefined
|
|
384
|
+
: (record.typedMetadata ?? inferred.typedMetadata),
|
|
385
|
+
metadataError:
|
|
386
|
+
record.metadataError ??
|
|
387
|
+
(record.typedMetadata ? undefined : inferred.metadataError),
|
|
379
388
|
recordKey: record.recordKey,
|
|
380
389
|
recordSourcePath: input.entry.relPath,
|
|
381
390
|
recordSourceLocator: record.sourceLocator,
|
package/src/ingestion/sync.ts
CHANGED
|
@@ -12,6 +12,7 @@ import { isAbsolute, join, relative, sep } from "node:path";
|
|
|
12
12
|
|
|
13
13
|
import type { NormalizedContentTypeRule } from "../config";
|
|
14
14
|
import type { Collection } from "../config/types";
|
|
15
|
+
import type { TypedMetadata } from "../core/typed-metadata";
|
|
15
16
|
import type {
|
|
16
17
|
ChunkInput,
|
|
17
18
|
DocLinkInput,
|
|
@@ -57,6 +58,7 @@ import {
|
|
|
57
58
|
} from "../core/links";
|
|
58
59
|
import { extractMemoryScopes } from "../core/memory-record";
|
|
59
60
|
import { normalizeTag, validateTag } from "../core/tags";
|
|
61
|
+
import { TYPED_METADATA_INGEST_VERSION } from "../core/typed-metadata";
|
|
60
62
|
import { defaultChunker } from "./chunker";
|
|
61
63
|
import { persistChunkLayout, prepareChunking } from "./chunking";
|
|
62
64
|
import {
|
|
@@ -81,6 +83,7 @@ import {
|
|
|
81
83
|
type SourceReadFailure,
|
|
82
84
|
} from "./source-availability";
|
|
83
85
|
import { getExcludedRanges } from "./strip";
|
|
86
|
+
import { extractTypedMetadata } from "./typed-metadata";
|
|
84
87
|
import { collectionToWalkConfig, DEFAULT_CHUNK_PARAMS } from "./types";
|
|
85
88
|
import { defaultWalker } from "./walker";
|
|
86
89
|
|
|
@@ -98,7 +101,7 @@ const MAX_CONCURRENCY = 16;
|
|
|
98
101
|
* Increment when ingestion adds new derived data (tags, metadata, etc.)
|
|
99
102
|
* Documents with ingestVersion < INGEST_VERSION will be re-processed.
|
|
100
103
|
*/
|
|
101
|
-
export const INGEST_VERSION =
|
|
104
|
+
export const INGEST_VERSION = TYPED_METADATA_INGEST_VERSION;
|
|
102
105
|
const EMPTY_CONTENT_TYPE_RULES_FINGERPRINT =
|
|
103
106
|
fingerprintContentTypeMetadataRules([]);
|
|
104
107
|
const NON_RETRYABLE_CONVERSION_ERROR_CODES = new Set([
|
|
@@ -211,6 +214,8 @@ function extractTags(markdown: string): string[] {
|
|
|
211
214
|
}
|
|
212
215
|
|
|
213
216
|
interface DocumentMetadata {
|
|
217
|
+
typedMetadata?: TypedMetadata;
|
|
218
|
+
metadataError?: string;
|
|
214
219
|
contentType?: string;
|
|
215
220
|
contentTypeSource: ContentTypeSource;
|
|
216
221
|
categories?: string[];
|
|
@@ -404,6 +409,7 @@ export function extractDocumentMetadata(
|
|
|
404
409
|
}
|
|
405
410
|
|
|
406
411
|
return {
|
|
412
|
+
...extractTypedMetadata(markdown),
|
|
407
413
|
contentType,
|
|
408
414
|
contentTypeSource,
|
|
409
415
|
categories: [...categories],
|
|
@@ -493,6 +499,8 @@ const preserveDocumentWithError = (
|
|
|
493
499
|
author: existing.author ?? undefined,
|
|
494
500
|
frontmatterDate: existing.frontmatterDate ?? undefined,
|
|
495
501
|
dateFields: existing.dateFields ?? undefined,
|
|
502
|
+
typedMetadata: existing.typedMetadata ?? undefined,
|
|
503
|
+
metadataError: existing.metadataError ?? undefined,
|
|
496
504
|
recordKey: existing.recordKey ?? undefined,
|
|
497
505
|
recordSourcePath: existing.recordSourcePath ?? undefined,
|
|
498
506
|
recordSourceLocator: existing.recordSourceLocator ?? undefined,
|
|
@@ -913,6 +921,8 @@ export class SyncService {
|
|
|
913
921
|
author: extractedMetadata.author,
|
|
914
922
|
frontmatterDate: extractedMetadata.frontmatterDate,
|
|
915
923
|
dateFields: extractedMetadata.dateFields,
|
|
924
|
+
typedMetadata: extractedMetadata.typedMetadata,
|
|
925
|
+
metadataError: extractedMetadata.metadataError,
|
|
916
926
|
contentTypeRulesFingerprint,
|
|
917
927
|
// Clear error fields on success (requires store to handle undefined → null)
|
|
918
928
|
lastErrorCode: undefined,
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { TypedMetadata } from "../core/typed-metadata";
|
|
2
|
+
|
|
3
|
+
import { METADATA_LIMITS, typedMetadataSchema } from "../core/typed-metadata";
|
|
4
|
+
|
|
5
|
+
const FRONTMATTER = /^---\r?\n([\s\S]*?)(?:\r?\n)?---(?:\r?\n|$)/;
|
|
6
|
+
const GNO_KEY = /(?:^|[{,]\s*)(?:gno|"gno"|'gno')\s*:/m;
|
|
7
|
+
export interface TypedMetadataExtraction {
|
|
8
|
+
typedMetadata?: TypedMetadata;
|
|
9
|
+
metadataError?: string;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export function extractTypedMetadata(
|
|
13
|
+
markdown: string
|
|
14
|
+
): TypedMetadataExtraction {
|
|
15
|
+
const yaml = FRONTMATTER.exec(markdown)?.[1];
|
|
16
|
+
if (yaml === undefined || !GNO_KEY.test(yaml)) return { typedMetadata: {} };
|
|
17
|
+
if (new TextEncoder().encode(yaml).length > METADATA_LIMITS.bytes) {
|
|
18
|
+
return {
|
|
19
|
+
metadataError: "frontmatter: exceeds 64 KiB typed metadata parsing limit",
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
// Bun.YAML aliases share object references rather than expanding copies.
|
|
23
|
+
// The flat typed validator below rejects nested/cyclic values.
|
|
24
|
+
let parsed: unknown;
|
|
25
|
+
try {
|
|
26
|
+
parsed = Bun.YAML.parse(yaml);
|
|
27
|
+
} catch {
|
|
28
|
+
return { metadataError: "gno.metadata: invalid YAML" };
|
|
29
|
+
}
|
|
30
|
+
if (!parsed || typeof parsed !== "object" || !Object.hasOwn(parsed, "gno"))
|
|
31
|
+
return { typedMetadata: {} };
|
|
32
|
+
const gno = (parsed as Record<string, unknown>).gno;
|
|
33
|
+
if (!gno || typeof gno !== "object" || Array.isArray(gno)) {
|
|
34
|
+
return { metadataError: "gno: expected a mapping" };
|
|
35
|
+
}
|
|
36
|
+
if (!Object.hasOwn(gno, "metadata")) return { typedMetadata: {} };
|
|
37
|
+
return validateRecordMetadata((gno as Record<string, unknown>).metadata);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function validateRecordMetadata(
|
|
41
|
+
value: unknown,
|
|
42
|
+
namespace = "gno.metadata"
|
|
43
|
+
): TypedMetadataExtraction {
|
|
44
|
+
const parsed = typedMetadataSchema.safeParse(value);
|
|
45
|
+
if (parsed.success) return { typedMetadata: parsed.data };
|
|
46
|
+
const issue = parsed.error.issues[0];
|
|
47
|
+
// Values may be private. Persist only the bounded field path, never input.
|
|
48
|
+
return {
|
|
49
|
+
metadataError:
|
|
50
|
+
`${namespace}.${issue?.path.map(String).join(".") ?? ""}: invalid typed metadata`.slice(
|
|
51
|
+
0,
|
|
52
|
+
256
|
|
53
|
+
),
|
|
54
|
+
};
|
|
55
|
+
}
|
package/src/mcp/tools/ask.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
/** MCP verified Ask tool over the shared closed-evidence application boundary. */
|
|
2
|
-
|
|
3
1
|
import { z } from "zod";
|
|
2
|
+
/** MCP verified Ask tool over the shared closed-evidence application boundary. */
|
|
4
3
|
|
|
5
4
|
import type { RetrievalTraceSession } from "../../core/retrieval-trace-session";
|
|
6
5
|
import type { QueryModeInput } from "../../pipeline/types";
|
|
@@ -16,6 +15,10 @@ import {
|
|
|
16
15
|
} from "../../core/retrieval-trace-request";
|
|
17
16
|
import { attachRetrievalTraceMetadata } from "../../core/retrieval-trace-session";
|
|
18
17
|
import { normalizeStructuredQueryInput } from "../../core/structured-query";
|
|
18
|
+
import {
|
|
19
|
+
metadataPredicateSchema,
|
|
20
|
+
normalizeMetadataPredicate,
|
|
21
|
+
} from "../../core/typed-metadata";
|
|
19
22
|
import { resolveModelUri } from "../../llm/registry";
|
|
20
23
|
import { answerTraceTerminalStatus } from "../../pipeline/answer";
|
|
21
24
|
import { createMcpModelPorts, type McpModelPortFactory } from "./context";
|
|
@@ -41,6 +44,7 @@ export const askInputSchema = z
|
|
|
41
44
|
candidateLimit: z.number().int().min(1).max(100).optional(),
|
|
42
45
|
exclude: z.array(z.string()).optional(),
|
|
43
46
|
queryModes: z.array(queryModeSchema).optional(),
|
|
47
|
+
filter: metadataPredicateSchema.optional(),
|
|
44
48
|
tagsAll: z.array(z.string()).optional(),
|
|
45
49
|
tagsAny: z.array(z.string()).optional(),
|
|
46
50
|
since: z.string().optional(),
|
|
@@ -163,6 +167,10 @@ export const handleAsk = (
|
|
|
163
167
|
);
|
|
164
168
|
const options = {
|
|
165
169
|
...askInput,
|
|
170
|
+
filter:
|
|
171
|
+
args.filter === undefined
|
|
172
|
+
? undefined
|
|
173
|
+
: normalizeMetadataPredicate(args.filter),
|
|
166
174
|
projectAffinity,
|
|
167
175
|
queryModes:
|
|
168
176
|
normalized.value.queryModes.length > 0
|
package/src/mcp/tools/context.ts
CHANGED
|
@@ -261,6 +261,7 @@ export const handleContext = (
|
|
|
261
261
|
until: parsed.input.until,
|
|
262
262
|
categories: parsed.input.categories,
|
|
263
263
|
author: parsed.input.author,
|
|
264
|
+
filter: parsed.input.filter,
|
|
264
265
|
graph: parsed.input.graph,
|
|
265
266
|
candidateLimit: parsed.input.candidateLimit,
|
|
266
267
|
queryModes: parsed.input.queryModes,
|
package/src/mcp/tools/index.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import type { McpServer } from "@modelcontextprotocol/server";
|
|
2
1
|
/**
|
|
3
2
|
* MCP tool registration and shared utilities.
|
|
4
3
|
*
|
|
5
4
|
* @module src/mcp/tools
|
|
6
5
|
*/
|
|
7
6
|
|
|
7
|
+
import type { McpServer } from "@modelcontextprotocol/server";
|
|
8
|
+
|
|
8
9
|
import { z } from "zod";
|
|
9
10
|
|
|
10
11
|
import type { ToolContext } from "../server";
|
|
@@ -17,6 +18,7 @@ import { CAPTURE_MAX_TEXT_BYTES } from "../../core/capture";
|
|
|
17
18
|
import { NOTE_PRESETS, type NotePresetId } from "../../core/note-presets";
|
|
18
19
|
import { RETRIEVAL_TRACE_METADATA } from "../../core/retrieval-trace-session";
|
|
19
20
|
import { normalizeTag } from "../../core/tags";
|
|
21
|
+
import { metadataPredicateSchema } from "../../core/typed-metadata";
|
|
20
22
|
import {
|
|
21
23
|
assertInferenceActive,
|
|
22
24
|
acquireInferencePermit,
|
|
@@ -258,6 +260,9 @@ export const searchInputSchema = z.object({
|
|
|
258
260
|
.string()
|
|
259
261
|
.optional()
|
|
260
262
|
.describe("Filter by author (case-insensitive substring match)"),
|
|
263
|
+
filter: metadataPredicateSchema
|
|
264
|
+
.optional()
|
|
265
|
+
.describe("Typed custom metadata predicate; intersects existing scope"),
|
|
261
266
|
tagsAll: z
|
|
262
267
|
.array(z.string())
|
|
263
268
|
.optional()
|
|
@@ -488,6 +493,9 @@ export const vsearchInputSchema = z.object({
|
|
|
488
493
|
.string()
|
|
489
494
|
.optional()
|
|
490
495
|
.describe("Filter by author (case-insensitive substring)"),
|
|
496
|
+
filter: metadataPredicateSchema
|
|
497
|
+
.optional()
|
|
498
|
+
.describe("Typed custom metadata predicate; intersects existing scope"),
|
|
491
499
|
tagsAll: z.array(z.string()).optional().describe("Require ALL of these tags"),
|
|
492
500
|
tagsAny: z.array(z.string()).optional().describe("Require ANY of these tags"),
|
|
493
501
|
});
|
|
@@ -614,6 +622,9 @@ export const queryInputSchema = z.object({
|
|
|
614
622
|
.boolean()
|
|
615
623
|
.optional()
|
|
616
624
|
.describe("Include deterministic stage and per-result scoring metadata"),
|
|
625
|
+
filter: metadataPredicateSchema
|
|
626
|
+
.optional()
|
|
627
|
+
.describe("Typed custom metadata predicate; intersects existing scope"),
|
|
617
628
|
tagsAll: z.array(z.string()).optional().describe("Require ALL of these tags"),
|
|
618
629
|
tagsAny: z.array(z.string()).optional().describe("Require ANY of these tags"),
|
|
619
630
|
});
|
package/src/mcp/tools/query.ts
CHANGED
|
@@ -38,6 +38,10 @@ import {
|
|
|
38
38
|
type RetrievalTraceSession,
|
|
39
39
|
} from "../../core/retrieval-trace-session";
|
|
40
40
|
import { normalizeStructuredQueryInput } from "../../core/structured-query";
|
|
41
|
+
import {
|
|
42
|
+
normalizeMetadataPredicate,
|
|
43
|
+
type MetadataPredicate,
|
|
44
|
+
} from "../../core/typed-metadata";
|
|
41
45
|
import { LlmAdapter } from "../../llm/nodeLlamaCpp/adapter";
|
|
42
46
|
import { resolveDownloadPolicy } from "../../llm/policy";
|
|
43
47
|
import { getActivePreset, resolveModelUri } from "../../llm/registry";
|
|
@@ -60,6 +64,7 @@ interface QueryInput {
|
|
|
60
64
|
until?: string;
|
|
61
65
|
categories?: string[];
|
|
62
66
|
author?: string;
|
|
67
|
+
filter?: MetadataPredicate;
|
|
63
68
|
queryModes?: QueryModeInput[];
|
|
64
69
|
fast?: boolean;
|
|
65
70
|
thorough?: boolean;
|
|
@@ -251,6 +256,10 @@ export function handleQuery(
|
|
|
251
256
|
until: args.until,
|
|
252
257
|
categories: args.categories,
|
|
253
258
|
author: args.author,
|
|
259
|
+
filter:
|
|
260
|
+
args.filter === undefined
|
|
261
|
+
? undefined
|
|
262
|
+
: normalizeMetadataPredicate(args.filter),
|
|
254
263
|
noExpand,
|
|
255
264
|
noRerank,
|
|
256
265
|
graph: args.graph,
|
|
@@ -524,6 +533,10 @@ export function handleQueryDiagnose(
|
|
|
524
533
|
until: args.until,
|
|
525
534
|
categories: args.categories,
|
|
526
535
|
author: args.author,
|
|
536
|
+
filter:
|
|
537
|
+
args.filter === undefined
|
|
538
|
+
? undefined
|
|
539
|
+
: normalizeMetadataPredicate(args.filter),
|
|
527
540
|
noExpand,
|
|
528
541
|
noRerank,
|
|
529
542
|
graph: args.graph,
|
package/src/mcp/tools/search.ts
CHANGED
|
@@ -19,6 +19,10 @@ import {
|
|
|
19
19
|
startRetrievalTraceRequest,
|
|
20
20
|
} from "../../core/retrieval-trace-request";
|
|
21
21
|
import { attachRetrievalTraceMetadata } from "../../core/retrieval-trace-session";
|
|
22
|
+
import {
|
|
23
|
+
normalizeMetadataPredicate,
|
|
24
|
+
type MetadataPredicate,
|
|
25
|
+
} from "../../core/typed-metadata";
|
|
22
26
|
import { searchBm25 } from "../../pipeline/search";
|
|
23
27
|
import { normalizeTagFilters, runTool, type ToolResult } from "./index";
|
|
24
28
|
|
|
@@ -35,6 +39,7 @@ interface SearchInput {
|
|
|
35
39
|
until?: string;
|
|
36
40
|
categories?: string[];
|
|
37
41
|
author?: string;
|
|
42
|
+
filter?: MetadataPredicate;
|
|
38
43
|
tagsAll?: string[];
|
|
39
44
|
tagsAny?: string[];
|
|
40
45
|
}
|
|
@@ -131,6 +136,10 @@ export function handleSearch(
|
|
|
131
136
|
until: args.until,
|
|
132
137
|
categories: args.categories,
|
|
133
138
|
author: args.author,
|
|
139
|
+
filter:
|
|
140
|
+
args.filter === undefined
|
|
141
|
+
? undefined
|
|
142
|
+
: normalizeMetadataPredicate(args.filter),
|
|
134
143
|
tagsAll: normalizeTagFilters(args.tagsAll),
|
|
135
144
|
tagsAny: normalizeTagFilters(args.tagsAny),
|
|
136
145
|
projectAffinity,
|
package/src/mcp/tools/status.ts
CHANGED
|
@@ -52,6 +52,10 @@ function formatStatus(status: IndexStatus): string {
|
|
|
52
52
|
`Total: ${status.activeDocuments} documents, ${status.totalChunks} chunks`
|
|
53
53
|
);
|
|
54
54
|
|
|
55
|
+
if (status.typedMetadata)
|
|
56
|
+
lines.push(
|
|
57
|
+
`Typed metadata: ${status.typedMetadata.pending} pending sync, ${status.typedMetadata.invalid} invalid`
|
|
58
|
+
);
|
|
55
59
|
if (status.embeddingBacklog > 0) {
|
|
56
60
|
lines.push(`Embedding backlog: ${status.embeddingBacklog} chunks`);
|
|
57
61
|
}
|
package/src/mcp/tools/vsearch.ts
CHANGED
|
@@ -22,6 +22,10 @@ import {
|
|
|
22
22
|
attachRetrievalTraceMetadata,
|
|
23
23
|
type RetrievalTraceSession,
|
|
24
24
|
} from "../../core/retrieval-trace-session";
|
|
25
|
+
import {
|
|
26
|
+
normalizeMetadataPredicate,
|
|
27
|
+
type MetadataPredicate,
|
|
28
|
+
} from "../../core/typed-metadata";
|
|
25
29
|
import { LlmAdapter } from "../../llm/nodeLlamaCpp/adapter";
|
|
26
30
|
import { resolveDownloadPolicy } from "../../llm/policy";
|
|
27
31
|
import { resolveModelUri } from "../../llm/registry";
|
|
@@ -46,6 +50,7 @@ interface VsearchInput {
|
|
|
46
50
|
until?: string;
|
|
47
51
|
categories?: string[];
|
|
48
52
|
author?: string;
|
|
53
|
+
filter?: MetadataPredicate;
|
|
49
54
|
tagsAll?: string[];
|
|
50
55
|
tagsAny?: string[];
|
|
51
56
|
}
|
|
@@ -151,6 +156,10 @@ export function handleVsearch(
|
|
|
151
156
|
until: args.until,
|
|
152
157
|
categories: args.categories,
|
|
153
158
|
author: args.author,
|
|
159
|
+
filter:
|
|
160
|
+
args.filter === undefined
|
|
161
|
+
? undefined
|
|
162
|
+
: normalizeMetadataPredicate(args.filter),
|
|
154
163
|
tagsAll: normalizeTagFilters(args.tagsAll),
|
|
155
164
|
tagsAny: normalizeTagFilters(args.tagsAny),
|
|
156
165
|
projectAffinity,
|
package/src/pipeline/diagnose.ts
CHANGED
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
normalizeContentTypes,
|
|
20
20
|
} from "../config";
|
|
21
21
|
import { resolveDocRef } from "../core/ref-parser";
|
|
22
|
+
import { normalizeMetadataPredicate } from "../core/typed-metadata";
|
|
22
23
|
import { err, ok } from "../store/types";
|
|
23
24
|
import {
|
|
24
25
|
getContentTypeBoostMetadata,
|
|
@@ -184,6 +185,11 @@ export async function diagnoseQueryTarget(
|
|
|
184
185
|
query: string,
|
|
185
186
|
options: QueryDiagnoseOptions
|
|
186
187
|
): Promise<StoreResult<QueryDiagnoseResult>> {
|
|
188
|
+
if (options.filter !== undefined)
|
|
189
|
+
options = {
|
|
190
|
+
...options,
|
|
191
|
+
filter: normalizeMetadataPredicate(options.filter),
|
|
192
|
+
};
|
|
187
193
|
const resolved = await resolveDocRef(deps.store, options.target);
|
|
188
194
|
if ("error" in resolved) {
|
|
189
195
|
return ok(buildBaseResult(query, options.target, "not_found", null));
|