@gmickel/gno 2.3.3 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -3
- package/assets/skill/SKILL.md +53 -0
- package/assets/skill/cli-reference.md +29 -0
- package/assets/skill/examples.md +30 -0
- package/assets/skill/mcp-reference.md +26 -0
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.3.3.zip → gno-browser-clipper-v2.5.1.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.5.1.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +41 -0
- package/spec/compiled-context.md +68 -0
- package/spec/db/schema.sql +2 -0
- package/spec/evals.md +48 -0
- package/spec/mcp.md +37 -0
- package/spec/output-schemas/ask.schema.json +12 -0
- package/spec/output-schemas/compiled-context-check.schema.json +44 -0
- package/spec/output-schemas/compiled-context-file.schema.json +165 -0
- package/spec/output-schemas/compiled-context-preview.schema.json +142 -0
- package/spec/output-schemas/context-capsule-v1.schema.json +205 -32
- package/spec/output-schemas/context-capsule-verification.schema.json +32 -9
- package/spec/output-schemas/evidence-fixtures.schema.json +176 -0
- package/spec/output-schemas/evidence-report.schema.json +322 -0
- package/spec/output-schemas/evidence-run.schema.json +277 -0
- package/spec/output-schemas/metadata-predicate.schema.json +460 -0
- package/spec/output-schemas/retrieval-trace-filters.schema.json +13 -4
- package/spec/output-schemas/search-results.schema.json +12 -0
- package/spec/output-schemas/status.schema.json +37 -6
- package/src/app/compiled-context-files.ts +361 -0
- package/src/app/compiled-context.ts +240 -0
- package/src/app/context-runtime-contract.ts +33 -2
- package/src/app/context-runtime-input.ts +17 -0
- package/src/app/context-runtime-types.ts +2 -0
- package/src/app/context-runtime.ts +27 -3
- package/src/app/context-surface.ts +7 -0
- package/src/app/verified-ask.ts +1 -0
- package/src/cli/commands/ask.ts +1 -0
- package/src/cli/commands/audit.ts +4 -0
- package/src/cli/commands/context-build.ts +1 -0
- package/src/cli/commands/context-compiled.ts +136 -0
- package/src/cli/commands/search.ts +1 -0
- package/src/cli/commands/status.ts +10 -0
- package/src/cli/errors.ts +10 -3
- package/src/cli/options.ts +19 -0
- package/src/cli/program.ts +82 -0
- package/src/converters/types.ts +2 -0
- package/src/core/compiled-context.ts +254 -0
- package/src/core/context-budget.ts +2 -10
- package/src/core/context-capsule-retrieval-schema.ts +8 -0
- package/src/core/context-capsule-schema.ts +47 -3
- package/src/core/context-capsule-validation.ts +14 -2
- package/src/core/context-capsule-verification.ts +1 -0
- package/src/core/context-capsule.ts +16 -2
- package/src/core/context-compiler.ts +3 -0
- package/src/core/file-lock.ts +22 -5
- package/src/core/folder-setup-planning.ts +2 -1
- package/src/core/network-boundary-inventory.ts +8 -0
- package/src/core/record-metadata.ts +1 -1
- package/src/core/retrieval-replay-candidate.ts +5 -0
- package/src/core/retrieval-trace-filter-normalization.ts +4 -0
- package/src/core/retrieval-trace-request.ts +1 -0
- package/src/core/retrieval-trace-session.ts +5 -1
- package/src/core/retrieval-trace.ts +2 -0
- package/src/core/setup-receipt.ts +27 -20
- package/src/core/typed-metadata.ts +228 -0
- package/src/core/validation.ts +9 -2
- package/src/core/windows-private-path.ts +96 -0
- package/src/index.ts +2 -2
- package/src/ingestion/compiled-context.ts +15 -0
- package/src/ingestion/record-adapter-canonical.ts +13 -2
- package/src/ingestion/record-container.ts +9 -0
- package/src/ingestion/sync.ts +51 -9
- package/src/ingestion/typed-metadata.ts +55 -0
- package/src/ingestion/walker.ts +5 -4
- package/src/llm/nodeLlamaCpp/simulator-install.ts +6 -2
- package/src/mcp/http-egress.ts +11 -3
- package/src/mcp/retrieval-warnings.ts +24 -0
- package/src/mcp/tools/ask.ts +23 -2
- package/src/mcp/tools/context.ts +47 -2
- package/src/mcp/tools/index.ts +58 -1
- package/src/mcp/tools/query.ts +16 -1
- package/src/mcp/tools/search.ts +12 -1
- package/src/mcp/tools/status.ts +4 -0
- package/src/mcp/tools/vsearch.ts +12 -1
- package/src/pipeline/diagnose.ts +6 -0
- package/src/pipeline/filters.ts +65 -0
- package/src/pipeline/graph-retrieval.ts +9 -1
- package/src/pipeline/hybrid.ts +22 -2
- package/src/pipeline/search.ts +11 -0
- package/src/pipeline/types.ts +7 -1
- package/src/pipeline/vsearch.ts +21 -1
- package/src/sdk/client.ts +98 -3
- package/src/sdk/index.ts +11 -0
- package/src/sdk/types.ts +35 -1
- package/src/serve/compiled-context.ts +84 -0
- package/src/serve/context-capsule.ts +1 -0
- package/src/serve/public/app.tsx +12 -1
- package/src/serve/public/components/MetadataFilter.tsx +186 -0
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/lib/metadata-filter.ts +28 -0
- package/src/serve/public/lib/retrieval-filters.ts +3 -0
- package/src/serve/public/lib/workspace-tabs.ts +2 -0
- package/src/serve/public/pages/Ask.tsx +48 -3
- package/src/serve/public/pages/CompiledContext.tsx +364 -0
- package/src/serve/public/pages/Dashboard.tsx +7 -0
- package/src/serve/public/pages/DocView.tsx +33 -0
- package/src/serve/public/pages/Search.tsx +44 -1
- package/src/serve/routes/api.ts +50 -0
- package/src/serve/server.ts +43 -2
- package/src/serve/spa-production-build.ts +6 -5
- package/src/store/migrations/030-typed-metadata.ts +13 -0
- package/src/store/migrations/index.ts +2 -0
- package/src/store/sqlite/adapter.ts +69 -5
- package/src/store/sqlite/eligibility.ts +12 -0
- package/src/store/sqlite/metadata-predicate.ts +71 -0
- package/src/store/types.ts +11 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.3.3.zip.sha256 +0 -1
|
@@ -7,11 +7,12 @@ import {
|
|
|
7
7
|
SlidersHorizontal,
|
|
8
8
|
XIcon,
|
|
9
9
|
} from "lucide-react";
|
|
10
|
-
import { useCallback, useEffect, useMemo, useState } from "react";
|
|
10
|
+
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
|
11
11
|
|
|
12
12
|
import { resolveDepthPolicy } from "../../../core/depth-policy";
|
|
13
13
|
import { normalizeStructuredQueryInput } from "../../../core/structured-query";
|
|
14
14
|
import { Loader } from "../components/ai-elements/loader";
|
|
15
|
+
import { MetadataFilter } from "../components/MetadataFilter";
|
|
15
16
|
import {
|
|
16
17
|
ThoroughnessSelector,
|
|
17
18
|
type Thoroughness,
|
|
@@ -37,6 +38,7 @@ import { apiFetch } from "../hooks/use-api";
|
|
|
37
38
|
import { useDocEvents } from "../hooks/use-doc-events";
|
|
38
39
|
import { useKeyboardShortcuts } from "../hooks/useKeyboardShortcuts";
|
|
39
40
|
import { buildDocDeepLink } from "../lib/deep-links";
|
|
41
|
+
import { parseMetadataFilter } from "../lib/metadata-filter";
|
|
40
42
|
import {
|
|
41
43
|
applyFiltersToUrl,
|
|
42
44
|
parseFiltersFromSearch,
|
|
@@ -111,6 +113,7 @@ interface SearchResult {
|
|
|
111
113
|
interface SearchResponse {
|
|
112
114
|
results: SearchResult[];
|
|
113
115
|
meta: {
|
|
116
|
+
warnings?: { code: string; message: string }[];
|
|
114
117
|
query: string;
|
|
115
118
|
mode: string;
|
|
116
119
|
totalResults: number;
|
|
@@ -152,6 +155,7 @@ export default function Search({ navigate }: PageProps) {
|
|
|
152
155
|
const [results, setResults] = useState<SearchResult[]>([]);
|
|
153
156
|
const [meta, setMeta] = useState<SearchResponse["meta"] | null>(null);
|
|
154
157
|
const [loading, setLoading] = useState(false);
|
|
158
|
+
const searchRequest = useRef(0);
|
|
155
159
|
const [error, setError] = useState<string | null>(null);
|
|
156
160
|
const [searched, setSearched] = useState(false);
|
|
157
161
|
const [capabilities, setCapabilities] = useState<ServerCapabilities | null>(
|
|
@@ -170,6 +174,7 @@ export default function Search({ navigate }: PageProps) {
|
|
|
170
174
|
initialFilters.until ||
|
|
171
175
|
initialFilters.category ||
|
|
172
176
|
initialFilters.author ||
|
|
177
|
+
initialFilters.filter ||
|
|
173
178
|
initialFilters.queryModes.length > 0
|
|
174
179
|
)
|
|
175
180
|
);
|
|
@@ -187,6 +192,9 @@ export default function Search({ navigate }: PageProps) {
|
|
|
187
192
|
const [since, setSince] = useState(initialFilters.since);
|
|
188
193
|
const [until, setUntil] = useState(initialFilters.until);
|
|
189
194
|
const [category, setCategory] = useState(initialFilters.category);
|
|
195
|
+
const [metadataFilter, setMetadataFilter] = useState(
|
|
196
|
+
initialFilters.filter ?? ""
|
|
197
|
+
);
|
|
190
198
|
const [author, setAuthor] = useState(initialFilters.author);
|
|
191
199
|
const [queryModes, setQueryModes] = useState<QueryModeEntry[]>(
|
|
192
200
|
initialFilters.queryModes
|
|
@@ -217,6 +225,7 @@ export default function Search({ navigate }: PageProps) {
|
|
|
217
225
|
useEffect(() => {
|
|
218
226
|
const url = new URL(window.location.href);
|
|
219
227
|
applyFiltersToUrl(url, {
|
|
228
|
+
filter: metadataFilter,
|
|
220
229
|
collection: selectedCollection,
|
|
221
230
|
intent,
|
|
222
231
|
candidateLimit,
|
|
@@ -232,6 +241,7 @@ export default function Search({ navigate }: PageProps) {
|
|
|
232
241
|
window.history.replaceState({}, "", url.toString());
|
|
233
242
|
}, [
|
|
234
243
|
activeTags,
|
|
244
|
+
metadataFilter,
|
|
235
245
|
author,
|
|
236
246
|
candidateLimit,
|
|
237
247
|
category,
|
|
@@ -319,10 +329,23 @@ export default function Search({ navigate }: PageProps) {
|
|
|
319
329
|
const handleSearch = useCallback(
|
|
320
330
|
async (e?: React.FormEvent) => {
|
|
321
331
|
e?.preventDefault();
|
|
332
|
+
const request = ++searchRequest.current;
|
|
322
333
|
if (!query.trim()) {
|
|
323
334
|
return;
|
|
324
335
|
}
|
|
336
|
+
const parsedFilter = parseMetadataFilter(metadataFilter);
|
|
337
|
+
if (parsedFilter.error) {
|
|
338
|
+
setLoading(false);
|
|
339
|
+
setResults([]);
|
|
340
|
+
setMeta(null);
|
|
341
|
+
setError(parsedFilter.error);
|
|
342
|
+
setShowAdvanced(true);
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
325
345
|
if (!structuredQueryState.ok) {
|
|
346
|
+
setLoading(false);
|
|
347
|
+
setResults([]);
|
|
348
|
+
setMeta(null);
|
|
326
349
|
setError(structuredQueryState.error.message);
|
|
327
350
|
return;
|
|
328
351
|
}
|
|
@@ -351,6 +374,7 @@ export default function Search({ navigate }: PageProps) {
|
|
|
351
374
|
limit: 20,
|
|
352
375
|
};
|
|
353
376
|
|
|
377
|
+
if (parsedFilter.filter) body.filter = parsedFilter.filter;
|
|
354
378
|
if (selectedCollection) {
|
|
355
379
|
body.collection = selectedCollection;
|
|
356
380
|
}
|
|
@@ -400,6 +424,7 @@ export default function Search({ navigate }: PageProps) {
|
|
|
400
424
|
}
|
|
401
425
|
);
|
|
402
426
|
|
|
427
|
+
if (request !== searchRequest.current) return;
|
|
403
428
|
setLoading(false);
|
|
404
429
|
if (fetchError) {
|
|
405
430
|
setError(fetchError);
|
|
@@ -412,6 +437,7 @@ export default function Search({ navigate }: PageProps) {
|
|
|
412
437
|
},
|
|
413
438
|
[
|
|
414
439
|
activeTags,
|
|
440
|
+
metadataFilter,
|
|
415
441
|
author,
|
|
416
442
|
candidateLimit,
|
|
417
443
|
activePreset,
|
|
@@ -438,6 +464,7 @@ export default function Search({ navigate }: PageProps) {
|
|
|
438
464
|
}, [
|
|
439
465
|
activeTags,
|
|
440
466
|
activePreset,
|
|
467
|
+
metadataFilter,
|
|
441
468
|
author,
|
|
442
469
|
candidateLimit,
|
|
443
470
|
category,
|
|
@@ -477,6 +504,7 @@ export default function Search({ navigate }: PageProps) {
|
|
|
477
504
|
since ? `since:${since}` : null,
|
|
478
505
|
until ? `until:${until}` : null,
|
|
479
506
|
category.trim() ? `category:${category.trim()}` : null,
|
|
507
|
+
metadataFilter.trim() ? "custom metadata" : null,
|
|
480
508
|
author.trim() ? `author:${author.trim()}` : null,
|
|
481
509
|
queryModes.length > 0 ? `${queryModes.length} query mode(s)` : null,
|
|
482
510
|
].filter((pill): pill is string => Boolean(pill));
|
|
@@ -493,6 +521,7 @@ export default function Search({ navigate }: PageProps) {
|
|
|
493
521
|
setUntil("");
|
|
494
522
|
setCategory("");
|
|
495
523
|
setAuthor("");
|
|
524
|
+
setMetadataFilter("");
|
|
496
525
|
setQueryModes([]);
|
|
497
526
|
setQueryModeText("");
|
|
498
527
|
setQueryModeError(null);
|
|
@@ -773,6 +802,11 @@ export default function Search({ navigate }: PageProps) {
|
|
|
773
802
|
</div>
|
|
774
803
|
</div>
|
|
775
804
|
|
|
805
|
+
<MetadataFilter
|
|
806
|
+
onChange={setMetadataFilter}
|
|
807
|
+
value={metadataFilter}
|
|
808
|
+
/>
|
|
809
|
+
|
|
776
810
|
<div className="flex flex-wrap items-center gap-2">
|
|
777
811
|
<span className="text-muted-foreground text-xs">
|
|
778
812
|
Tag match:
|
|
@@ -954,6 +988,15 @@ export default function Search({ navigate }: PageProps) {
|
|
|
954
988
|
</div>
|
|
955
989
|
)}
|
|
956
990
|
|
|
991
|
+
{meta?.warnings?.map((warning) => (
|
|
992
|
+
<p
|
|
993
|
+
className="mb-3 rounded border border-secondary/30 bg-secondary/10 p-3 text-sm"
|
|
994
|
+
key={warning.code}
|
|
995
|
+
role="status"
|
|
996
|
+
>
|
|
997
|
+
{warning.message}
|
|
998
|
+
</p>
|
|
999
|
+
))}
|
|
957
1000
|
{error && (
|
|
958
1001
|
<Card className="mb-6 border-destructive bg-destructive/10">
|
|
959
1002
|
<CardContent className="py-4 text-destructive">
|
package/src/serve/routes/api.ts
CHANGED
|
@@ -124,6 +124,11 @@ import {
|
|
|
124
124
|
parseAndValidateTagFilter,
|
|
125
125
|
validateTag,
|
|
126
126
|
} from "../../core/tags";
|
|
127
|
+
import {
|
|
128
|
+
metadataPredicateSchema,
|
|
129
|
+
normalizeMetadataPredicate,
|
|
130
|
+
type MetadataPredicate,
|
|
131
|
+
} from "../../core/typed-metadata";
|
|
127
132
|
import { validateRelPath } from "../../core/validation";
|
|
128
133
|
import { writeLeasePath } from "../../core/write-lease";
|
|
129
134
|
import {
|
|
@@ -287,6 +292,7 @@ export interface SearchRequestBody {
|
|
|
287
292
|
/** Comma-separated category filters */
|
|
288
293
|
category?: string;
|
|
289
294
|
author?: string;
|
|
295
|
+
filter?: MetadataPredicate;
|
|
290
296
|
/** Comma-separated tags - filter to docs having ALL (AND) */
|
|
291
297
|
tagsAll?: string;
|
|
292
298
|
/** Comma-separated tags - filter to docs having ANY (OR) */
|
|
@@ -308,6 +314,7 @@ export interface QueryRequestBody {
|
|
|
308
314
|
/** Comma-separated category filters */
|
|
309
315
|
category?: string;
|
|
310
316
|
author?: string;
|
|
317
|
+
filter?: MetadataPredicate;
|
|
311
318
|
queryModes?: QueryModeInput[];
|
|
312
319
|
noExpand?: boolean;
|
|
313
320
|
noRerank?: boolean;
|
|
@@ -339,6 +346,7 @@ export interface AskRequestBody {
|
|
|
339
346
|
/** Comma-separated category filters */
|
|
340
347
|
category?: string;
|
|
341
348
|
author?: string;
|
|
349
|
+
filter?: MetadataPredicate;
|
|
342
350
|
maxAnswerTokens?: number;
|
|
343
351
|
verify?: boolean;
|
|
344
352
|
contextBudgetTokens?: number;
|
|
@@ -369,6 +377,7 @@ const ASK_REQUEST_KEYS = new Set<keyof AskRequestBody>([
|
|
|
369
377
|
"until",
|
|
370
378
|
"category",
|
|
371
379
|
"author",
|
|
380
|
+
"filter",
|
|
372
381
|
"maxAnswerTokens",
|
|
373
382
|
"verify",
|
|
374
383
|
"contextBudgetTokens",
|
|
@@ -586,6 +595,29 @@ function errorResponse(
|
|
|
586
595
|
);
|
|
587
596
|
}
|
|
588
597
|
|
|
598
|
+
function parseRestMetadataFilter(
|
|
599
|
+
value: unknown
|
|
600
|
+
):
|
|
601
|
+
| { ok: true; filter: MetadataPredicate | undefined }
|
|
602
|
+
| { ok: false; response: Response } {
|
|
603
|
+
if (value === undefined) return { ok: true, filter: undefined };
|
|
604
|
+
const parsed = metadataPredicateSchema.safeParse(value);
|
|
605
|
+
if (!parsed.success) {
|
|
606
|
+
const issue = parsed.error.issues[0];
|
|
607
|
+
const path = ["filter", ...(issue?.path ?? [])].join(".");
|
|
608
|
+
return {
|
|
609
|
+
ok: false,
|
|
610
|
+
response: errorResponse(
|
|
611
|
+
"VALIDATION",
|
|
612
|
+
`${path}: ${issue?.message ?? "Invalid predicate"}`,
|
|
613
|
+
400,
|
|
614
|
+
{ field: path }
|
|
615
|
+
),
|
|
616
|
+
};
|
|
617
|
+
}
|
|
618
|
+
return { ok: true, filter: normalizeMetadataPredicate(parsed.data) };
|
|
619
|
+
}
|
|
620
|
+
|
|
589
621
|
function fileRefactorHttpErrorResponse(error: FileRefactorHttpError): Response {
|
|
590
622
|
return errorResponse(error.code, error.message, error.status, error.details);
|
|
591
623
|
}
|
|
@@ -2090,6 +2122,8 @@ export async function handleDoc(
|
|
|
2090
2122
|
title: doc.title,
|
|
2091
2123
|
content,
|
|
2092
2124
|
contentAvailable,
|
|
2125
|
+
typedMetadata: doc.typedMetadata ?? null,
|
|
2126
|
+
metadataError: doc.metadataError ?? null,
|
|
2093
2127
|
collection: doc.collection,
|
|
2094
2128
|
relPath,
|
|
2095
2129
|
tags,
|
|
@@ -4194,6 +4228,9 @@ export async function handleSearch(
|
|
|
4194
4228
|
? parseCommaSeparatedValues(body.exclude)
|
|
4195
4229
|
: undefined;
|
|
4196
4230
|
const author = body.author?.trim() || undefined;
|
|
4231
|
+
const parsedFilter = parseRestMetadataFilter(body.filter);
|
|
4232
|
+
if (!parsedFilter.ok) return parsedFilter.response;
|
|
4233
|
+
const filter = parsedFilter.filter;
|
|
4197
4234
|
let projectAffinity: ProjectAffinityScoringInput | undefined;
|
|
4198
4235
|
try {
|
|
4199
4236
|
projectAffinity = context
|
|
@@ -4221,6 +4258,7 @@ export async function handleSearch(
|
|
|
4221
4258
|
until: body.until,
|
|
4222
4259
|
categories,
|
|
4223
4260
|
author,
|
|
4261
|
+
filter,
|
|
4224
4262
|
projectAffinity,
|
|
4225
4263
|
contentTypeRules: context
|
|
4226
4264
|
? normalizeContentTypes(context.config.contentTypes ?? []).rules
|
|
@@ -4388,6 +4426,9 @@ export async function handleQuery(
|
|
|
4388
4426
|
? parseCommaSeparatedValues(body.exclude)
|
|
4389
4427
|
: undefined;
|
|
4390
4428
|
const author = body.author?.trim() || undefined;
|
|
4429
|
+
const parsedFilter = parseRestMetadataFilter(body.filter);
|
|
4430
|
+
if (!parsedFilter.ok) return parsedFilter.response;
|
|
4431
|
+
const filter = parsedFilter.filter;
|
|
4391
4432
|
let projectAffinity: ProjectAffinityScoringInput | undefined;
|
|
4392
4433
|
try {
|
|
4393
4434
|
projectAffinity = await resolveRemoteProjectAffinity(
|
|
@@ -4425,6 +4466,7 @@ export async function handleQuery(
|
|
|
4425
4466
|
until: body.until,
|
|
4426
4467
|
categories,
|
|
4427
4468
|
author,
|
|
4469
|
+
filter,
|
|
4428
4470
|
projectAffinity,
|
|
4429
4471
|
explain: body.explain,
|
|
4430
4472
|
};
|
|
@@ -4603,6 +4645,9 @@ export async function handleQueryDiagnose(
|
|
|
4603
4645
|
? parseCommaSeparatedValues(body.exclude)
|
|
4604
4646
|
: undefined;
|
|
4605
4647
|
const author = body.author?.trim() || undefined;
|
|
4648
|
+
const parsedFilter = parseRestMetadataFilter(body.filter);
|
|
4649
|
+
if (!parsedFilter.ok) return parsedFilter.response;
|
|
4650
|
+
const filter = parsedFilter.filter;
|
|
4606
4651
|
const contentTypeRules = normalizeContentTypes(
|
|
4607
4652
|
ctx.config.contentTypes ?? []
|
|
4608
4653
|
).rules;
|
|
@@ -4654,6 +4699,7 @@ export async function handleQueryDiagnose(
|
|
|
4654
4699
|
until: body.until,
|
|
4655
4700
|
categories,
|
|
4656
4701
|
author,
|
|
4702
|
+
filter,
|
|
4657
4703
|
projectAffinity,
|
|
4658
4704
|
contentTypeRules,
|
|
4659
4705
|
contentTypeRulesFingerprint:
|
|
@@ -4847,6 +4893,9 @@ export async function handleAsk(
|
|
|
4847
4893
|
? parseCommaSeparatedValues(body.exclude)
|
|
4848
4894
|
: undefined;
|
|
4849
4895
|
const author = body.author?.trim() || undefined;
|
|
4896
|
+
const parsedFilter = parseRestMetadataFilter(body.filter);
|
|
4897
|
+
if (!parsedFilter.ok) return parsedFilter.response;
|
|
4898
|
+
const filter = parsedFilter.filter;
|
|
4850
4899
|
let projectAffinity: ProjectAffinityScoringInput | undefined;
|
|
4851
4900
|
try {
|
|
4852
4901
|
projectAffinity = await resolveRemoteProjectAffinity(
|
|
@@ -4885,6 +4934,7 @@ export async function handleAsk(
|
|
|
4885
4934
|
until: body.until,
|
|
4886
4935
|
categories,
|
|
4887
4936
|
author,
|
|
4937
|
+
filter,
|
|
4888
4938
|
verify: body.verify,
|
|
4889
4939
|
contextBudgetTokens: body.contextBudgetTokens,
|
|
4890
4940
|
contextBudgetBytes: body.contextBudgetBytes,
|
package/src/serve/server.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { HttpGatewayOverrides } from "../mcp/http-security";
|
|
1
2
|
/**
|
|
2
3
|
* Bun.serve() web server for GNO web UI.
|
|
3
4
|
* Uses Bun's fullstack dev server with HTML imports.
|
|
@@ -5,8 +6,6 @@
|
|
|
5
6
|
*
|
|
6
7
|
* @module src/serve/server
|
|
7
8
|
*/
|
|
8
|
-
|
|
9
|
-
import type { HttpGatewayOverrides } from "../mcp/http-security";
|
|
10
9
|
import type { RequestPeerServer } from "./request-locality";
|
|
11
10
|
import type { ResidentRuntime } from "./resident-runtime";
|
|
12
11
|
import type { ContextHolder } from "./routes/api";
|
|
@@ -16,6 +15,7 @@ import {
|
|
|
16
15
|
resolveHttpGatewayConfig,
|
|
17
16
|
} from "../mcp/http-security";
|
|
18
17
|
import { startBackgroundRuntime } from "./background-runtime";
|
|
18
|
+
import { handleCompiledContext } from "./compiled-context";
|
|
19
19
|
import { handleContextBuild, handleContextVerify } from "./context-capsule";
|
|
20
20
|
import { DocumentEventBus } from "./doc-events";
|
|
21
21
|
import {
|
|
@@ -566,6 +566,7 @@ export async function startServer(
|
|
|
566
566
|
"/collections": spaPageRoute,
|
|
567
567
|
"/connectors": spaPageRoute,
|
|
568
568
|
"/traces": spaPageRoute,
|
|
569
|
+
"/context/compiled": spaPageRoute,
|
|
569
570
|
"/ask": spaPageRoute,
|
|
570
571
|
"/graph": spaPageRoute,
|
|
571
572
|
"/clipper/pair": spaPageRoute,
|
|
@@ -1164,6 +1165,46 @@ export async function startServer(
|
|
|
1164
1165
|
);
|
|
1165
1166
|
},
|
|
1166
1167
|
},
|
|
1168
|
+
"/api/context/compiled/preview": {
|
|
1169
|
+
POST: async (req: Request, server: RequestPeerServer) => {
|
|
1170
|
+
if (!isRequestAllowed(req, port))
|
|
1171
|
+
return withSecurityHeaders(forbiddenResponse(), isDev);
|
|
1172
|
+
return withSecurityHeaders(
|
|
1173
|
+
await handleResidentRead(
|
|
1174
|
+
runtime as ResidentRuntime,
|
|
1175
|
+
req,
|
|
1176
|
+
(signal) =>
|
|
1177
|
+
handleCompiledContext(
|
|
1178
|
+
ctxHolder.current,
|
|
1179
|
+
new Request(req, { signal }),
|
|
1180
|
+
{ requestIP: () => server.requestIP(req) },
|
|
1181
|
+
"preview"
|
|
1182
|
+
)
|
|
1183
|
+
),
|
|
1184
|
+
isDev
|
|
1185
|
+
);
|
|
1186
|
+
},
|
|
1187
|
+
},
|
|
1188
|
+
"/api/context/compiled/check": {
|
|
1189
|
+
POST: async (req: Request, server: RequestPeerServer) => {
|
|
1190
|
+
if (!isRequestAllowed(req, port))
|
|
1191
|
+
return withSecurityHeaders(forbiddenResponse(), isDev);
|
|
1192
|
+
return withSecurityHeaders(
|
|
1193
|
+
await handleResidentRead(
|
|
1194
|
+
runtime as ResidentRuntime,
|
|
1195
|
+
req,
|
|
1196
|
+
(signal) =>
|
|
1197
|
+
handleCompiledContext(
|
|
1198
|
+
ctxHolder.current,
|
|
1199
|
+
new Request(req, { signal }),
|
|
1200
|
+
{ requestIP: () => server.requestIP(req) },
|
|
1201
|
+
"check"
|
|
1202
|
+
)
|
|
1203
|
+
),
|
|
1204
|
+
isDev
|
|
1205
|
+
);
|
|
1206
|
+
},
|
|
1207
|
+
},
|
|
1167
1208
|
"/api/context/verify": {
|
|
1168
1209
|
POST: async (req: Request) => {
|
|
1169
1210
|
if (!isRequestAllowed(req, port)) {
|
|
@@ -79,12 +79,13 @@ const contentTypeFor = (path: string): string => {
|
|
|
79
79
|
return "application/octet-stream";
|
|
80
80
|
};
|
|
81
81
|
|
|
82
|
-
export const
|
|
83
|
-
|
|
84
|
-
|
|
82
|
+
export const isBunfsPath = (path: string): boolean => {
|
|
83
|
+
const normalized = path.replaceAll("\\", "/");
|
|
84
|
+
return normalized.includes("/$bunfs/") || normalized.startsWith("B:/~BUN/");
|
|
85
|
+
};
|
|
85
86
|
|
|
86
|
-
export const
|
|
87
|
-
|
|
87
|
+
export const isStandaloneExecutable = (): boolean =>
|
|
88
|
+
isBunfsPath(import.meta.path);
|
|
88
89
|
|
|
89
90
|
const rewriteProductionHtml = (html: string, jsEntryPath: string): string => {
|
|
90
91
|
const script = `<script type="module" src="/${basename(jsEntryPath)}"></script>`;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Migration } from "./runner";
|
|
2
|
+
|
|
3
|
+
export const migration: Migration = {
|
|
4
|
+
version: 30,
|
|
5
|
+
name: "typed_metadata",
|
|
6
|
+
up(db): void {
|
|
7
|
+
db.exec("ALTER TABLE documents ADD COLUMN typed_metadata TEXT");
|
|
8
|
+
db.exec("ALTER TABLE documents ADD COLUMN metadata_error TEXT");
|
|
9
|
+
},
|
|
10
|
+
down(): void {
|
|
11
|
+
// Derived columns are retained; SQLite rollback must not rewrite source rows.
|
|
12
|
+
},
|
|
13
|
+
};
|
|
@@ -43,6 +43,7 @@ import { migration as m026 } from "./026-file-refactor-recovery-journal";
|
|
|
43
43
|
import { migration as m027 } from "./027-memory-scopes";
|
|
44
44
|
import { migration as m028 } from "./028-vector-variants";
|
|
45
45
|
import { migration as m029 } from "./029-graph-reference-state";
|
|
46
|
+
import { migration as m030 } from "./030-typed-metadata";
|
|
46
47
|
|
|
47
48
|
/** All migrations in order */
|
|
48
49
|
export const migrations = [
|
|
@@ -75,4 +76,5 @@ export const migrations = [
|
|
|
75
76
|
m027,
|
|
76
77
|
m028,
|
|
77
78
|
m029,
|
|
79
|
+
m030,
|
|
78
80
|
];
|
|
@@ -141,6 +141,10 @@ import {
|
|
|
141
141
|
import { buildWikiBestMatchSubquery } from "../../core/graph-resolver";
|
|
142
142
|
import { buildContentPrefilterNeedles } from "../../core/link-relevance";
|
|
143
143
|
import { normalizeWikiName, stripWikiMdExt } from "../../core/links";
|
|
144
|
+
import {
|
|
145
|
+
TYPED_METADATA_INGEST_VERSION,
|
|
146
|
+
typedMetadataSchema,
|
|
147
|
+
} from "../../core/typed-metadata";
|
|
144
148
|
import {
|
|
145
149
|
parseActivationReceipt,
|
|
146
150
|
serializeActivationReceipt,
|
|
@@ -665,7 +669,20 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
665
669
|
async close(): Promise<void> {
|
|
666
670
|
this.fenceForShutdown();
|
|
667
671
|
if (this.db) {
|
|
668
|
-
|
|
672
|
+
try {
|
|
673
|
+
// Current Bun finalizes all statements and releases Windows handles now.
|
|
674
|
+
this.db.close(true);
|
|
675
|
+
} catch (cause) {
|
|
676
|
+
// Older Bun only finalizes cached statements. Its native SQLITE_BUSY
|
|
677
|
+
// error has no code field; retain deferred close for that exact case.
|
|
678
|
+
if (
|
|
679
|
+
!(cause instanceof Error) ||
|
|
680
|
+
cause.message !== "database is locked"
|
|
681
|
+
) {
|
|
682
|
+
throw cause;
|
|
683
|
+
}
|
|
684
|
+
this.db.close(false);
|
|
685
|
+
}
|
|
669
686
|
this.db = null;
|
|
670
687
|
}
|
|
671
688
|
}
|
|
@@ -1427,6 +1444,30 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1427
1444
|
// Documents
|
|
1428
1445
|
// ─────────────────────────────────────────────────────────────────────────
|
|
1429
1446
|
|
|
1447
|
+
async getTypedMetadataCoverage(
|
|
1448
|
+
options: import("../types").DocumentEligibilityOptions
|
|
1449
|
+
): Promise<StoreResult<{ pending: number; invalid: number }>> {
|
|
1450
|
+
try {
|
|
1451
|
+
const db = this.ensureOpen();
|
|
1452
|
+
const eligible = buildEligibleDocumentQuery(
|
|
1453
|
+
{ ...options, filter: undefined },
|
|
1454
|
+
db
|
|
1455
|
+
);
|
|
1456
|
+
const row = db
|
|
1457
|
+
.query<{ pending: number; invalid: number }, (string | number)[]>(`
|
|
1458
|
+
SELECT COALESCE(SUM(CASE WHEN d.ingest_version IS NULL OR d.ingest_version < ${TYPED_METADATA_INGEST_VERSION} OR (d.typed_metadata IS NULL AND d.metadata_error IS NULL) THEN 1 ELSE 0 END),0) AS pending,
|
|
1459
|
+
COALESCE(SUM(CASE WHEN d.metadata_error IS NOT NULL THEN 1 ELSE 0 END),0) AS invalid
|
|
1460
|
+
FROM documents d WHERE d.mirror_hash IS NOT NULL AND d.id IN (SELECT id FROM (${eligible.sql}))
|
|
1461
|
+
`)
|
|
1462
|
+
.get(...eligible.params);
|
|
1463
|
+
return ok(row ?? { pending: 0, invalid: 0 });
|
|
1464
|
+
} catch (cause) {
|
|
1465
|
+
return err("QUERY_FAILED", "Cannot determine typed metadata coverage", {
|
|
1466
|
+
cause,
|
|
1467
|
+
});
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1430
1471
|
async upsertDocument(
|
|
1431
1472
|
doc: DocumentInput
|
|
1432
1473
|
): Promise<StoreResult<UpsertDocumentResult>> {
|
|
@@ -1457,13 +1498,13 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1457
1498
|
collection, rel_path, source_hash, source_mime, source_ext,
|
|
1458
1499
|
source_size, source_mtime, source_ctime, docid, uri, title, mirror_hash,
|
|
1459
1500
|
converter_id, converter_version, language_hint, content_type, categories,
|
|
1460
|
-
content_type_source, author, frontmatter_date, date_fields,
|
|
1501
|
+
content_type_source, author, frontmatter_date, date_fields, typed_metadata, metadata_error,
|
|
1461
1502
|
record_key, record_source_path, record_source_locator, record_metadata, record_anchors,
|
|
1462
1503
|
record_adapter_fingerprint,
|
|
1463
1504
|
content_type_rules_fingerprint,
|
|
1464
1505
|
active, indexed_at, last_error_code, last_error_message, last_error_at,
|
|
1465
1506
|
ingest_version, updated_at
|
|
1466
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, datetime('now'), ?, ?, ?, ?, datetime('now'))
|
|
1507
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, datetime('now'), ?, ?, ?, ?, datetime('now'))
|
|
1467
1508
|
ON CONFLICT(collection, rel_path) DO UPDATE SET
|
|
1468
1509
|
source_hash = excluded.source_hash,
|
|
1469
1510
|
source_mime = excluded.source_mime,
|
|
@@ -1484,6 +1525,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1484
1525
|
author = excluded.author,
|
|
1485
1526
|
frontmatter_date = excluded.frontmatter_date,
|
|
1486
1527
|
date_fields = excluded.date_fields,
|
|
1528
|
+
typed_metadata = excluded.typed_metadata,
|
|
1529
|
+
metadata_error = excluded.metadata_error,
|
|
1487
1530
|
record_key = excluded.record_key,
|
|
1488
1531
|
record_source_path = excluded.record_source_path,
|
|
1489
1532
|
record_source_locator = excluded.record_source_locator,
|
|
@@ -1521,6 +1564,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1521
1564
|
doc.author ?? null,
|
|
1522
1565
|
doc.frontmatterDate ?? null,
|
|
1523
1566
|
doc.dateFields ? JSON.stringify(doc.dateFields) : null,
|
|
1567
|
+
doc.typedMetadata
|
|
1568
|
+
? JSON.stringify(typedMetadataSchema.parse(doc.typedMetadata))
|
|
1569
|
+
: null,
|
|
1570
|
+
doc.metadataError ?? null,
|
|
1524
1571
|
doc.recordKey ?? null,
|
|
1525
1572
|
doc.recordSourcePath ?? null,
|
|
1526
1573
|
doc.recordSourceLocator ?? null,
|
|
@@ -2109,6 +2156,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2109
2156
|
options: {
|
|
2110
2157
|
collection?: string;
|
|
2111
2158
|
activeOnly?: boolean;
|
|
2159
|
+
eligibility?: import("../types").DocumentEligibilityOptions;
|
|
2112
2160
|
} = {}
|
|
2113
2161
|
): Promise<StoreResult<DocumentRow[]>> {
|
|
2114
2162
|
try {
|
|
@@ -2131,7 +2179,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2131
2179
|
const batch = uniqueDocids.slice(i, i + SQL_PARAM_LIMIT);
|
|
2132
2180
|
const placeholders = batch.map(() => "?").join(",");
|
|
2133
2181
|
const clauses = [`docid IN (${placeholders})`];
|
|
2134
|
-
const params: string[] = [...batch];
|
|
2182
|
+
const params: (string | number)[] = [...batch];
|
|
2135
2183
|
|
|
2136
2184
|
if (options.activeOnly ?? true) {
|
|
2137
2185
|
clauses.push("active = 1");
|
|
@@ -2141,8 +2189,15 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2141
2189
|
params.push(options.collection);
|
|
2142
2190
|
}
|
|
2143
2191
|
|
|
2192
|
+
if (options.eligibility) {
|
|
2193
|
+
const eligible = buildEligibleDocumentQuery(options.eligibility, db);
|
|
2194
|
+
clauses.push(`id IN (SELECT id FROM (${eligible.sql}))`);
|
|
2195
|
+
params.push(...eligible.params);
|
|
2196
|
+
}
|
|
2144
2197
|
const sql = `SELECT * FROM documents WHERE ${clauses.join(" AND ")} ORDER BY id`;
|
|
2145
|
-
rows.push(
|
|
2198
|
+
rows.push(
|
|
2199
|
+
...db.query<DbDocumentRow, (string | number)[]>(sql).all(...params)
|
|
2200
|
+
);
|
|
2146
2201
|
}
|
|
2147
2202
|
|
|
2148
2203
|
return ok(rows.map(mapDocumentRow));
|
|
@@ -5960,7 +6015,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
5960
6015
|
const recentErrors = recentErrorsRow?.count ?? 0;
|
|
5961
6016
|
const healthy = recentErrors === 0;
|
|
5962
6017
|
|
|
6018
|
+
const metadataCoverage = await this.getTypedMetadataCoverage({});
|
|
6019
|
+
if (!metadataCoverage.ok) return metadataCoverage;
|
|
5963
6020
|
return ok({
|
|
6021
|
+
typedMetadata: metadataCoverage.value,
|
|
5964
6022
|
version,
|
|
5965
6023
|
indexName,
|
|
5966
6024
|
configPath: this.configPath,
|
|
@@ -6286,6 +6344,8 @@ interface DbDocumentRow {
|
|
|
6286
6344
|
author: string | null;
|
|
6287
6345
|
frontmatter_date: string | null;
|
|
6288
6346
|
date_fields: string | null;
|
|
6347
|
+
typed_metadata: string | null;
|
|
6348
|
+
metadata_error: string | null;
|
|
6289
6349
|
record_key: string | null;
|
|
6290
6350
|
record_source_path: string | null;
|
|
6291
6351
|
record_source_locator: string | null;
|
|
@@ -6454,6 +6514,10 @@ function mapDocumentRow(row: DbDocumentRow): DocumentRow {
|
|
|
6454
6514
|
author: row.author,
|
|
6455
6515
|
frontmatterDate: row.frontmatter_date,
|
|
6456
6516
|
dateFields,
|
|
6517
|
+
typedMetadata: row.typed_metadata
|
|
6518
|
+
? typedMetadataSchema.parse(JSON.parse(row.typed_metadata))
|
|
6519
|
+
: null,
|
|
6520
|
+
metadataError: row.metadata_error,
|
|
6457
6521
|
recordKey: row.record_key,
|
|
6458
6522
|
recordSourcePath: row.record_source_path,
|
|
6459
6523
|
recordSourceLocator: row.record_source_locator,
|
|
@@ -2,7 +2,9 @@ import type { Database } from "bun:sqlite";
|
|
|
2
2
|
|
|
3
3
|
import type { DocumentEligibilityOptions } from "../types";
|
|
4
4
|
|
|
5
|
+
import { TYPED_METADATA_INGEST_VERSION } from "../../core/typed-metadata";
|
|
5
6
|
import { matchesExcludedText } from "../../pipeline/exclude";
|
|
7
|
+
import { compileMetadataPredicate } from "./metadata-predicate";
|
|
6
8
|
|
|
7
9
|
/** Unbounded owner selection; compose with FTS rowids or chunk mirror ownership
|
|
8
10
|
* before ranking/LIMIT. EXISTS keeps multiple tags/scopes from duplicating owners.
|
|
@@ -14,6 +16,16 @@ export function buildEligibleDocumentQuery(
|
|
|
14
16
|
// Build tag filter conditions using EXISTS subqueries
|
|
15
17
|
const conditions: string[] = ["d.active = 1"];
|
|
16
18
|
const params: (string | number)[] = [];
|
|
19
|
+
if (options.filter) {
|
|
20
|
+
const predicate = compileMetadataPredicate(options.filter);
|
|
21
|
+
conditions.push(
|
|
22
|
+
"d.ingest_version >= ?",
|
|
23
|
+
"d.metadata_error IS NULL",
|
|
24
|
+
"d.typed_metadata IS NOT NULL",
|
|
25
|
+
predicate.sql
|
|
26
|
+
);
|
|
27
|
+
params.push(TYPED_METADATA_INGEST_VERSION, ...predicate.params);
|
|
28
|
+
}
|
|
17
29
|
if (options.chunkLanguage) {
|
|
18
30
|
conditions.push(
|
|
19
31
|
"EXISTS (SELECT 1 FROM content_chunks lc WHERE lc.mirror_hash = d.mirror_hash AND lc.language = ?)"
|