@gmickel/gno 1.18.0 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -7
- package/assets/skill/SKILL.md +54 -12
- package/assets/skill/mcp-reference.md +7 -2
- package/assets/skill/recipes/citation-and-provenance.md +32 -9
- package/package.json +2 -1
- package/spec/AGENTS.md +83 -0
- package/spec/CLAUDE.md +83 -0
- package/spec/bench-fixture.schema.json +137 -0
- package/spec/cli.md +2919 -0
- package/spec/db/schema.sql +442 -0
- package/spec/evals-agentic.md +592 -0
- package/spec/evals.md +1106 -0
- package/spec/mcp.md +2279 -0
- package/spec/output-schemas/activation-verification.schema.json +515 -0
- package/spec/output-schemas/ask.schema.json +564 -0
- package/spec/output-schemas/backlinks.schema.json +131 -0
- package/spec/output-schemas/bench-result.schema.json +120 -0
- package/spec/output-schemas/capture-receipt.schema.json +143 -0
- package/spec/output-schemas/claim-verification.schema.json +291 -0
- package/spec/output-schemas/collection-list.schema.json +45 -0
- package/spec/output-schemas/context-capsule-v1.schema.json +726 -0
- package/spec/output-schemas/context-capsule-verification.schema.json +1338 -0
- package/spec/output-schemas/context-list.schema.json +21 -0
- package/spec/output-schemas/doctor.schema.json +313 -0
- package/spec/output-schemas/error.schema.json +30 -0
- package/spec/output-schemas/expansion.schema.json +37 -0
- package/spec/output-schemas/get.schema.json +140 -0
- package/spec/output-schemas/graph-query.schema.json +99 -0
- package/spec/output-schemas/graph.schema.json +371 -0
- package/spec/output-schemas/links-list.schema.json +186 -0
- package/spec/output-schemas/mcp-add-collection-result.schema.json +23 -0
- package/spec/output-schemas/mcp-capture-result.schema.json +152 -0
- package/spec/output-schemas/mcp-http-error.schema.json +30 -0
- package/spec/output-schemas/mcp-job-list.schema.json +58 -0
- package/spec/output-schemas/mcp-job-status.schema.json +224 -0
- package/spec/output-schemas/mcp-remove-result.schema.json +39 -0
- package/spec/output-schemas/mcp-sync-result.schema.json +41 -0
- package/spec/output-schemas/mcp-tag-result.schema.json +33 -0
- package/spec/output-schemas/models-list.schema.json +93 -0
- package/spec/output-schemas/multi-get.schema.json +103 -0
- package/spec/output-schemas/process-status.schema.json +119 -0
- package/spec/output-schemas/query-diagnose.schema.json +123 -0
- package/spec/output-schemas/resident-status.schema.json +154 -0
- package/spec/output-schemas/retrieval-trace-common.schema.json +492 -0
- package/spec/output-schemas/retrieval-trace-delete.schema.json +16 -0
- package/spec/output-schemas/retrieval-trace-export.schema.json +61 -0
- package/spec/output-schemas/retrieval-trace-filters.schema.json +139 -0
- package/spec/output-schemas/retrieval-trace-judgment.schema.json +15 -0
- package/spec/output-schemas/retrieval-trace-list.schema.json +18 -0
- package/spec/output-schemas/retrieval-trace-payloads.schema.json +178 -0
- package/spec/output-schemas/retrieval-trace-purge.schema.json +31 -0
- package/spec/output-schemas/retrieval-trace-qrels.schema.json +303 -0
- package/spec/output-schemas/retrieval-trace-replay.schema.json +286 -0
- package/spec/output-schemas/retrieval-trace-show.schema.json +69 -0
- package/spec/output-schemas/retrieval-trace-summary.schema.json +65 -0
- package/spec/output-schemas/search-result.schema.json +154 -0
- package/spec/output-schemas/search-results.schema.json +338 -0
- package/spec/output-schemas/similar.schema.json +84 -0
- package/spec/output-schemas/status.schema.json +676 -0
- package/spec/output-schemas/tags-list.schema.json +48 -0
- package/src/app/context-runtime-contract.ts +10 -5
- package/src/app/context-runtime-input.ts +29 -1
- package/src/app/context-runtime-types.ts +7 -0
- package/src/app/context-runtime.ts +20 -2
- package/src/app/context-surface.ts +4 -0
- package/src/app/verified-ask.ts +291 -0
- package/src/cli/commands/ask-format.ts +255 -0
- package/src/cli/commands/ask.ts +144 -183
- package/src/cli/commands/context-build.ts +56 -9
- package/src/cli/commands/get.ts +64 -3
- package/src/cli/commands/query.ts +62 -23
- package/src/cli/commands/replay.ts +140 -0
- package/src/cli/commands/search.ts +48 -3
- package/src/cli/commands/shared.ts +3 -1
- package/src/cli/commands/trace.ts +200 -0
- package/src/cli/commands/vsearch.ts +75 -53
- package/src/cli/program.ts +287 -1
- package/src/config/index.ts +9 -0
- package/src/config/retrieval-traces.ts +56 -0
- package/src/config/types.ts +4 -0
- package/src/core/context-budget.ts +6 -0
- package/src/core/context-capsule-retrieval-schema.ts +4 -0
- package/src/core/context-capsule-schema.ts +17 -0
- package/src/core/context-capsule-validation.ts +3 -2
- package/src/core/context-capsule.ts +18 -0
- package/src/core/context-compiler.ts +44 -25
- package/src/core/context-evidence.ts +6 -0
- package/src/core/retrieval-qrels.ts +405 -0
- package/src/core/retrieval-replay-candidate.ts +368 -0
- package/src/core/retrieval-replay-types.ts +109 -0
- package/src/core/retrieval-replay-validation.ts +89 -0
- package/src/core/retrieval-replay.ts +441 -0
- package/src/core/retrieval-trace-evidence-origin.ts +178 -0
- package/src/core/retrieval-trace-export.ts +113 -0
- package/src/core/retrieval-trace-filter-normalization.ts +27 -0
- package/src/core/retrieval-trace-filters.ts +19 -0
- package/src/core/retrieval-trace-management-helpers.ts +247 -0
- package/src/core/retrieval-trace-management-types.ts +132 -0
- package/src/core/retrieval-trace-management.ts +422 -0
- package/src/core/retrieval-trace-request.ts +141 -0
- package/src/core/retrieval-trace-session.ts +507 -0
- package/src/core/retrieval-trace.ts +472 -0
- package/src/llm/errors.ts +10 -1
- package/src/llm/httpGeneration.ts +11 -1
- package/src/llm/nodeLlamaCpp/generation.ts +54 -10
- package/src/llm/types.ts +6 -0
- package/src/mcp/tools/ask.ts +228 -0
- package/src/mcp/tools/context.ts +87 -15
- package/src/mcp/tools/get.ts +35 -1
- package/src/mcp/tools/index.ts +83 -0
- package/src/mcp/tools/query.ts +95 -64
- package/src/mcp/tools/search.ts +36 -13
- package/src/mcp/tools/trace.ts +143 -0
- package/src/mcp/tools/vsearch.ts +71 -38
- package/src/pipeline/answer.ts +167 -26
- package/src/pipeline/claim-verification-schema.ts +235 -0
- package/src/pipeline/claim-verification.ts +487 -0
- package/src/pipeline/claim-verifier.ts +474 -0
- package/src/pipeline/graph-retrieval.ts +15 -1
- package/src/pipeline/hybrid.ts +151 -43
- package/src/pipeline/search.ts +36 -3
- package/src/pipeline/trace-metadata.ts +47 -0
- package/src/pipeline/types.ts +68 -0
- package/src/pipeline/vsearch.ts +101 -38
- package/src/sdk/client.ts +415 -73
- package/src/sdk/documents.ts +48 -1
- package/src/sdk/index.ts +17 -0
- package/src/sdk/types.ts +28 -0
- package/src/serve/context-capsule.ts +67 -8
- package/src/serve/public/app.tsx +12 -1
- package/src/serve/public/components/AskVerificationPanel.tsx +189 -0
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/lib/workspace-tabs.ts +2 -0
- package/src/serve/public/pages/Ask.tsx +42 -4
- package/src/serve/public/pages/Dashboard.tsx +10 -0
- package/src/serve/public/pages/TraceHistory.tsx +478 -0
- package/src/serve/public/pages/trace-history-detail.tsx +224 -0
- package/src/serve/retrieval-trace.ts +28 -0
- package/src/serve/routes/api.ts +508 -68
- package/src/serve/routes/traces.ts +156 -0
- package/src/serve/server.ts +87 -2
- package/src/store/index.ts +31 -0
- package/src/store/migrations/014-retrieval-traces.ts +303 -0
- package/src/store/migrations/index.ts +2 -0
- package/src/store/retrieval-trace-codec.ts +384 -0
- package/src/store/sqlite/adapter.ts +153 -1
- package/src/store/sqlite/retrieval-trace-management-store.ts +341 -0
- package/src/store/sqlite/retrieval-trace-retention.ts +349 -0
- package/src/store/sqlite/retrieval-trace-rows.ts +267 -0
- package/src/store/sqlite/retrieval-trace-store.ts +515 -0
- package/src/store/types.ts +297 -0
- package/src/store/vector/sqlite-vec.ts +76 -1
- package/src/store/vector/types.ts +1 -1
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/** Loopback REST handlers for private retrieval trace management. */
|
|
2
|
+
|
|
3
|
+
import type {
|
|
4
|
+
RetrievalTraceExportRequest,
|
|
5
|
+
RetrievalTraceLabelRequest,
|
|
6
|
+
} from "../../core/retrieval-trace-management";
|
|
7
|
+
import type { StoreError, StorePort, StoreResult } from "../../store/types";
|
|
8
|
+
|
|
9
|
+
import { RetrievalTraceManagementService } from "../../core/retrieval-trace-management";
|
|
10
|
+
|
|
11
|
+
const errorStatus = (error: StoreError): number => {
|
|
12
|
+
if (error.code === "NOT_FOUND") return 404;
|
|
13
|
+
if (error.code === "CONSTRAINT_VIOLATION") return 409;
|
|
14
|
+
if (error.code === "INVALID_INPUT") return 400;
|
|
15
|
+
return 500;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
const response = <T>(result: StoreResult<T>): Response => {
|
|
19
|
+
if (result.ok) return Response.json(result.value);
|
|
20
|
+
return Response.json(
|
|
21
|
+
{
|
|
22
|
+
error: {
|
|
23
|
+
code: result.error.code,
|
|
24
|
+
message: result.error.message,
|
|
25
|
+
},
|
|
26
|
+
},
|
|
27
|
+
{ status: errorStatus(result.error) }
|
|
28
|
+
);
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
const parseJsonObject = async (
|
|
32
|
+
request: Request
|
|
33
|
+
): Promise<Record<string, unknown> | Response> => {
|
|
34
|
+
try {
|
|
35
|
+
const value = (await request.json()) as unknown;
|
|
36
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
37
|
+
return Response.json(
|
|
38
|
+
{
|
|
39
|
+
error: {
|
|
40
|
+
code: "INVALID_INPUT",
|
|
41
|
+
message: "Request body must be a JSON object",
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
{ status: 400 }
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
return value as Record<string, unknown>;
|
|
48
|
+
} catch {
|
|
49
|
+
return Response.json(
|
|
50
|
+
{
|
|
51
|
+
error: {
|
|
52
|
+
code: "INVALID_INPUT",
|
|
53
|
+
message: "Request body must be valid JSON",
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
{ status: 400 }
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
const parseOptionalInteger = (
|
|
62
|
+
value: string | null
|
|
63
|
+
): number | undefined | null => {
|
|
64
|
+
if (value === null) return undefined;
|
|
65
|
+
if (!/^\d+$/.test(value)) return null;
|
|
66
|
+
const parsed = Number(value);
|
|
67
|
+
return Number.isSafeInteger(parsed) ? parsed : null;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
export const handleTraceList = async (
|
|
71
|
+
store: StorePort,
|
|
72
|
+
request: Request
|
|
73
|
+
): Promise<Response> => {
|
|
74
|
+
const url = new URL(request.url);
|
|
75
|
+
const limit = parseOptionalInteger(url.searchParams.get("limit"));
|
|
76
|
+
if (limit === null) {
|
|
77
|
+
return Response.json(
|
|
78
|
+
{
|
|
79
|
+
error: {
|
|
80
|
+
code: "INVALID_INPUT",
|
|
81
|
+
message: "limit must be a positive integer",
|
|
82
|
+
},
|
|
83
|
+
},
|
|
84
|
+
{ status: 400 }
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
return response(
|
|
88
|
+
await new RetrievalTraceManagementService(store).list({
|
|
89
|
+
limit,
|
|
90
|
+
cursor: url.searchParams.get("cursor") ?? undefined,
|
|
91
|
+
})
|
|
92
|
+
);
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
export const handleTraceShow = async (
|
|
96
|
+
store: StorePort,
|
|
97
|
+
traceId: string,
|
|
98
|
+
request: Request
|
|
99
|
+
): Promise<Response> => {
|
|
100
|
+
const detailLimit = parseOptionalInteger(
|
|
101
|
+
new URL(request.url).searchParams.get("detailLimit")
|
|
102
|
+
);
|
|
103
|
+
if (detailLimit === null) {
|
|
104
|
+
return Response.json(
|
|
105
|
+
{
|
|
106
|
+
error: {
|
|
107
|
+
code: "INVALID_INPUT",
|
|
108
|
+
message: "detailLimit must be a positive integer",
|
|
109
|
+
},
|
|
110
|
+
},
|
|
111
|
+
{ status: 400 }
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
return response(
|
|
115
|
+
await new RetrievalTraceManagementService(store).show(traceId, {
|
|
116
|
+
detailLimit,
|
|
117
|
+
})
|
|
118
|
+
);
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
export const handleTraceLabel = async (
|
|
122
|
+
store: StorePort,
|
|
123
|
+
traceId: string,
|
|
124
|
+
request: Request
|
|
125
|
+
): Promise<Response> => {
|
|
126
|
+
const body = await parseJsonObject(request);
|
|
127
|
+
if (body instanceof Response) return body;
|
|
128
|
+
return response(
|
|
129
|
+
await new RetrievalTraceManagementService(store).label({
|
|
130
|
+
...body,
|
|
131
|
+
traceId,
|
|
132
|
+
} as unknown as RetrievalTraceLabelRequest)
|
|
133
|
+
);
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
export const handleTraceExport = async (
|
|
137
|
+
store: StorePort,
|
|
138
|
+
request: Request
|
|
139
|
+
): Promise<Response> => {
|
|
140
|
+
const body = await parseJsonObject(request);
|
|
141
|
+
if (body instanceof Response) return body;
|
|
142
|
+
return response(
|
|
143
|
+
await new RetrievalTraceManagementService(store).export(
|
|
144
|
+
body as unknown as RetrievalTraceExportRequest
|
|
145
|
+
)
|
|
146
|
+
);
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
export const handleTraceDelete = async (
|
|
150
|
+
store: StorePort,
|
|
151
|
+
traceId: string
|
|
152
|
+
): Promise<Response> =>
|
|
153
|
+
response(await new RetrievalTraceManagementService(store).delete(traceId));
|
|
154
|
+
|
|
155
|
+
export const handleTracePurge = async (store: StorePort): Promise<Response> =>
|
|
156
|
+
response(await new RetrievalTraceManagementService(store).purge());
|
package/src/serve/server.ts
CHANGED
|
@@ -76,6 +76,14 @@ import {
|
|
|
76
76
|
handleDocSimilar,
|
|
77
77
|
} from "./routes/links";
|
|
78
78
|
import { createMcpHttpGateway } from "./routes/mcp";
|
|
79
|
+
import {
|
|
80
|
+
handleTraceDelete,
|
|
81
|
+
handleTraceExport,
|
|
82
|
+
handleTraceLabel,
|
|
83
|
+
handleTraceList,
|
|
84
|
+
handleTracePurge,
|
|
85
|
+
handleTraceShow,
|
|
86
|
+
} from "./routes/traces";
|
|
79
87
|
import { forbiddenResponse, isRequestAllowed } from "./security";
|
|
80
88
|
|
|
81
89
|
export interface ServeOptions extends HttpGatewayOverrides {
|
|
@@ -246,6 +254,7 @@ export async function startServer(
|
|
|
246
254
|
"/edit": homepage,
|
|
247
255
|
"/collections": homepage,
|
|
248
256
|
"/connectors": homepage,
|
|
257
|
+
"/traces": homepage,
|
|
249
258
|
"/ask": homepage,
|
|
250
259
|
"/graph": homepage,
|
|
251
260
|
|
|
@@ -265,6 +274,82 @@ export async function startServer(
|
|
|
265
274
|
isDev
|
|
266
275
|
),
|
|
267
276
|
},
|
|
277
|
+
"/api/traces/export": {
|
|
278
|
+
POST: async (req: Request) => {
|
|
279
|
+
if (!isRequestAllowed(req, port)) {
|
|
280
|
+
return withSecurityHeaders(forbiddenResponse(), isDev);
|
|
281
|
+
}
|
|
282
|
+
return withSecurityHeaders(
|
|
283
|
+
await handleResidentRead(runtime as ResidentRuntime, req, () =>
|
|
284
|
+
handleTraceExport(store, req)
|
|
285
|
+
),
|
|
286
|
+
isDev
|
|
287
|
+
);
|
|
288
|
+
},
|
|
289
|
+
},
|
|
290
|
+
"/api/traces/:traceId/judgments": {
|
|
291
|
+
POST: async (req: Request) => {
|
|
292
|
+
if (!isRequestAllowed(req, port)) {
|
|
293
|
+
return withSecurityHeaders(forbiddenResponse(), isDev);
|
|
294
|
+
}
|
|
295
|
+
const traceId = decodeURIComponent(
|
|
296
|
+
new URL(req.url).pathname.split("/")[3] ?? ""
|
|
297
|
+
);
|
|
298
|
+
return withSecurityHeaders(
|
|
299
|
+
await handleResidentRead(runtime as ResidentRuntime, req, () =>
|
|
300
|
+
handleTraceLabel(store, traceId, req)
|
|
301
|
+
),
|
|
302
|
+
isDev
|
|
303
|
+
);
|
|
304
|
+
},
|
|
305
|
+
},
|
|
306
|
+
"/api/traces/:traceId": {
|
|
307
|
+
GET: async (req: Request) => {
|
|
308
|
+
const traceId = decodeURIComponent(
|
|
309
|
+
new URL(req.url).pathname.split("/")[3] ?? ""
|
|
310
|
+
);
|
|
311
|
+
return withSecurityHeaders(
|
|
312
|
+
await handleResidentRead(runtime as ResidentRuntime, req, () =>
|
|
313
|
+
handleTraceShow(store, traceId, req)
|
|
314
|
+
),
|
|
315
|
+
isDev
|
|
316
|
+
);
|
|
317
|
+
},
|
|
318
|
+
DELETE: async (req: Request) => {
|
|
319
|
+
if (!isRequestAllowed(req, port)) {
|
|
320
|
+
return withSecurityHeaders(forbiddenResponse(), isDev);
|
|
321
|
+
}
|
|
322
|
+
const traceId = decodeURIComponent(
|
|
323
|
+
new URL(req.url).pathname.split("/")[3] ?? ""
|
|
324
|
+
);
|
|
325
|
+
return withSecurityHeaders(
|
|
326
|
+
await handleResidentRead(runtime as ResidentRuntime, req, () =>
|
|
327
|
+
handleTraceDelete(store, traceId)
|
|
328
|
+
),
|
|
329
|
+
isDev
|
|
330
|
+
);
|
|
331
|
+
},
|
|
332
|
+
},
|
|
333
|
+
"/api/traces": {
|
|
334
|
+
GET: async (req: Request) =>
|
|
335
|
+
withSecurityHeaders(
|
|
336
|
+
await handleResidentRead(runtime as ResidentRuntime, req, () =>
|
|
337
|
+
handleTraceList(store, req)
|
|
338
|
+
),
|
|
339
|
+
isDev
|
|
340
|
+
),
|
|
341
|
+
DELETE: async (req: Request) => {
|
|
342
|
+
if (!isRequestAllowed(req, port)) {
|
|
343
|
+
return withSecurityHeaders(forbiddenResponse(), isDev);
|
|
344
|
+
}
|
|
345
|
+
return withSecurityHeaders(
|
|
346
|
+
await handleResidentRead(runtime as ResidentRuntime, req, () =>
|
|
347
|
+
handleTracePurge(store)
|
|
348
|
+
),
|
|
349
|
+
isDev
|
|
350
|
+
);
|
|
351
|
+
},
|
|
352
|
+
},
|
|
268
353
|
"/api/resident/status": {
|
|
269
354
|
GET: () =>
|
|
270
355
|
withSecurityHeaders(
|
|
@@ -581,7 +666,7 @@ export async function startServer(
|
|
|
581
666
|
const url = new URL(req.url);
|
|
582
667
|
return withSecurityHeaders(
|
|
583
668
|
await handleResidentRead(runtime as ResidentRuntime, req, () =>
|
|
584
|
-
handleDoc(store, ctxHolder.config, url)
|
|
669
|
+
handleDoc(store, ctxHolder.config, url, req)
|
|
585
670
|
),
|
|
586
671
|
isDev
|
|
587
672
|
);
|
|
@@ -624,7 +709,7 @@ export async function startServer(
|
|
|
624
709
|
}
|
|
625
710
|
return withSecurityHeaders(
|
|
626
711
|
await handleResidentRead(runtime as ResidentRuntime, req, () =>
|
|
627
|
-
handleSearch(
|
|
712
|
+
handleSearch(ctxHolder.current, req)
|
|
628
713
|
),
|
|
629
714
|
isDev
|
|
630
715
|
);
|
package/src/store/index.ts
CHANGED
|
@@ -41,6 +41,37 @@ export type {
|
|
|
41
41
|
IngestErrorInput,
|
|
42
42
|
IngestErrorRow,
|
|
43
43
|
MigrationResult,
|
|
44
|
+
RetrievalTraceAppendResult,
|
|
45
|
+
RetrievalTraceBundle,
|
|
46
|
+
RetrievalTraceBoundedBundle,
|
|
47
|
+
RetrievalTraceBundleTotals,
|
|
48
|
+
RetrievalTraceCursor,
|
|
49
|
+
RetrievalTraceDeleteCounts,
|
|
50
|
+
RetrievalTraceEventInput,
|
|
51
|
+
RetrievalTraceEventKind,
|
|
52
|
+
RetrievalTraceEventRow,
|
|
53
|
+
RetrievalTraceExportInput,
|
|
54
|
+
RetrievalTraceExportManifestInput,
|
|
55
|
+
RetrievalTraceExportManifestRow,
|
|
56
|
+
RetrievalTraceExportFormat,
|
|
57
|
+
RetrievalTraceExportRow,
|
|
58
|
+
RetrievalTraceFingerprints,
|
|
59
|
+
RetrievalTraceInput,
|
|
60
|
+
RetrievalTraceJudgmentInput,
|
|
61
|
+
RetrievalTraceJudgmentLabel,
|
|
62
|
+
RetrievalTraceJudgmentRow,
|
|
63
|
+
RetrievalTraceJudgmentTargetKind,
|
|
64
|
+
RetrievalTracePhysicalCleanupStatus,
|
|
65
|
+
RetrievalTracePurgeResult,
|
|
66
|
+
RetrievalTraceRedactionMode,
|
|
67
|
+
RetrievalTraceRetentionPolicy,
|
|
68
|
+
RetrievalTraceRetentionResult,
|
|
69
|
+
RetrievalTraceRow,
|
|
70
|
+
RetrievalTraceRunInput,
|
|
71
|
+
RetrievalTraceRunKind,
|
|
72
|
+
RetrievalTraceRunRow,
|
|
73
|
+
RetrievalTraceStatus,
|
|
74
|
+
RetrievalTraceTerminalStatus,
|
|
44
75
|
StoreError,
|
|
45
76
|
StoreErrorCode,
|
|
46
77
|
StorePort,
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration: private, bounded retrieval trace receipts.
|
|
3
|
+
*
|
|
4
|
+
* Trace rows own all subordinate runs, events, judgments, and export receipts.
|
|
5
|
+
* Foreign-key cascades make per-trace deletion and full purge atomic.
|
|
6
|
+
*
|
|
7
|
+
* @module src/store/migrations/014-retrieval-traces
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { Database } from "bun:sqlite";
|
|
11
|
+
|
|
12
|
+
import type { Migration } from "./runner";
|
|
13
|
+
|
|
14
|
+
export const migration: Migration = {
|
|
15
|
+
version: 14,
|
|
16
|
+
name: "retrieval_traces",
|
|
17
|
+
|
|
18
|
+
up(db: Database): void {
|
|
19
|
+
db.exec(`
|
|
20
|
+
CREATE TABLE retrieval_traces (
|
|
21
|
+
trace_id TEXT PRIMARY KEY,
|
|
22
|
+
schema_version TEXT NOT NULL CHECK (schema_version = '1.0'),
|
|
23
|
+
redaction_mode TEXT NOT NULL
|
|
24
|
+
CHECK (redaction_mode IN ('metadata', 'replay')),
|
|
25
|
+
replay_capable INTEGER NOT NULL
|
|
26
|
+
CHECK (replay_capable IN (0, 1)),
|
|
27
|
+
query_text TEXT,
|
|
28
|
+
query_digest TEXT,
|
|
29
|
+
query_shape_json TEXT NOT NULL,
|
|
30
|
+
goal_text TEXT,
|
|
31
|
+
goal_digest TEXT,
|
|
32
|
+
goal_shape_json TEXT NOT NULL,
|
|
33
|
+
filters_json TEXT NOT NULL,
|
|
34
|
+
pipeline_fingerprint TEXT NOT NULL
|
|
35
|
+
CHECK (length(pipeline_fingerprint) = 64 AND pipeline_fingerprint NOT GLOB '*[^0-9a-f]*'),
|
|
36
|
+
model_fingerprint TEXT NOT NULL
|
|
37
|
+
CHECK (length(model_fingerprint) = 64 AND model_fingerprint NOT GLOB '*[^0-9a-f]*'),
|
|
38
|
+
config_fingerprint TEXT NOT NULL
|
|
39
|
+
CHECK (length(config_fingerprint) = 64 AND config_fingerprint NOT GLOB '*[^0-9a-f]*'),
|
|
40
|
+
index_fingerprint TEXT NOT NULL
|
|
41
|
+
CHECK (length(index_fingerprint) = 64 AND index_fingerprint NOT GLOB '*[^0-9a-f]*'),
|
|
42
|
+
status TEXT NOT NULL
|
|
43
|
+
CHECK (status IN ('open', 'completed', 'partial', 'failed', 'cancelled')),
|
|
44
|
+
created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0),
|
|
45
|
+
updated_at_ms INTEGER NOT NULL CHECK (updated_at_ms >= created_at_ms),
|
|
46
|
+
expires_at_ms INTEGER NOT NULL CHECK (expires_at_ms > created_at_ms),
|
|
47
|
+
byte_size INTEGER NOT NULL CHECK (byte_size >= 0),
|
|
48
|
+
creation_digest TEXT NOT NULL
|
|
49
|
+
CHECK (length(creation_digest) = 64 AND creation_digest NOT GLOB '*[^0-9a-f]*'),
|
|
50
|
+
CHECK (length(CAST(query_shape_json AS BLOB)) <= 1024),
|
|
51
|
+
CHECK (length(CAST(goal_shape_json AS BLOB)) <= 1024),
|
|
52
|
+
CHECK (length(CAST(filters_json AS BLOB)) <= 16384),
|
|
53
|
+
CHECK (query_text IS NULL OR length(CAST(query_text AS BLOB)) <= 8192),
|
|
54
|
+
CHECK (goal_text IS NULL OR length(CAST(goal_text AS BLOB)) <= 8192),
|
|
55
|
+
CHECK (length(trace_id) BETWEEN 1 AND 128),
|
|
56
|
+
CHECK (
|
|
57
|
+
byte_size =
|
|
58
|
+
length(CAST(COALESCE(query_text, '') AS BLOB))
|
|
59
|
+
+ length(CAST(query_shape_json AS BLOB))
|
|
60
|
+
+ length(CAST(COALESCE(goal_text, '') AS BLOB))
|
|
61
|
+
+ length(CAST(goal_shape_json AS BLOB))
|
|
62
|
+
+ length(CAST(filters_json AS BLOB))
|
|
63
|
+
),
|
|
64
|
+
CHECK (
|
|
65
|
+
(
|
|
66
|
+
redaction_mode = 'metadata'
|
|
67
|
+
AND replay_capable = 0
|
|
68
|
+
AND query_text IS NULL
|
|
69
|
+
AND query_digest IS NULL
|
|
70
|
+
AND goal_text IS NULL
|
|
71
|
+
AND goal_digest IS NULL
|
|
72
|
+
)
|
|
73
|
+
OR
|
|
74
|
+
(
|
|
75
|
+
redaction_mode = 'replay'
|
|
76
|
+
AND replay_capable = 1
|
|
77
|
+
AND query_text IS NOT NULL
|
|
78
|
+
AND length(query_digest) = 64
|
|
79
|
+
AND query_digest NOT GLOB '*[^0-9a-f]*'
|
|
80
|
+
AND (
|
|
81
|
+
(goal_text IS NULL AND goal_digest IS NULL)
|
|
82
|
+
OR
|
|
83
|
+
(
|
|
84
|
+
goal_text IS NOT NULL
|
|
85
|
+
AND length(goal_digest) = 64
|
|
86
|
+
AND goal_digest NOT GLOB '*[^0-9a-f]*'
|
|
87
|
+
)
|
|
88
|
+
)
|
|
89
|
+
)
|
|
90
|
+
)
|
|
91
|
+
);
|
|
92
|
+
|
|
93
|
+
CREATE INDEX idx_retrieval_traces_retention
|
|
94
|
+
ON retrieval_traces(expires_at_ms, created_at_ms, trace_id);
|
|
95
|
+
|
|
96
|
+
CREATE TABLE retrieval_trace_runs (
|
|
97
|
+
run_id TEXT PRIMARY KEY,
|
|
98
|
+
trace_id TEXT NOT NULL,
|
|
99
|
+
idempotency_key TEXT NOT NULL,
|
|
100
|
+
kind TEXT NOT NULL
|
|
101
|
+
CHECK (kind IN ('retrieval', 'context', 'get')),
|
|
102
|
+
payload_json TEXT NOT NULL,
|
|
103
|
+
payload_bytes INTEGER NOT NULL CHECK (payload_bytes >= 0),
|
|
104
|
+
canonical_digest TEXT NOT NULL
|
|
105
|
+
CHECK (length(canonical_digest) = 64 AND canonical_digest NOT GLOB '*[^0-9a-f]*'),
|
|
106
|
+
created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0),
|
|
107
|
+
CHECK (length(idempotency_key) BETWEEN 1 AND 256),
|
|
108
|
+
CHECK (length(run_id) BETWEEN 1 AND 128),
|
|
109
|
+
CHECK (length(trace_id) BETWEEN 1 AND 128),
|
|
110
|
+
CHECK (payload_bytes = length(CAST(payload_json AS BLOB))),
|
|
111
|
+
CHECK (payload_bytes <= 65536),
|
|
112
|
+
UNIQUE (trace_id, idempotency_key),
|
|
113
|
+
UNIQUE (trace_id, run_id),
|
|
114
|
+
FOREIGN KEY (trace_id) REFERENCES retrieval_traces(trace_id) ON DELETE CASCADE
|
|
115
|
+
);
|
|
116
|
+
|
|
117
|
+
CREATE INDEX idx_retrieval_trace_runs_trace
|
|
118
|
+
ON retrieval_trace_runs(trace_id, created_at_ms, run_id);
|
|
119
|
+
|
|
120
|
+
CREATE TABLE retrieval_trace_events (
|
|
121
|
+
event_id TEXT PRIMARY KEY,
|
|
122
|
+
trace_id TEXT NOT NULL,
|
|
123
|
+
run_id TEXT,
|
|
124
|
+
idempotency_key TEXT NOT NULL,
|
|
125
|
+
kind TEXT NOT NULL
|
|
126
|
+
CHECK (kind IN ('query', 'retrieval', 'context', 'get', 'open', 'cite', 'pin', 'capability', 'complete')),
|
|
127
|
+
payload_json TEXT NOT NULL,
|
|
128
|
+
payload_bytes INTEGER NOT NULL CHECK (payload_bytes >= 0),
|
|
129
|
+
canonical_digest TEXT NOT NULL
|
|
130
|
+
CHECK (length(canonical_digest) = 64 AND canonical_digest NOT GLOB '*[^0-9a-f]*'),
|
|
131
|
+
created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0),
|
|
132
|
+
CHECK (length(idempotency_key) BETWEEN 1 AND 256),
|
|
133
|
+
CHECK (length(event_id) BETWEEN 1 AND 128),
|
|
134
|
+
CHECK (run_id IS NULL OR length(run_id) BETWEEN 1 AND 128),
|
|
135
|
+
CHECK (length(trace_id) BETWEEN 1 AND 128),
|
|
136
|
+
CHECK (payload_bytes = length(CAST(payload_json AS BLOB))),
|
|
137
|
+
CHECK (payload_bytes <= 65536),
|
|
138
|
+
UNIQUE (trace_id, idempotency_key),
|
|
139
|
+
FOREIGN KEY (trace_id) REFERENCES retrieval_traces(trace_id) ON DELETE CASCADE,
|
|
140
|
+
FOREIGN KEY (trace_id, run_id)
|
|
141
|
+
REFERENCES retrieval_trace_runs(trace_id, run_id) ON DELETE CASCADE
|
|
142
|
+
);
|
|
143
|
+
|
|
144
|
+
CREATE INDEX idx_retrieval_trace_events_trace
|
|
145
|
+
ON retrieval_trace_events(trace_id, created_at_ms, event_id);
|
|
146
|
+
|
|
147
|
+
CREATE TABLE retrieval_trace_judgments (
|
|
148
|
+
judgment_id TEXT PRIMARY KEY,
|
|
149
|
+
trace_id TEXT NOT NULL,
|
|
150
|
+
run_id TEXT,
|
|
151
|
+
idempotency_key TEXT NOT NULL,
|
|
152
|
+
label TEXT NOT NULL
|
|
153
|
+
CHECK (label IN ('relevant', 'irrelevant', 'missing_expected')),
|
|
154
|
+
target_kind TEXT NOT NULL
|
|
155
|
+
CHECK (target_kind IN ('document', 'chunk', 'span', 'query')),
|
|
156
|
+
target_ref TEXT NOT NULL,
|
|
157
|
+
target_json TEXT NOT NULL,
|
|
158
|
+
target_bytes INTEGER NOT NULL CHECK (target_bytes >= 0),
|
|
159
|
+
canonical_digest TEXT NOT NULL
|
|
160
|
+
CHECK (length(canonical_digest) = 64 AND canonical_digest NOT GLOB '*[^0-9a-f]*'),
|
|
161
|
+
created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0),
|
|
162
|
+
CHECK (length(idempotency_key) BETWEEN 1 AND 256),
|
|
163
|
+
CHECK (length(judgment_id) BETWEEN 1 AND 128),
|
|
164
|
+
CHECK (run_id IS NULL OR length(run_id) BETWEEN 1 AND 128),
|
|
165
|
+
CHECK (length(trace_id) BETWEEN 1 AND 128),
|
|
166
|
+
CHECK (length(CAST(target_ref AS BLOB)) BETWEEN 1 AND 4096),
|
|
167
|
+
CHECK (target_bytes = length(CAST(target_json AS BLOB))),
|
|
168
|
+
CHECK (target_bytes <= 16384),
|
|
169
|
+
UNIQUE (trace_id, idempotency_key),
|
|
170
|
+
FOREIGN KEY (trace_id) REFERENCES retrieval_traces(trace_id) ON DELETE CASCADE,
|
|
171
|
+
FOREIGN KEY (trace_id, run_id)
|
|
172
|
+
REFERENCES retrieval_trace_runs(trace_id, run_id) ON DELETE CASCADE
|
|
173
|
+
);
|
|
174
|
+
|
|
175
|
+
CREATE INDEX idx_retrieval_trace_judgments_trace
|
|
176
|
+
ON retrieval_trace_judgments(trace_id, created_at_ms, judgment_id);
|
|
177
|
+
|
|
178
|
+
CREATE TABLE retrieval_trace_exports (
|
|
179
|
+
export_id TEXT PRIMARY KEY,
|
|
180
|
+
format TEXT NOT NULL CHECK (format IN ('agentic-receipt', 'qrels')),
|
|
181
|
+
artifact_hash TEXT NOT NULL
|
|
182
|
+
CHECK (length(artifact_hash) = 64 AND artifact_hash NOT GLOB '*[^0-9a-f]*'),
|
|
183
|
+
created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0),
|
|
184
|
+
CHECK (length(export_id) BETWEEN 1 AND 128),
|
|
185
|
+
UNIQUE (format, artifact_hash)
|
|
186
|
+
);
|
|
187
|
+
|
|
188
|
+
CREATE TABLE retrieval_trace_export_traces (
|
|
189
|
+
export_id TEXT NOT NULL,
|
|
190
|
+
trace_id TEXT NOT NULL,
|
|
191
|
+
PRIMARY KEY (export_id, trace_id),
|
|
192
|
+
FOREIGN KEY (export_id) REFERENCES retrieval_trace_exports(export_id) ON DELETE CASCADE,
|
|
193
|
+
FOREIGN KEY (trace_id) REFERENCES retrieval_traces(trace_id) ON DELETE CASCADE
|
|
194
|
+
);
|
|
195
|
+
|
|
196
|
+
CREATE INDEX idx_retrieval_trace_export_traces_trace
|
|
197
|
+
ON retrieval_trace_export_traces(trace_id, export_id);
|
|
198
|
+
|
|
199
|
+
CREATE TRIGGER delete_empty_retrieval_trace_export
|
|
200
|
+
AFTER DELETE ON retrieval_trace_export_traces
|
|
201
|
+
WHEN NOT EXISTS (
|
|
202
|
+
SELECT 1 FROM retrieval_trace_export_traces
|
|
203
|
+
WHERE export_id = OLD.export_id
|
|
204
|
+
)
|
|
205
|
+
BEGIN
|
|
206
|
+
DELETE FROM retrieval_trace_exports WHERE export_id = OLD.export_id;
|
|
207
|
+
END;
|
|
208
|
+
|
|
209
|
+
CREATE TRIGGER cap_retrieval_trace_runs
|
|
210
|
+
BEFORE INSERT ON retrieval_trace_runs
|
|
211
|
+
WHEN (
|
|
212
|
+
(SELECT COUNT(*) FROM retrieval_trace_runs WHERE trace_id = NEW.trace_id)
|
|
213
|
+
+ (SELECT COUNT(*) FROM retrieval_trace_events WHERE trace_id = NEW.trace_id)
|
|
214
|
+
+ (SELECT COUNT(*) FROM retrieval_trace_judgments WHERE trace_id = NEW.trace_id)
|
|
215
|
+
+ (SELECT COUNT(*) FROM retrieval_trace_export_traces WHERE trace_id = NEW.trace_id)
|
|
216
|
+
) >= 100000
|
|
217
|
+
AND NOT EXISTS (
|
|
218
|
+
SELECT 1 FROM retrieval_trace_runs existing
|
|
219
|
+
WHERE existing.run_id = NEW.run_id
|
|
220
|
+
OR (
|
|
221
|
+
existing.trace_id = NEW.trace_id
|
|
222
|
+
AND existing.idempotency_key = NEW.idempotency_key
|
|
223
|
+
)
|
|
224
|
+
)
|
|
225
|
+
BEGIN
|
|
226
|
+
SELECT RAISE(ABORT, 'retrieval trace record cap exceeded');
|
|
227
|
+
END;
|
|
228
|
+
|
|
229
|
+
CREATE TRIGGER cap_retrieval_trace_events
|
|
230
|
+
BEFORE INSERT ON retrieval_trace_events
|
|
231
|
+
WHEN (
|
|
232
|
+
(SELECT COUNT(*) FROM retrieval_trace_runs WHERE trace_id = NEW.trace_id)
|
|
233
|
+
+ (SELECT COUNT(*) FROM retrieval_trace_events WHERE trace_id = NEW.trace_id)
|
|
234
|
+
+ (SELECT COUNT(*) FROM retrieval_trace_judgments WHERE trace_id = NEW.trace_id)
|
|
235
|
+
+ (SELECT COUNT(*) FROM retrieval_trace_export_traces WHERE trace_id = NEW.trace_id)
|
|
236
|
+
) >= 100000
|
|
237
|
+
AND NOT EXISTS (
|
|
238
|
+
SELECT 1 FROM retrieval_trace_events existing
|
|
239
|
+
WHERE existing.event_id = NEW.event_id
|
|
240
|
+
OR (
|
|
241
|
+
existing.trace_id = NEW.trace_id
|
|
242
|
+
AND existing.idempotency_key = NEW.idempotency_key
|
|
243
|
+
)
|
|
244
|
+
)
|
|
245
|
+
BEGIN
|
|
246
|
+
SELECT RAISE(ABORT, 'retrieval trace record cap exceeded');
|
|
247
|
+
END;
|
|
248
|
+
|
|
249
|
+
CREATE TRIGGER cap_retrieval_trace_judgments
|
|
250
|
+
BEFORE INSERT ON retrieval_trace_judgments
|
|
251
|
+
WHEN (
|
|
252
|
+
(SELECT COUNT(*) FROM retrieval_trace_runs WHERE trace_id = NEW.trace_id)
|
|
253
|
+
+ (SELECT COUNT(*) FROM retrieval_trace_events WHERE trace_id = NEW.trace_id)
|
|
254
|
+
+ (SELECT COUNT(*) FROM retrieval_trace_judgments WHERE trace_id = NEW.trace_id)
|
|
255
|
+
+ (SELECT COUNT(*) FROM retrieval_trace_export_traces WHERE trace_id = NEW.trace_id)
|
|
256
|
+
) >= 100000
|
|
257
|
+
AND NOT EXISTS (
|
|
258
|
+
SELECT 1 FROM retrieval_trace_judgments existing
|
|
259
|
+
WHERE existing.judgment_id = NEW.judgment_id
|
|
260
|
+
OR (
|
|
261
|
+
existing.trace_id = NEW.trace_id
|
|
262
|
+
AND existing.idempotency_key = NEW.idempotency_key
|
|
263
|
+
)
|
|
264
|
+
)
|
|
265
|
+
BEGIN
|
|
266
|
+
SELECT RAISE(ABORT, 'retrieval trace record cap exceeded');
|
|
267
|
+
END;
|
|
268
|
+
|
|
269
|
+
CREATE TRIGGER cap_retrieval_trace_export_links
|
|
270
|
+
BEFORE INSERT ON retrieval_trace_export_traces
|
|
271
|
+
WHEN (
|
|
272
|
+
(SELECT COUNT(*) FROM retrieval_trace_runs WHERE trace_id = NEW.trace_id)
|
|
273
|
+
+ (SELECT COUNT(*) FROM retrieval_trace_events WHERE trace_id = NEW.trace_id)
|
|
274
|
+
+ (SELECT COUNT(*) FROM retrieval_trace_judgments WHERE trace_id = NEW.trace_id)
|
|
275
|
+
+ (SELECT COUNT(*) FROM retrieval_trace_export_traces WHERE trace_id = NEW.trace_id)
|
|
276
|
+
) >= 100000
|
|
277
|
+
AND NOT EXISTS (
|
|
278
|
+
SELECT 1 FROM retrieval_trace_export_traces existing
|
|
279
|
+
WHERE existing.export_id = NEW.export_id
|
|
280
|
+
AND existing.trace_id = NEW.trace_id
|
|
281
|
+
)
|
|
282
|
+
BEGIN
|
|
283
|
+
SELECT RAISE(ABORT, 'retrieval trace record cap exceeded');
|
|
284
|
+
END;
|
|
285
|
+
`);
|
|
286
|
+
},
|
|
287
|
+
|
|
288
|
+
down(db: Database): void {
|
|
289
|
+
db.exec(`
|
|
290
|
+
DROP TRIGGER IF EXISTS cap_retrieval_trace_export_links;
|
|
291
|
+
DROP TRIGGER IF EXISTS cap_retrieval_trace_judgments;
|
|
292
|
+
DROP TRIGGER IF EXISTS cap_retrieval_trace_events;
|
|
293
|
+
DROP TRIGGER IF EXISTS cap_retrieval_trace_runs;
|
|
294
|
+
DROP TRIGGER IF EXISTS delete_empty_retrieval_trace_export;
|
|
295
|
+
DROP TABLE IF EXISTS retrieval_trace_export_traces;
|
|
296
|
+
DROP TABLE IF EXISTS retrieval_trace_exports;
|
|
297
|
+
DROP TABLE IF EXISTS retrieval_trace_judgments;
|
|
298
|
+
DROP TABLE IF EXISTS retrieval_trace_events;
|
|
299
|
+
DROP TABLE IF EXISTS retrieval_trace_runs;
|
|
300
|
+
DROP TABLE IF EXISTS retrieval_traces;
|
|
301
|
+
`);
|
|
302
|
+
},
|
|
303
|
+
};
|
|
@@ -27,6 +27,7 @@ import { migration as m010 } from "./010-typed-edges";
|
|
|
27
27
|
import { migration as m011 } from "./011-doc-edge-traversal-indexes";
|
|
28
28
|
import { migration as m012 } from "./012-activation-receipts";
|
|
29
29
|
import { migration as m013 } from "./013-fts-sync-marker";
|
|
30
|
+
import { migration as m014 } from "./014-retrieval-traces";
|
|
30
31
|
|
|
31
32
|
/** All migrations in order */
|
|
32
33
|
export const migrations = [
|
|
@@ -43,4 +44,5 @@ export const migrations = [
|
|
|
43
44
|
m011,
|
|
44
45
|
m012,
|
|
45
46
|
m013,
|
|
47
|
+
m014,
|
|
46
48
|
];
|