@gmickel/gno 1.28.0 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -20
- package/assets/skill/SKILL.md +33 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.29.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.29.0.zip.sha256 +1 -0
- package/browser-extension/dist/chunk-b2zm0jjd.js +50 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/browser-extension/dist/preview.html +1 -1
- package/browser-extension/dist/service-worker.js +6 -6
- package/package.json +1 -1
- package/spec/cli.md +41 -1
- package/spec/db/schema.sql +19 -0
- package/spec/mcp.md +66 -0
- package/spec/output-schemas/ask.schema.json +31 -1
- package/spec/output-schemas/browser-clip-preview.schema.json +30 -0
- package/spec/output-schemas/collection-egress-check.schema.json +91 -0
- package/spec/output-schemas/collection-egress-policy-set.schema.json +56 -0
- package/spec/output-schemas/collection-egress-policy.schema.json +41 -0
- package/spec/output-schemas/context-capsule-v1.schema.json +89 -2
- package/spec/output-schemas/context-capsule-verification.schema.json +1 -1
- package/spec/output-schemas/egress-audit-management.schema.json +88 -0
- package/spec/output-schemas/mcp-http-error.schema.json +113 -2
- package/spec/output-schemas/publish-artifact.schema.json +32 -0
- package/spec/output-schemas/record-import.schema.json +30 -0
- package/spec/output-schemas/retrieval-trace-common.schema.json +31 -0
- package/spec/output-schemas/retrieval-trace-export.schema.json +3 -0
- package/spec/output-schemas/retrieval-trace-qrels.schema.json +3 -0
- package/spec/output-schemas/retrieval-trace-summary.schema.json +4 -0
- package/spec/output-schemas/search-results.schema.json +30 -0
- package/src/app/context-runtime-contract.ts +16 -7
- package/src/cli/commands/ask.ts +7 -0
- package/src/cli/commands/collection/index.ts +5 -0
- package/src/cli/commands/collection/policy.ts +126 -0
- package/src/cli/commands/context-build.ts +5 -0
- package/src/cli/commands/daemon.ts +45 -3
- package/src/cli/commands/egress-audit.ts +68 -0
- package/src/cli/commands/embed.ts +2 -0
- package/src/cli/commands/query.ts +12 -0
- package/src/cli/commands/replay.ts +13 -3
- package/src/cli/commands/vsearch.ts +3 -1
- package/src/cli/program.ts +123 -0
- package/src/config/index.ts +9 -0
- package/src/config/types.ts +50 -0
- package/src/core/browser-clip.ts +16 -1
- package/src/core/collection-egress-policy-projection.ts +28 -0
- package/src/core/collection-egress-policy-service.ts +443 -0
- package/src/core/collection-egress-policy-validation.ts +242 -0
- package/src/core/config-mutation.ts +31 -19
- package/src/core/context-capsule-schema.ts +111 -1
- package/src/core/context-capsule-validation.ts +4 -4
- package/src/core/context-capsule-verification.ts +5 -1
- package/src/core/context-capsule.ts +177 -132
- package/src/core/context-evidence.ts +86 -14
- package/src/core/destination-classifier.ts +402 -0
- package/src/core/egress-audit.ts +239 -0
- package/src/core/egress-authorization.ts +50 -0
- package/src/core/egress-enforcement.ts +264 -0
- package/src/core/egress-policy.ts +440 -0
- package/src/core/egress-provenance.ts +336 -0
- package/src/core/job-manager.ts +37 -0
- package/src/core/network-boundary-inventory.ts +261 -0
- package/src/core/retrieval-qrels.ts +6 -0
- package/src/core/retrieval-replay.ts +18 -4
- package/src/core/retrieval-trace-export.ts +23 -4
- package/src/core/retrieval-trace-management-helpers.ts +1 -0
- package/src/core/retrieval-trace-management-types.ts +3 -0
- package/src/core/retrieval-trace-management.ts +9 -1
- package/src/core/retrieval-trace-session.ts +94 -0
- package/src/core/retrieval-trace.ts +22 -0
- package/src/ingestion/record-adapter.ts +12 -1
- package/src/ingestion/record-container.ts +38 -22
- package/src/ingestion/sync.ts +15 -0
- package/src/ingestion/types.ts +2 -0
- package/src/llm/errors.ts +10 -0
- package/src/llm/http-inference.ts +175 -0
- package/src/llm/http-policy.ts +537 -0
- package/src/llm/httpEmbedding.ts +47 -28
- package/src/llm/httpGeneration.ts +31 -18
- package/src/llm/httpRerank.ts +30 -18
- package/src/llm/index.ts +1 -0
- package/src/llm/nodeLlamaCpp/adapter.ts +55 -20
- package/src/llm/pinned-http-connection.ts +177 -0
- package/src/mcp/context.ts +53 -1
- package/src/mcp/http-egress.ts +220 -0
- package/src/mcp/http-security.ts +32 -21
- package/src/mcp/http-session.ts +13 -2
- package/src/mcp/http-transport.ts +96 -6
- package/src/mcp/sync-egress.ts +24 -0
- package/src/mcp/tools/add-collection.ts +4 -0
- package/src/mcp/tools/ask.ts +1 -1
- package/src/mcp/tools/context.ts +9 -2
- package/src/mcp/tools/egress.ts +247 -0
- package/src/mcp/tools/embed.ts +1 -0
- package/src/mcp/tools/index-cmd.ts +7 -0
- package/src/mcp/tools/index.ts +98 -0
- package/src/mcp/tools/query.ts +10 -0
- package/src/mcp/tools/remove-collection.ts +4 -0
- package/src/mcp/tools/sync.ts +6 -0
- package/src/mcp/tools/trace.ts +7 -1
- package/src/mcp/tools/vsearch.ts +1 -0
- package/src/pipeline/egress-lineage.ts +124 -0
- package/src/pipeline/hybrid.ts +14 -0
- package/src/pipeline/search.ts +12 -0
- package/src/pipeline/types.ts +3 -0
- package/src/pipeline/vsearch.ts +41 -20
- package/src/publish/artifact.ts +31 -3
- package/src/publish/export-service.ts +25 -0
- package/src/sdk/client.ts +119 -2
- package/src/sdk/embed.ts +1 -0
- package/src/sdk/types.ts +33 -0
- package/src/serve/clipper-capture.ts +5 -1
- package/src/serve/closed-json.ts +61 -0
- package/src/serve/config-sync.ts +4 -1
- package/src/serve/context.ts +1 -0
- package/src/serve/doc-events.ts +110 -39
- package/src/serve/public/components/CollectionEgressCheckPanel.tsx +411 -0
- package/src/serve/public/components/CollectionEgressPolicyEditor.tsx +78 -0
- package/src/serve/public/components/CollectionModelDialog.tsx +93 -9
- package/src/serve/public/components/EgressAuditPanel.tsx +278 -0
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/pages/Collections.tsx +3 -0
- package/src/serve/resident-request.ts +80 -3
- package/src/serve/resident-runtime.ts +63 -1
- package/src/serve/routes/api.ts +174 -0
- package/src/serve/routes/clipper.ts +64 -1
- package/src/serve/routes/mcp.ts +6 -0
- package/src/serve/routes/traces.ts +27 -4
- package/src/serve/server.ts +118 -7
- package/src/store/migrations/023-collection-egress-policy.ts +61 -0
- package/src/store/migrations/024-egress-derived-lineage.ts +121 -0
- package/src/store/migrations/025-collection-egress-policy-revision.ts +40 -0
- package/src/store/migrations/index.ts +6 -0
- package/src/store/retrieval-trace-codec.ts +15 -0
- package/src/store/sqlite/adapter.ts +170 -7
- package/src/store/sqlite/change-journal-store.ts +41 -2
- package/src/store/sqlite/egress-audit-store.ts +485 -0
- package/src/store/sqlite/retrieval-trace-management-store.ts +47 -6
- package/src/store/sqlite/retrieval-trace-retention.ts +2 -0
- package/src/store/sqlite/retrieval-trace-rows.ts +8 -1
- package/src/store/sqlite/retrieval-trace-store.ts +104 -5
- package/src/store/types.ts +126 -1
- package/browser-extension/artifacts/gno-browser-clipper-v1.28.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.28.0.zip.sha256 +0 -1
- package/browser-extension/dist/chunk-vn5f663b.js +0 -50
|
@@ -7,10 +7,12 @@ import type {
|
|
|
7
7
|
RetrievalTraceTerminalStatus,
|
|
8
8
|
StoreResult,
|
|
9
9
|
} from "../store/types";
|
|
10
|
+
import type { EgressLineage } from "./egress-provenance";
|
|
10
11
|
import type { EvidenceTarget } from "./retrieval-trace-management-helpers";
|
|
11
12
|
|
|
12
13
|
import { hashTraceCanonical } from "../store/retrieval-trace-codec";
|
|
13
14
|
import { err, ok } from "../store/types";
|
|
15
|
+
import { mergeEgressLineages } from "./egress-provenance";
|
|
14
16
|
import { parseRetrievalTraceFilters } from "./retrieval-trace-filters";
|
|
15
17
|
import { stableTarget, targetKey } from "./retrieval-trace-management-helpers";
|
|
16
18
|
|
|
@@ -85,6 +87,7 @@ export interface RetrievalQrelsCase {
|
|
|
85
87
|
export interface RetrievalTraceQrelsArtifact {
|
|
86
88
|
schemaVersion: "1.0";
|
|
87
89
|
format: "qrels";
|
|
90
|
+
egressLineage: EgressLineage;
|
|
88
91
|
cases: RetrievalQrelsCase[];
|
|
89
92
|
}
|
|
90
93
|
|
|
@@ -400,6 +403,9 @@ export const buildRetrievalQrelsArtifact = (
|
|
|
400
403
|
return ok({
|
|
401
404
|
schemaVersion: "1.0",
|
|
402
405
|
format: "qrels",
|
|
406
|
+
egressLineage: mergeEgressLineages(
|
|
407
|
+
bundles.map(({ trace }) => trace.egressLineage)
|
|
408
|
+
),
|
|
403
409
|
cases: cases.sort((a, b) => a.caseId.localeCompare(b.caseId)),
|
|
404
410
|
});
|
|
405
411
|
};
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
/** Read-only comparison of persisted retrieval baselines with one candidate. */
|
|
2
2
|
|
|
3
3
|
import type { SearchResult } from "../pipeline/types";
|
|
4
|
-
import type {
|
|
4
|
+
import type {
|
|
5
|
+
DocumentRow,
|
|
6
|
+
RetrievalTraceExportManifestRow,
|
|
7
|
+
StoreResult,
|
|
8
|
+
} from "../store/types";
|
|
5
9
|
import type {
|
|
6
10
|
RetrievalQrelsCase,
|
|
11
|
+
RetrievalTraceQrelsArtifact,
|
|
7
12
|
RetrievalQrel,
|
|
8
13
|
RetrievalQrelsEvidence,
|
|
9
14
|
} from "./retrieval-qrels";
|
|
@@ -23,6 +28,7 @@ import { computeRetrievalMetrics } from "../bench/metrics";
|
|
|
23
28
|
import { diagnoseQueryTarget } from "../pipeline/diagnose";
|
|
24
29
|
import { hashTraceCanonical } from "../store/retrieval-trace-codec";
|
|
25
30
|
import { ok } from "../store/types";
|
|
31
|
+
import { isMigratedLegacyEgressLineage } from "./egress-provenance";
|
|
26
32
|
import { buildRetrievalQrelsArtifact } from "./retrieval-qrels";
|
|
27
33
|
import {
|
|
28
34
|
buildRetrievalReplaySearchOptions,
|
|
@@ -74,6 +80,16 @@ const reconstructionReason = (message: string): RetrievalReplayReason | null =>
|
|
|
74
80
|
message.includes(`${reason}:`)
|
|
75
81
|
) ?? null;
|
|
76
82
|
|
|
83
|
+
const artifactMatchesManifest = (
|
|
84
|
+
artifact: RetrievalTraceQrelsArtifact,
|
|
85
|
+
manifest: RetrievalTraceExportManifestRow
|
|
86
|
+
): boolean => {
|
|
87
|
+
if (hashTraceCanonical(artifact) === manifest.artifactHash) return true;
|
|
88
|
+
if (!isMigratedLegacyEgressLineage(manifest.egressLineage)) return false;
|
|
89
|
+
const { egressLineage: _legacyProjection, ...legacyArtifact } = artifact;
|
|
90
|
+
return hashTraceCanonical(legacyArtifact) === manifest.artifactHash;
|
|
91
|
+
};
|
|
92
|
+
|
|
77
93
|
const targetMatchesResult = (
|
|
78
94
|
qrel: RetrievalQrel,
|
|
79
95
|
result: SearchResult
|
|
@@ -376,9 +392,7 @@ export const replayRetrievalTraces = async (
|
|
|
376
392
|
)
|
|
377
393
|
);
|
|
378
394
|
}
|
|
379
|
-
if (
|
|
380
|
-
hashTraceCanonical(artifact.value) !== stored.value.manifest.artifactHash
|
|
381
|
-
) {
|
|
395
|
+
if (!artifactMatchesManifest(artifact.value, stored.value.manifest)) {
|
|
382
396
|
return ok(unreplayable(validatedInput, "manifest_hash_mismatch"));
|
|
383
397
|
}
|
|
384
398
|
let currentFingerprints;
|
|
@@ -6,6 +6,7 @@ import type {
|
|
|
6
6
|
StorePort,
|
|
7
7
|
StoreResult,
|
|
8
8
|
} from "../store/types";
|
|
9
|
+
import type { EgressLineage } from "./egress-provenance";
|
|
9
10
|
import type {
|
|
10
11
|
ExportRetrievalTracesInput,
|
|
11
12
|
ExportRetrievalTracesResult,
|
|
@@ -14,6 +15,7 @@ import type {
|
|
|
14
15
|
|
|
15
16
|
import { hashTraceCanonical } from "../store/retrieval-trace-codec";
|
|
16
17
|
import { err, ok } from "../store/types";
|
|
18
|
+
import { mergeEgressLineages } from "./egress-provenance";
|
|
17
19
|
import { buildRetrievalQrelsArtifact } from "./retrieval-qrels";
|
|
18
20
|
|
|
19
21
|
const validateTraceIds = (traceIds: unknown): StoreResult<string[]> => {
|
|
@@ -38,6 +40,9 @@ const agenticArtifact = (
|
|
|
38
40
|
): RetrievalTraceArtifact => ({
|
|
39
41
|
schemaVersion: "1.0",
|
|
40
42
|
format: "agentic-receipt",
|
|
43
|
+
egressLineage: mergeEgressLineages(
|
|
44
|
+
bundles.map(({ trace }) => trace.egressLineage)
|
|
45
|
+
),
|
|
41
46
|
traces: bundles.map(({ exports: _exports, ...trace }) => trace),
|
|
42
47
|
});
|
|
43
48
|
|
|
@@ -54,7 +59,10 @@ export const exportRetrievalTraces = async <
|
|
|
54
59
|
>(
|
|
55
60
|
store: StorePort,
|
|
56
61
|
clock: () => number,
|
|
57
|
-
input: ExportRetrievalTracesInput<Format
|
|
62
|
+
input: ExportRetrievalTracesInput<Format>,
|
|
63
|
+
deps: {
|
|
64
|
+
authorize?: (lineage: EgressLineage) => Promise<StoreResult<EgressLineage>>;
|
|
65
|
+
} = {}
|
|
58
66
|
): Promise<StoreResult<ExportRetrievalTracesResult<Format>>> => {
|
|
59
67
|
const format = input.format ?? "agentic-receipt";
|
|
60
68
|
if (!["agentic-receipt", "qrels"].includes(format)) {
|
|
@@ -77,13 +85,20 @@ export const exportRetrievalTraces = async <
|
|
|
77
85
|
}
|
|
78
86
|
const built = buildArtifact(format, bundles);
|
|
79
87
|
if (!built.ok) return built;
|
|
80
|
-
const
|
|
88
|
+
const authorized = await deps.authorize?.(built.value.egressLineage);
|
|
89
|
+
if (authorized && !authorized.ok) return authorized;
|
|
90
|
+
const artifact = {
|
|
91
|
+
...built.value,
|
|
92
|
+
egressLineage: authorized?.value ?? built.value.egressLineage,
|
|
93
|
+
} as RetrievalTraceArtifact;
|
|
94
|
+
const artifactHash = hashTraceCanonical(artifact);
|
|
81
95
|
const exportId = `trace-export-${artifactHash.slice(0, 40)}`;
|
|
82
96
|
const appended = await store.appendRetrievalTraceExportManifest({
|
|
83
97
|
exportId,
|
|
84
98
|
traceIds: traceIds.value,
|
|
85
99
|
format,
|
|
86
100
|
artifactHash,
|
|
101
|
+
egressLineage: artifact.egressLineage,
|
|
87
102
|
createdAtMs: clock(),
|
|
88
103
|
});
|
|
89
104
|
if (!appended.ok) return appended;
|
|
@@ -94,9 +109,13 @@ export const exportRetrievalTraces = async <
|
|
|
94
109
|
}
|
|
95
110
|
const reconstructed = buildArtifact(format, complete.value.traces);
|
|
96
111
|
if (!reconstructed.ok) return reconstructed;
|
|
112
|
+
const reconstructedCurrent = {
|
|
113
|
+
...reconstructed.value,
|
|
114
|
+
egressLineage: complete.value.manifest.egressLineage,
|
|
115
|
+
} as RetrievalTraceArtifact;
|
|
97
116
|
if (
|
|
98
117
|
complete.value.manifest.traceIds.join("\0") !== traceIds.value.join("\0") ||
|
|
99
|
-
hashTraceCanonical(
|
|
118
|
+
hashTraceCanonical(reconstructedCurrent) !==
|
|
100
119
|
complete.value.manifest.artifactHash
|
|
101
120
|
) {
|
|
102
121
|
return err(
|
|
@@ -108,6 +127,6 @@ export const exportRetrievalTraces = async <
|
|
|
108
127
|
schemaVersion: "1.0",
|
|
109
128
|
result: appended.value,
|
|
110
129
|
manifest: complete.value.manifest,
|
|
111
|
-
artifact:
|
|
130
|
+
artifact: reconstructedCurrent,
|
|
112
131
|
} as ExportRetrievalTracesResult<Format>);
|
|
113
132
|
};
|
|
@@ -44,6 +44,7 @@ export const summaryOf = (
|
|
|
44
44
|
queryShape: trace.queryShape,
|
|
45
45
|
goalShape: trace.goalShape,
|
|
46
46
|
fingerprints: trace.fingerprints,
|
|
47
|
+
egressLineage: trace.egressLineage,
|
|
47
48
|
createdAtMs: trace.createdAtMs,
|
|
48
49
|
updatedAtMs: trace.updatedAtMs,
|
|
49
50
|
expiresAtMs: trace.expiresAtMs,
|
|
@@ -10,6 +10,7 @@ import type {
|
|
|
10
10
|
RetrievalTracePurgeResult as StoredRetrievalTracePurgeResult,
|
|
11
11
|
RetrievalTraceRow,
|
|
12
12
|
} from "../store/types";
|
|
13
|
+
import type { EgressLineage } from "./egress-provenance";
|
|
13
14
|
import type { RetrievalTraceQrelsArtifact } from "./retrieval-qrels";
|
|
14
15
|
|
|
15
16
|
export interface RetrievalTraceSummary {
|
|
@@ -21,6 +22,7 @@ export interface RetrievalTraceSummary {
|
|
|
21
22
|
queryShape: RetrievalTraceRow["queryShape"];
|
|
22
23
|
goalShape: RetrievalTraceRow["goalShape"];
|
|
23
24
|
fingerprints: RetrievalTraceRow["fingerprints"];
|
|
25
|
+
egressLineage: EgressLineage;
|
|
24
26
|
createdAtMs: number;
|
|
25
27
|
updatedAtMs: number;
|
|
26
28
|
expiresAtMs: number;
|
|
@@ -87,6 +89,7 @@ export interface LabelRetrievalTraceResult {
|
|
|
87
89
|
export interface RetrievalTraceAgenticArtifact {
|
|
88
90
|
schemaVersion: "1.0";
|
|
89
91
|
format: "agentic-receipt";
|
|
92
|
+
egressLineage: EgressLineage;
|
|
90
93
|
traces: Array<Omit<RetrievalTraceBundle, "exports">>;
|
|
91
94
|
}
|
|
92
95
|
|
|
@@ -6,6 +6,7 @@ import type {
|
|
|
6
6
|
StorePort,
|
|
7
7
|
StoreResult,
|
|
8
8
|
} from "../store/types";
|
|
9
|
+
import type { EgressLineage } from "./egress-provenance";
|
|
9
10
|
import type {
|
|
10
11
|
ReplayRetrievalTraceInput,
|
|
11
12
|
ReplayRetrievalTraceResult,
|
|
@@ -56,16 +57,21 @@ const DOCID_PATTERN = /^#[a-f0-9]{6,}$/;
|
|
|
56
57
|
|
|
57
58
|
interface ManagementDeps {
|
|
58
59
|
clock?: () => number;
|
|
60
|
+
authorizeExport?: (
|
|
61
|
+
lineage: EgressLineage
|
|
62
|
+
) => Promise<StoreResult<EgressLineage>>;
|
|
59
63
|
}
|
|
60
64
|
|
|
61
65
|
export class RetrievalTraceManagementService {
|
|
62
66
|
private readonly clock: () => number;
|
|
67
|
+
private readonly authorizeExport?: ManagementDeps["authorizeExport"];
|
|
63
68
|
|
|
64
69
|
constructor(
|
|
65
70
|
private readonly store: StorePort,
|
|
66
71
|
deps: ManagementDeps = {}
|
|
67
72
|
) {
|
|
68
73
|
this.clock = deps.clock ?? Date.now;
|
|
74
|
+
this.authorizeExport = deps.authorizeExport;
|
|
69
75
|
}
|
|
70
76
|
|
|
71
77
|
async list(
|
|
@@ -219,7 +225,9 @@ export class RetrievalTraceManagementService {
|
|
|
219
225
|
async export<Format extends RetrievalTraceExportFormat = "agentic-receipt">(
|
|
220
226
|
input: ExportRetrievalTracesInput<Format>
|
|
221
227
|
): Promise<StoreResult<ExportRetrievalTracesResult<Format>>> {
|
|
222
|
-
return exportRetrievalTraces(this.store, this.clock, input
|
|
228
|
+
return exportRetrievalTraces(this.store, this.clock, input, {
|
|
229
|
+
authorize: this.authorizeExport,
|
|
230
|
+
});
|
|
223
231
|
}
|
|
224
232
|
|
|
225
233
|
async replay(
|
|
@@ -16,6 +16,13 @@ import {
|
|
|
16
16
|
SEARCH_RESULTS_TRACE_METADATA,
|
|
17
17
|
} from "../pipeline/types";
|
|
18
18
|
import { err, ok } from "../store/types";
|
|
19
|
+
import { contextCapsuleEgressLineage } from "./context-capsule";
|
|
20
|
+
import {
|
|
21
|
+
createEgressLineage,
|
|
22
|
+
legacyLocalOnlyEgressLineage,
|
|
23
|
+
mergeEgressLineages,
|
|
24
|
+
resolveEgressLineage,
|
|
25
|
+
} from "./egress-provenance";
|
|
19
26
|
import {
|
|
20
27
|
MIN_RETRIEVAL_TRACE_RECORDS,
|
|
21
28
|
RetrievalTraceRecorder,
|
|
@@ -140,6 +147,26 @@ const evidenceFromCapsule = (
|
|
|
140
147
|
: { graphExpanded: item.graphExpanded }),
|
|
141
148
|
}));
|
|
142
149
|
|
|
150
|
+
const requestedCollectionScope = (
|
|
151
|
+
filters: Record<string, unknown> | undefined
|
|
152
|
+
): string[] | undefined => {
|
|
153
|
+
const collections = filters?.collections;
|
|
154
|
+
if (Array.isArray(collections) && collections.length > 0) {
|
|
155
|
+
if (!collections.every((value) => typeof value === "string")) {
|
|
156
|
+
throw new TypeError("Trace collection scope must contain only names");
|
|
157
|
+
}
|
|
158
|
+
return collections;
|
|
159
|
+
}
|
|
160
|
+
const collection = filters?.collection;
|
|
161
|
+
if (collection === undefined || collection === null || collection === "") {
|
|
162
|
+
return undefined;
|
|
163
|
+
}
|
|
164
|
+
if (typeof collection !== "string") {
|
|
165
|
+
throw new TypeError("Trace collection scope must be a name");
|
|
166
|
+
}
|
|
167
|
+
return [collection];
|
|
168
|
+
};
|
|
169
|
+
|
|
143
170
|
export class RetrievalTraceSession {
|
|
144
171
|
readonly traceId: string;
|
|
145
172
|
|
|
@@ -154,6 +181,7 @@ export class RetrievalTraceSession {
|
|
|
154
181
|
private readonly recorder: RetrievalTraceRecorder,
|
|
155
182
|
traceId: string,
|
|
156
183
|
private readonly clock: () => number,
|
|
184
|
+
private egressLineage: ReturnType<typeof createEgressLineage>,
|
|
157
185
|
private readonly operationId = traceId,
|
|
158
186
|
private readonly maxRecords = 100_000
|
|
159
187
|
) {
|
|
@@ -180,11 +208,37 @@ export class RetrievalTraceSession {
|
|
|
180
208
|
clock,
|
|
181
209
|
idFactory: input.idFactory,
|
|
182
210
|
});
|
|
211
|
+
const collections = await input.store.getCollections();
|
|
212
|
+
if (!collections.ok) return collections;
|
|
213
|
+
let egressLineage;
|
|
214
|
+
try {
|
|
215
|
+
const requestedScope = requestedCollectionScope(input.filters);
|
|
216
|
+
egressLineage =
|
|
217
|
+
collections.value.length === 0
|
|
218
|
+
? legacyLocalOnlyEgressLineage()
|
|
219
|
+
: resolveEgressLineage(
|
|
220
|
+
collections.value.map((collection) => ({
|
|
221
|
+
collection: collection.name,
|
|
222
|
+
policy: collection.egressPolicy,
|
|
223
|
+
source: collection.egressPolicySource,
|
|
224
|
+
})),
|
|
225
|
+
requestedScope
|
|
226
|
+
);
|
|
227
|
+
} catch (cause) {
|
|
228
|
+
return err(
|
|
229
|
+
"INVALID_INPUT",
|
|
230
|
+
cause instanceof Error
|
|
231
|
+
? cause.message
|
|
232
|
+
: "Invalid retrieval trace collection scope",
|
|
233
|
+
cause
|
|
234
|
+
);
|
|
235
|
+
}
|
|
183
236
|
const started = await recorder.start({
|
|
184
237
|
query: input.query,
|
|
185
238
|
goal: input.goal,
|
|
186
239
|
filters: input.filters,
|
|
187
240
|
fingerprints: await input.fingerprints(),
|
|
241
|
+
egressLineage,
|
|
188
242
|
});
|
|
189
243
|
if (!started.ok) return started;
|
|
190
244
|
if (!started.value.recorded) return ok(null);
|
|
@@ -192,6 +246,7 @@ export class RetrievalTraceSession {
|
|
|
192
246
|
recorder,
|
|
193
247
|
started.value.traceId,
|
|
194
248
|
clock,
|
|
249
|
+
egressLineage,
|
|
195
250
|
started.value.traceId,
|
|
196
251
|
input.config.retention.maxRecordsPerTrace
|
|
197
252
|
);
|
|
@@ -233,6 +288,7 @@ export class RetrievalTraceSession {
|
|
|
233
288
|
new RetrievalTraceRecorder(input.store, input.config, { clock }),
|
|
234
289
|
input.traceId,
|
|
235
290
|
clock,
|
|
291
|
+
stored.value.trace.egressLineage,
|
|
236
292
|
crypto.randomUUID(),
|
|
237
293
|
input.config.retention.maxRecordsPerTrace
|
|
238
294
|
);
|
|
@@ -257,6 +313,13 @@ export class RetrievalTraceSession {
|
|
|
257
313
|
latencyMs?: number
|
|
258
314
|
): Promise<StoreResult<"inserted" | "duplicate" | "disabled">> {
|
|
259
315
|
if (!this.hasCapacity(2)) return ok("disabled");
|
|
316
|
+
const resultLineages = result.results.flatMap((item) =>
|
|
317
|
+
item.egressLineage ? [item.egressLineage] : []
|
|
318
|
+
);
|
|
319
|
+
if (resultLineages.length > 0) {
|
|
320
|
+
const widened = await this.ensureLineageCoverage(resultLineages);
|
|
321
|
+
if (!widened.ok) return widened;
|
|
322
|
+
}
|
|
260
323
|
const ranked = result.results.flatMap((item, index) => {
|
|
261
324
|
const evidence = evidenceFromResult(item, index + 1);
|
|
262
325
|
return evidence ? [evidence] : [];
|
|
@@ -317,6 +380,12 @@ export class RetrievalTraceSession {
|
|
|
317
380
|
latencyMs?: number
|
|
318
381
|
): Promise<StoreResult<"inserted" | "duplicate" | "disabled">> {
|
|
319
382
|
if (!this.hasCapacity(2)) return ok("disabled");
|
|
383
|
+
if (capsule.schemaVersion === "1.0" || capsule.schemaVersion === "1.1") {
|
|
384
|
+
const widened = await this.ensureLineageCoverage([
|
|
385
|
+
contextCapsuleEgressLineage(capsule),
|
|
386
|
+
]);
|
|
387
|
+
if (!widened.ok) return widened;
|
|
388
|
+
}
|
|
320
389
|
const payload = {
|
|
321
390
|
evidence: evidenceFromCapsule(capsule),
|
|
322
391
|
capsuleId: capsule.capsuleId,
|
|
@@ -449,6 +518,31 @@ export class RetrievalTraceSession {
|
|
|
449
518
|
return appended;
|
|
450
519
|
}
|
|
451
520
|
|
|
521
|
+
private async ensureLineageCoverage(
|
|
522
|
+
lineages: ReturnType<typeof createEgressLineage>[]
|
|
523
|
+
): Promise<StoreResult<"inserted" | "duplicate">> {
|
|
524
|
+
let merged;
|
|
525
|
+
try {
|
|
526
|
+
merged = mergeEgressLineages([this.egressLineage, ...lineages]);
|
|
527
|
+
} catch (cause) {
|
|
528
|
+
return err(
|
|
529
|
+
"INVALID_INPUT",
|
|
530
|
+
cause instanceof Error
|
|
531
|
+
? cause.message
|
|
532
|
+
: "Conflicting retrieval trace policy lineage",
|
|
533
|
+
cause
|
|
534
|
+
);
|
|
535
|
+
}
|
|
536
|
+
if (merged.digest === this.egressLineage.digest) return ok("duplicate");
|
|
537
|
+
const persisted = await this.recorder.mergeEgressLineage(
|
|
538
|
+
this.traceId,
|
|
539
|
+
merged
|
|
540
|
+
);
|
|
541
|
+
if (!persisted.ok) return persisted;
|
|
542
|
+
this.egressLineage = merged;
|
|
543
|
+
return persisted;
|
|
544
|
+
}
|
|
545
|
+
|
|
452
546
|
private hasCapacity(records: number): boolean {
|
|
453
547
|
return this.persistedRecords + records + 1 <= this.maxRecords;
|
|
454
548
|
}
|
|
@@ -16,6 +16,7 @@ import type {
|
|
|
16
16
|
StorePort,
|
|
17
17
|
StoreResult,
|
|
18
18
|
} from "../store/types";
|
|
19
|
+
import type { EgressLineage } from "./egress-provenance";
|
|
19
20
|
|
|
20
21
|
import {
|
|
21
22
|
parseRetrievalTraceEventInput,
|
|
@@ -23,6 +24,10 @@ import {
|
|
|
23
24
|
parseRetrievalTraceRunInput,
|
|
24
25
|
} from "../store/retrieval-trace-codec";
|
|
25
26
|
import { err, ok } from "../store/types";
|
|
27
|
+
import {
|
|
28
|
+
egressLineageSchema,
|
|
29
|
+
legacyLocalOnlyEgressLineage,
|
|
30
|
+
} from "./egress-provenance";
|
|
26
31
|
import { canonicalizeRetrievalTraceFilters } from "./retrieval-trace-filter-normalization";
|
|
27
32
|
|
|
28
33
|
const sha256Schema = z.string().regex(/^[a-f0-9]{64}$/);
|
|
@@ -80,6 +85,7 @@ const startTraceSchema = z
|
|
|
80
85
|
index: sha256Schema,
|
|
81
86
|
})
|
|
82
87
|
.strict(),
|
|
88
|
+
egressLineage: egressLineageSchema.default(legacyLocalOnlyEgressLineage()),
|
|
83
89
|
})
|
|
84
90
|
.strict();
|
|
85
91
|
|
|
@@ -186,6 +192,7 @@ export interface StartRetrievalTraceInput {
|
|
|
186
192
|
goal?: string;
|
|
187
193
|
filters?: z.input<typeof traceFiltersSchema>;
|
|
188
194
|
fingerprints: RetrievalTraceFingerprints;
|
|
195
|
+
egressLineage?: EgressLineage;
|
|
189
196
|
}
|
|
190
197
|
|
|
191
198
|
export type RetrievalTraceWriteResult =
|
|
@@ -318,6 +325,7 @@ export class RetrievalTraceRecorder {
|
|
|
318
325
|
goalDigest: replay && goal !== undefined ? sha256(goal) : null,
|
|
319
326
|
goalShape: textShape(goal),
|
|
320
327
|
filters: replay ? filters : projectMetadataObject(filters),
|
|
328
|
+
egressLineage: parsed.data.egressLineage,
|
|
321
329
|
fingerprints: parsed.data.fingerprints,
|
|
322
330
|
status: "open",
|
|
323
331
|
createdAtMs: nowMs,
|
|
@@ -371,6 +379,20 @@ export class RetrievalTraceRecorder {
|
|
|
371
379
|
}
|
|
372
380
|
}
|
|
373
381
|
|
|
382
|
+
async mergeEgressLineage(
|
|
383
|
+
traceId: string,
|
|
384
|
+
lineage: EgressLineage
|
|
385
|
+
): Promise<StoreResult<RetrievalTraceAppendResult>> {
|
|
386
|
+
if (!this.config?.enabled) return ok("duplicate");
|
|
387
|
+
if (!this.store.mergeRetrievalTraceEgressLineage) {
|
|
388
|
+
return err(
|
|
389
|
+
"CONSTRAINT_VIOLATION",
|
|
390
|
+
"Store cannot widen retrieval trace policy lineage"
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
return this.store.mergeRetrievalTraceEgressLineage(traceId, lineage);
|
|
394
|
+
}
|
|
395
|
+
|
|
374
396
|
async appendEvent(
|
|
375
397
|
input: RetrievalTraceEventInput
|
|
376
398
|
): Promise<StoreResult<RetrievalTraceAppendResult | "disabled">> {
|
|
@@ -5,7 +5,12 @@ import type {
|
|
|
5
5
|
RecordAdapterEvent,
|
|
6
6
|
RecordAdapterInput,
|
|
7
7
|
} from "../converters/types";
|
|
8
|
+
import type { EgressLineage } from "../core/egress-provenance";
|
|
8
9
|
|
|
10
|
+
import {
|
|
11
|
+
egressLineageSchema,
|
|
12
|
+
legacyLocalOnlyEgressLineage,
|
|
13
|
+
} from "../core/egress-provenance";
|
|
9
14
|
import {
|
|
10
15
|
type AccountedCanonicalRecord,
|
|
11
16
|
adapterFailureMessage,
|
|
@@ -27,6 +32,7 @@ export interface RecordAdapterRunResult {
|
|
|
27
32
|
adapterId: string;
|
|
28
33
|
adapterVersion: string;
|
|
29
34
|
adapterFingerprint: string;
|
|
35
|
+
egressLineage: EgressLineage;
|
|
30
36
|
records: CanonicalRecord[];
|
|
31
37
|
failures: RecordAdapterFailure[];
|
|
32
38
|
failedRecordKeys: string[];
|
|
@@ -83,10 +89,14 @@ const closeIterator = async (
|
|
|
83
89
|
/** Consume one adapter snapshot while centrally enforcing every global cap. */
|
|
84
90
|
export async function runRecordAdapter(
|
|
85
91
|
adapter: RecordAdapter,
|
|
86
|
-
input: RecordAdapterInput
|
|
92
|
+
input: RecordAdapterInput,
|
|
93
|
+
options: { egressLineage?: EgressLineage } = {}
|
|
87
94
|
): Promise<RecordAdapterRunResult> {
|
|
88
95
|
const adapterIdentity = normalizeRecordAdapterIdentity(adapter);
|
|
89
96
|
const adapterFingerprint = recordAdapterFingerprint(adapter);
|
|
97
|
+
const egressLineage = egressLineageSchema.parse(
|
|
98
|
+
options.egressLineage ?? legacyLocalOnlyEgressLineage(input.collection)
|
|
99
|
+
);
|
|
90
100
|
const accepted = new Map<string, AccountedCanonicalRecord>();
|
|
91
101
|
const seenKeys = new Set<string>();
|
|
92
102
|
const failedRecordKeys = new Set<string>();
|
|
@@ -418,6 +428,7 @@ export async function runRecordAdapter(
|
|
|
418
428
|
adapterId: adapterIdentity.id,
|
|
419
429
|
adapterVersion: adapterIdentity.version,
|
|
420
430
|
adapterFingerprint,
|
|
431
|
+
egressLineage,
|
|
421
432
|
records: [...accepted.values()]
|
|
422
433
|
.sort((left, right) => left.recordKey.localeCompare(right.recordKey))
|
|
423
434
|
.map(
|
|
@@ -15,11 +15,13 @@ import type {
|
|
|
15
15
|
WalkEntry,
|
|
16
16
|
} from "./types";
|
|
17
17
|
|
|
18
|
+
import { resolveConfiguredEgressPolicy } from "../config/types";
|
|
18
19
|
import { DEFAULT_RECORD_ADAPTER_LIMITS } from "../converters/types";
|
|
19
20
|
import {
|
|
20
21
|
diffDocumentStructure,
|
|
21
22
|
extractDocumentStructure,
|
|
22
23
|
} from "../core/change-diff";
|
|
24
|
+
import { createEgressLineage } from "../core/egress-provenance";
|
|
23
25
|
import { normalizeTag, validateTag } from "../core/tags";
|
|
24
26
|
import { runRecordAdapter } from "./record-adapter";
|
|
25
27
|
import { recordVirtualPath } from "./record-path";
|
|
@@ -432,29 +434,42 @@ const recordProvenanceChanged = (
|
|
|
432
434
|
export async function processRecordContainer(
|
|
433
435
|
input: RecordContainerInput
|
|
434
436
|
): Promise<FileSyncResult> {
|
|
435
|
-
const
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
437
|
+
const effectiveEgress = resolveConfiguredEgressPolicy(input.collection);
|
|
438
|
+
const snapshot = await runRecordAdapter(
|
|
439
|
+
input.adapter,
|
|
440
|
+
{
|
|
441
|
+
sourcePath: input.entry.absPath,
|
|
442
|
+
relativePath: input.entry.relPath,
|
|
443
|
+
collection: input.collection.name,
|
|
444
|
+
mime: input.mime,
|
|
445
|
+
ext: input.ext,
|
|
446
|
+
open: (signal) => sourceStream(input.entry.absPath, signal),
|
|
447
|
+
limits: {
|
|
448
|
+
...DEFAULT_RECORD_ADAPTER_LIMITS,
|
|
449
|
+
timeoutMs: Math.min(
|
|
450
|
+
DEFAULT_RECORD_ADAPTER_LIMITS.timeoutMs ?? 60_000,
|
|
451
|
+
input.options.limits?.timeoutMs ?? Number.POSITIVE_INFINITY
|
|
452
|
+
),
|
|
453
|
+
maxSourceBytes: Math.min(
|
|
454
|
+
DEFAULT_RECORD_ADAPTER_LIMITS.maxSourceBytes,
|
|
455
|
+
input.options.limits?.maxBytes ?? Number.POSITIVE_INFINITY
|
|
456
|
+
),
|
|
457
|
+
maxTotalChars: Math.min(
|
|
458
|
+
DEFAULT_RECORD_ADAPTER_LIMITS.maxTotalChars,
|
|
459
|
+
input.options.limits?.maxOutputChars ?? Number.POSITIVE_INFINITY
|
|
460
|
+
),
|
|
461
|
+
},
|
|
456
462
|
},
|
|
457
|
-
|
|
463
|
+
{
|
|
464
|
+
egressLineage: createEgressLineage([
|
|
465
|
+
{
|
|
466
|
+
collection: input.collection.name,
|
|
467
|
+
policy: effectiveEgress.policy,
|
|
468
|
+
source: effectiveEgress.source,
|
|
469
|
+
},
|
|
470
|
+
]),
|
|
471
|
+
}
|
|
472
|
+
);
|
|
458
473
|
|
|
459
474
|
const reconcileAndApply = async (): Promise<AppliedRecordReconciliation> => {
|
|
460
475
|
const priorDocuments = mustOk(
|
|
@@ -615,6 +630,7 @@ export async function processRecordContainer(
|
|
|
615
630
|
adapterId: snapshot.adapterId,
|
|
616
631
|
adapterVersion: snapshot.adapterVersion,
|
|
617
632
|
adapterFingerprint: snapshot.adapterFingerprint,
|
|
633
|
+
egressLineage: snapshot.egressLineage,
|
|
618
634
|
snapshotState: snapshot.snapshotState,
|
|
619
635
|
authoritative: snapshot.authoritative,
|
|
620
636
|
stoppedByCap: snapshot.stoppedByCap,
|
package/src/ingestion/sync.ts
CHANGED
|
@@ -53,6 +53,7 @@ import {
|
|
|
53
53
|
normalizeRelationEdgeType,
|
|
54
54
|
normalizeRelationTarget,
|
|
55
55
|
} from "../core/change-diff";
|
|
56
|
+
import { enforceCollectionEgress } from "../core/egress-enforcement";
|
|
56
57
|
import {
|
|
57
58
|
normalizeMarkdownPath,
|
|
58
59
|
normalizeWikiName,
|
|
@@ -1598,10 +1599,24 @@ export class SyncService {
|
|
|
1598
1599
|
|
|
1599
1600
|
// 1. Run preflight commands
|
|
1600
1601
|
if (options.runUpdateCmd !== false && collection.updateCmd) {
|
|
1602
|
+
enforceCollectionEgress({
|
|
1603
|
+
collections: [collection],
|
|
1604
|
+
action: "export",
|
|
1605
|
+
destinationZone: "remote",
|
|
1606
|
+
caller: { authenticated: true, operationAuthorized: true },
|
|
1607
|
+
contentClass: "source",
|
|
1608
|
+
});
|
|
1601
1609
|
await runUpdateCmd(collection.path, collection.updateCmd);
|
|
1602
1610
|
}
|
|
1603
1611
|
|
|
1604
1612
|
if (options.gitPull && (await isGitRepo(collection.path))) {
|
|
1613
|
+
enforceCollectionEgress({
|
|
1614
|
+
collections: [collection],
|
|
1615
|
+
action: "export",
|
|
1616
|
+
destinationZone: "remote",
|
|
1617
|
+
caller: { authenticated: true, operationAuthorized: true },
|
|
1618
|
+
contentClass: "metadata",
|
|
1619
|
+
});
|
|
1605
1620
|
await gitPull(collection.path);
|
|
1606
1621
|
}
|
|
1607
1622
|
|
package/src/ingestion/types.ts
CHANGED
|
@@ -11,6 +11,7 @@ import type {
|
|
|
11
11
|
RecordAdapterFailure,
|
|
12
12
|
RecordAttachmentInventoryItem,
|
|
13
13
|
} from "../converters/types";
|
|
14
|
+
import type { EgressLineage } from "../core/egress-provenance";
|
|
14
15
|
|
|
15
16
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
16
17
|
// Walker Types
|
|
@@ -203,6 +204,7 @@ export interface FileSyncResult {
|
|
|
203
204
|
adapterId: string;
|
|
204
205
|
adapterVersion: string;
|
|
205
206
|
adapterFingerprint: string;
|
|
207
|
+
egressLineage?: EgressLineage;
|
|
206
208
|
snapshotState: "complete" | "partial";
|
|
207
209
|
authoritative: boolean;
|
|
208
210
|
stoppedByCap: boolean;
|
package/src/llm/errors.ts
CHANGED
|
@@ -18,6 +18,7 @@ export type LlmErrorCode =
|
|
|
18
18
|
| "INVALID_MODEL_FILE"
|
|
19
19
|
| "MODEL_DOWNLOAD_INTERCEPTED"
|
|
20
20
|
| "INFERENCE_FAILED"
|
|
21
|
+
| "EGRESS_DENIED"
|
|
21
22
|
| "TIMEOUT"
|
|
22
23
|
| "OUT_OF_MEMORY"
|
|
23
24
|
| "INVALID_URI"
|
|
@@ -202,6 +203,15 @@ export function inferenceFailedError(uri: string, cause?: unknown): LlmError {
|
|
|
202
203
|
});
|
|
203
204
|
}
|
|
204
205
|
|
|
206
|
+
export function egressDeniedInferenceError(cause?: unknown): LlmError {
|
|
207
|
+
return llmError("EGRESS_DENIED", {
|
|
208
|
+
message: "Operation blocked by collection egress policy",
|
|
209
|
+
retryable: false,
|
|
210
|
+
cause,
|
|
211
|
+
suggestion: "Review the collection egress policy and model destination.",
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
|
|
205
215
|
export function timeoutError(
|
|
206
216
|
uri: string,
|
|
207
217
|
operation: "load" | "inference",
|