@gmickel/gno 1.28.0 → 1.29.1

package/src/serve/routes/traces.ts

@@ -1,17 +1,20 @@
 /** Loopback REST handlers for private retrieval trace management. */
 
+import type { Collection } from "../../config/types";
 import type {
   RetrievalTraceExportRequest,
   RetrievalTraceLabelRequest,
 } from "../../core/retrieval-trace-management";
 import type { StoreError, StorePort, StoreResult } from "../../store/types";
 
+import { authorizeCurrentEgress } from "../../core/egress-authorization";
 import { RetrievalTraceManagementService } from "../../core/retrieval-trace-management";
 
 const errorStatus = (error: StoreError): number => {
   if (error.code === "NOT_FOUND") return 404;
   if (error.code === "CONSTRAINT_VIOLATION") return 409;
   if (error.code === "INVALID_INPUT") return 400;
+  if (error.code === "EGRESS_DENIED") return 403;
   return 500;
 };
 
@@ -135,14 +138,34 @@ export const handleTraceLabel = async (
 
 export const handleTraceExport = async (
   store: StorePort,
-  request: Request
+  request: Request,
+  collections: readonly Collection[] = []
 ): Promise<Response> => {
   const body = await parseJsonObject(request);
   if (body instanceof Response) return body;
   return response(
-    await new RetrievalTraceManagementService(store).export(
-      body as unknown as RetrievalTraceExportRequest
-    )
+    await new RetrievalTraceManagementService(store, {
+      authorizeExport: async (lineage) => {
+        const hostname = new URL(request.url).hostname;
+        const destinationZone = ["localhost", "127.0.0.1", "::1"].includes(
+          hostname
+        )
+          ? "local_process"
+          : "remote";
+        return authorizeCurrentEgress({
+          store,
+          config: { collections },
+          lineage,
+          action: "export",
+          destinationZone,
+          caller: {
+            authenticated: true,
+            operationAuthorized: true,
+          },
+          contentClass: "retrieval_trace",
+        });
+      },
+    }).export(body as unknown as RetrievalTraceExportRequest)
   );
 };
 

package/src/serve/server.ts

@@ -26,6 +26,8 @@ import {
   handleBrowseTree,
   handleCapabilities,
   handleClearCollectionEmbeddings,
+  handleCollectionEgressCheck,
+  handleCollectionEgressPolicy,
   handleCollections,
   handleConnectors,
   handleCreateFolder,
@@ -43,6 +45,11 @@ import {
   handleDuplicateDoc,
   handleEmbed,
   handleEmbedStatus,
+  handleEgressAuditDelete,
+  handleEgressAuditList,
+  handleEgressAuditPurge,
+  handleEgressAuditShow,
+  handleEgressAuditStatus,
   handleHealth,
   handleImportPreview,
   handleInstallConnector,
@@ -66,6 +73,7 @@ import {
   handleTags,
   handleTrashDoc,
   handleUpdateCollection,
+  handleUpdateCollectionEgressPolicy,
   handleUpdateDoc,
   handleVerifyConnector,
 } from "./routes/api";
@@ -304,7 +312,7 @@ export async function startServer(
             }
             return withSecurityHeaders(
               await handleResidentRead(runtime as ResidentRuntime, req, () =>
-                handleTraceExport(store, req)
+                handleTraceExport(store, req, ctxHolder.config.collections)
               ),
               isDev
             );
@@ -398,6 +406,53 @@ export async function startServer(
             );
           },
         },
+        "/api/egress/check": {
+          POST: async (req: Request) =>
+            withSecurityHeaders(
+              await handleCollectionEgressCheck(ctxHolder, req),
+              isDev
+            ),
+        },
+        "/api/egress/audits/status": {
+          GET: async () =>
+            withSecurityHeaders(await handleEgressAuditStatus(store), isDev),
+        },
+        "/api/egress/audits": {
+          GET: async (req: Request) =>
+            withSecurityHeaders(await handleEgressAuditList(store, req), isDev),
+          DELETE: async (req: Request) => {
+            if (!isRequestAllowed(req, port)) {
+              return withSecurityHeaders(forbiddenResponse(), isDev);
+            }
+            return withSecurityHeaders(
+              await handleEgressAuditPurge(store),
+              isDev
+            );
+          },
+        },
+        "/api/egress/audits/:auditId": {
+          GET: async (req: Request) => {
+            const auditId = decodeURIComponent(
+              new URL(req.url).pathname.split("/").at(-1) ?? ""
+            );
+            return withSecurityHeaders(
+              await handleEgressAuditShow(store, auditId),
+              isDev
+            );
+          },
+          DELETE: async (req: Request) => {
+            if (!isRequestAllowed(req, port)) {
+              return withSecurityHeaders(forbiddenResponse(), isDev);
+            }
+            const auditId = decodeURIComponent(
+              new URL(req.url).pathname.split("/").at(-1) ?? ""
+            );
+            return withSecurityHeaders(
+              await handleEgressAuditDelete(store, auditId),
+              isDev
+            );
+          },
+        },
         "/api/connectors": {
           GET: async (req: Request) =>
             withSecurityHeaders(
@@ -707,12 +762,42 @@ export async function startServer(
           },
         },
         "/api/events": {
-          GET: () =>
-            withSecurityHeaders(
-              runtime.eventBus?.createResponse() ??
-                new Response("event stream unavailable", { status: 503 }),
-              isDev
-            ),
+          GET: (req: Request) => {
+            const residentRuntime = runtime as ResidentRuntime;
+            const admitted = residentRuntime.admitRequest(req.signal);
+            if (!admitted || !residentRuntime.eventBus) {
+              admitted?.finish();
+              return withSecurityHeaders(
+                Response.json(
+                  {
+                    error: {
+                      code: "UNAVAILABLE",
+                      message: "Document event stream unavailable",
+                    },
+                  },
+                  { status: 503 }
+                ),
+                isDev
+              );
+            }
+            try {
+              return withSecurityHeaders(
+                residentRuntime.eventBus.createResponse({
+                  authorizationEpoch:
+                    admitted.authorizationEpoch ??
+                    residentRuntime.authorizationEpoch,
+                  isAuthorizationEpochCurrent: () =>
+                    admitted.isAuthorizationEpochCurrent?.() ?? true,
+                  onClose: () => admitted.finish(),
+                  signal: admitted.signal,
+                }),
+                isDev
+              );
+            } catch (error) {
+              admitted.finish();
+              throw error;
+            }
+          },
         },
         "/api/tags": {
           GET: async (req: Request) => {
@@ -893,6 +978,32 @@ export async function startServer(
             );
           },
         },
+        "/api/collections/:name/egress-policy": {
+          GET: (req: Request) => {
+            const parts = new URL(req.url).pathname.split("/");
+            const name = decodeURIComponent(parts.at(-2) ?? "");
+            return withSecurityHeaders(
+              handleCollectionEgressPolicy(ctxHolder, name),
+              isDev
+            );
+          },
+          PUT: async (req: Request) => {
+            if (!isRequestAllowed(req, port)) {
+              return withSecurityHeaders(forbiddenResponse(), isDev);
+            }
+            const parts = new URL(req.url).pathname.split("/");
+            const name = decodeURIComponent(parts.at(-2) ?? "");
+            return withSecurityHeaders(
+              await handleUpdateCollectionEgressPolicy(
+                ctxHolder,
+                store,
+                name,
+                req
+              ),
+              isDev
+            );
+          },
+        },
         "/api/collections/:name/embeddings/clear": {
           POST: async (req: Request) => {
             if (!isRequestAllowed(req, port)) {

package/src/store/migrations/023-collection-egress-policy.ts

@@ -0,0 +1,61 @@
+/**
+ * Add fail-closed collection egress policy state.
+ *
+ * Existing rows become local-only with legacy_default provenance. Config sync
+ * can later replace that value with config_default or an explicit user choice
+ * without affecting local retrieval/indexing.
+ */
+import type { Migration } from "./runner";
+
+export const migration: Migration = {
+  version: 23,
+  name: "collection_egress_policy",
+
+  up(db): void {
+    const columns = new Set(
+      db
+        .query<{ name: string }, []>("PRAGMA table_info(collections)")
+        .all()
+        .map((row) => row.name)
+    );
+    if (!columns.has("egress_policy")) {
+      db.exec(`
+        ALTER TABLE collections
+        ADD COLUMN egress_policy TEXT NOT NULL DEFAULT 'local_only'
+          CHECK (egress_policy IN ('local_only', 'lan', 'remote'))
+      `);
+    }
+    if (!columns.has("egress_policy_source")) {
+      db.exec(`
+        ALTER TABLE collections
+        ADD COLUMN egress_policy_source TEXT NOT NULL DEFAULT 'legacy_default'
+          CHECK (
+            egress_policy_source IN (
+              'explicit',
+              'config_default',
+              'legacy_default'
+            )
+          )
+          CHECK (
+            egress_policy_source = 'explicit'
+            OR egress_policy = 'local_only'
+          )
+      `);
+    }
+  },
+
+  down(db): void {
+    const columns = new Set(
+      db
+        .query<{ name: string }, []>("PRAGMA table_info(collections)")
+        .all()
+        .map((row) => row.name)
+    );
+    if (columns.has("egress_policy_source")) {
+      db.exec("ALTER TABLE collections DROP COLUMN egress_policy_source");
+    }
+    if (columns.has("egress_policy")) {
+      db.exec("ALTER TABLE collections DROP COLUMN egress_policy");
+    }
+  },
+};

package/src/store/migrations/024-egress-derived-lineage.ts

@@ -0,0 +1,121 @@
+/** Persist derived egress lineage and bounded redacted audit receipts. */
+
+import type { Database } from "bun:sqlite";
+
+import type { Migration } from "./runner";
+
+const LEGACY_LINEAGE_JSON =
+  '{"digest":"87b249b76459c91c172da6be6cbe3f93b61c44869eb196f3470ec36ebb50b8b0","effectivePolicy":"local_only","sources":[{"collection":"legacy","policy":"local_only","source":"legacy_default"}]}';
+const LEGACY_LINEAGE_DIGEST =
+  "87b249b76459c91c172da6be6cbe3f93b61c44869eb196f3470ec36ebb50b8b0";
+
+export const migration: Migration = {
+  version: 24,
+  name: "egress_derived_lineage",
+
+  up(db: Database): void {
+    const lineageBytes = new TextEncoder().encode(
+      LEGACY_LINEAGE_JSON
+    ).byteLength;
+    db.exec(`
+      ALTER TABLE retrieval_traces
+      ADD COLUMN effective_egress_policy TEXT NOT NULL DEFAULT 'local_only'
+        CHECK (effective_egress_policy IN ('local_only', 'lan', 'remote'));
+      ALTER TABLE retrieval_traces
+      ADD COLUMN egress_lineage_digest TEXT NOT NULL DEFAULT '${LEGACY_LINEAGE_DIGEST}'
+        CHECK (length(egress_lineage_digest) = 64 AND egress_lineage_digest NOT GLOB '*[^0-9a-f]*');
+      ALTER TABLE retrieval_traces
+      ADD COLUMN egress_lineage_json TEXT NOT NULL DEFAULT '${LEGACY_LINEAGE_JSON}';
+      ALTER TABLE retrieval_traces
+      ADD COLUMN egress_lineage_bytes INTEGER NOT NULL DEFAULT ${lineageBytes}
+        CHECK (egress_lineage_bytes > 0 AND egress_lineage_bytes <= 32768);
+      ALTER TABLE retrieval_traces
+      ADD COLUMN creation_digest_version INTEGER NOT NULL DEFAULT 0
+        CHECK (creation_digest_version IN (0, 1));
+
+      ALTER TABLE retrieval_trace_exports
+      ADD COLUMN effective_egress_policy TEXT NOT NULL DEFAULT 'local_only'
+        CHECK (effective_egress_policy IN ('local_only', 'lan', 'remote'));
+      ALTER TABLE retrieval_trace_exports
+      ADD COLUMN egress_lineage_digest TEXT NOT NULL DEFAULT '${LEGACY_LINEAGE_DIGEST}'
+        CHECK (length(egress_lineage_digest) = 64 AND egress_lineage_digest NOT GLOB '*[^0-9a-f]*');
+      ALTER TABLE retrieval_trace_exports
+      ADD COLUMN egress_lineage_json TEXT NOT NULL DEFAULT '${LEGACY_LINEAGE_JSON}';
+      ALTER TABLE retrieval_trace_exports
+      ADD COLUMN egress_lineage_bytes INTEGER NOT NULL DEFAULT ${lineageBytes}
+        CHECK (egress_lineage_bytes > 0 AND egress_lineage_bytes <= 32768);
+
+      ALTER TABLE document_changes
+      ADD COLUMN effective_egress_policy TEXT NOT NULL DEFAULT 'local_only'
+        CHECK (effective_egress_policy IN ('local_only', 'lan', 'remote'));
+      ALTER TABLE document_changes
+      ADD COLUMN egress_lineage_digest TEXT NOT NULL DEFAULT '${LEGACY_LINEAGE_DIGEST}'
+        CHECK (length(egress_lineage_digest) = 64 AND egress_lineage_digest NOT GLOB '*[^0-9a-f]*');
+      ALTER TABLE document_changes
+      ADD COLUMN egress_lineage_json TEXT NOT NULL DEFAULT '${LEGACY_LINEAGE_JSON}';
+      ALTER TABLE document_changes
+      ADD COLUMN egress_lineage_bytes INTEGER NOT NULL DEFAULT ${lineageBytes}
+        CHECK (egress_lineage_bytes > 0 AND egress_lineage_bytes <= 32768);
+
+      UPDATE document_changes
+      SET byte_size = byte_size + egress_lineage_bytes;
+      UPDATE document_change_journal_state
+      SET retained_bytes = (
+        SELECT COALESCE(SUM(byte_size), 0) FROM document_changes
+      );
+
+      CREATE TABLE egress_audit_receipts (
+        audit_id TEXT PRIMARY KEY,
+        decision TEXT NOT NULL CHECK (decision IN ('allow', 'deny')),
+        action TEXT NOT NULL
+          CHECK (action IN ('retrieve', 'serve', 'publish', 'remote_inference', 'export', 'clip_write')),
+        destination_zone TEXT NOT NULL
+          CHECK (destination_zone IN ('local_process', 'loopback', 'lan', 'remote')),
+        content_class TEXT NOT NULL
+          CHECK (content_class IN ('source', 'snippet', 'metadata', 'attachment', 'embedding', 'capsule', 'audit_log', 'retrieval_trace')),
+        effective_policy TEXT NOT NULL
+          CHECK (effective_policy IN ('local_only', 'lan', 'remote')),
+        reason_code TEXT NOT NULL,
+        lineage_digest TEXT NOT NULL
+          CHECK (length(lineage_digest) = 64 AND lineage_digest NOT GLOB '*[^0-9a-f]*'),
+        created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0),
+        expires_at_ms INTEGER NOT NULL CHECK (expires_at_ms > created_at_ms),
+        byte_size INTEGER NOT NULL CHECK (byte_size > 0 AND byte_size <= 4096),
+        CHECK (length(audit_id) BETWEEN 1 AND 128),
+        CHECK (length(reason_code) BETWEEN 1 AND 64)
+      );
+
+      CREATE INDEX idx_egress_audit_retention
+      ON egress_audit_receipts(expires_at_ms, created_at_ms, audit_id);
+    `);
+  },
+
+  down(db: Database): void {
+    db.exec(`
+      DROP TABLE egress_audit_receipts;
+
+      UPDATE document_changes
+      SET byte_size = byte_size - egress_lineage_bytes;
+      UPDATE document_change_journal_state
+      SET retained_bytes = (
+        SELECT COALESCE(SUM(byte_size), 0) FROM document_changes
+      );
+
+      ALTER TABLE document_changes DROP COLUMN egress_lineage_bytes;
+      ALTER TABLE document_changes DROP COLUMN egress_lineage_json;
+      ALTER TABLE document_changes DROP COLUMN egress_lineage_digest;
+      ALTER TABLE document_changes DROP COLUMN effective_egress_policy;
+
+      ALTER TABLE retrieval_trace_exports DROP COLUMN egress_lineage_bytes;
+      ALTER TABLE retrieval_trace_exports DROP COLUMN egress_lineage_json;
+      ALTER TABLE retrieval_trace_exports DROP COLUMN egress_lineage_digest;
+      ALTER TABLE retrieval_trace_exports DROP COLUMN effective_egress_policy;
+
+      ALTER TABLE retrieval_traces DROP COLUMN egress_lineage_bytes;
+      ALTER TABLE retrieval_traces DROP COLUMN egress_lineage_json;
+      ALTER TABLE retrieval_traces DROP COLUMN egress_lineage_digest;
+      ALTER TABLE retrieval_traces DROP COLUMN effective_egress_policy;
+      ALTER TABLE retrieval_traces DROP COLUMN creation_digest_version;
+    `);
+  },
+};

package/src/store/migrations/025-collection-egress-policy-revision.ts

@@ -0,0 +1,40 @@
+/**
+ * Add a durable monotonic collection egress policy revision.
+ *
+ * Legacy rows start at zero. Canonical config mutations advance the revision
+ * whenever the effective policy or its provenance changes.
+ */
+import type { Migration } from "./runner";
+
+export const migration: Migration = {
+  version: 25,
+  name: "collection_egress_policy_revision",
+
+  up(db): void {
+    const columns = new Set(
+      db
+        .query<{ name: string }, []>("PRAGMA table_info(collections)")
+        .all()
+        .map((row) => row.name)
+    );
+    if (!columns.has("egress_policy_revision")) {
+      db.exec(`
+        ALTER TABLE collections
+        ADD COLUMN egress_policy_revision INTEGER NOT NULL DEFAULT 0
+          CHECK (egress_policy_revision >= 0)
+      `);
+    }
+  },
+
+  down(db): void {
+    const columns = new Set(
+      db
+        .query<{ name: string }, []>("PRAGMA table_info(collections)")
+        .all()
+        .map((row) => row.name)
+    );
+    if (columns.has("egress_policy_revision")) {
+      db.exec("ALTER TABLE collections DROP COLUMN egress_policy_revision");
+    }
+  },
+};

package/src/store/migrations/index.ts

@@ -36,6 +36,9 @@ import { migration as m019 } from "./019-saved-capsule-registration-generation";
 import { migration as m020 } from "./020-browser-clipper-security";
 import { migration as m021 } from "./021-multi-context-identity";
 import { migration as m022 } from "./022-record-export-lineage";
+import { migration as m023 } from "./023-collection-egress-policy";
+import { migration as m024 } from "./024-egress-derived-lineage";
+import { migration as m025 } from "./025-collection-egress-policy-revision";
 
 /** All migrations in order */
 export const migrations = [
@@ -61,4 +64,7 @@ export const migrations = [
   m020,
   m021,
   m022,
+  m023,
+  m024,
+  m025,
 ];

package/src/store/retrieval-trace-codec.ts

@@ -10,6 +10,7 @@ import type {
   RetrievalTraceRunInput,
 } from "./types";
 
+import { egressLineageSchema } from "../core/egress-provenance";
 const MAX_ID_LENGTH = 128;
 const MAX_IDEMPOTENCY_KEY_LENGTH = 256;
 const sha256Schema = z.string().regex(/^[a-f0-9]{64}$/);
@@ -134,6 +135,7 @@ const traceInputBaseSchema = z
     goalDigest: sha256Schema.nullable(),
     goalShape: queryShapeSchema,
     filters: traceJsonObjectSchema,
+    egressLineage: egressLineageSchema,
     fingerprints: z
       .object({
         pipeline: sha256Schema,
@@ -362,6 +364,19 @@ export const hashRetrievalTraceCreation = (
   return hashTraceCanonical(immutable);
 };
 
+/** Exact pre-v24 creation projection, used only for marked migrated rows. */
+export const hashLegacyRetrievalTraceCreation = (
+  trace: RetrievalTraceInput
+): string => {
+  const {
+    status: _status,
+    updatedAtMs: _updatedAtMs,
+    egressLineage: _egressLineage,
+    ...immutable
+  } = trace;
+  return hashTraceCanonical(immutable);
+};
+
 export const parseRetrievalTraceInput = (
   input: RetrievalTraceInput
 ): RetrievalTraceInput => retrievalTraceInputSchema.parse(input);