@gmickel/gno 1.18.0 → 1.20.0

package/src/store/sqlite/retrieval-trace-management-store.ts

@@ -0,0 +1,341 @@
+/** SQLite primitives used by local retrieval-trace management surfaces. */
+
+import type { Database } from "bun:sqlite";
+
+import type {
+  RetrievalTraceAppendResult,
+  RetrievalTraceBoundedBundle,
+  RetrievalTraceExportBundle,
+  RetrievalTraceExportManifestInput,
+  RetrievalTraceExportManifestRow,
+  StoreResult,
+} from "../types";
+
+import { ok } from "../types";
+import {
+  type DbRetrievalTraceEventRow,
+  type DbRetrievalTraceExportRow,
+  type DbRetrievalTraceJudgmentRow,
+  type DbRetrievalTraceRow,
+  type DbRetrievalTraceRunRow,
+  mapEventRow,
+  mapExportRow,
+  mapJudgmentRow,
+  mapRunRow,
+  mapTraceRow,
+  RetrievalTraceConflictError,
+  traceReadError,
+  traceWriteError,
+  validateTraceId,
+} from "./retrieval-trace-rows";
+import { getTrace } from "./retrieval-trace-store";
+
+const SHA256_PATTERN = /^[a-f0-9]{64}$/;
+const REDACTION_SECRET_KEY = "retrieval_trace_redaction_secret_v1";
+
+const randomSecret = (): string => {
+  const bytes = crypto.getRandomValues(new Uint8Array(32));
+  return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(
+    ""
+  );
+};
+
+const countForTrace = (db: Database, table: string, traceId: string): number =>
+  db
+    .query<{ count: number }, [string]>(
+      `SELECT COUNT(*) AS count FROM ${table} WHERE trace_id = ?`
+    )
+    .get(traceId)?.count ?? 0;
+
+export const getBoundedTrace = (
+  db: Database,
+  traceId: string,
+  detailLimit: number
+): StoreResult<RetrievalTraceBoundedBundle | null> => {
+  try {
+    validateTraceId(traceId, "traceId");
+    if (
+      !Number.isSafeInteger(detailLimit) ||
+      detailLimit < 1 ||
+      detailLimit > 10_000
+    ) {
+      throw new RangeError("Trace detail limit must be from 1 to 10000");
+    }
+    const transaction = db.transaction(
+      (): RetrievalTraceBoundedBundle | null => {
+        const trace = db
+          .query<DbRetrievalTraceRow, [string]>(
+            "SELECT * FROM retrieval_traces WHERE trace_id = ?"
+          )
+          .get(traceId);
+        if (!trace) return null;
+        const runs = db
+          .query<DbRetrievalTraceRunRow, [string, number]>(
+            `SELECT * FROM retrieval_trace_runs WHERE trace_id = ?
+             ORDER BY created_at_ms, run_id LIMIT ?`
+          )
+          .all(traceId, detailLimit);
+        const events = db
+          .query<DbRetrievalTraceEventRow, [string, number]>(
+            `SELECT * FROM retrieval_trace_events WHERE trace_id = ?
+             ORDER BY created_at_ms, event_id LIMIT ?`
+          )
+          .all(traceId, detailLimit);
+        const judgments = db
+          .query<DbRetrievalTraceJudgmentRow, [string, number]>(
+            `SELECT * FROM retrieval_trace_judgments WHERE trace_id = ?
+             ORDER BY created_at_ms, judgment_id LIMIT ?`
+          )
+          .all(traceId, detailLimit);
+        const exports = db
+          .query<DbRetrievalTraceExportRow, [string, number]>(
+            `SELECT x.*, et.trace_id FROM retrieval_trace_exports x
+             JOIN retrieval_trace_export_traces et USING (export_id)
+             WHERE et.trace_id = ?
+             ORDER BY x.created_at_ms, x.export_id LIMIT ?`
+          )
+          .all(traceId, detailLimit);
+        return {
+          bundle: {
+            trace: mapTraceRow(trace),
+            runs: runs.map(mapRunRow),
+            events: events.map(mapEventRow),
+            judgments: judgments.map(mapJudgmentRow),
+            exports: exports.map(mapExportRow),
+          },
+          totals: {
+            runs: countForTrace(db, "retrieval_trace_runs", traceId),
+            events: countForTrace(db, "retrieval_trace_events", traceId),
+            judgments: countForTrace(db, "retrieval_trace_judgments", traceId),
+            exports: countForTrace(
+              db,
+              "retrieval_trace_export_traces",
+              traceId
+            ),
+          },
+        };
+      }
+    );
+    return ok(transaction());
+  } catch (cause) {
+    return traceReadError(cause, "Failed to get bounded retrieval trace");
+  }
+};
+
+const normalizeManifest = (
+  input: RetrievalTraceExportManifestInput
+): RetrievalTraceExportManifestInput => {
+  validateTraceId(input.exportId, "exportId");
+  if (!["agentic-receipt", "qrels"].includes(input.format)) {
+    throw new RangeError("Invalid retrieval trace export format");
+  }
+  if (!SHA256_PATTERN.test(input.artifactHash)) {
+    throw new RangeError("artifactHash must be a lowercase SHA-256 digest");
+  }
+  if (!Number.isSafeInteger(input.createdAtMs) || input.createdAtMs < 0) {
+    throw new RangeError("createdAtMs must be epoch milliseconds");
+  }
+  if (input.traceIds.length < 1 || input.traceIds.length > 10_000) {
+    throw new RangeError("Export manifests require from 1 to 10000 traces");
+  }
+  const traceIds = [...new Set(input.traceIds)].sort();
+  for (const traceId of traceIds) validateTraceId(traceId, "traceId");
+  return { ...input, traceIds };
+};
+
+const readManifest = (
+  db: Database,
+  exportId: string
+): RetrievalTraceExportManifestRow | null => {
+  const row = db
+    .query<
+      {
+        export_id: string;
+        format: RetrievalTraceExportManifestRow["format"];
+        artifact_hash: string;
+        created_at_ms: number;
+      },
+      [string]
+    >(
+      `SELECT export_id, format, artifact_hash, created_at_ms
+       FROM retrieval_trace_exports WHERE export_id = ?`
+    )
+    .get(exportId);
+  if (!row) return null;
+  const traceIds = db
+    .query<{ trace_id: string }, [string]>(
+      `SELECT trace_id FROM retrieval_trace_export_traces
+       WHERE export_id = ? ORDER BY trace_id`
+    )
+    .all(exportId)
+    .map(({ trace_id }) => trace_id);
+  return {
+    exportId: row.export_id,
+    traceIds,
+    format: row.format,
+    artifactHash: row.artifact_hash,
+    createdAtMs: row.created_at_ms,
+  };
+};
+
+export const getOrCreateRedactionSecret = (
+  db: Database
+): StoreResult<string> => {
+  try {
+    const candidate = randomSecret();
+    const transaction = db.transaction((): string => {
+      db.run(
+        `INSERT OR IGNORE INTO schema_meta (key, value, updated_at)
+         VALUES (?, ?, datetime('now'))`,
+        [REDACTION_SECRET_KEY, candidate]
+      );
+      const stored = db
+        .query<{ value: string }, [string]>(
+          "SELECT value FROM schema_meta WHERE key = ?"
+        )
+        .get(REDACTION_SECRET_KEY)?.value;
+      if (!stored || !SHA256_PATTERN.test(stored)) {
+        throw new RetrievalTraceConflictError(
+          "Invalid retrieval trace redaction secret"
+        );
+      }
+      return stored;
+    });
+    return ok(transaction());
+  } catch (cause) {
+    return traceWriteError(
+      cause,
+      "Failed to load retrieval trace redaction secret"
+    );
+  }
+};
+
+export const appendExportManifest = (
+  db: Database,
+  input: RetrievalTraceExportManifestInput
+): StoreResult<RetrievalTraceAppendResult> => {
+  try {
+    const manifest = normalizeManifest(input);
+    const transaction = db.transaction((): RetrievalTraceAppendResult => {
+      const hashOwner = db
+        .query<{ export_id: string }, [string, string]>(
+          `SELECT export_id FROM retrieval_trace_exports
+           WHERE format = ? AND artifact_hash = ?`
+        )
+        .get(manifest.format, manifest.artifactHash)?.export_id;
+      if (hashOwner && hashOwner !== manifest.exportId) {
+        throw new RetrievalTraceConflictError(
+          "Export artifact already belongs to a different manifest"
+        );
+      }
+      for (const traceId of manifest.traceIds) {
+        const trace = db
+          .query<{ status: string }, [string]>(
+            "SELECT status FROM retrieval_traces WHERE trace_id = ?"
+          )
+          .get(traceId);
+        if (!trace) {
+          throw new RetrievalTraceConflictError(
+            "Export manifest references a missing retrieval trace"
+          );
+        }
+        if (trace.status === "open") {
+          throw new RetrievalTraceConflictError(
+            "Open retrieval traces cannot be exported"
+          );
+        }
+      }
+      const inserted = db.run(
+        `INSERT OR IGNORE INTO retrieval_trace_exports
+         (export_id, format, artifact_hash, created_at_ms) VALUES (?, ?, ?, ?)`,
+        [
+          manifest.exportId,
+          manifest.format,
+          manifest.artifactHash,
+          manifest.createdAtMs,
+        ]
+      );
+      const existing = readManifest(db, manifest.exportId);
+      if (
+        !existing ||
+        existing.format !== manifest.format ||
+        existing.artifactHash !== manifest.artifactHash
+      ) {
+        throw new RetrievalTraceConflictError(
+          "Export ID already exists with different content"
+        );
+      }
+      for (const traceId of manifest.traceIds) {
+        db.run(
+          `INSERT OR IGNORE INTO retrieval_trace_export_traces
+           (export_id, trace_id) VALUES (?, ?)`,
+          [manifest.exportId, traceId]
+        );
+      }
+      const stored = readManifest(db, manifest.exportId);
+      if (
+        !stored ||
+        stored.traceIds.length !== manifest.traceIds.length ||
+        stored.traceIds.some(
+          (traceId, index) => traceId !== manifest.traceIds[index]
+        )
+      ) {
+        throw new RetrievalTraceConflictError(
+          "Export manifest membership conflicts with stored content"
+        );
+      }
+      return inserted.changes > 0 ? "inserted" : "duplicate";
+    });
+    return ok(transaction());
+  } catch (cause) {
+    return traceWriteError(
+      cause,
+      "Failed to append retrieval trace export manifest"
+    );
+  }
+};
+
+export const getExportManifest = (
+  db: Database,
+  exportId: string
+): StoreResult<RetrievalTraceExportManifestRow | null> => {
+  try {
+    validateTraceId(exportId, "exportId");
+    return ok(readManifest(db, exportId));
+  } catch (cause) {
+    return traceReadError(cause, "Failed to get retrieval trace export");
+  }
+};
+
+export const getExportBundle = (
+  db: Database,
+  exportId: string
+): StoreResult<RetrievalTraceExportBundle | null> => {
+  try {
+    validateTraceId(exportId, "exportId");
+    const transaction = db.transaction(
+      (): StoreResult<RetrievalTraceExportBundle | null> => {
+        const manifest = readManifest(db, exportId);
+        if (!manifest) return ok(null);
+        const traces = [];
+        for (const traceId of manifest.traceIds) {
+          const trace = getTrace(db, traceId);
+          if (!trace.ok) return trace;
+          if (!trace.value) {
+            throw new RetrievalTraceConflictError(
+              "Export manifest references a missing retrieval trace"
+            );
+          }
+          traces.push(trace.value);
+        }
+        return ok({ manifest, traces });
+      }
+    );
+    return transaction();
+  } catch (cause) {
+    return traceReadError(
+      cause,
+      "Failed to get complete retrieval trace export"
+    );
+  }
+};

package/src/store/sqlite/retrieval-trace-retention.ts

@@ -0,0 +1,349 @@
+/** Secure deletion, full purge, and deterministic trace retention for SQLite. */
+
+import type { Database } from "bun:sqlite";
+
+import type {
+  RetrievalTraceDeleteCounts,
+  RetrievalTracePurgeResult,
+  RetrievalTraceRetentionPolicy,
+  RetrievalTraceRetentionResult,
+  StoreResult,
+} from "../types";
+
+import { ok } from "../types";
+import {
+  countTraceContent,
+  traceWriteError,
+  validateTraceId,
+} from "./retrieval-trace-rows";
+
+const DAY_MS = 86_400_000;
+const compareCodeUnits = (left: string, right: string): number =>
+  left < right ? -1 : left > right ? 1 : 0;
+
+interface StorageRow {
+  trace_id: string;
+  created_at_ms: number;
+  expires_at_ms: number;
+  total_bytes: number;
+  record_count: number;
+}
+
+const storageRows = (db: Database): StorageRow[] =>
+  db
+    .query<StorageRow, []>(
+      `SELECT
+         t.trace_id,
+         t.created_at_ms,
+         t.expires_at_ms,
+         (
+           length(CAST(t.trace_id AS BLOB))
+           + length(CAST(t.schema_version AS BLOB))
+           + length(CAST(t.redaction_mode AS BLOB))
+           + length(CAST(COALESCE(t.query_text, '') AS BLOB))
+           + length(CAST(COALESCE(t.query_digest, '') AS BLOB))
+           + length(CAST(t.query_shape_json AS BLOB))
+           + length(CAST(COALESCE(t.goal_text, '') AS BLOB))
+           + length(CAST(COALESCE(t.goal_digest, '') AS BLOB))
+           + length(CAST(t.goal_shape_json AS BLOB))
+           + length(CAST(t.filters_json AS BLOB))
+           + length(CAST(t.pipeline_fingerprint AS BLOB))
+           + length(CAST(t.model_fingerprint AS BLOB))
+           + length(CAST(t.config_fingerprint AS BLOB))
+           + length(CAST(t.index_fingerprint AS BLOB))
+           + length(CAST(t.status AS BLOB))
+           + length(CAST(t.creation_digest AS BLOB))
+           + 40
+         )
+           + COALESCE((
+               SELECT SUM(
+                 length(CAST(r.run_id AS BLOB))
+                   + length(CAST(r.trace_id AS BLOB))
+                   + length(CAST(r.idempotency_key AS BLOB))
+                   + length(CAST(r.kind AS BLOB))
+                   + r.payload_bytes
+                   + length(CAST(r.canonical_digest AS BLOB))
+                   + 8
+               ) FROM retrieval_trace_runs r
+               WHERE r.trace_id = t.trace_id
+             ), 0)
+           + COALESCE((
+               SELECT SUM(
+                 length(CAST(e.event_id AS BLOB))
+                   + length(CAST(e.trace_id AS BLOB))
+                   + length(CAST(COALESCE(e.run_id, '') AS BLOB))
+                   + length(CAST(e.idempotency_key AS BLOB))
+                   + length(CAST(e.kind AS BLOB))
+                   + e.payload_bytes
+                   + length(CAST(e.canonical_digest AS BLOB))
+                   + 8
+               ) FROM retrieval_trace_events e
+               WHERE e.trace_id = t.trace_id
+             ), 0)
+           + COALESCE((
+               SELECT SUM(
+                 length(CAST(j.judgment_id AS BLOB))
+                   + length(CAST(j.trace_id AS BLOB))
+                   + length(CAST(COALESCE(j.run_id, '') AS BLOB))
+                   + length(CAST(j.idempotency_key AS BLOB))
+                   + length(CAST(j.label AS BLOB))
+                   + length(CAST(j.target_kind AS BLOB))
+                   + length(CAST(j.target_ref AS BLOB))
+                   + j.target_bytes
+                   + length(CAST(j.canonical_digest AS BLOB))
+                   + 8
+               ) FROM retrieval_trace_judgments j
+               WHERE j.trace_id = t.trace_id
+             ), 0)
+           + COALESCE((
+               SELECT SUM(
+                 length(CAST(x.export_id AS BLOB))
+                   + length(CAST(et.trace_id AS BLOB))
+                   + length(CAST(x.format AS BLOB))
+                   + length(CAST(x.artifact_hash AS BLOB))
+                   + 8
+               )
+               FROM retrieval_trace_export_traces et
+               JOIN retrieval_trace_exports x USING (export_id)
+               WHERE et.trace_id = t.trace_id
+             ), 0) AS total_bytes,
+         (
+           SELECT COUNT(*) FROM retrieval_trace_runs r
+           WHERE r.trace_id = t.trace_id
+         ) + (
+           SELECT COUNT(*) FROM retrieval_trace_events e
+           WHERE e.trace_id = t.trace_id
+         ) + (
+           SELECT COUNT(*) FROM retrieval_trace_judgments j
+           WHERE j.trace_id = t.trace_id
+         ) + (
+           SELECT COUNT(*) FROM retrieval_trace_export_traces et
+           WHERE et.trace_id = t.trace_id
+         ) AS record_count
+       FROM retrieval_traces t
+       ORDER BY t.created_at_ms ASC, t.trace_id ASC`
+    )
+    .all();
+
+const validatePolicy = (
+  policy: RetrievalTraceRetentionPolicy,
+  nowMs: number
+): void => {
+  if (
+    !Number.isSafeInteger(nowMs) ||
+    nowMs < 0 ||
+    !Number.isSafeInteger(policy.maxAgeDays) ||
+    policy.maxAgeDays < 1 ||
+    policy.maxAgeDays > 3650 ||
+    !Number.isSafeInteger(policy.maxTraces) ||
+    policy.maxTraces < 1 ||
+    policy.maxTraces > 1_000_000 ||
+    !Number.isSafeInteger(policy.maxRecordsPerTrace) ||
+    policy.maxRecordsPerTrace < 1 ||
+    policy.maxRecordsPerTrace > 100_000 ||
+    !Number.isSafeInteger(policy.maxBytes) ||
+    policy.maxBytes < 65_536 ||
+    policy.maxBytes > 1024 * 1024 * 1024
+  ) {
+    throw new RangeError("Invalid retrieval trace retention policy");
+  }
+};
+
+const subtractCounts = (
+  before: RetrievalTraceDeleteCounts,
+  after: RetrievalTraceDeleteCounts
+): RetrievalTraceDeleteCounts => ({
+  traces: before.traces - after.traces,
+  runs: before.runs - after.runs,
+  events: before.events - after.events,
+  judgments: before.judgments - after.judgments,
+  exports: before.exports - after.exports,
+  exportLinks: before.exportLinks - after.exportLinks,
+});
+
+const readSecureDelete = (db: Database): number =>
+  db.query<{ secure_delete: number }, []>("PRAGMA secure_delete").get()
+    ?.secure_delete ?? 0;
+
+const restoreSecureDelete = (db: Database, prior: number): void => {
+  const mode = prior === 0 ? "OFF" : prior === 2 ? "FAST" : "ON";
+  db.exec(`PRAGMA secure_delete = ${mode}`);
+};
+
+export const deleteTrace = (
+  db: Database,
+  traceId: string
+): StoreResult<RetrievalTraceDeleteCounts> => {
+  const prior = readSecureDelete(db);
+  try {
+    validateTraceId(traceId, "traceId");
+    db.exec("PRAGMA secure_delete = ON");
+    const transaction = db.transaction(() => {
+      const counts = countTraceContent(db, traceId);
+      db.run("DELETE FROM retrieval_traces WHERE trace_id = ?", [traceId]);
+      return counts;
+    });
+    return ok(transaction());
+  } catch (cause) {
+    return traceWriteError(cause, "Failed to delete retrieval trace");
+  } finally {
+    restoreSecureDelete(db, prior);
+  }
+};
+
+export const purgeTraces = (
+  db: Database
+): StoreResult<RetrievalTracePurgeResult> => {
+  const prior = readSecureDelete(db);
+  try {
+    db.exec("PRAGMA secure_delete = ON");
+    const transaction = db.transaction(() => {
+      const counts = countTraceContent(db);
+      db.run("DELETE FROM retrieval_traces");
+      db.run(
+        `DELETE FROM retrieval_trace_exports WHERE NOT EXISTS (
+           SELECT 1 FROM retrieval_trace_export_traces et
+           WHERE et.export_id = retrieval_trace_exports.export_id
+         )`
+      );
+      return counts;
+    });
+    const counts = transaction();
+    let checkpoint:
+      | { busy: number; log: number; checkpointed: number }
+      | undefined;
+    try {
+      checkpoint =
+        db
+          .query<{ busy: number; log: number; checkpointed: number }, []>(
+            "PRAGMA wal_checkpoint(TRUNCATE)"
+          )
+          .get() ?? undefined;
+    } catch {
+      return ok({
+        ...counts,
+        physicalCleanup: "failed",
+        checkpointedFrames: 0,
+        remainingWalFrames: -1,
+      });
+    }
+    const physicalCleanup =
+      checkpoint?.busy === 0 && (checkpoint.log === 0 || checkpoint.log === -1)
+        ? "completed"
+        : "wal_busy";
+    return ok({
+      ...counts,
+      physicalCleanup,
+      checkpointedFrames: checkpoint?.checkpointed ?? 0,
+      remainingWalFrames: checkpoint?.log ?? -1,
+    });
+  } catch (cause) {
+    return traceWriteError(cause, "Failed to purge retrieval traces");
+  } finally {
+    restoreSecureDelete(db, prior);
+  }
+};
+
+export const enforceRetention = (
+  db: Database,
+  policy: RetrievalTraceRetentionPolicy,
+  nowMs: number
+): StoreResult<RetrievalTraceRetentionResult> => {
+  const prior = readSecureDelete(db);
+  try {
+    validatePolicy(policy, nowMs);
+    db.exec("PRAGMA secure_delete = ON");
+    db.exec("BEGIN IMMEDIATE");
+    try {
+      const before = countTraceContent(db);
+      const rows = storageRows(db);
+      const ageMs = policy.maxAgeDays * DAY_MS;
+      const byExpiry = [...rows].sort((left, right) => {
+        const leftExpiry = Math.min(
+          left.expires_at_ms,
+          left.created_at_ms + ageMs
+        );
+        const rightExpiry = Math.min(
+          right.expires_at_ms,
+          right.created_at_ms + ageMs
+        );
+        return (
+          leftExpiry - rightExpiry ||
+          left.created_at_ms - right.created_at_ms ||
+          compareCodeUnits(left.trace_id, right.trace_id)
+        );
+      });
+      const deleted = new Set<string>();
+      const deletedTraceIds: string[] = [];
+      let remainingTraces = rows.length;
+      let remainingBytes = rows.reduce(
+        (total, row) => total + row.total_bytes,
+        0
+      );
+
+      for (const row of byExpiry) {
+        const effectiveExpiry = Math.min(
+          row.expires_at_ms,
+          row.created_at_ms + ageMs
+        );
+        if (effectiveExpiry > nowMs) break;
+        db.run("DELETE FROM retrieval_traces WHERE trace_id = ?", [
+          row.trace_id,
+        ]);
+        deleted.add(row.trace_id);
+        deletedTraceIds.push(row.trace_id);
+        remainingTraces -= 1;
+        remainingBytes -= row.total_bytes;
+      }
+
+      for (const row of rows) {
+        if (
+          deleted.has(row.trace_id) ||
+          row.record_count <= policy.maxRecordsPerTrace
+        ) {
+          continue;
+        }
+        db.run("DELETE FROM retrieval_traces WHERE trace_id = ?", [
+          row.trace_id,
+        ]);
+        deleted.add(row.trace_id);
+        deletedTraceIds.push(row.trace_id);
+        remainingTraces -= 1;
+        remainingBytes -= row.total_bytes;
+      }
+
+      for (const row of rows) {
+        if (
+          remainingTraces <= policy.maxTraces &&
+          remainingBytes <= policy.maxBytes
+        ) {
+          break;
+        }
+        if (deleted.has(row.trace_id)) continue;
+        db.run("DELETE FROM retrieval_traces WHERE trace_id = ?", [
+          row.trace_id,
+        ]);
+        deleted.add(row.trace_id);
+        deletedTraceIds.push(row.trace_id);
+        remainingTraces -= 1;
+        remainingBytes -= row.total_bytes;
+      }
+
+      const after = countTraceContent(db);
+      db.exec("COMMIT");
+      return ok({
+        deleted: subtractCounts(before, after),
+        deletedTraceIds,
+        remainingTraces,
+        remainingBytes,
+      });
+    } catch (cause) {
+      db.exec("ROLLBACK");
+      throw cause;
+    }
+  } catch (cause) {
+    return traceWriteError(cause, "Failed to enforce trace retention");
+  } finally {
+    restoreSecureDelete(db, prior);
+  }
+};