@gmickel/gno 1.17.0 → 1.19.0

package/src/store/retrieval-trace-codec.ts

@@ -0,0 +1,384 @@
+/** Runtime validation and deterministic encoding for local retrieval receipts. */
+
+import { z } from "zod";
+
+import type {
+  RetrievalTraceEventInput,
+  RetrievalTraceExportInput,
+  RetrievalTraceInput,
+  RetrievalTraceJudgmentInput,
+  RetrievalTraceRunInput,
+} from "./types";
+
+const MAX_ID_LENGTH = 128;
+const MAX_IDEMPOTENCY_KEY_LENGTH = 256;
+const sha256Schema = z.string().regex(/^[a-f0-9]{64}$/);
+const idSchema = z.string().min(1).max(MAX_ID_LENGTH);
+const idempotencyKeySchema = z.string().min(1).max(MAX_IDEMPOTENCY_KEY_LENGTH);
+const epochMsSchema = z.number().int().safe().nonnegative();
+const DANGEROUS_JSON_KEYS = new Set(["__proto__", "constructor", "prototype"]);
+
+const jsonValueSchema: z.ZodType<unknown> = z.lazy(() =>
+  z.union([
+    z.string(),
+    z.number().finite(),
+    z.boolean(),
+    z.null(),
+    z.array(jsonValueSchema),
+    z.record(z.string(), jsonValueSchema),
+  ])
+);
+
+export const traceJsonObjectSchema = z.record(z.string(), jsonValueSchema);
+
+const SAFE_RECEIPT_STRING_KEYS = new Set([
+  "capability",
+  "capsuleId",
+  "collection",
+  "code",
+  "docid",
+  "filterFingerprint",
+  "heading",
+  "id",
+  "kind",
+  "mirrorHash",
+  "mode",
+  "name",
+  "outcome",
+  "passageHash",
+  "reasonCode",
+  "ref",
+  "sourceHash",
+  "status",
+  "type",
+  "uri",
+]);
+const SAFE_RECEIPT_STRING_LIST_KEYS = new Set([
+  "capabilities",
+  "categories",
+  "collections",
+  "fallbackCodes",
+  "fallbacks",
+  "itemTypes",
+  "sources",
+  "tags",
+]);
+
+const validateReceiptStrings = (
+  value: unknown,
+  context: z.RefinementCtx,
+  key = "",
+  path: Array<string | number> = []
+): void => {
+  if (typeof value === "string") {
+    if (
+      !(
+        SAFE_RECEIPT_STRING_KEYS.has(key) ||
+        SAFE_RECEIPT_STRING_LIST_KEYS.has(key)
+      )
+    ) {
+      context.addIssue({
+        code: "custom",
+        path,
+        message: `String field ${key || "<root>"} is not allowed in trace receipt payloads`,
+      });
+    } else if (key === "uri" && !value.startsWith("gno://")) {
+      context.addIssue({
+        code: "custom",
+        path,
+        message: "Trace evidence URI must use gno://",
+      });
+    } else if (traceUtf8Bytes(value) > 4096) {
+      context.addIssue({
+        code: "custom",
+        path,
+        message: `String field ${key} exceeds 4096 UTF-8 bytes`,
+      });
+    }
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (const [index, child] of value.entries()) {
+      validateReceiptStrings(child, context, key, [...path, index]);
+    }
+    return;
+  }
+  if (value !== null && typeof value === "object") {
+    for (const [childKey, child] of Object.entries(value)) {
+      validateReceiptStrings(child, context, childKey, [...path, childKey]);
+    }
+  }
+};
+
+const traceReceiptPayloadSchema = traceJsonObjectSchema.superRefine(
+  (value, context) => validateReceiptStrings(value, context)
+);
+
+const queryShapeSchema = z
+  .object({
+    characters: z.number().int().nonnegative(),
+    terms: z.number().int().nonnegative(),
+  })
+  .strict();
+
+const traceInputBaseSchema = z
+  .object({
+    traceId: idSchema,
+    schemaVersion: z.literal("1.0"),
+    redactionMode: z.enum(["metadata", "replay"]),
+    replayCapable: z.boolean(),
+    queryText: z.string().max(8192).nullable(),
+    queryDigest: sha256Schema.nullable(),
+    queryShape: queryShapeSchema,
+    goalText: z.string().max(8192).nullable(),
+    goalDigest: sha256Schema.nullable(),
+    goalShape: queryShapeSchema,
+    filters: traceJsonObjectSchema,
+    fingerprints: z
+      .object({
+        pipeline: sha256Schema,
+        model: sha256Schema,
+        config: sha256Schema,
+        index: sha256Schema,
+      })
+      .strict(),
+    status: z.literal("open"),
+    createdAtMs: epochMsSchema,
+    updatedAtMs: epochMsSchema,
+    expiresAtMs: epochMsSchema,
+  })
+  .strict();
+
+export const retrievalTraceInputSchema = traceInputBaseSchema.superRefine(
+  (value, context) => {
+    if (value.updatedAtMs < value.createdAtMs) {
+      context.addIssue({
+        code: "custom",
+        path: ["updatedAtMs"],
+        message: "updatedAtMs must not precede createdAtMs",
+      });
+    }
+    if (value.updatedAtMs !== value.createdAtMs) {
+      context.addIssue({
+        code: "custom",
+        path: ["updatedAtMs"],
+        message: "new traces must start with updatedAtMs equal to createdAtMs",
+      });
+    }
+    if (value.expiresAtMs <= value.createdAtMs) {
+      context.addIssue({
+        code: "custom",
+        path: ["expiresAtMs"],
+        message: "expiresAtMs must be later than createdAtMs",
+      });
+    }
+    const metadataInvariant =
+      value.redactionMode === "metadata" &&
+      !value.replayCapable &&
+      value.queryText === null &&
+      value.queryDigest === null &&
+      value.goalText === null &&
+      value.goalDigest === null;
+    const replayInvariant =
+      value.redactionMode === "replay" &&
+      value.replayCapable &&
+      value.queryText !== null &&
+      value.queryDigest !== null &&
+      ((value.goalText === null && value.goalDigest === null) ||
+        (value.goalText !== null && value.goalDigest !== null));
+    if (!(metadataInvariant || replayInvariant)) {
+      context.addIssue({
+        code: "custom",
+        message:
+          "metadata traces cannot retain query material; replay traces require explicit replay inputs",
+      });
+    }
+    if (value.redactionMode === "replay" && value.queryText !== null) {
+      const expectedDigest = new Bun.CryptoHasher("sha256")
+        .update(value.queryText)
+        .digest("hex");
+      if (value.queryDigest !== expectedDigest) {
+        context.addIssue({
+          code: "custom",
+          path: ["queryDigest"],
+          message: "queryDigest does not match queryText",
+        });
+      }
+      const expectedShape = shapeTraceText(value.queryText);
+      if (
+        value.queryShape.characters !== expectedShape.characters ||
+        value.queryShape.terms !== expectedShape.terms
+      ) {
+        context.addIssue({
+          code: "custom",
+          path: ["queryShape"],
+          message: "queryShape does not match queryText",
+        });
+      }
+    }
+    if (value.redactionMode === "replay" && value.goalText !== null) {
+      const expectedDigest = new Bun.CryptoHasher("sha256")
+        .update(value.goalText)
+        .digest("hex");
+      if (value.goalDigest !== expectedDigest) {
+        context.addIssue({
+          code: "custom",
+          path: ["goalDigest"],
+          message: "goalDigest does not match goalText",
+        });
+      }
+      const expectedShape = shapeTraceText(value.goalText);
+      if (
+        value.goalShape.characters !== expectedShape.characters ||
+        value.goalShape.terms !== expectedShape.terms
+      ) {
+        context.addIssue({
+          code: "custom",
+          path: ["goalShape"],
+          message: "goalShape does not match goalText",
+        });
+      }
+    }
+  }
+);
+
+export const retrievalTraceRunInputSchema = z
+  .object({
+    runId: idSchema,
+    traceId: idSchema,
+    idempotencyKey: idempotencyKeySchema,
+    kind: z.enum(["retrieval", "context", "get"]),
+    payload: traceReceiptPayloadSchema,
+    createdAtMs: epochMsSchema,
+  })
+  .strict();
+
+export const retrievalTraceEventInputSchema = z
+  .object({
+    eventId: idSchema,
+    traceId: idSchema,
+    runId: idSchema.nullable(),
+    idempotencyKey: idempotencyKeySchema,
+    kind: z.enum([
+      "query",
+      "retrieval",
+      "context",
+      "get",
+      "open",
+      "cite",
+      "pin",
+      "capability",
+      "complete",
+    ]),
+    payload: traceReceiptPayloadSchema,
+    createdAtMs: epochMsSchema,
+  })
+  .strict();
+
+export const retrievalTraceJudgmentInputSchema = z
+  .object({
+    judgmentId: idSchema,
+    traceId: idSchema,
+    runId: idSchema.nullable(),
+    idempotencyKey: idempotencyKeySchema,
+    label: z.enum(["relevant", "irrelevant", "missing_expected"]),
+    targetKind: z.enum(["document", "chunk", "span", "query"]),
+    targetRef: z.string().min(1).max(4096),
+    target: traceReceiptPayloadSchema,
+    createdAtMs: epochMsSchema,
+  })
+  .strict()
+  .superRefine((value, context) => {
+    if (value.label === "missing_expected" && value.targetKind !== "document") {
+      context.addIssue({
+        code: "custom",
+        path: ["targetKind"],
+        message: "missing_expected judgments must target a document",
+      });
+    }
+  });
+
+export const retrievalTraceExportInputSchema = z
+  .object({
+    exportId: idSchema,
+    traceId: idSchema,
+    format: z.enum(["agentic-receipt", "qrels"]),
+    artifactHash: sha256Schema,
+    createdAtMs: epochMsSchema,
+  })
+  .strict();
+
+const compareCodeUnits = (left: string, right: string): number =>
+  left < right ? -1 : left > right ? 1 : 0;
+
+const canonicalizeJsonValue = (value: unknown): unknown => {
+  if (Array.isArray(value)) return value.map(canonicalizeJsonValue);
+  if (value !== null && typeof value === "object") {
+    const sorted: Record<string, unknown> = Object.create(null) as Record<
+      string,
+      unknown
+    >;
+    for (const key of Object.keys(value).sort(compareCodeUnits)) {
+      if (DANGEROUS_JSON_KEYS.has(key)) {
+        throw new Error(`Canonical JSON rejects dangerous key ${key}`);
+      }
+      const child = (value as Record<string, unknown>)[key];
+      if (child === undefined) {
+        throw new Error(`Canonical JSON rejects undefined at ${key}`);
+      }
+      sorted[key] = canonicalizeJsonValue(child);
+    }
+    return sorted;
+  }
+  if (typeof value === "number" && !Number.isFinite(value)) {
+    throw new Error("Canonical JSON rejects non-finite numbers");
+  }
+  return value;
+};
+
+const shapeTraceText = (
+  value: string
+): { characters: number; terms: number } => ({
+  characters: Array.from(value).length,
+  terms: value.trim() ? value.trim().split(/\s+/u).length : 0,
+});
+
+export const canonicalTraceJson = (value: unknown): string =>
+  JSON.stringify(canonicalizeJsonValue(value));
+
+export const traceUtf8Bytes = (value: string): number =>
+  new TextEncoder().encode(value).byteLength;
+
+export const hashTraceCanonical = (value: unknown): string =>
+  new Bun.CryptoHasher("sha256")
+    .update(canonicalTraceJson(value))
+    .digest("hex");
+
+/** Stable trace identity excludes terminal status and mutable updatedAtMs. */
+export const hashRetrievalTraceCreation = (
+  trace: RetrievalTraceInput
+): string => {
+  const { status: _status, updatedAtMs: _updatedAtMs, ...immutable } = trace;
+  return hashTraceCanonical(immutable);
+};
+
+export const parseRetrievalTraceInput = (
+  input: RetrievalTraceInput
+): RetrievalTraceInput => retrievalTraceInputSchema.parse(input);
+
+export const parseRetrievalTraceRunInput = (
+  input: RetrievalTraceRunInput
+): RetrievalTraceRunInput => retrievalTraceRunInputSchema.parse(input);
+
+export const parseRetrievalTraceEventInput = (
+  input: RetrievalTraceEventInput
+): RetrievalTraceEventInput => retrievalTraceEventInputSchema.parse(input);
+
+export const parseRetrievalTraceJudgmentInput = (
+  input: RetrievalTraceJudgmentInput
+): RetrievalTraceJudgmentInput =>
+  retrievalTraceJudgmentInputSchema.parse(input);
+
+export const parseRetrievalTraceExportInput = (
+  input: RetrievalTraceExportInput
+): RetrievalTraceExportInput => retrievalTraceExportInputSchema.parse(input);

package/src/store/sqlite/adapter.ts

@@ -10,6 +10,8 @@
 // CRITICAL: Import setup FIRST to configure custom SQLite before any Database use
 import "./setup";
 import { Database } from "bun:sqlite";
+// node:async_hooks binds nested transactions to one async request; Bun has no separate native equivalent.
+import { AsyncLocalStorage } from "node:async_hooks";
 // node:path basename: no Bun path utilities.
 import { basename } from "node:path";
 
@@ -50,6 +52,24 @@ import type {
   IngestErrorInput,
   IngestErrorRow,
   MigrationResult,
+  RetrievalTraceAppendResult,
+  RetrievalTraceBundle,
+  RetrievalTraceCursor,
+  RetrievalTraceBoundedBundle,
+  RetrievalTraceDeleteCounts,
+  RetrievalTraceEventInput,
+  RetrievalTraceExportInput,
+  RetrievalTraceExportBundle,
+  RetrievalTraceExportManifestInput,
+  RetrievalTraceExportManifestRow,
+  RetrievalTraceInput,
+  RetrievalTraceJudgmentInput,
+  RetrievalTracePurgeResult,
+  RetrievalTraceRetentionPolicy,
+  RetrievalTraceRetentionResult,
+  RetrievalTraceRow,
+  RetrievalTraceRunInput,
+  RetrievalTraceTerminalStatus,
   StorePort,
   StoreResult,
   TagCount,
@@ -76,6 +96,28 @@ import { err, ok } from "../types";
 import { getStoredEmbeddingFingerprint } from "../vector/freshness";
 import { modelTableName } from "../vector/sqlite-vec";
 import { loadFts5Snowball } from "./fts5-snowball";
+import {
+  appendExportManifest as appendStoredTraceExportManifest,
+  getBoundedTrace as getBoundedStoredTrace,
+  getExportBundle as getStoredTraceExportBundle,
+  getExportManifest as getStoredTraceExportManifest,
+  getOrCreateRedactionSecret as getOrCreateStoredTraceRedactionSecret,
+} from "./retrieval-trace-management-store";
+import {
+  deleteTrace as deleteStoredTrace,
+  enforceRetention as enforceStoredTraceRetention,
+  purgeTraces as purgeStoredTraces,
+} from "./retrieval-trace-retention";
+import {
+  appendEvent as appendStoredTraceEvent,
+  appendExport as appendStoredTraceExport,
+  appendJudgment as appendStoredTraceJudgment,
+  appendRun as appendStoredTraceRun,
+  createTrace as createStoredTrace,
+  finalizeTrace as finalizeStoredTrace,
+  getTrace as getStoredTrace,
+  listTraces as listStoredTraces,
+} from "./retrieval-trace-store";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // FTS5 Query Escaping
@@ -312,8 +354,9 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
   private dbPath = "";
   private ftsTokenizer: FtsTokenizer = "unicode61";
   private configPath = ""; // Set by CLI layer for status output
-  private txDepth = 0; // Transaction nesting depth
   private txCounter = 0; // Savepoint counter for unique names
+  private readonly txContext = new AsyncLocalStorage<{ depth: number }>();
+  private txTail: Promise<void> = Promise.resolve();
   private contextGeneration = 0;
 
   // ─────────────────────────────────────────────────────────────────────────
@@ -407,9 +450,12 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
    */
   async withTransaction<T>(fn: () => Promise<T>): Promise<StoreResult<T>> {
     const db = this.ensureOpen();
-
-    const isOuter = this.txDepth === 0;
+    const parent = this.txContext.getStore();
+    const isOuter = parent === undefined;
     const savepoint = `sp_${++this.txCounter}`;
+    const releaseWriter = isOuter
+      ? await this.acquireTransactionWriter()
+      : null;
 
     try {
       if (isOuter) {
@@ -419,9 +465,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
         db.exec(`SAVEPOINT ${savepoint}`);
       }
 
-      this.txDepth += 1;
-      const value = await fn();
-      this.txDepth -= 1;
+      const value = await this.txContext.run(
+        { depth: (parent?.depth ?? 0) + 1 },
+        fn
+      );
 
       if (isOuter) {
         db.exec("COMMIT");
@@ -431,8 +478,6 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
 
       return ok(value);
     } catch (cause) {
-      this.txDepth = Math.max(0, this.txDepth - 1);
-
       try {
         if (isOuter) {
           db.exec("ROLLBACK");
@@ -447,9 +492,21 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
       const message =
         cause instanceof Error ? cause.message : "Transaction failed";
       return err("TRANSACTION_FAILED", message, cause);
+    } finally {
+      releaseWriter?.();
     }
   }
 
+  private async acquireTransactionWriter(): Promise<() => void> {
+    const previous = this.txTail;
+    let release!: () => void;
+    this.txTail = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+    await previous;
+    return release;
+  }
+
   /**
    * Set config path for status output (called by CLI layer).
    */
@@ -765,6 +822,107 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     }
   }
 
+  async createRetrievalTrace(
+    input: RetrievalTraceInput
+  ): Promise<StoreResult<RetrievalTraceAppendResult>> {
+    return createStoredTrace(this.ensureOpen(), input);
+  }
+
+  async getRetrievalTrace(
+    traceId: string
+  ): Promise<StoreResult<RetrievalTraceBundle | null>> {
+    return getStoredTrace(this.ensureOpen(), traceId);
+  }
+
+  async getBoundedRetrievalTrace(
+    traceId: string,
+    detailLimit: number
+  ): Promise<StoreResult<RetrievalTraceBoundedBundle | null>> {
+    return getBoundedStoredTrace(this.ensureOpen(), traceId, detailLimit);
+  }
+
+  async listRetrievalTraces(
+    limit: number,
+    cursor?: RetrievalTraceCursor
+  ): Promise<StoreResult<RetrievalTraceRow[]>> {
+    return listStoredTraces(this.ensureOpen(), limit, cursor);
+  }
+
+  async finalizeRetrievalTrace(
+    traceId: string,
+    status: RetrievalTraceTerminalStatus,
+    updatedAtMs: number
+  ): Promise<StoreResult<RetrievalTraceAppendResult>> {
+    return finalizeStoredTrace(this.ensureOpen(), traceId, status, updatedAtMs);
+  }
+
+  async appendRetrievalTraceRun(
+    input: RetrievalTraceRunInput
+  ): Promise<StoreResult<RetrievalTraceAppendResult>> {
+    return appendStoredTraceRun(this.ensureOpen(), input);
+  }
+
+  async appendRetrievalTraceEvent(
+    input: RetrievalTraceEventInput
+  ): Promise<StoreResult<RetrievalTraceAppendResult>> {
+    return appendStoredTraceEvent(this.ensureOpen(), input);
+  }
+
+  async appendRetrievalTraceJudgment(
+    input: RetrievalTraceJudgmentInput
+  ): Promise<StoreResult<RetrievalTraceAppendResult>> {
+    return appendStoredTraceJudgment(this.ensureOpen(), input);
+  }
+
+  async appendRetrievalTraceExport(
+    input: RetrievalTraceExportInput
+  ): Promise<StoreResult<RetrievalTraceAppendResult>> {
+    return appendStoredTraceExport(this.ensureOpen(), input);
+  }
+
+  async appendRetrievalTraceExportManifest(
+    input: RetrievalTraceExportManifestInput
+  ): Promise<StoreResult<RetrievalTraceAppendResult>> {
+    return appendStoredTraceExportManifest(this.ensureOpen(), input);
+  }
+
+  async getRetrievalTraceExportManifest(
+    exportId: string
+  ): Promise<StoreResult<RetrievalTraceExportManifestRow | null>> {
+    return getStoredTraceExportManifest(this.ensureOpen(), exportId);
+  }
+
+  async getRetrievalTraceExportBundle(
+    exportId: string
+  ): Promise<StoreResult<RetrievalTraceExportBundle | null>> {
+    return getStoredTraceExportBundle(this.ensureOpen(), exportId);
+  }
+
+  async getOrCreateRetrievalTraceRedactionSecret(): Promise<
+    StoreResult<string>
+  > {
+    return getOrCreateStoredTraceRedactionSecret(this.ensureOpen());
+  }
+
+  async deleteRetrievalTrace(
+    traceId: string
+  ): Promise<StoreResult<RetrievalTraceDeleteCounts>> {
+    return deleteStoredTrace(this.ensureOpen(), traceId);
+  }
+
+  async purgeRetrievalTraces(): Promise<
+    StoreResult<RetrievalTracePurgeResult>
+  > {
+    return purgeStoredTraces(this.ensureOpen());
+  }
+
+  async enforceRetrievalTraceRetention(
+    policy: RetrievalTraceRetentionPolicy,
+    nowMs: number
+  ): Promise<StoreResult<RetrievalTraceRetentionResult>> {
+    return enforceStoredTraceRetention(this.ensureOpen(), policy, nowMs);
+  }
+
   getContextGeneration(): number {
     return this.contextGeneration;
   }
@@ -1635,6 +1793,11 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
             SELECT id FROM documents
             WHERE active = 1
             ${options.collection ? "AND collection = ?" : ""}
+            ${
+              options.relPathPrefix !== undefined
+                ? "AND (rel_path = ? OR substr(rel_path, 1, length(?) + 1) = ? || '/')"
+                : ""
+            }
           )
           ORDER BY score
           LIMIT ?
@@ -1688,6 +1851,13 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
       const queryParams = [
         builtQuery.query,
         ...(options.collection ? [options.collection] : []),
+        ...(options.relPathPrefix !== undefined
+          ? [
+              options.relPathPrefix,
+              options.relPathPrefix,
+              options.relPathPrefix,
+            ]
+          : []),
         ftsLimit,
         ...params,
       ];
@@ -4232,7 +4402,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
       // Last updated (max updated_at from documents)
       const lastUpdatedRow = db
         .query<{ last_updated: string | null }, []>(
-          "SELECT MAX(updated_at) as last_updated FROM documents"
+          "SELECT strftime('%Y-%m-%dT%H:%M:%fZ', MAX(updated_at)) as last_updated FROM documents"
         )
         .get();
 
@@ -4605,7 +4775,6 @@ interface DbIngestErrorRow {
   details_json: string | null;
 }
 
-// ─────────────────────────────────────────────────────────────────────────────
 // Row Mappers (snake_case -> camelCase)
 // ─────────────────────────────────────────────────────────────────────────────