dpdp-erasure-cli 1.0.0

package/src/modules/network/outbox/store.ts

@@ -0,0 +1,346 @@
+import { sha256HexDigest } from "@/lib";
+import type { Sql, Tsql } from "@/types";
+import { canonicalJsonStringify } from "@/utils";
+import { asWorkerError, CODE, fail } from "@/errors";
+import type { OutboxEvent } from "./types";
+import { calculateRetryDelayMs } from "./shared";
+
+const DEFAULT_CHAIN_FINALIZATION_LIMIT = 1000;
+
+interface ChainTailRow {
+  current_hash: string;
+}
+
+interface UnfinalizedOutboxRow {
+  id: string;
+  idempotency_key: string;
+  payload: unknown;
+}
+
+async function finalizePendingOutboxChain(
+  tx: Tsql,
+  engineSchema: string,
+  limit: number
+): Promise<void> {
+  const [tail] = await tx<ChainTailRow[]>`
+    SELECT current_hash
+    FROM ${tx(engineSchema)}.outbox
+    WHERE chain_status = 'finalized'
+    ORDER BY created_at DESC, id DESC
+    LIMIT 1
+  `;
+  let previousHash = tail?.current_hash ?? "GENESIS";
+
+  const rows = await tx<UnfinalizedOutboxRow[]>`
+    SELECT id, idempotency_key, payload
+    FROM ${tx(engineSchema)}.outbox
+    WHERE chain_status = 'pending'
+    ORDER BY created_at ASC, id ASC
+    LIMIT ${limit}
+    FOR UPDATE
+  `;
+
+  for (const row of rows) {
+    const currentHash = await sha256HexDigest(
+      `${previousHash}${canonicalJsonStringify(row.payload)}${row.idempotency_key}`
+    );
+
+    await tx`
+      UPDATE ${tx(engineSchema)}.outbox
+      SET previous_hash = ${previousHash},
+          current_hash = ${currentHash},
+          chain_status = 'finalized',
+          updated_at = NOW()
+      WHERE id = ${row.id}
+    `;
+    previousHash = currentHash;
+  }
+}
+
+/**
+ * Claims one contiguous WORM-chain batch of due outbox events.
+ *
+ * The dispatcher cannot reorder events without invalidating the Control Plane audit chain.
+ * A short advisory lock serializes lease selection across worker containers, then the query
+ * walks `previous_hash -> current_hash` from the last processed event.
+ *
+ * @param sql - Postgres pool owning the outbox table.
+ * @param engineSchema - Worker engine schema.
+ * @param batchSize - Maximum rows to claim.
+ * @param leaseSeconds - Lease duration in seconds.
+ * @param now - Lease anchor timestamp.
+ * @returns Lease token plus claimed events.
+ */
+export async function claimOutboxBatch(
+  sql: Sql,
+  engineSchema: string,
+  batchSize: number,
+  leaseSeconds: number,
+  now: Date
+): Promise<{ leaseToken: string; events: OutboxEvent[] }> {
+  return sql.begin(async (tx) => {
+    const leaseToken = globalThis.crypto.randomUUID();
+    const leaseExpiresAt = new Date(now.getTime() + leaseSeconds * 1000);
+    const events: OutboxEvent[] = [];
+
+    await tx`
+      SELECT pg_advisory_xact_lock(hashtext(${`${engineSchema}.outbox.dispatch_chain`}))
+    `;
+
+    await finalizePendingOutboxChain(
+      tx,
+      engineSchema,
+      Math.max(batchSize, DEFAULT_CHAIN_FINALIZATION_LIMIT)
+    );
+
+    const [activeLease] = await tx<{ count: string }[]>`
+      SELECT COUNT(*)::TEXT AS count
+      FROM ${tx(engineSchema)}.outbox
+      WHERE status = 'leased'
+        AND lease_expires_at IS NOT NULL
+        AND lease_expires_at > ${now}
+    `;
+    if (Number(activeLease?.count ?? "0") > 0) {
+      return {
+        leaseToken,
+        events,
+      };
+    }
+
+    const [tail] = await tx<ChainTailRow[]>`
+      SELECT current_hash
+      FROM ${tx(engineSchema)}.outbox
+      WHERE status = 'processed'
+        AND chain_status = 'finalized'
+      ORDER BY created_at DESC, id DESC
+      LIMIT 1
+    `;
+    let expectedPreviousHash = tail?.current_hash ?? "GENESIS";
+
+    for (let index = 0; index < batchSize; index += 1) {
+      const [event] = await tx<OutboxEvent[]>`
+        SELECT *
+        FROM ${tx(engineSchema)}.outbox
+        WHERE status IN ('pending', 'leased')
+          AND chain_status = 'finalized'
+          AND previous_hash = ${expectedPreviousHash}
+          AND next_attempt_at <= ${now}
+          AND (status = 'pending' OR lease_expires_at IS NULL OR lease_expires_at <= ${now})
+        ORDER BY created_at ASC, id ASC
+        LIMIT 1
+        FOR UPDATE SKIP LOCKED
+      `;
+
+      if (!event) {
+        break;
+      }
+
+      await tx`
+        UPDATE ${tx(engineSchema)}.outbox
+        SET status = 'leased',
+            lease_token = ${leaseToken},
+            lease_expires_at = ${leaseExpiresAt},
+            updated_at = ${now}
+        WHERE id = ${event.id}
+      `;
+
+      event.status = "leased";
+      event.lease_token = leaseToken;
+      event.lease_expires_at = leaseExpiresAt;
+      event.updated_at = now;
+      events.push(event);
+      expectedPreviousHash = event.current_hash;
+    }
+
+    return {
+      leaseToken,
+      events,
+    };
+  });
+}
+
+/**
+ * Marks a leased outbox event as processed.
+ *
+ * @param sql - Postgres pool owning the outbox table.
+ * @param engineSchema - Worker engine schema.
+ * @param eventId - Outbox event id.
+ * @param leaseToken - Current lease token.
+ * @param now - Update timestamp.
+ */
+export async function markOutboxEventProcessed(
+  sql: Sql,
+  engineSchema: string,
+  eventId: string,
+  leaseToken: string,
+  now: Date
+): Promise<void> {
+  const updated = await sql`
+    UPDATE ${sql(engineSchema)}.outbox
+    SET status = 'processed',
+        processed_at = ${now},
+        lease_token = NULL,
+        lease_expires_at = NULL,
+        last_error = NULL,
+        updated_at = ${now}
+    WHERE id = ${eventId}
+      AND lease_token = ${leaseToken}
+      AND status = 'leased'
+    RETURNING id
+  `;
+
+  if (updated.length === 0) {
+    fail({
+      code: CODE.OUTBOX_LEASE_LOST,
+      detail: `Outbox lease for event ${eventId} was lost before it could be marked processed.`,
+      context: { eventId },
+    });
+  }
+}
+
+/**
+ * Extends active leases for the current contiguous outbox batch.
+ *
+ * Long API/WORM append latency can make later events in a leased batch expire before
+ * the owning worker reaches them. Renewing the remaining batch before every dispatch
+ * prevents another worker from reclaiming the same chain segment and creating a
+ * false previous-hash conflict.
+ *
+ * @param sql - Postgres pool owning the outbox table.
+ * @param engineSchema - Worker engine schema.
+ * @param eventIds - Remaining leased event ids in the current batch.
+ * @param currentEventId - Event that is about to be dispatched.
+ * @param leaseToken - Current lease token.
+ * @param leaseExpiresAt - New lease expiry timestamp.
+ * @param now - Update timestamp.
+ */
+export async function extendOutboxLeases(
+  sql: Sql,
+  engineSchema: string,
+  eventIds: readonly string[],
+  currentEventId: string,
+  leaseToken: string,
+  leaseExpiresAt: Date,
+  now: Date
+): Promise<void> {
+  if (eventIds.length === 0) {
+    return;
+  }
+
+  const rows = await sql<{ id: string }[]>`
+    UPDATE ${sql(engineSchema)}.outbox
+    SET lease_expires_at = ${leaseExpiresAt},
+        updated_at = ${now}
+    WHERE id = ANY(${eventIds})
+      AND lease_token = ${leaseToken}
+      AND status = 'leased'
+    RETURNING id
+  `;
+
+  if (!rows.some((row) => row.id === currentEventId)) {
+    fail({
+      code: CODE.OUTBOX_LEASE_LOST,
+      detail: `Outbox lease for event ${currentEventId} was lost before dispatch.`,
+      context: { eventId: currentEventId },
+    });
+  }
+}
+
+/**
+ * Marks a leased outbox event as failed or dead-lettered.
+ *
+ * @param sql - Postgres pool owning the outbox table.
+ * @param engineSchema - Worker engine schema.
+ * @param event - Leased outbox event.
+ * @param leaseToken - Current lease token.
+ * @param now - Update timestamp.
+ * @param maxAttempts - Retry ceiling before dead-lettering.
+ * @param baseBackoffMs - Initial exponential backoff.
+ * @param error - Original delivery error.
+ * @returns Resulting queue state.
+ */
+export async function markOutboxEventFailed(
+  sql: Sql,
+  engineSchema: string,
+  event: OutboxEvent,
+  leaseToken: string,
+  now: Date,
+  maxAttempts: number,
+  baseBackoffMs: number,
+  error: unknown
+): Promise<"pending" | "dead_letter"> {
+  const nextAttemptCount = event.attempt_count + 1;
+  const deadLetter = nextAttemptCount >= maxAttempts;
+  const nextAttemptAt = new Date(
+    now.getTime() + calculateRetryDelayMs(nextAttemptCount, baseBackoffMs)
+  );
+  const errorMessage = error instanceof Error ? error.message : String(error);
+
+  const updated = await sql`
+    UPDATE ${sql(engineSchema)}.outbox
+    SET status = ${deadLetter ? "dead_letter" : "pending"},
+        attempt_count = ${nextAttemptCount},
+        lease_token = NULL,
+        lease_expires_at = NULL,
+        next_attempt_at = ${deadLetter ? now : nextAttemptAt},
+        last_error = ${errorMessage.slice(0, 1024)},
+        updated_at = ${now}
+    WHERE id = ${event.id}
+      AND lease_token = ${leaseToken}
+      AND status = 'leased'
+    RETURNING id
+  `;
+
+  if (updated.length === 0) {
+    fail({
+      code: CODE.OUTBOX_LEASE_LOST,
+      detail: `Outbox lease for event ${event.id} was lost before it could be retried.`,
+      context: { eventId: event.id },
+    });
+  }
+
+  return deadLetter ? "dead_letter" : "pending";
+}
+
+/**
+ * Releases a leased outbox event back to pending state after a fatal delivery failure.
+ *
+ * @param sql - Postgres pool owning the outbox table.
+ * @param engineSchema - Worker engine schema.
+ * @param eventId - Outbox event id.
+ * @param leaseToken - Current lease token.
+ * @param now - Update timestamp.
+ * @param error - Fatal delivery error.
+ */
+export async function releaseOutboxLease(
+  sql: Sql,
+  engineSchema: string,
+  eventId: string,
+  leaseToken: string,
+  now: Date,
+  error: unknown
+): Promise<void> {
+  const normalized = asWorkerError(error);
+
+  const updated = await sql`
+    UPDATE ${sql(engineSchema)}.outbox
+    SET status = 'pending',
+        lease_token = NULL,
+        lease_expires_at = NULL,
+        next_attempt_at = ${now},
+        last_error = ${normalized.detail.slice(0, 1024)},
+        updated_at = ${now}
+    WHERE id = ${eventId}
+      AND lease_token = ${leaseToken}
+      AND status = 'leased'
+    RETURNING id
+  `;
+
+  if (updated.length === 0) {
+    fail({
+      code: CODE.OUTBOX_LEASE_LOST,
+      detail: `Outbox lease for event ${eventId} was lost before it could be released.`,
+      context: { eventId },
+    });
+  }
+}

package/src/modules/network/outbox/types.ts

@@ -0,0 +1,54 @@
+// HTTP dispatcher configuration for pushing outbox events to the Control Plane.
+export interface FetchDispatcherOptions {
+  url: string;
+  token?: string;
+  clientId?: string;
+  requestSigningSecret?: string;
+  timeoutMs?: number;
+}
+
+// Worker-local outbox row appended inside mutation transactions.
+export interface OutboxRow {
+  id: string;
+  idempotency_key: string;
+  user_uuid_hash: string;
+  event_type: string;
+  payload: unknown;
+  previous_hash: string;
+  current_hash: string;
+  chain_status: "pending" | "finalized";
+  status: "pending" | "leased" | "processed" | "dead_letter";
+  attempt_count: number;
+  lease_token: string | null;
+  lease_expires_at: Date | null;
+  next_attempt_at: Date;
+  processed_at: Date | null;
+  last_error: string | null;
+  created_at: Date;
+  updated_at: Date;
+}
+
+/**
+ * Aggregated counters from one outbox processing cycle.
+ */
+export interface ProcessOutboxResult {
+  claimed: number;
+  processed: number;
+  failed: number;
+  deadLettered: number;
+}
+
+/**
+ * Runtime controls for outbox claim and retry behavior.
+ */
+export interface ProcessOutboxOptions {
+  engineSchema?: string;
+  batchSize?: number;
+  leaseSeconds?: number;
+  maxAttempts?: number;
+  baseBackoffMs?: number;
+  now?: Date;
+}
+
+// Outbox row type exposed by the relay pipeline.
+export interface OutboxEvent extends OutboxRow { };

package/src/modules/network/request-signing.ts

@@ -0,0 +1,61 @@
+const textEncoder = new TextEncoder();
+const signingKeyCache = new Map<string, Promise<CryptoKey>>();
+const MAX_SIGNING_KEY_CACHE_ENTRIES = 128;
+
+function bytesToHex(bytes: Uint8Array): string {
+  return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+async function importSigningKey(secret: string): Promise<CryptoKey> {
+  const cached = signingKeyCache.get(secret);
+  if (cached) {
+    return cached;
+  }
+
+  if (signingKeyCache.size >= MAX_SIGNING_KEY_CACHE_ENTRIES) {
+    signingKeyCache.delete(signingKeyCache.keys().next().value as string);
+  }
+
+  const imported = globalThis.crypto.subtle.importKey(
+    "raw",
+    textEncoder.encode(secret),
+    {
+      name: "HMAC",
+      hash: "SHA-256",
+    },
+    false,
+    ["sign"]
+  );
+  signingKeyCache.set(secret, imported);
+  return imported;
+}
+
+/**
+ * Computes the canonical worker/API HMAC request signature.
+ *
+ * @param secret - Shared HMAC secret.
+ * @param method - HTTP method.
+ * @param path - URL pathname.
+ * @param clientId - Worker client identifier.
+ * @param timestamp - Unix epoch milliseconds string.
+ * @param nonce - Optional per-request nonce used to prevent same-millisecond multi-worker collisions.
+ * @param bodyText - Exact request body text.
+ * @returns Lowercase hex digest.
+ */
+export async function computeRequestSignature(
+  secret: string,
+  method: string,
+  path: string,
+  clientId: string,
+  timestamp: string,
+  bodyText: string,
+  nonce: string = ""
+): Promise<string> {
+  const key = await importSigningKey(secret);
+  const parts = nonce
+    ? [method.toUpperCase(), path, clientId, timestamp, nonce, bodyText]
+    : [method.toUpperCase(), path, clientId, timestamp, bodyText];
+  const payload = textEncoder.encode(parts.join("\n"));
+  const signature = await globalThis.crypto.subtle.sign("HMAC", key, payload);
+  return bytesToHex(new Uint8Array(signature));
+}

package/src/modules/worker/index.ts

@@ -0,0 +1,2 @@
+export * from "./types";
+export * from "./worker";

package/src/modules/worker/tasks.ts

@@ -0,0 +1,58 @@
+import { fail, workerError } from "@/errors";
+import type { ApiClient, TaskAckPayload, WorkerTask } from "./types";
+
+/**
+ * Resolves a valid subject identifier from a worker task payload.
+ * Prioritizes a non-empty string `subject_opaque_id` first, falling back to a positive integer `userId`.
+ * 
+ * @param task - The worker task containing payload.
+ * @returns The resolved identifier as a string or a number.
+ * @throws Invokes `fail()` if no valid identifier is found.
+ */
+export function resolveTaskSubject(task: WorkerTask): string | number {
+  if (
+    typeof task.payload.subject_opaque_id === "string"
+    && task.payload.subject_opaque_id.trim().length > 0
+  ) {
+    return task.payload.subject_opaque_id.trim();
+  }
+
+  if (
+    typeof task.payload.userId === "number"
+    && Number.isInteger(task.payload.userId)
+    && task.payload.userId > 0
+  ) {
+    return task.payload.userId;
+  }
+
+  fail({
+    code: "TASK_PAYLOAD_INVALID",
+    title: "Invalid task payload",
+    detail: `Task ${task.id} requires a non-empty subject_opaque_id or numeric userId for ${task.task_type}.`,
+    category: "validation",
+    retryable: false,
+    context: { taskId: task.id, taskType: task.task_type },
+  });
+}
+
+export async function acknowledgeTask(
+  apiClient: ApiClient,
+  taskId: string,
+  status: "completed" | "failed",
+  result: TaskAckPayload
+): Promise<void> {
+  const acknowledged = await apiClient.ackTask(taskId, status, result);
+  if (!acknowledged) {
+    throw workerError({
+      code: "TASK_ACK_FAILED",
+      title: "Task acknowledgement failed",
+      detail: `Control Plane did not acknowledge task ${taskId}.`,
+      category: "network",
+      retryable: true,
+      context: {
+        taskId,
+        status,
+      },
+    });
+  }
+}

package/src/modules/worker/types.ts

@@ -0,0 +1,89 @@
+import type { Sql } from "@/types";
+import type { WorkerProblemDetails } from "@/errors";
+import type {
+  DispatchNoticeResult,
+  ShredUserResult,
+  VaultUserResult,
+  WorkerSecrets,
+  MockMailer
+} from "@modules/engine";
+import type { OutboxEvent, S3Client } from "@modules/network";
+import type { WorkerConfig } from "@modules/config";
+
+/**
+ * Normalizes task payload accepted from Control Plane. 
+ */
+export interface WorkerTaskPayload {
+  request_id?: string;
+  subject_opaque_id?: string;
+  idempotency_key?: string;
+  trigger_source?: string;
+  actor_opaque_id?: string;
+  legal_framework?: string;
+  request_timestamp?: string;
+  tenant_id?: string;
+  cooldown_days?: number;
+  shadow_mode?: boolean;
+  webhook_url?: string;
+  userId?: number;
+  now?: string;
+  shadowMode?: boolean;
+}
+
+/**
+ * Leased task envelope returned by Control Plan sync.
+ */
+export interface WorkerTask {
+  id: string;
+  task_type: "COMPILE_DAG" | "VAULT_USER" | "NOTIFY_USER" | "SHRED_USER" | string;
+  payload: WorkerTaskPayload;
+}
+/**
+ * Control Plane sync response shape.
+ */
+export interface SyncTaskResponse {
+  pending: boolean;
+  task?: WorkerTask;
+}
+
+export interface CompileDagResult {
+  action: "compiled_dag";
+  userHash: null;
+  dryRun: false;
+  compiledTargetCount: number;
+}
+
+export type TaskExecutionResult = CompileDagResult | VaultUserResult | DispatchNoticeResult | ShredUserResult;
+
+/**
+ * Failed-task acknowledgement payload.
+ */
+export interface TaskFailureResult {
+  error: WorkerProblemDetails;
+}
+
+export type TaskAckPayload = TaskExecutionResult | TaskFailureResult;
+
+/**
+ * Network contract required by the worker loop.
+ */
+export interface ApiClient {
+  syncTask(): Promise<SyncTaskResponse>;
+  ackTask(taskId: string, status: "completed" | "failed", result: TaskAckPayload): Promise<boolean>;
+  heartbeatTask?(taskId: string): Promise<boolean>;
+  pushOutboxEvent(event: OutboxEvent): Promise<boolean>;
+}
+
+/**
+ * Dependencies required to construct the compliance worker.
+ */
+export interface ComplianceWorkerOptions {
+  sql: Sql;
+  sqlReplica?: Sql;
+  secrets: WorkerSecrets;
+  config: WorkerConfig;
+  apiClient: ApiClient;
+  mailer: MockMailer;
+  s3Client?: S3Client;
+  taskHeartbeatIntervalMs?: number;
+}