dpdp-erasure-cli 1.0.0

package/src/modules/network/object-store/aws/index.ts

@@ -0,0 +1,2 @@
+export * from "./client";
+export * from "./credentials";

package/src/modules/network/object-store/aws/sigv4.ts

@@ -0,0 +1,190 @@
+import { bytesToHex } from "@/lib";
+import type { S3AwsCredentials } from "./type";
+
+const textEncoder = new TextEncoder();
+const signingKeyCache = new Map<string, Promise<CryptoKey>>();
+const MAX_SIGNING_KEY_CACHE_ENTRIES = 256;
+
+
+export interface AwsSignedRequestInput {
+  method: string;
+  url: URL;
+  region: string;
+  service: string;
+  headers: Headers;
+  body?: Uint8Array | string;
+  credentials: S3AwsCredentials;
+  now?: Date;
+}
+
+function copyBytes(bytes: Uint8Array): Uint8Array {
+  const copy = new Uint8Array(bytes.length);
+  copy.set(bytes);
+  return copy;
+}
+
+function encodeBody(body: Uint8Array | string | undefined): Uint8Array {
+  if (body === undefined) {
+    return new Uint8Array(0);
+  }
+
+  return typeof body === "string" ? copyBytes(textEncoder.encode(body)) : copyBytes(body);
+}
+
+async function sha256Hex(input: Uint8Array | string): Promise<string> {
+  const bytes = typeof input === "string" ? textEncoder.encode(input) : input;
+  const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes.slice().buffer as ArrayBuffer);
+  return bytesToHex(new Uint8Array(digest));
+}
+
+async function hmacSha256(key: Uint8Array, value: string): Promise<Uint8Array> {
+  const keyBytes = key.slice();
+  const cryptoKey = await globalThis.crypto.subtle.importKey(
+    "raw",
+    keyBytes.buffer as ArrayBuffer,
+    { name: "HMAC", hash: "SHA-256" },
+    false,
+    ["sign"]
+  );
+  const data = textEncoder.encode(value).slice();
+  const signature = await globalThis.crypto.subtle.sign("HMAC", cryptoKey, data.buffer as ArrayBuffer);
+  return new Uint8Array(signature);
+}
+
+async function hmacSha256WithKey(key: CryptoKey, value: string): Promise<Uint8Array> {
+  const data = textEncoder.encode(value);
+  const signature = await globalThis.crypto.subtle.sign("HMAC", key, data);
+  return new Uint8Array(signature);
+}
+
+function signingCacheKey(input: AwsSignedRequestInput, dateStamp: string): string {
+  return [
+    input.credentials.accessKeyId,
+    input.credentials.sessionToken ?? "",
+    input.credentials.expiration?.getTime() ?? "",
+    dateStamp,
+    input.region,
+    input.service,
+  ].join("|");
+}
+
+async function importSigningKey(bytes: Uint8Array): Promise<CryptoKey> {
+  const keyBytes = bytes.slice();
+  try {
+    return await globalThis.crypto.subtle.importKey(
+      "raw",
+      keyBytes,
+      { name: "HMAC", hash: "SHA-256" },
+      false,
+      ["sign"]
+    );
+  } finally {
+    keyBytes.fill(0);
+  }
+}
+
+async function deriveSigningKey(input: AwsSignedRequestInput, dateStamp: string): Promise<CryptoKey> {
+  const cacheKey = signingCacheKey(input, dateStamp);
+  const cached = signingKeyCache.get(cacheKey);
+  if (cached) {
+    return cached;
+  }
+
+  if (signingKeyCache.size >= MAX_SIGNING_KEY_CACHE_ENTRIES) {
+    signingKeyCache.delete(signingKeyCache.keys().next().value as string);
+  }
+
+  const derived = (async () => {
+    const secretSeed = textEncoder.encode(`AWS4${input.credentials.secretAccessKey}`);
+    let dateKey: Uint8Array = new Uint8Array(0);
+    let regionKey: Uint8Array = new Uint8Array(0);
+    let serviceKey: Uint8Array = new Uint8Array(0);
+    let signingKey: Uint8Array = new Uint8Array(0);
+    try {
+      dateKey = await hmacSha256(secretSeed, dateStamp);
+      regionKey = await hmacSha256(dateKey, input.region);
+      serviceKey = await hmacSha256(regionKey, input.service);
+      signingKey = await hmacSha256(serviceKey, "aws4_request");
+      return importSigningKey(signingKey);
+    } finally {
+      secretSeed.fill(0);
+      dateKey.fill(0);
+      regionKey.fill(0);
+      serviceKey.fill(0);
+      signingKey.fill(0);
+    }
+  })();
+  signingKeyCache.set(cacheKey, derived);
+  return derived;
+}
+
+function buildCanonicalQuery(searchParams: URLSearchParams): string {
+  return Array.from(searchParams.entries())
+    .sort(([leftName, leftValue], [rightName, rightValue]) => {
+      const byName = leftName.localeCompare(rightName);
+      return byName === 0 ? leftValue.localeCompare(rightValue) : byName;
+    })
+    .map(([name, value]) => `${encodeURIComponent(name)}=${encodeURIComponent(value)}`)
+    .join("&");
+}
+
+function normalizeAmzDate(now: Date): { amzDate: string; dateStamp: string } {
+  const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, "");
+  return { amzDate, dateStamp: amzDate.slice(0, 8) };
+}
+
+/**
+ * Signs an AWS REST request with Signature Version 4 using Web Crypto HMAC-SHA256.
+ *
+ * @param input - Request method, URL, headers, body, service, region, and credentials.
+ * @returns Headers containing SigV4 authorization fields.
+ */
+export async function signAwsRequest(input: AwsSignedRequestInput): Promise<Headers> {
+  const bodyBytes = encodeBody(input.body);
+  const payloadHash = await sha256Hex(bodyBytes);
+  const { amzDate, dateStamp } = normalizeAmzDate(input.now ?? new Date());
+  const headers = new Headers(input.headers);
+
+  headers.set("host", input.url.host);
+  headers.set("x-amz-content-sha256", payloadHash);
+  headers.set("x-amz-date", amzDate);
+  if (input.credentials.sessionToken) {
+    headers.set("x-amz-security-token", input.credentials.sessionToken);
+  }
+
+  const sortedHeaders = Array.from(headers.entries())
+    .map(([name, value]) => [name.toLowerCase(), value.trim().replace(/\s+/g, " ")] as const)
+    .sort(([left], [right]) => left.localeCompare(right));
+  const canonicalHeaders = sortedHeaders.map(([name, value]) => `${name}:${value}\n`).join("");
+  const signedHeaders = sortedHeaders.map(([name]) => name).join(";");
+  const canonicalRequest = [
+    input.method.toUpperCase(),
+    input.url.pathname || "/",
+    buildCanonicalQuery(input.url.searchParams),
+    canonicalHeaders,
+    signedHeaders,
+    payloadHash,
+  ].join("\n");
+
+  const credentialScope = `${dateStamp}/${input.region}/${input.service}/aws4_request`;
+  const stringToSign = [
+    "AWS4-HMAC-SHA256",
+    amzDate,
+    credentialScope,
+    await sha256Hex(canonicalRequest),
+  ].join("\n");
+
+  let signatureBytes: Uint8Array = new Uint8Array(0);
+
+  try {
+    const signingKey = await deriveSigningKey(input, dateStamp);
+    signatureBytes = await hmacSha256WithKey(signingKey, stringToSign);
+    headers.set(
+      "authorization",
+      `AWS4-HMAC-SHA256 Credential=${input.credentials.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${bytesToHex(signatureBytes)}`
+    );
+    return headers;
+  } finally {
+    signatureBytes.fill(0);
+  }
+}

package/src/modules/network/object-store/aws/type.ts

@@ -0,0 +1,6 @@
+export interface S3AwsCredentials {
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken?: string;
+  expiration?: Date;
+}

package/src/modules/network/object-store/index.ts

@@ -0,0 +1 @@
+export * from "./aws";

package/src/modules/network/outbox/dispatcher.ts

@@ -0,0 +1,183 @@
+import { outboxLogger } from "@/utils";
+import { fail, workerError } from "@/errors";
+import { computeRequestSignature } from "../request-signing";
+import type { FetchDispatcherOptions, OutboxEvent } from "./types";
+
+interface ControlPlaneOutboxPayload {
+  request_id?: string | null;
+  subject_opaque_id?: string | null;
+  event_timestamp?: string | null;
+  [key: string]: unknown;
+}
+
+
+function isRetryableProblemBody(body: unknown): boolean {
+  if (!body || typeof body !== "object") {
+    return false;
+  }
+
+  const record = body as Record<string, unknown>;
+  if (record.retryable === true || record.code === "API_OUTBOX_PREVIOUS_HASH_INVALID") {
+    return true;
+  }
+
+  const error = record.error;
+  if (!error || typeof error !== "object") {
+    return false;
+  }
+
+  const errorRecord = error as Record<string, unknown>;
+  return errorRecord.retryable === true || errorRecord.code === "API_OUTBOX_PREVIOUS_HASH_INVALID";
+}
+
+async function readRetryableProblem(response: Response): Promise<boolean> {
+  try {
+    const body = await response.clone().json() as unknown;
+    return isRetryableProblemBody(body);
+  } catch {
+    return false;
+  }
+}
+
+function buildControlPlaneRequestBody(event: OutboxEvent) {
+  if (!event.payload || typeof event.payload !== "object" || Array.isArray(event.payload)) {
+    fail({
+      code: "OUTBOX_PAYLOAD_INVALID",
+      title: "Invalid outbox payload",
+      detail: `Outbox payload for event ${event.id} must be an object.`,
+      category: "integrity",
+      retryable: false,
+      fatal: true,
+      context: { eventId: event.id },
+    });
+  }
+
+  const payload = event.payload as ControlPlaneOutboxPayload;
+  if (!payload.request_id || !payload.subject_opaque_id || !payload.event_timestamp) {
+    fail({
+      code: "OUTBOX_PROTOCOL_REJECTED",
+      title: "Outbox payload missing control-plane envelope",
+      detail: `Outbox event ${event.id} is missing request_id, subject_opaque_id, or event_timestamp.`,
+      category: "integrity",
+      retryable: false,
+      fatal: true,
+      context: { eventId: event.id },
+    });
+  }
+
+  return {
+    idempotency_key: event.idempotency_key,
+    request_id: payload.request_id,
+    subject_opaque_id: payload.subject_opaque_id,
+    event_type: event.event_type,
+    payload,
+    previous_hash: event.previous_hash,
+    current_hash: event.current_hash,
+    event_timestamp: payload.event_timestamp,
+  };
+}
+
+/**
+ * No-op dispatcher used by tests and local execution when no HTTP transport is injected.
+ *
+ * @param event - Outbox event to "send".
+ * @returns Always `true` after logging.
+ */
+export async function sendToAPI(event: OutboxEvent): Promise<boolean> {
+  outboxLogger.info({ eventId: event.id, eventType: event.event_type }, "Outbox event synced");
+  return true;
+}
+
+/**
+ * Creates an HTTP dispatcher that publishes worker outbox events to the Control Plane.
+ *
+ * @param options - Endpoint URL, auth headers, and timeout configuration.
+ * @returns Dispatcher function compatible with `processOutbox`.
+ */
+export function createFetchDispatcher(options: FetchDispatcherOptions) {
+  const timeoutMs = options.timeoutMs ?? 10_000;
+
+  return async function dispatch(event: OutboxEvent): Promise<boolean> {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    const body = buildControlPlaneRequestBody(event);
+
+    try {
+      const bodyText = JSON.stringify(body);
+      const requestSigningSecret = (options as FetchDispatcherOptions).requestSigningSecret;
+      const clientId = options.clientId ?? fail({
+        code: "DISPATCHER_CONFIG_INVALID",
+        title: "Missing Client Identifier",
+        detail: "Cannot create fetch dispatcher because clientId is missing from options.",
+        category: "integrity",
+        retryable: false,
+        fatal: true,
+      });
+
+      const signingHeaders = requestSigningSecret
+        ? (async () => {
+          const timestamp = String(Date.now());
+          return await computeRequestSignature(
+            requestSigningSecret,
+            "POST",
+            new URL(options.url).pathname,
+            clientId,
+            timestamp,
+            bodyText
+          ).then((signature) => ({
+            "x-dpdp-timestamp": timestamp,
+            "x-dpdp-signature": signature
+          }));
+        })()
+        : Promise.resolve({});
+
+      const signedHeaders = await signingHeaders;
+      const response = await fetch(options.url, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          ...(options.clientId ? { "x-client-id": options.clientId } : {}),
+          ...(options.token ? { authorization: `Bearer ${options.token}` } : {}),
+          ...signedHeaders,
+        },
+        body: bodyText,
+        signal: controller.signal,
+        redirect: "error",
+      });
+
+      if (!response.ok) {
+        const retryableProblem = await readRetryableProblem(response);
+        const retryable = response.status >= 500 || response.status === 429 || retryableProblem;
+        throw workerError({
+          code:
+            response.status === 401 || response.status === 403
+              ? "OUTBOX_AUTH_REJECTED"
+              : retryable
+                ? "OUTBOX_DELIVERY_FAILED"
+                : "OUTBOX_PROTOCOL_REJECTED",
+          title:
+            response.status === 401 || response.status === 403
+              ? "Control Plane authentication rejected outbox event"
+              : "Control Plane rejected outbox event",
+          detail: `Brain API responded with HTTP ${response.status}.`,
+          category:
+            response.status === 401 || response.status === 403
+              ? "configuration"
+              : retryable
+                ? "network"
+                : "external",
+          retryable,
+          fatal: !retryable,
+          context: {
+            status: response.status,
+            url: options.url,
+          },
+        });
+      }
+
+      return true;
+    } finally {
+      clearTimeout(timer);
+    }
+  };
+}

package/src/modules/network/outbox/index.ts

@@ -0,0 +1,3 @@
+export * from "./dispatcher";
+export * from "./types";
+export * from "./process";

package/src/modules/network/outbox/process.ts

@@ -0,0 +1,133 @@
+import type { Sql } from "@/types";
+import { workerError } from "@/errors";
+import { assertIdentifier, logError, outboxLogger } from "@/utils";
+import type { OutboxEvent, ProcessOutboxOptions, ProcessOutboxResult } from "./types";
+import {
+  DEFAULT_BASE_BACKOFF_MS,
+  DEFAULT_BATCH_SIZE,
+  DEFAULT_ENGINE_SCHEMA,
+  DEFAULT_LEASE_SECONDS,
+  DEFAULT_MAX_ATTEMPTS,
+  resolvePositiveInteger
+} from "./shared";
+import { sendToAPI } from "./dispatcher";
+import { claimOutboxBatch, extendOutboxLeases, markOutboxEventFailed, markOutboxEventProcessed, releaseOutboxLease } from "./store";
+
+/**
+ * Claims due outbox events, dispatches them, and applies processed/retry/dead-letter state transitions.
+ *
+ * Fatal delivery failures are rethrown after lease release so the worker loop can fail closed.
+ *
+ * @param sql - Postgres pool owning the outbox table.
+ * @param syncFn - Event delivery function, usually an HTTP dispatcher.
+ * @param options - Lease and retry tuning values.
+ * @returns Aggregate processing counters for the claimed batch.
+ * @throws {WorkerError} On fatal protocol or configuration errors, or lease invariants.
+ */
+export async function processOutbox(
+  sql: Sql,
+  syncFn: (event: OutboxEvent) => Promise<boolean> = sendToAPI,
+  options: ProcessOutboxOptions = {}
+): Promise<ProcessOutboxResult> {
+  const engineSchema = assertIdentifier(
+    options.engineSchema ?? DEFAULT_ENGINE_SCHEMA,
+    "engine schema name"
+  );
+  const batchSize = resolvePositiveInteger(
+    options.batchSize,
+    DEFAULT_BATCH_SIZE,
+    "batchSize"
+  );
+  const leaseSeconds = resolvePositiveInteger(
+    options.leaseSeconds,
+    DEFAULT_LEASE_SECONDS,
+    "leaseSeconds"
+  );
+  const maxAttempts = resolvePositiveInteger(
+    options.maxAttempts,
+    DEFAULT_MAX_ATTEMPTS,
+    "maxAttempts"
+  );
+  const baseBackoffMs = resolvePositiveInteger(
+    options.baseBackoffMs,
+    DEFAULT_BASE_BACKOFF_MS,
+    "baseBackoffMs"
+  );
+  const clock = () => (options.now ? new Date(options.now) : new Date());
+  const now = clock();
+
+  const { leaseToken, events } = await claimOutboxBatch(
+    sql,
+    engineSchema,
+    batchSize,
+    leaseSeconds,
+    now
+  );
+
+  const result: ProcessOutboxResult = {
+    claimed: events.length,
+    processed: 0,
+    failed: 0,
+    deadLettered: 0,
+  };
+
+  for (let index = 0; index < events.length; index += 1) {
+    const event = events[index]!;
+    try {
+      const leaseNow = clock();
+      await extendOutboxLeases(
+        sql,
+        engineSchema,
+        events.slice(index).map((candidate) => candidate.id),
+        event.id,
+        leaseToken,
+        new Date(leaseNow.getTime() + leaseSeconds * 1000),
+        leaseNow
+      );
+
+      const delivered = await syncFn(event);
+      if (!delivered) {
+        throw workerError({
+          code: "OUTBOX_DELIVERY_RESULT_INVALID",
+          title: "Outbox dispatcher returned an invalid result",
+          detail: `Dispatcher returned a falsy delivery result for event ${event.id}.`,
+          category: "network",
+          retryable: true,
+          context: { eventId: event.id },
+        });
+      }
+
+      await markOutboxEventProcessed(sql, engineSchema, event.id, leaseToken, clock());
+      result.processed += 1;
+    } catch (error) {
+      const normalized = logError(outboxLogger, error, "Failed to sync outbox event", {
+        eventId: event.id,
+        eventType: event.event_type,
+      });
+
+      if (normalized.fatal) {
+        await releaseOutboxLease(sql, engineSchema, event.id, leaseToken, clock(), normalized);
+        throw normalized;
+      }
+
+      const failureState = await markOutboxEventFailed(
+        sql,
+        engineSchema,
+        event,
+        leaseToken,
+        clock(),
+        maxAttempts,
+        baseBackoffMs,
+        error
+      );
+
+      result.failed += 1;
+      if (failureState === "dead_letter") {
+        result.deadLettered += 1;
+      }
+    }
+  }
+
+  return result;
+}
+

package/src/modules/network/outbox/shared.ts

@@ -0,0 +1,56 @@
+import { fail } from "@/errors";
+
+export const DEFAULT_ENGINE_SCHEMA = "dpdp_engine";
+export const DEFAULT_BATCH_SIZE = 10;
+export const DEFAULT_LEASE_SECONDS = 60;
+export const DEFAULT_MAX_ATTEMPTS = 10;
+export const DEFAULT_BASE_BACKOFF_MS = 1000;
+
+/**
+ * Validates that an outbox tuning value is a positive integer.
+ *
+ * @param value - Optional runtime override.
+ * @param fallback - Default value when the override is absent.
+ * @param label - Human-readable option name for error details.
+ * @returns Validated positive integer.
+ */
+export function resolvePositiveInteger(
+  value: number | undefined,
+  fallback: number,
+  label: string
+): number {
+  if (value === undefined) {
+    return fallback;
+  }
+
+  if (!Number.isInteger(value) || value < 1) {
+    fail({
+      code: "OUTBOX_OPTION_INVALID",
+      title: "Invalid outbox option",
+      detail: `${label} must be an integer greater than 0.`,
+      category: "validation",
+      retryable: false,
+      context: { label },
+    });
+  }
+
+  return value;
+}
+
+/**
+ * Computes exponential retry delay capped at five minutes.
+ *
+ * @param attemptNumber - 1-based attempt count.
+ * @param baseBackoffMs - Initial backoff duration in milliseconds.
+ * @returns Retry delay in milliseconds.
+ */
+export function calculateRetryDelayMs(
+  attemptNumber: number,
+  baseBackoffMs: number = DEFAULT_BASE_BACKOFF_MS
+): number {
+  return Math.min(
+    baseBackoffMs * 2 ** Math.max(0, attemptNumber - 1),
+    5 * 60 * 1000
+  );
+}
+