dpdp-erasure-cli 1.0.0

package/src/modules/worker/worker.ts

@@ -0,0 +1,243 @@
+import {
+  dispatchPreErasureNotice,
+  shredUser,
+  vaultUser,
+  type MockMailer,
+  type WorkerSecrets
+} from "@modules/engine";
+import type { WorkerConfig } from "@modules/config";
+import { processOutbox, type ProcessOutboxResult, type S3Client } from "@modules/network";
+import type { Sql } from "@/types";
+import { fail, serializeWorkerError } from "@/errors";
+import { logError, workerLogger } from "@/utils";
+import type { ApiClient, ComplianceWorkerOptions, TaskExecutionResult, WorkerTask } from "./types";
+import { acknowledgeTask, resolveTaskSubject } from "./tasks";
+
+/**
+ * Orchestrates Control Plane tasks and enforces fail-closed execution semantics.
+ */
+export class ComplianceWorker {
+  private readonly sql: Sql;
+  private readonly sqlReplica?: Sql;
+  private readonly secrets: WorkerSecrets;
+  private readonly config: WorkerConfig;
+  private readonly apiClient: ApiClient;
+  private readonly mailer: MockMailer;
+  private readonly s3Client?: S3Client;
+  private readonly taskHeartbeatIntervalMs: number;
+
+  constructor(options: ComplianceWorkerOptions) {
+    this.sql = options.sql;
+    this.sqlReplica = options.sqlReplica;
+    this.secrets = options.secrets;
+    this.config = options.config;
+    this.apiClient = options.apiClient;
+    this.mailer = options.mailer;
+    this.s3Client = options.s3Client;
+    this.taskHeartbeatIntervalMs = Math.max(1_000, options.taskHeartbeatIntervalMs ?? 30_000);
+  }
+
+  private startTaskHeartbeat(task: WorkerTask): () => void {
+    if (!this.apiClient.heartbeatTask) {
+      return () => undefined;
+    }
+
+    const heartbeat = this.apiClient.heartbeatTask;
+    const timer = setInterval(() => {
+      void heartbeat(task.id).catch((error) => {
+        logError(
+          workerLogger.child({ taskId: task.id, taskType: task.task_type }),
+          error,
+          "Task heartbeat failed"
+        );
+      });
+    }, this.taskHeartbeatIntervalMs);
+
+    return () => clearInterval(timer);
+  }
+
+  private async executeTask(task: WorkerTask, now: Date): Promise<TaskExecutionResult> {
+    switch (task.task_type) {
+      case "COMPILE_DAG":
+        return {
+          action: "compiled_dag",
+          userHash: null,
+          dryRun: false,
+          compiledTargetCount: this.config.rules?.[0]?.targets.length ?? 0,
+        };
+
+      case "VAULT_USER":
+        return vaultUser(
+          this.sql,
+          task.payload.subject_opaque_id ?? task.payload.userId ?? "",
+          this.secrets,
+          {
+            appSchema: this.config.database.app_schema,
+            engineSchema: this.config.database.engine_schema,
+            defaultRetentionYears: this.config.compliance_policy.default_retention_years,
+            noticeWindowHours: this.config.compliance_policy.notice_window_hours,
+            graphMaxDepth: this.config.graph.max_depth,
+            rootTable: this.config.graph.root_table,
+            rootIdColumn: this.config.graph.root_id_column,
+            rootPiiColumns: this.config.graph.root_pii_columns,
+            satelliteTargets: this.config.satellite_targets,
+            blobTargets: this.config.blob_targets,
+            compiledTargets: this.config.rules?.[0]?.targets,
+            retentionRules: this.config.compliance_policy.retention_rules,
+            tenantId: task.payload.tenant_id,
+            requestId: task.payload.request_id,
+            subjectOpaqueId: task.payload.subject_opaque_id,
+            triggerSource: task.payload.trigger_source,
+            actorOpaqueId: task.payload.actor_opaque_id,
+            legalFramework: task.payload.legal_framework,
+            requestTimestamp: task.payload.request_timestamp,
+            shadowMode: task.payload.shadow_mode ?? task.payload.shadowMode,
+            sqlReplica: this.sqlReplica,
+            s3Client: this.s3Client,
+            now,
+          }
+        );
+
+      case "NOTIFY_USER":
+        return dispatchPreErasureNotice(this.sql, resolveTaskSubject(task), this.secrets, this.mailer, {
+          appSchema: this.config.database.app_schema,
+          engineSchema: this.config.database.engine_schema,
+          rootTable: this.config.graph.root_table,
+          notificationLeaseSeconds: this.config.security.notification_lease_seconds,
+          noticeEmailColumn: this.config.graph.notice_email_column,
+          noticeNameColumn: this.config.graph.notice_name_column,
+          rootPiiColumns: this.config.graph.root_pii_columns,
+          now,
+        });
+
+      case "SHRED_USER":
+        return shredUser(this.sql, resolveTaskSubject(task), {
+          appSchema: this.config.database.app_schema,
+          engineSchema: this.config.database.engine_schema,
+          rootTable: this.config.graph.root_table,
+          hmacKey: this.secrets.hmacKey,
+          s3Client: this.s3Client,
+          now,
+        });
+
+      default:
+        fail({
+          code: "TASK_TYPE_UNKNOWN",
+          title: "Unknown task type",
+          detail: `Unknown task type: ${task.task_type}.`,
+          category: "validation",
+          retryable: false,
+          context: {
+            taskId: task.id,
+            taskType: task.task_type,
+          },
+        });
+    }
+  }
+
+  /**
+   * Processes at most one leased task from the Control Plane.
+   *
+   * Retryable/fatal errors are rethrown to preserve lease recovery behavior in the caller loop.
+   *
+   * @returns `true` when a task was claimed (completed or failed-ack), `false` when no task was pending.
+   * @throws {WorkerError} On retryable/fatal execution failures.
+   */
+  async processNextTask(): Promise<boolean> {
+    const { pending, task } = await this.apiClient.syncTask();
+    if (!pending || !task) {
+      return false
+    };
+
+    const taskLogger = workerLogger.child({
+      taskId: task.id,
+      taskType: task.task_type
+    });
+
+    try {
+      const now = task.payload.now ? new Date(task.payload.now) : new Date();
+      const stopHeartbeat = this.startTaskHeartbeat(task);
+      let result: TaskExecutionResult;
+
+      try {
+        result = await this.executeTask(task, now);
+      } finally {
+        stopHeartbeat();
+      }
+
+      await acknowledgeTask(this.apiClient, task.id, "completed", result)
+      taskLogger.info({ action: result.action, uesrHash: result.userHash })
+
+      return true;
+    } catch (error) {
+      const normalized = logError(taskLogger, error, 'Task execution failed');
+
+      if (normalized.fatal || normalized.retryable) {
+        throw normalized;
+      }
+
+      await acknowledgeTask(this.apiClient, task.id, "failed", {
+        error: serializeWorkerError(normalized, `task:${task.id}`),
+      });
+      taskLogger.warn({ code: normalized.code }, "Task acknowledged as failed");
+      return true;
+    }
+  }
+
+  /**
+   * Processes a bounded number of independent Control Plane tasks concurrently.
+   *
+   * Each task still owns its own database transaction, lease, heartbeat, and acknowledgement.
+   * Fatal failures are rethrown after all in-flight tasks settle so one bad task cannot orphan
+   * sibling leases in the same worker loop iteration.
+   *
+   * @param concurrency - Maximum task claims to attempt in this pass.
+   * @returns Number of claimed tasks that completed or were acknowledged as failed.
+   * @throws {WorkerError} If any in-flight task reports a fatal/retryable loop-level failure.
+   */
+  async processTaskBatch(concurrency: number): Promise<number> {
+    const width = Math.max(1, Math.floor(concurrency));
+    if (width === 1) {
+      return await this.processNextTask() ? 1 : 0;
+    }
+
+    const results = await Promise.allSettled(
+      Array.from({ length: width }, () => this.processNextTask())
+    );
+    let processed = 0;
+    let firstFailure: unknown;
+
+    for (const result of results) {
+      if (result.status === "fulfilled") {
+        if (result.value) {
+          processed += 1;
+        }
+        continue;
+      }
+
+      firstFailure ??= result.reason;
+    }
+
+    if (firstFailure) {
+      throw firstFailure;
+    }
+
+    return processed;
+  }
+
+  /**
+     * Flushes the local transactional outbox to the Control Plane endpoint.
+     *
+     * @returns Promise resolved after one outbox processing pass.
+     * @throws {WorkerError} When outbox processing detects fatal delivery/protocol errors.
+     */
+  async flushOutbox(): Promise<ProcessOutboxResult> {
+    return processOutbox(this.sql, async (event) => this.apiClient.pushOutboxEvent(event), {
+      engineSchema: this.config.database.engine_schema,
+      batchSize: this.config.outbox.batch_size,
+      leaseSeconds: this.config.outbox.lease_seconds,
+      maxAttempts: this.config.outbox.max_attempts,
+      baseBackoffMs: this.config.outbox.base_backoff_ms,
+    });
+  }
+}

package/src/secrets/index.ts

@@ -0,0 +1,4 @@
+export * from "./kms";
+export * from "./reader";
+export * from "./repository";
+export * from "./resolvers";

package/src/secrets/kms/index.ts

@@ -0,0 +1,2 @@
+export * from "./signature";
+export * from "./validation";

package/src/secrets/kms/signature.ts

@@ -0,0 +1,82 @@
+import { bytesToHex } from "@/lib";
+import { hmacSha256, seedSecret, sha256Hex } from "../repository"
+
+export interface KMSAwsCredentials {
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken?: string;
+}
+
+export async function signAwsKmsRequest(
+  endpoint: URL,
+  region: string,
+  body: string,
+  credentials: KMSAwsCredentials,
+  now: Date = new Date()
+): Promise<Headers> {
+  const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, "");
+  const dateStamp = amzDate.slice(0, 8);
+  const payloadHash = await sha256Hex(body);
+  const headers = new Headers({
+    "content-type": "application/x-amz-json-1.1",
+    host: endpoint.host,
+    "x-amz-content-sha256": payloadHash,
+    "x-amz-date": amzDate,
+    "x-amz-target": "TrentService.Decrypt",
+  });
+
+  if (credentials.sessionToken) {
+    headers.set("x-amz-security-token", credentials.sessionToken);
+  }
+
+  const sortedHeaders = Array.from(headers.entries()).sort(([left], [right]) => left.localeCompare(right));
+  const canonicalHeaders = sortedHeaders
+    .map(([name, value]) => `${name.toLowerCase()}:${value.trim().replace(/\s+/g, " ")}\n`)
+    .join("");
+  const signedHeaders = sortedHeaders.map(([name]) => name.toLowerCase()).join(";");
+  const canonicalRequest = [
+    "POST",
+    endpoint.pathname || "/",
+    endpoint.search.length > 1 ? endpoint.search.slice(1) : "",
+    canonicalHeaders,
+    signedHeaders,
+    payloadHash,
+  ].join("\n");
+  const credentialScope = `${dateStamp}/${region}/kms/aws4_request`;
+  const stringToSign = [
+    "AWS4-HMAC-SHA256",
+    amzDate,
+    credentialScope,
+    await sha256Hex(canonicalRequest),
+  ].join("\n");
+
+  const secretSeed = seedSecret(credentials.secretAccessKey)
+  let dateKey: Uint8Array = new Uint8Array(0);
+  let regionKey: Uint8Array = new Uint8Array(0);
+  let serviceKey: Uint8Array = new Uint8Array(0);
+  let signingKey: Uint8Array = new Uint8Array(0);
+  let signatureBytes: Uint8Array = new Uint8Array(0);
+
+  try {
+    dateKey = await hmacSha256(secretSeed, dateStamp);
+    regionKey = await hmacSha256(dateKey, region);
+    serviceKey = await hmacSha256(regionKey, "kms");
+    signingKey = await hmacSha256(serviceKey, "aws4_request");
+    signatureBytes = await hmacSha256(signingKey, stringToSign);
+    const signature = bytesToHex(signatureBytes);
+
+    headers.set(
+      "authorization",
+      `AWS4-HMAC-SHA256 Credential=${credentials.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`
+    );
+  } finally {
+    secretSeed.fill(0);
+    dateKey.fill(0);
+    regionKey.fill(0);
+    serviceKey.fill(0);
+    signingKey.fill(0);
+    signatureBytes.fill(0);
+  }
+
+  return headers;
+}

package/src/secrets/kms/validation.ts

@@ -0,0 +1,64 @@
+import { z } from "zod";
+
+const envSourceSchema = z
+  .object({
+    provider: z.literal("env"),
+    env: z.string().min(1),
+  })
+  .strict();
+
+const fileSourceSchema = z
+  .object({
+    provider: z.literal("file"),
+    path: z.string().min(1),
+  })
+
+const awsKmsSourceSchema = z
+  .object({
+    provider: z.literal("aws_kms"),
+    region: z.string().min(1),
+    ciphertext_blob_base64: z.string().min(1),
+    key_id: z.string().min(1).optional(),
+    encryption_context: z.record(z.string(), z.string()).optional(),
+    endpoint: z.url().optional(),
+    access_key_id_env: z.string().min(1).default("AWS_ACCESS_KEY_ID"),
+    secret_access_key_env: z.string().min(1).default("AWS_SECRET_ACCESS_KEY"),
+    session_token_env: z.string().min(1).default("AWS_SESSION_TOKEN"),
+  })
+  .strict();
+
+const gcpSecretManagerSourceSchema = z
+  .object({
+    provider: z.literal("gcp_secret_manager"),
+    secret_version: z.string().min(1),
+    endpoint: z.url().optional(),
+    access_token_env: z.string().min(1).default("GCP_ACCESS_TOKEN"),
+    metadata_token_url: z
+      .url()
+      .default("http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"),
+  })
+  .strict();
+
+const vaultKvV2SourceSchema = z
+  .object({
+    provider: z.literal("hashicorp_vault"),
+    address: z.url().optional(),
+    address_env: z.string().min(1).default("VAULT_ADDR"),
+    token_env: z.string().min(1).default("VAULT_TOKEN"),
+    namespace_env: z.string().min(1).default("VAULT_NAMESPACE"),
+    mount: z.string().min(1),
+    path: z.string().min(1),
+    field: z.string().min(1),
+    version: z.number().int().positive().optional(),
+  })
+  .strict();
+
+export const keySourceSchema = z.discriminatedUnion("provider", [
+  envSourceSchema,
+  fileSourceSchema,
+  awsKmsSourceSchema,
+  gcpSecretManagerSourceSchema,
+  vaultKvV2SourceSchema
+])
+
+export type KeySourceConfig = z.infer<typeof keySourceSchema>;

package/src/secrets/reader.ts

@@ -0,0 +1,42 @@
+import type { EnvType } from "@/types";
+import { decodeKeyMaterial } from "./repository";
+import type { ResolveKeyOptions } from "./resolvers";
+
+/**
+ * Resolves a runtime secret from an environment variable or it's moutned-file companion
+ * 
+ * If `FOO_FILE` is present, the worker reads the secret from that path. This supports
+ * Kubernetes secret volumes, Vault agent injection, and CSI-mounted secret providers
+ * without changing the YAML contract that names logical secret identifiers. 
+ * 
+ * @param env - Raw environment variable
+ * @param envName - Logical environment variable name declared in yaml
+ * @returns Resolved secret value or an empty string when no source is configured
+ */
+export async function readRuntimeSecret(
+  env: EnvType,
+  envName: string
+): Promise<string> {
+  const directValue = env[envName];
+  if (directValue && directValue.trim().length > 0) {
+    return directValue.trim()
+  };
+
+  const filePath = env[`${envName}_FILE`];
+  if (!filePath || filePath.trim().length === 0) {
+    return "";
+  }
+
+  return (await Bun.file(filePath).text()).trim();
+}
+
+export async function readLegacyEnvKey(options: ResolveKeyOptions): Promise<Uint8Array> {
+  const value =
+    await readRuntimeSecret(options.env, options.legacyEnvName)
+    || (
+      options.fallbackLegacyEnvName
+        ? await readRuntimeSecret(options.env, options.fallbackLegacyEnvName)
+        : ""
+    );
+  return decodeKeyMaterial(value, options.keyName);
+}

package/src/secrets/repository/crypto.ts

@@ -0,0 +1,89 @@
+import { fail, CODE } from "@/errors";
+import { base64ToBytes, bytesToHex, copyBytes, hexToBytes } from "@/lib";
+
+const KEY_LENGTH = 32;
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+
+function ensureKeyLength(bytes: Uint8Array, keyName: string): Uint8Array {
+  if (bytes.length === KEY_LENGTH) {
+    return new Uint8Array(bytes);
+  }
+
+  fail({
+    code: CODE.SECRET_ENV_INVALID,
+    data: { keyName, KEY_LENGTH },
+    context: { keyName },
+  })
+}
+
+export function normalizeBase64(value: string): string {
+  return value.trim().replace(/-/g, "+").replace(/_/g, "/");
+}
+
+/**
+ * Decodes configured key material from raw bytes or textual hex/base64.
+ *
+ * @param rawValue - Runtime key value returned by env, file, KMS, Secret Manager, or Vault.
+ * @param keyName - Human-readable key label used in fail-closed error details.
+ * @returns A defensive copy of the 32-byte key.
+ * @throws {WorkerError} If the value cannot be decoded into a 256-bit key.
+ */
+export function decodeKeyMaterial(rawValue: string | Uint8Array, keyName: string): Uint8Array {
+  if (rawValue instanceof Uint8Array) {
+    if (rawValue.length === KEY_LENGTH) {
+      return new Uint8Array(rawValue);
+    }
+    rawValue = textDecoder.decode(rawValue);
+  }
+
+  const value = rawValue.trim();
+  if (value.length === 0) {
+    fail({
+      code: CODE.SECRET_ENV_MISSING,
+      data: { keyName },
+      context: { keyName }
+    })
+  }
+
+  const normalizedHex = value.startsWith("hex:") ? value.slice(4) : value;
+  if (/^[0-9a-fA-F]+$/.test(normalizedHex) && normalizedHex.length === KEY_LENGTH * 2) {
+    return hexToBytes(normalizedHex);
+  }
+
+  const normalizedBase64 = value.startsWith("base64:") ? value.slice(7) : value;
+  try {
+    return ensureKeyLength(base64ToBytes(normalizeBase64(normalizedBase64)), keyName)
+  } catch (error) {
+    fail({
+      code: CODE.SECRET_ENV_INVALID,
+      data: { keyName, KEY_LENGTH },
+      context: { keyName }
+    })
+  }
+}
+
+export async function sha256Hex(value: string): Promise<string> {
+  const bytes = textEncoder.encode(value);
+  // Ensure we pass a regular ArrayBuffer, as SharedArrayBuffer is not allowed for crypto.
+  const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes.slice().buffer as ArrayBuffer);
+  return bytesToHex(new Uint8Array(digest));
+}
+
+export function seedSecret(secretAccessKey: string): Uint8Array {
+  return copyBytes(textEncoder.encode(`AWS4${secretAccessKey}`));
+}
+
+export async function hmacSha256(key: Uint8Array, value: string | Uint8Array): Promise<Uint8Array> {
+  const keyBytes = key.slice();
+  const cryptoKey = await globalThis.crypto.subtle.importKey(
+    "raw",
+    keyBytes.buffer as ArrayBuffer,
+    { name: "HMAC", hash: "SHA-256" },
+    false,
+    ["sign"]
+  );
+  const data = typeof value === "string" ? textEncoder.encode(value).slice() : value.slice();
+  const signature = await globalThis.crypto.subtle.sign("HMAC", cryptoKey, data.buffer as ArrayBuffer);
+  return new Uint8Array(signature);
+}

package/src/secrets/repository/index.ts

@@ -0,0 +1,2 @@
+export * from "./crypto";
+export * from "./methods";

package/src/secrets/repository/methods.ts

@@ -0,0 +1,37 @@
+import { fail } from "@/errors";
+
+export const fetchJson = async (
+  fetchFn: typeof fetch,
+  url: string | URL,
+  init: RequestInit,
+  provider: string
+): Promise<unknown> => {
+  const response = await fetchFn(url, {
+    ...init,
+    redirect: "error"
+  });
+  if (!response.ok) {
+    fail({
+      code: "KMS_PROVIDER_FAILED",
+      title: "Key provider request failed",
+      detail: `${provider} responded with HTTP ${response.status}.`,
+      category: "external",
+      retryable: response.status >= 500 || response.status === 429,
+      fatal: response.status >= 400 && response.status < 500 && response.status !== 429,
+      context: { provider, status: response.status },
+    });
+  }
+  return response.json();
+};
+
+export const isRecord = (value: unknown): value is Record<string, unknown> => {
+  return typeof value === "object" && value !== null;
+}
+
+export function encodeVaultPathSegment(value: string): string {
+  return value
+    .split("/")
+    .filter((part) => part.length > 0)
+    .map((part) => encodeURIComponent(part))
+    .join("/");
+}