dpdp-erasure-cli 1.0.0

package/src/modules/engine/blob/types.ts

@@ -0,0 +1,44 @@
+import type { BlobTarget } from "@modules/config";
+import type { S3Client } from "@modules/network";
+
+export type BlobAction = BlobTarget["action"];
+
+export interface BlobProtectionResult {
+  sourceTable: string;
+  sourceColumn: string;
+  provider: "aws_s3";
+  action: BlobAction;
+  objectRefHash: string;
+  versionIdHash: string;
+  legalHoldApplied: boolean;
+  overwriteApplied: boolean;
+};
+
+export interface DiscoveredBlobObject {
+  target: BlobTarget;
+  sourceTable: string;
+  sourceColumn: string;
+  originalValue: string;
+  maskedValue: string;
+  bucket: string;
+  key: string;
+  versionId: string;
+  eTag: string | null;
+  overwriteETag: string | null;
+  overwriteVersionId: string | null;
+}
+
+export interface BlobShredReceipt {
+  provider: "aws_s3";
+  action: BlobAction;
+  objectRefHash: string;
+  versionCount: number;
+  deletedVersionIdHashes: string[];
+  retainedVersionIdHashes: string[];
+  status: "purged" | "captured_version_deleted" | "retained_by_policy";
+}
+
+export interface BlobWorkflowOptions {
+  s3Client?: S3Client;
+}
+

package/src/modules/engine/helpers/identity.ts

@@ -0,0 +1,47 @@
+import { bytesToBase64 } from "@/lib";
+import { generateHMAC } from "@modules/crypto/hmac";
+
+/**
+ * Produces a deterministic subject hash used as the worker's irreversible lookup key.
+ *
+ * @param rootId - Subject identifier in the source schema.
+ * @param appSchema - Source schema name.
+ * @param rootTable - Source root table name.
+ * @param hmacKey - HMAC key bytes.
+ * @param tenantId - Optional tenant discriminator.
+ * @returns Stable HMAC-SHA256 hex digest.
+ */
+export async function createUserHash(
+  rootId: string | number,
+  appSchema: string,
+  rootTable: string,
+  hmacKey: Uint8Array,
+  tenantId?: string
+): Promise<string> {
+  return generateHMAC(
+    `${appSchema}:${rootTable}:${tenantId ?? ""}:${rootId}`,
+    bytesToBase64(hmacKey)
+  );
+}
+
+/**
+ * Derives an irreversible synthetic email for downstream systems that still require an address.
+ *
+ * @param userId - Source subject identifier.
+ * @param email - Original email value from the root payload.
+ * @param salt - Per-row salt stored in vault metadata.
+ * @param hmacKey - HMAC key bytes.
+ * @returns Pseudonymous `dpdp_...@dpdp.invalid` address.
+ */
+export async function createPseudonym(
+  userId: string | number,
+  email: string,
+  salt: string,
+  hmacKey: Uint8Array
+): Promise<string> {
+  const digest = await generateHMAC(
+    `${userId}:${email}`,
+    `${salt}:${bytesToBase64(hmacKey)}`
+  );
+  return `dpdp_${digest.slice(0, 24)}@dpdp.invalid`;
+}

package/src/modules/engine/helpers/index.ts

@@ -0,0 +1,4 @@
+export * from "./runtime";
+export * from "./types";
+export * from "./identity";
+export * from "./outbox";

package/src/modules/engine/helpers/outbox.ts

@@ -0,0 +1,118 @@
+import { fail } from "@/errors";
+import type { OutboxRow } from "@modules/network";
+import type { SqlExecutor } from "@/types";
+import { canonicalJsonStringify } from "@/utils";
+import type { JSONValue } from "postgres";
+
+const UNFINALIZED_PREVIOUS_HASH = "UNFINALIZED";
+const UNFINALIZED_CURRENT_HASH = "0000000000000000000000000000000000000000000000000000000000000000";
+
+/**
+ * Enqueues a tamper-evident outbox event inside the current transaction scope.
+ *
+ * The function is idempotent by `idempotency_key` and inserts an unfinalized chain record.
+ * Hash-chain finalization is intentionally performed by the relay in a short read-committed
+ * transaction after commit, avoiding repeatable-read snapshot forks under multi-worker load.
+ *
+ * @param sql - Postgres pool or transaction.
+ * @param engineSchema - Worker engine schema.
+ * @param userHash - Subject hash associated with the event.
+ * @param eventType - Outbox event type.
+ * @param payload - JSON-serializable event payload.
+ * @param idempotencyKey - Global idempotency key for replay safety.
+ * @param now - Event creation timestamp.
+ * @returns Existing or newly inserted outbox row.
+ * @throws {WorkerError} When payload is non-serializable or insert invariants are violated.
+ */
+export async function enqueueOutboxEvent(
+  sql: SqlExecutor,
+  engineSchema: string,
+  userHash: string,
+  eventType: string,
+  payload: unknown,
+  idempotencyKey: string,
+  now: Date
+): Promise<OutboxRow> {
+  const jsonPayload = payload as JSONValue;
+  try {
+    canonicalJsonStringify(jsonPayload);
+  } catch {
+    fail({
+      code: "OUTBOX_PAYLOAD_INVALID",
+      title: "Invalid outbox payload",
+      detail: "Outbox payload must be JSON-serializable.",
+      category: "validation",
+      retryable: false,
+    });
+  }
+
+  const [existing] = await sql<OutboxRow[]>`
+    SELECT *
+    FROM ${sql(engineSchema)}.outbox
+    WHERE idempotency_key = ${idempotencyKey}
+    LIMIT 1
+  `;
+
+  if (existing) {
+    return existing;
+  }
+
+  const [inserted] = await sql<OutboxRow[]>`
+    INSERT INTO ${sql(engineSchema)}.outbox (
+      idempotency_key,
+      user_uuid_hash,
+      event_type,
+      payload,
+      previous_hash,
+      current_hash,
+      chain_status,
+      status,
+      attempt_count,
+      next_attempt_at,
+      created_at,
+      updated_at
+    )
+    VALUES (
+      ${idempotencyKey},
+      ${userHash},
+      ${eventType},
+      ${sql.json(jsonPayload)},
+      ${UNFINALIZED_PREVIOUS_HASH},
+      ${UNFINALIZED_CURRENT_HASH},
+      'pending',
+      'pending',
+      0,
+      ${now},
+      ${now},
+      ${now}
+    )
+    ON CONFLICT (idempotency_key) DO NOTHING
+    RETURNING *
+  `;
+
+  if (inserted) {
+    return inserted;
+  }
+
+  const [stored] = await sql<OutboxRow[]>`
+    SELECT *
+    FROM ${sql(engineSchema)}.outbox
+    WHERE idempotency_key = ${idempotencyKey}
+    LIMIT 1
+  `;
+
+  if (!stored) {
+    fail({
+      code: "OUTBOX_INSERT_INVARIANT_BROKEN",
+      title: "Outbox insert invariant broken",
+      detail: `Outbox insert for ${idempotencyKey} completed without returning a row.`,
+      category: "database",
+      retryable: false,
+      fatal: true,
+      context: { idempotencyKey },
+    });
+  }
+
+  return stored;
+}
+

package/src/modules/engine/helpers/runtime.ts

@@ -0,0 +1,115 @@
+import { assertIdentifier } from "@/utils";
+import { DEFAULT_APP_SCHEMA, DEFAULT_ENGINE_SCHEMA, DEFAULT_NOTICE_WINDOW_HOURS, DEFAULT_RETENTION_YEARS } from "./types";
+import type { WorkerSchemas, WorkerSecrets } from "../types";
+import { fail } from "@/errors";
+
+/**
+ * Resolves and validates application and engine schema identifiers.
+ *
+ * @param input - Optional schema overrides from operation options.
+ * @returns Canonical schema names safe for dynamic identifier interpolation.
+ * @throws {WorkerError} When any schema name fails identifier validation.
+ */
+export function resolveSchemas(input: WorkerSchemas = {}) {
+  return {
+    appSchema: assertIdentifier(
+      input.appSchema ?? DEFAULT_APP_SCHEMA,
+      "application schema name"
+    ),
+    engineSchema: assertIdentifier(
+      input.engineSchema ?? DEFAULT_ENGINE_SCHEMA,
+      "engine schema name"
+    ),
+  };
+}
+
+/**
+ * Validates worker cryptographic material before any vaulting operation begins.
+ *
+ * `hmacKey` falls back to `kek` when not provided, preserving deterministic pseudonymization.
+ *
+ * @param secrets - Worker key material loaded from config or env.
+ * @returns Normalized key pair safe for downstream crypto helpers.
+ * @throws {WorkerError} When KEK length is not 32 bytes or HMAC key is empty.
+ */
+export function assertWorkerSecrets(
+  secrets: WorkerSecrets
+): { kek: Uint8Array; hmacKey: Uint8Array } {
+  if (secrets.kek.length !== 32) {
+    fail({
+      code: "KEK_INVALID_LENGTH",
+      title: "Invalid KEK length",
+      detail: `Invalid KEK length. Expected 32 bytes, got ${secrets.kek.length}.`,
+      category: "configuration",
+      retryable: false,
+      fatal: true,
+    })
+  }
+
+  const hmacKey = secrets.hmacKey ?? secrets.kek;
+  if (hmacKey.length === 0) {
+    fail({
+      code: "HMAC_KEY_EMPTY",
+      title: "Invalid HMAC key",
+      detail: "HMAC key must not be empty.",
+      category: "configuration",
+      retryable: false,
+      fatal: true,
+    });
+  }
+
+  return {
+    kek: secrets.kek,
+    hmacKey,
+  };
+}
+
+/**
+ * Normalizes retention years with strict non-negative validation.
+ *
+ * @param years - Optional retention duration in years.
+ * @returns Validated retention duration.
+ * @throws {WorkerError} When `years` is non-integer or negative.
+ */
+export function resolveRetentionYears(years?: number): number {
+  if (years === undefined) {
+    return DEFAULT_RETENTION_YEARS;
+  }
+
+  if (!Number.isInteger(years) || years < 0) {
+    fail({
+      code: "RETENTION_YEARS_INVALID",
+      title: "Invalid retention period",
+      detail: "retentionYears must be an integer greater than or equal to 0.",
+      category: "validation",
+      retryable: false,
+    });
+  }
+
+  return years;
+}
+
+/**
+ * Normalizes notice-window configuration while enforcing the legal minimum of one hour.
+ *
+ * @param hours - Optional notice window in hours.
+ * @returns Validated notice window value.
+ * @throws {WorkerError} When `hours` is non-integer or less than 1.
+ */
+export function resolveNoticeWindowHours(hours?: number): number {
+  if (hours === undefined) {
+    return DEFAULT_NOTICE_WINDOW_HOURS;
+  }
+
+  if (!Number.isInteger(hours) || hours < 1) {
+    fail({
+      code: "NOTICE_WINDOW_INVALID",
+      title: "Invalid notice window",
+      detail: "noticeWindowHours must be an integer greater than 0.",
+      category: "validation",
+      retryable: false,
+    });
+  }
+
+  return hours;
+}

package/src/modules/engine/helpers/types.ts

@@ -0,0 +1,61 @@
+/**
+ * Default application schema used by tests and local worker bootstrap.
+ */
+export const DEFAULT_APP_SCHEMA = "mock_app";
+
+/**
+ * Default worker-owned engine schema.
+ */
+export const DEFAULT_ENGINE_SCHEMA = "dpdp_engine";
+
+/**
+ * Default notice lead time in hours.
+ */
+export const DEFAULT_NOTICE_WINDOW_HOURS = 48;
+
+/**
+ * Default retention period when no evidence rule matches.
+ */
+export const DEFAULT_RETENTION_YEARS = 0;
+
+/**
+ * Default recursive graph traversal limit.
+ */
+export const DEFAULT_GRAPH_MAX_DEPTH = 32;
+
+/**
+ * Sentinel payload stored after cryptographic shredding.
+ */
+export const DESTROYED_PII_SENTINEL = Object.freeze({ v: 1, destroyed: true });
+
+/**
+ * Durable record shape stored in `${engineSchema}.pii_vault`.
+ *
+ * Legal metadata is treated as append-only, while lifecycle timestamps and lease fields remain
+ * mutable as the state machine advances.
+ */
+export interface VaultRecord {
+  user_uuid_hash: string;
+  request_id: string | null;
+  tenant_id: string;
+  root_schema: string;
+  root_table: string;
+  root_id: string;
+  pseudonym: string;
+  encrypted_pii: { v?: number; data?: string; destroyed?: boolean };
+  salt: string;
+  dependency_count: number;
+  trigger_source: string | null;
+  legal_framework: string | null;
+  actor_opaque_id: string | null;
+  applied_rule_name: string | null;
+  applied_rule_citation: string | null;
+  retention_expiry: Date;
+  notification_due_at: Date;
+  notification_sent_at: Date | null;
+  notification_lock_id: string | null;
+  notification_lock_expires_at: Date | null;
+  shredded_at: Date | null;
+  created_at: Date;
+  updated_at: Date;
+}

package/src/modules/engine/index.ts

@@ -0,0 +1,6 @@
+export * from "./vault";
+export * from "./helpers";
+export * from "./types";
+export * from "./blob";
+export * from "./notifier";
+export * from "./shredder";

package/src/modules/engine/notifier/config.ts

@@ -0,0 +1,147 @@
+import { fail } from "@/errors";
+import { assertIdentifier } from "@/utils";
+import type { DispatchNoticeOptions } from "../types";
+
+/**
+ * Explicit root payload columns used to build the pre-erasure notice.
+ */
+export interface NoticeColumns {
+  emailColumn: string;
+  nameColumn?: string;
+}
+
+/**
+ * Validates the short-lived notification lease duration.
+ *
+ * @param value - Optional lease duration in seconds.
+ * @returns Lease duration in seconds.
+ * @throws {WorkerError} When the value is non-integer or less than one second.
+ */
+export function resolveNotificationLeaseSeconds(value?: number): number {
+  if (value === undefined) {
+    return 120;
+  }
+
+  if (!Number.isInteger(value) || value < 1) {
+    fail({
+      code: "NOTIFICATION_LEASE_INVALID",
+      title: "Invalid notification lease",
+      detail: "notificationLeaseSeconds must be an integer greater than 0.",
+      category: "validation",
+      retryable: false,
+    });
+  }
+
+  return value;
+}
+
+/**
+ * Builds the deterministic mail idempotency key used by the transport.
+ *
+ * @param vault - Reserved vault row.
+ * @returns Stable idempotency key for the notice delivery.
+ */
+export function buildNotificationIdempotencyKey(vault: {
+  request_id: string | null;
+  root_schema: string;
+  root_table: string;
+  root_id: string;
+  notification_due_at: Date;
+}): string {
+  return vault.request_id
+    ? `notice:${vault.request_id}:${vault.notification_due_at.toISOString()}`
+    : `notice:${vault.root_schema}:${vault.root_table}:${vault.root_id}:${vault.notification_due_at.toISOString()}`;
+}
+
+/**
+ * Resolves the root payload columns that should be used to build the legal notice.
+ *
+ * @param options - Runtime notice options.
+ * @returns Validated email/name column mapping.
+ * @throws {WorkerError} When no explicit or compatible email mapping can be derived.
+ */
+export function resolveNoticeColumns(options: DispatchNoticeOptions): NoticeColumns {
+  if (options.noticeEmailColumn) {
+    return {
+      emailColumn: assertIdentifier(
+        options.noticeEmailColumn,
+        "graph notice email column"
+      ),
+      nameColumn: options.noticeNameColumn
+        ? assertIdentifier(options.noticeNameColumn, "graph notice name column")
+        : undefined,
+    };
+  }
+
+  const configuredColumns = new Set(Object.keys(options.rootPiiColumns ?? {}));
+  if (configuredColumns.size === 0) {
+    return {
+      emailColumn: "email",
+      nameColumn: "full_name",
+    };
+  }
+
+  if (configuredColumns.has("email")) {
+    return {
+      emailColumn: "email",
+      nameColumn: configuredColumns.has("full_name") ? "full_name" : undefined,
+    };
+  }
+
+  fail({
+    code: "NOTIFICATION_EMAIL_COLUMN_MISSING",
+    title: "Missing notice email column mapping",
+    detail:
+      "noticeEmailColumn is required when root_pii_columns does not contain 'email'. Configure graph.notice_email_column in compliance.worker.yml.",
+    category: "configuration",
+    retryable: false,
+    fatal: true,
+  });
+}
+
+/**
+ * Produces the human-readable dry-run plan for a pre-erasure notice.
+ *
+ * @param appSchema - Source application schema.
+ * @param engineSchema - Worker engine schema.
+ * @param rootTable - Root table name.
+ * @param subjectId - Root identifier.
+ * @param userHash - Worker-side subject hash.
+ * @param notificationDueAt - Notice eligibility timestamp.
+ * @param retentionExpiry - Retention expiry timestamp.
+ * @returns Dry-run plan describing the lease, decrypt, mail, and outbox flow.
+ */
+export function buildNoticeDryRunPlan(
+  appSchema: string,
+  engineSchema: string,
+  rootTable: string,
+  subjectId: string | number,
+  userHash: string,
+  notificationDueAt: Date,
+  retentionExpiry: Date
+) {
+  return {
+    mode: "dry-run" as const,
+    summary: `Would attempt the pre-erasure notice for root row ${subjectId} (${userHash}).`,
+    checks: [
+      `Read ${engineSchema}.pii_vault using (${appSchema}, ${rootTable}, ${subjectId}) as the lookup key.`,
+      `Verify that now is between notification_due_at (${notificationDueAt.toISOString()}) and retention_expiry (${retentionExpiry.toISOString()}).`,
+      "Acquire a short notification lease before decrypting or sending mail.",
+      "Use a deterministic mail idempotency key so retries do not duplicate sends.",
+      "Write the outbox event only after the mailer succeeds.",
+    ],
+    cryptoSteps: [
+      "Unwrap the stored DEK with the worker KEK.",
+      "Decrypt the vaulted JSON payload in memory only.",
+      "Null and overwrite temporary buffers after the email path completes.",
+    ],
+    sqlSteps: [
+      `SELECT * FROM ${engineSchema}.pii_vault WHERE root_schema = '${appSchema}' AND root_table = '${rootTable}' AND root_id = '${subjectId}' FOR UPDATE;`,
+      "UPDATE pii_vault SET notification_lock_id = '<uuid>', notification_lock_expires_at = '<lease-expiry>';",
+      "SELECT encrypted_dek FROM user_keys WHERE user_uuid_hash = '<user-hash>';",
+      "UPDATE pii_vault SET notification_sent_at = '<timestamp>', notification_lock_id = NULL, notification_lock_expires_at = NULL;",
+      `INSERT INTO ${engineSchema}.outbox (...) VALUES (... 'NOTIFICATION_SENT' ...);`,
+    ],
+  };
+}
+