dpdp-erasure-cli 1.0.0

package/src/modules/engine/blob/s3.ts

@@ -0,0 +1,455 @@
+import type { BlobTarget } from "@modules/config";
+import { createS3Client, parseS3ObjectUrl, type S3Client, type S3ObjectVersion } from "@modules/network";
+import type { Tsql } from "@/types";
+import type { BlobProtectionResult, BlobShredReceipt, DiscoveredBlobObject } from "./types";
+import { fail } from "@/errors";
+import { yieldWorkerEventLoop } from "../vault/satellite-mutation";
+import { generateHMAC } from "@modules/crypto";
+import { bytesToBase64 } from "@/lib";
+import { countOtherActiveBlobReferences, getPendingBlobObjectsForUser, insertBlobObject, markBlobObjectShredded, type BlobObjectRow } from "./store";
+
+
+export interface ProtectBlobTargetsInput {
+  tx: Tsql;
+  appSchema: string;
+  engineSchema: string;
+  rootTable: string;
+  rootIdColumn: string;
+  rootId: string | number;
+  userHash: string;
+  requestId?: string | null;
+  tenantId?: string;
+  targets: readonly BlobTarget[];
+  lockedRootRow: Record<string, unknown>;
+  hmacKey: Uint8Array;
+  s3Client?: S3Client;
+  shadowMode?: boolean;
+  now: Date;
+}
+
+interface ProtectedBlobTargetsResult {
+  rootColumnMasks: Record<string, string>;
+  receipts: BlobProtectionResult[];
+}
+
+interface BlobCandidate {
+  target: BlobTarget;
+  sourceTable: string;
+  sourceColumn: string;
+  originalValue: string;
+}
+
+interface ProtectedBlobTargetsResult {
+  rootColumnMasks: Record<string, string>;
+  receipts: BlobProtectionResult[];
+}
+
+
+async function hmacBlobField(prefix: string, value: string, hmacKey: Uint8Array): Promise<string> {
+  return generateHMAC(`${prefix}:${value}`, bytesToBase64(hmacKey));
+}
+
+async function buildObjectRefHash(
+  bucket: string,
+  key: string,
+  versionId: string,
+  hmacKey: Uint8Array
+): Promise<string> {
+  return hmacBlobField("s3-object-ref", `${bucket}\n${key}\n${versionId}`, hmacKey);
+}
+
+async function buildVersionHash(versionId: string, hmacKey: Uint8Array): Promise<string> {
+  return hmacBlobField("s3-version-id", versionId, hmacKey);
+}
+
+
+async function updateBlobSourceColumn(
+  tx: Tsql,
+  appSchema: string,
+  rootId: string | number,
+  tenantId: string | undefined,
+  candidate: BlobCandidate,
+  maskedValue: string
+): Promise<void> {
+  if (!candidate.target.lookup_column) {
+    return;
+  }
+
+  const tenantFilter = tenantId ? tx` AND tenant_id = ${tenantId}` : tx``;
+  await tx`
+    UPDATE ${tx(appSchema)}.${tx(candidate.target.table)}
+    SET ${tx(candidate.target.column)} = ${maskedValue}
+    WHERE ${tx(candidate.target.lookup_column)} = ${rootId}
+      AND ${tx(candidate.target.column)} = ${candidate.originalValue}
+      ${tenantFilter}
+  `;
+}
+
+async function loadMaskingBlob(target: BlobTarget): Promise<Uint8Array> {
+  if (!target.masking_blob_path) {
+    fail({
+      code: "BLOB_MASKING_ASSET_MISSING",
+      title: "Blob masking asset missing",
+      detail: `Blob target ${target.table}.${target.column} requires masking_blob_path for overwrite mode.`,
+      category: "configuration",
+      retryable: false,
+      fatal: true,
+    });
+  }
+
+  return new Uint8Array(await Bun.file(target.masking_blob_path).arrayBuffer());
+}
+
+
+function normalizeBlobCell(value: unknown): string | null {
+  if (typeof value !== "string") {
+    return null;
+  }
+
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+/**
+ * Checks whether configured blob targets contain any S3 URL values for the locked subject.
+ *
+ * @param input - Active vault transaction and target metadata.
+ * @returns `true` when at least one configured blob cell contains a non-empty URL.
+ */
+export async function hasBlobTargetValues(input: ProtectBlobTargetsInput): Promise<boolean> {
+  return (await collectBlobCandidates(input)).length > 0;
+}
+
+async function collectBlobCandidates(input: ProtectBlobTargetsInput): Promise<BlobCandidate[]> {
+  const candidates: BlobCandidate[] = [];
+
+  for (const target of input.targets) {
+    if (target.table === input.rootTable) {
+      const originalValue = normalizeBlobCell(input.lockedRootRow[target.column]);
+      if (originalValue) {
+        candidates.push({
+          target,
+          sourceTable: target.table,
+          sourceColumn: target.column,
+          originalValue,
+        });
+      }
+      continue;
+    }
+
+    if (!target.lookup_column) {
+      fail({
+        code: "BLOB_LOOKUP_COLUMN_MISSING",
+        title: "Blob target lookup column missing",
+        detail: `Blob target ${target.table}.${target.column} requires lookup_column because it is not the root table.`,
+        category: "configuration",
+        retryable: false,
+        fatal: true,
+      });
+    }
+
+    const tenantFilter = input.tenantId ? input.tx` AND tenant_id = ${input.tenantId}` : input.tx``;
+    const rows = await input.tx<{ value: string | null }[]>`
+      SELECT ${input.tx(target.column)}::text AS value
+      FROM ${input.tx(input.appSchema)}.${input.tx(target.table)}
+      WHERE ${input.tx(target.lookup_column)} = ${input.rootId}
+      ${tenantFilter}
+      FOR UPDATE
+    `;
+
+    for (const row of rows) {
+      const originalValue = normalizeBlobCell(row.value);
+      if (!originalValue) {
+        continue;
+      }
+
+      candidates.push({
+        target,
+        sourceTable: target.table,
+        sourceColumn: target.column,
+        originalValue,
+      });
+    }
+  }
+
+  return candidates;
+}
+
+/**
+ * Applies S3 legal holds, optional sanitized overwrites, and local DB URL masking for configured blob targets.
+ *
+ * External S3 side effects are skipped in shadow mode so a shadow task cannot mutate object storage
+ * while its Postgres transaction rolls back.
+ *
+ * @param input - Active vault transaction, config, locked root row, S3 client, and crypto key.
+ * @returns Root-column masks to merge into the root update plus sanitized outbox receipts.
+ */
+export async function protectBlobTargets(input: ProtectBlobTargetsInput): Promise<ProtectedBlobTargetsResult> {
+  if (input.targets.length === 0) {
+    return { rootColumnMasks: {}, receipts: [] };
+  }
+
+  const candidates = await collectBlobCandidates(input);
+  if (candidates.length === 0) {
+    return { rootColumnMasks: {}, receipts: [] };
+  }
+
+  const s3Client = input.shadowMode ? undefined : input.s3Client ?? createS3Client();
+  const rootColumnMasks: Record<string, string> = {};
+  const receipts: BlobProtectionResult[] = [];
+
+  for (const candidate of candidates) {
+    const pointer = parseS3ObjectUrl(candidate.originalValue);
+    const maskedValue = await hmacBlobField(
+      `blob-url:${input.appSchema}:${candidate.sourceTable}:${candidate.sourceColumn}`,
+      candidate.originalValue,
+      input.hmacKey
+    );
+
+    let versionId = pointer.versionId;
+    let eTag: string | null = null;
+    let overwriteETag: string | null = null;
+    let overwriteVersionId: string | null = null;
+
+    if (!input.shadowMode) {
+      const head = await s3Client!.headObject({
+        bucket: pointer.bucket,
+        key: pointer.key,
+        versionId: pointer.versionId,
+        region: candidate.target.region,
+        expectedBucketOwner: candidate.target.expected_bucket_owner,
+      });
+      versionId = versionId ?? head.versionId ?? undefined;
+      eTag = head.eTag;
+
+      if (candidate.target.require_version_id && !versionId) {
+        fail({
+          code: "BLOB_VERSION_ID_MISSING",
+          title: "S3 object version id missing",
+          detail: `Blob target ${candidate.sourceTable}.${candidate.sourceColumn} resolved to an S3 object without a version id.`,
+          category: "integrity",
+          retryable: false,
+          fatal: true,
+        });
+      }
+
+      await s3Client!.putObjectLegalHold({
+        bucket: pointer.bucket,
+        key: pointer.key,
+        versionId,
+        region: candidate.target.region,
+        expectedBucketOwner: candidate.target.expected_bucket_owner,
+        status: "ON",
+      });
+
+      if (candidate.target.action === "overwrite") {
+        const overwriteReceipt = await s3Client!.putObject({
+          bucket: pointer.bucket,
+          key: pointer.key,
+          region: candidate.target.region,
+          expectedBucketOwner: candidate.target.expected_bucket_owner,
+          body: await loadMaskingBlob(candidate.target),
+          contentType: candidate.target.masking_blob_path?.endsWith(".pdf")
+            ? "application/pdf"
+            : "application/octet-stream",
+        });
+        overwriteETag = overwriteReceipt.eTag;
+        overwriteVersionId = overwriteReceipt.versionId;
+      }
+    } else {
+      versionId = versionId ?? "SHADOW_VERSION";
+    }
+
+    const resolvedVersionId = versionId ?? "null";
+    const discovered: DiscoveredBlobObject = {
+      target: candidate.target,
+      sourceTable: candidate.sourceTable,
+      sourceColumn: candidate.sourceColumn,
+      originalValue: candidate.originalValue,
+      maskedValue,
+      bucket: pointer.bucket,
+      key: pointer.key,
+      versionId: resolvedVersionId,
+      eTag,
+      overwriteETag,
+      overwriteVersionId,
+    };
+
+    if (candidate.target.table === input.rootTable) {
+      rootColumnMasks[candidate.target.column] = maskedValue;
+    } else {
+      await updateBlobSourceColumn(
+        input.tx,
+        input.appSchema,
+        input.rootId,
+        input.tenantId,
+        candidate,
+        maskedValue
+      );
+    }
+
+    await insertBlobObject(input.tx, {
+      engineSchema: input.engineSchema,
+      userHash: input.userHash,
+      requestId: input.requestId,
+      tenantId: input.tenantId,
+      rootSchema: input.appSchema,
+      rootTable: input.rootTable,
+      rootId: String(input.rootId),
+      discovered,
+      now: input.now,
+    });
+
+    receipts.push({
+      sourceTable: candidate.sourceTable,
+      sourceColumn: candidate.sourceColumn,
+      provider: "aws_s3",
+      action: candidate.target.action,
+      objectRefHash: await buildObjectRefHash(pointer.bucket, pointer.key, resolvedVersionId, input.hmacKey),
+      versionIdHash: await buildVersionHash(resolvedVersionId, input.hmacKey),
+      legalHoldApplied: !input.shadowMode,
+      overwriteApplied: Boolean(overwriteVersionId),
+    });
+
+    await yieldWorkerEventLoop();
+  }
+
+  return { rootColumnMasks, receipts };
+}
+
+function filterVersionsForDeletion(row: BlobObjectRow, versions: S3ObjectVersion[]): S3ObjectVersion[] {
+  if (row.action !== "overwrite" || !row.overwrite_version_id) {
+    return versions;
+  }
+
+  return versions.filter((version) => version.versionId !== row.overwrite_version_id);
+}
+
+async function buildReceipt(
+  row: BlobObjectRow,
+  deletedVersions: readonly string[],
+  retainedVersions: readonly string[],
+  hmacKey: Uint8Array,
+  status: BlobShredReceipt["status"]
+): Promise<BlobShredReceipt> {
+  return {
+    provider: "aws_s3",
+    action: row.action,
+    objectRefHash: await buildObjectRefHash(row.bucket, row.object_key, row.version_id, hmacKey),
+    versionCount: deletedVersions.length + retainedVersions.length,
+    deletedVersionIdHashes: await Promise.all(deletedVersions.map((versionId) => buildVersionHash(versionId, hmacKey))),
+    retainedVersionIdHashes: await Promise.all(retainedVersions.map((versionId) => buildVersionHash(versionId, hmacKey))),
+    status,
+  };
+}
+
+/**
+ * Removes legal holds and purges all configured S3 object versions for a shredded subject.
+ *
+ * Raw bucket names and keys remain only in the worker-local database. Returned receipts are HMACed
+ * so Control Plane CoEs can prove deletion without receiving object paths that may contain PII.
+ *
+ * @param tx - Active shred transaction.
+ * @param engineSchema - Worker engine schema.
+ * @param userHash - Subject hash in the local vault.
+ * @param hmacKey - Worker HMAC key used to sanitize receipt identifiers.
+ * @param now - Shred timestamp.
+ * @param s3Client - Optional S3 client override for tests.
+ * @returns Sanitized deletion receipts safe for the Control Plane outbox.
+ */
+export async function shredBlobObjects(
+  tx: Tsql,
+  engineSchema: string,
+  userHash: string,
+  hmacKey: Uint8Array,
+  now: Date,
+  s3Client: S3Client = createS3Client()
+): Promise<BlobShredReceipt[]> {
+  const rows = await getPendingBlobObjectsForUser(tx, engineSchema, userHash);
+  const receipts: BlobShredReceipt[] = [];
+
+  for (const row of rows) {
+    if (row.action === "legal_hold_only") {
+      const receipt = await buildReceipt(row, [], [row.version_id], hmacKey, "retained_by_policy");
+      await markBlobObjectShredded(tx, engineSchema, row.id, receipt, now);
+      receipts.push(receipt);
+      continue;
+    }
+
+    const otherReferences = await countOtherActiveBlobReferences(tx, engineSchema, row);
+    if (otherReferences > 0) {
+      fail({
+        code: "BLOB_SHARED_OBJECT_CONFLICT",
+        title: "Shared S3 object deletion refused",
+        detail: "The worker refuses to delete an S3 object that is still referenced by another unshredded subject.",
+        category: "integrity",
+        retryable: false,
+        fatal: true,
+        context: {
+          provider: row.provider,
+          bucket: row.bucket,
+          objectKeyHash: await buildObjectRefHash(row.bucket, row.object_key, row.version_id, hmacKey),
+          otherReferences,
+        },
+      });
+    }
+
+    if (row.legal_hold_status === "ON") {
+      await s3Client.putObjectLegalHold({
+        bucket: row.bucket,
+        key: row.object_key,
+        versionId: row.version_id,
+        region: row.region,
+        expectedBucketOwner: row.expected_bucket_owner ?? undefined,
+        status: "OFF",
+      });
+    }
+
+    if (row.action === "hard_delete") {
+      const deleted = await s3Client.deleteObjectVersion({
+        bucket: row.bucket,
+        key: row.object_key,
+        versionId: row.version_id === "null" ? undefined : row.version_id,
+        region: row.region,
+        expectedBucketOwner: row.expected_bucket_owner ?? undefined,
+        bypassGovernanceRetention: row.retention_mode === "governance",
+      });
+      const deletedVersionId = deleted.versionId ?? row.version_id;
+      const receipt = await buildReceipt(row, [deletedVersionId], [], hmacKey, "captured_version_deleted");
+      await markBlobObjectShredded(tx, engineSchema, row.id, receipt, now);
+      receipts.push(receipt);
+      await yieldWorkerEventLoop();
+      continue;
+    }
+
+    const versions = await s3Client.listObjectVersions({
+      bucket: row.bucket,
+      key: row.object_key,
+      region: row.region,
+      expectedBucketOwner: row.expected_bucket_owner ?? undefined,
+    });
+    const versionsToDelete = filterVersionsForDeletion(row, versions);
+    const deletedVersionIds: string[] = [];
+
+    for (const version of versionsToDelete) {
+      await s3Client.deleteObjectVersion({
+        bucket: row.bucket,
+        key: row.object_key,
+        versionId: version.versionId,
+        region: row.region,
+        expectedBucketOwner: row.expected_bucket_owner ?? undefined,
+        bypassGovernanceRetention: row.retention_mode === "governance",
+      });
+      deletedVersionIds.push(version.versionId);
+      await yieldWorkerEventLoop();
+    }
+
+    const retainedVersionIds =
+      row.action === "overwrite" && row.overwrite_version_id ? [row.overwrite_version_id] : [];
+    const receipt = await buildReceipt(row, deletedVersionIds, retainedVersionIds, hmacKey, "purged");
+    await markBlobObjectShredded(tx, engineSchema, row.id, receipt, now);
+    receipts.push(receipt);
+  }
+
+  return receipts;
+}

package/src/modules/engine/blob/store.ts

@@ -0,0 +1,236 @@
+import type { JSONValue } from "postgres";
+import type { BlobAction, DiscoveredBlobObject, BlobShredReceipt } from "./types";
+import type { Tsql } from "@/types";
+
+export interface BlobObjectRow {
+  id: string;
+  user_uuid_hash: string;
+  request_id: string | null;
+  tenant_id: string;
+  root_schema: string;
+  root_table: string;
+  root_id: string;
+  source_table: string;
+  source_column: string;
+  provider: "aws_s3";
+  action: BlobAction;
+  retention_mode: "governance" | "compliance";
+  region: string;
+  expected_bucket_owner: string | null;
+  bucket: string;
+  object_key: string;
+  version_id: string;
+  e_tag: string | null;
+  masked_value: string;
+  legal_hold_status: "ON" | "OFF" | "not_supported";
+  legal_hold_applied_at: Date | null;
+  overwrite_status: "not_requested" | "applied";
+  overwrite_e_tag: string | null;
+  overwrite_version_id: string | null;
+  overwrite_applied_at: Date | null;
+  shred_status: "pending" | "purged" | "captured_version_deleted" | "retained_by_policy";
+  shred_receipt: BlobShredReceipt | null;
+  shredded_at: Date | null;
+  created_at: Date;
+  updated_at: Date;
+}
+
+export interface InsertBlobObjectInput {
+  engineSchema: string;
+  userHash: string;
+  requestId?: string | null;
+  tenantId?: string;
+  rootSchema: string;
+  rootTable: string;
+  rootId: string;
+  discovered: DiscoveredBlobObject;
+  now: Date;
+}
+
+/**
+ * Persists one protected S3 object receipt in the worker-local vault ledger.
+ *
+ * @param tx - Active vault transaction.
+ * @param input - Blob metadata discovered from the local database and S3.
+ * @returns Inserted or replayed blob object row.
+ */
+export async function insertBlobObject(
+  tx: Tsql,
+  input: InsertBlobObjectInput
+): Promise<BlobObjectRow> {
+  const [row] = await tx<BlobObjectRow[]>`
+    INSERT INTO ${tx(input.engineSchema)}.blob_objects (
+      user_uuid_hash,
+      request_id,
+      tenant_id,
+      root_schema,
+      root_table,
+      root_id,
+      source_table,
+      source_column,
+      provider,
+      action,
+      retention_mode,
+      region,
+      expected_bucket_owner,
+      bucket,
+      object_key,
+      version_id,
+      e_tag,
+      masked_value,
+      legal_hold_status,
+      legal_hold_applied_at,
+      overwrite_status,
+      overwrite_e_tag,
+      overwrite_version_id,
+      overwrite_applied_at,
+      shred_status,
+      created_at,
+      updated_at
+    )
+    VALUES (
+      ${input.userHash},
+      ${input.requestId ?? null},
+      ${input.tenantId ?? ""},
+      ${input.rootSchema},
+      ${input.rootTable},
+      ${input.rootId},
+      ${input.discovered.sourceTable},
+      ${input.discovered.sourceColumn},
+      'aws_s3',
+      ${input.discovered.target.action},
+      ${input.discovered.target.retention_mode},
+      ${input.discovered.target.region},
+      ${input.discovered.target.expected_bucket_owner ?? null},
+      ${input.discovered.bucket},
+      ${input.discovered.key},
+      ${input.discovered.versionId},
+      ${input.discovered.eTag},
+      ${input.discovered.maskedValue},
+      'ON',
+      ${input.now},
+      ${input.discovered.overwriteVersionId ? "applied" : "not_requested"},
+      ${input.discovered.overwriteETag},
+      ${input.discovered.overwriteVersionId},
+      ${input.discovered.overwriteVersionId ? input.now : null},
+      'pending',
+      ${input.now},
+      ${input.now}
+    )
+    ON CONFLICT (user_uuid_hash, source_table, source_column, bucket, object_key, version_id)
+    DO UPDATE
+      SET legal_hold_status = EXCLUDED.legal_hold_status,
+          expected_bucket_owner = EXCLUDED.expected_bucket_owner,
+          legal_hold_applied_at = EXCLUDED.legal_hold_applied_at,
+          overwrite_status = EXCLUDED.overwrite_status,
+          overwrite_e_tag = EXCLUDED.overwrite_e_tag,
+          overwrite_version_id = EXCLUDED.overwrite_version_id,
+          overwrite_applied_at = EXCLUDED.overwrite_applied_at,
+          updated_at = EXCLUDED.updated_at
+    RETURNING *
+  `;
+
+  return row!;
+}
+
+/**
+ * Lists pending blob objects attached to one vaulted subject.
+ *
+ * @param tx - Active shred transaction.
+ * @param engineSchema - Worker engine schema.
+ * @param userHash - Vault subject hash.
+ * @returns Pending blob object rows.
+ */
+export async function getPendingBlobObjectsForUser(
+  tx: Tsql,
+  engineSchema: string,
+  userHash: string
+): Promise<BlobObjectRow[]> {
+  return tx<BlobObjectRow[]>`
+    SELECT *
+    FROM ${tx(engineSchema)}.blob_objects
+    WHERE user_uuid_hash = ${userHash}
+      AND shred_status = 'pending'
+    ORDER BY created_at ASC, id ASC
+    FOR UPDATE
+  `;
+}
+
+/**
+ * Counts pending blob objects for a subject without claiming them.
+ *
+ * @param tx - Active shred transaction.
+ * @param engineSchema - Worker engine schema.
+ * @param userHash - Vault subject hash.
+ * @returns Number of pending blob object rows.
+ */
+export async function countPendingBlobObjectsForUser(
+  tx: Tsql,
+  engineSchema: string,
+  userHash: string
+): Promise<number> {
+  const [row] = await tx<{ total: number }[]>`
+    SELECT COUNT(*)::int AS total
+    FROM ${tx(engineSchema)}.blob_objects
+    WHERE user_uuid_hash = ${userHash}
+      AND shred_status = 'pending'
+  `;
+
+  return row?.total ?? 0;
+}
+
+/**
+ * Counts active references to the same physical S3 object held by other subjects.
+ *
+ * @param tx - Active shred transaction.
+ * @param engineSchema - Worker engine schema.
+ * @param row - Blob object being considered for destructive S3 deletion.
+ * @returns Number of non-purged references owned by other subjects.
+ */
+export async function countOtherActiveBlobReferences(
+  tx: Tsql,
+  engineSchema: string,
+  row: BlobObjectRow
+): Promise<number> {
+  const [result] = await tx<{ total: number }[]>`
+    SELECT COUNT(*)::int AS total
+    FROM ${tx(engineSchema)}.blob_objects
+    WHERE provider = ${row.provider}
+      AND bucket = ${row.bucket}
+      AND object_key = ${row.object_key}
+      AND user_uuid_hash <> ${row.user_uuid_hash}
+      AND shred_status IN ('pending', 'retained_by_policy')
+  `;
+
+  return result?.total ?? 0;
+}
+
+/**
+ * Marks a blob object as shredded or explicitly retained by policy.
+ *
+ * @param tx - Active shred transaction.
+ * @param engineSchema - Worker engine schema.
+ * @param rowId - Blob object row id.
+ * @param receipt - Sanitized non-PII deletion receipt.
+ * @param now - Completion timestamp.
+ */
+export async function markBlobObjectShredded(
+  tx: Tsql,
+  engineSchema: string,
+  rowId: string,
+  receipt: BlobShredReceipt,
+  now: Date
+): Promise<void> {
+  await tx`
+    UPDATE ${tx(engineSchema)}.blob_objects
+    SET shred_status = ${receipt.status},
+        shred_receipt = ${tx.json(receipt as unknown as JSONValue)},
+        legal_hold_status = CASE
+          WHEN ${receipt.status === "retained_by_policy"} THEN legal_hold_status
+          ELSE 'OFF'
+        END,
+        shredded_at = ${now},
+        updated_at = ${now}
+    WHERE id = ${rowId}
+  `;
+}