dpdp-erasure-cli 1.0.0

package/tests/s3-client.test.ts

@@ -0,0 +1,110 @@
+import { describe, expect, it, vi } from "vitest";
+import { createS3Client, parseS3ObjectUrl } from "@modules/network";
+
+describe("S3 client", () => {
+  it("parses S3 URLs without losing version ids", () => {
+    expect(parseS3ObjectUrl("s3://kyc-bucket/kyc/john-doe-aadhar.pdf?versionId=v123")).toEqual({
+      bucket: "kyc-bucket",
+      key: "kyc/john-doe-aadhar.pdf",
+      versionId: "v123",
+    });
+
+    expect(parseS3ObjectUrl("https://kyc-bucket.s3.ap-south-1.amazonaws.com/kyc/doc.pdf?versionId=v9")).toEqual({
+      bucket: "kyc-bucket",
+      key: "kyc/doc.pdf",
+      versionId: "v9",
+    });
+  });
+
+  it("signs native S3 requests and crawls object versions", async () => {
+    const calls: Array<{ url: string; method: string; authorization: string | null }> = [];
+    const fetchFn = vi.fn(async (input: string | URL | Request, init?: RequestInit) => {
+      const url = String(input);
+      const headers = new Headers(init?.headers);
+      calls.push({
+        url,
+        method: init?.method ?? "GET",
+        authorization: headers.get("authorization"),
+      });
+
+      if (url.includes("169.254.170.2")) {
+        return new Response(JSON.stringify({
+          AccessKeyId: "AKIATEST",
+          SecretAccessKey: "secret",
+          Token: "session",
+        }), { status: 200 });
+      }
+
+      if (url.includes("versions")) {
+        return new Response(`<?xml version="1.0" encoding="UTF-8"?>
+          <ListVersionsResult>
+            <IsTruncated>false</IsTruncated>
+            <Version>
+              <Key>kyc/doc.pdf</Key>
+              <VersionId>v1</VersionId>
+              <ETag>"etag-1"</ETag>
+            </Version>
+            <DeleteMarker>
+              <Key>kyc/doc.pdf</Key>
+              <VersionId>marker-1</VersionId>
+            </DeleteMarker>
+          </ListVersionsResult>`, { status: 200 });
+      }
+
+      return new Response(null, {
+        status: 200,
+        headers: {
+          "x-amz-version-id": "v1",
+          etag: "\"etag-1\"",
+        },
+      });
+    });
+
+    const client = createS3Client({
+      env: {
+        AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: "/v2/credentials/task",
+        AWS_EC2_METADATA_DISABLED: "true",
+      },
+      fetchFn: fetchFn as unknown as typeof fetch,
+    });
+
+    const head = await client.headObject({
+      bucket: "kyc-bucket",
+      key: "kyc/doc.pdf",
+      region: "ap-south-1",
+    });
+    const versions = await client.listObjectVersions({
+      bucket: "kyc-bucket",
+      key: "kyc/doc.pdf",
+      region: "ap-south-1",
+    });
+
+    expect(head).toMatchObject({ versionId: "v1", eTag: "etag-1" });
+    expect(versions).toEqual([
+      { key: "kyc/doc.pdf", versionId: "v1", eTag: "etag-1", isDeleteMarker: false },
+      { key: "kyc/doc.pdf", versionId: "marker-1", eTag: null, isDeleteMarker: true },
+    ]);
+    expect(calls.some((call) => call.authorization?.startsWith("AWS4-HMAC-SHA256"))).toBe(true);
+    expect(calls.some((call) => call.url.includes("169.254.170.2/v2/credentials/task"))).toBe(true);
+  });
+
+  it("rejects unsafe HTTP container credential endpoints before any S3 request is sent", async () => {
+    const fetchFn = vi.fn(async () => new Response(null, { status: 500 }));
+    const client = createS3Client({
+      env: {
+        AWS_CONTAINER_CREDENTIALS_FULL_URI: "http://metadata.evil.local/creds",
+        AWS_EC2_METADATA_DISABLED: "true",
+      },
+      fetchFn: fetchFn as unknown as typeof fetch,
+    });
+
+    await expect(client.headObject({
+      bucket: "kyc-bucket",
+      key: "kyc/doc.pdf",
+      region: "ap-south-1",
+    })).rejects.toMatchObject({
+      code: "AWS_CREDENTIALS_URI_REJECTED",
+    });
+    expect(fetchFn).not.toHaveBeenCalled();
+  });
+});

package/tests/satellite.test.ts

@@ -0,0 +1,119 @@
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { redactSatelliteTable } from "@modules/engine/vault/satellite";
+import { createTestSql, dropSchemas, uniqueSchema } from "./helpers";
+import type { Sql } from "@/types";
+
+describe("Satellite Table Chunking", () => {
+  let sql: Sql;
+  const schemasToDrop: string[] = [];
+
+  beforeAll(() => {
+    sql = createTestSql();
+  });
+
+  afterAll(async () => {
+    await dropSchemas(sql, ...schemasToDrop);
+    await sql.end();
+  });
+
+  async function prepare() {
+    const schema = uniqueSchema("satellite_app");
+    schemasToDrop.push(schema);
+
+    await dropSchemas(sql, schema);
+    await sql`CREATE SCHEMA ${sql(schema)}`;
+    await sql`
+      CREATE TABLE ${sql(schema)}.orders (
+        id SERIAL PRIMARY KEY,
+        user_ref TEXT NOT NULL,
+        amount NUMERIC NOT NULL
+      )
+    `;
+
+    return { schema };
+  }
+
+  it("redacts matching satellite rows in batches until exhaustion", async () => {
+    const { schema } = await prepare();
+
+    for (let index = 0; index < 5; index += 1) {
+      await sql`
+        INSERT INTO ${sql(schema)}.orders (user_ref, amount)
+        VALUES ('legacy-user', ${index + 1})
+      `;
+    }
+
+    await sql`
+      INSERT INTO ${sql(schema)}.orders (user_ref, amount)
+      VALUES ('other-user', 99)
+    `;
+
+    const redacted = await sql.begin((tx) =>
+      redactSatelliteTable(tx, `${schema}.orders`, "user_ref", "legacy-user", "hmac-user", 2)
+    );
+
+    expect(redacted).toBe(5);
+
+    const rows = await sql`
+      SELECT user_ref
+      FROM ${sql(schema)}.orders
+      ORDER BY id ASC
+    `;
+
+    expect(rows.map((row) => row.user_ref)).toEqual([
+      "hmac-user",
+      "hmac-user",
+      "hmac-user",
+      "hmac-user",
+      "hmac-user",
+      "other-user",
+    ]);
+  });
+
+  it("returns zero when no rows match the lookup value", async () => {
+    const { schema } = await prepare();
+
+    const redacted = await sql.begin((tx) =>
+      redactSatelliteTable(tx, `${schema}.orders`, "user_ref", "missing-user", "hmac-user", 100)
+    );
+
+    expect(redacted).toBe(0);
+  });
+
+  it("yields to the Bun event loop between satellite batches", async () => {
+    const { schema } = await prepare();
+
+    for (let index = 0; index < 3; index += 1) {
+      await sql`
+        INSERT INTO ${sql(schema)}.orders (user_ref, amount)
+        VALUES ('yield-user', ${index + 1})
+      `;
+    }
+
+    const runtime = globalThis as typeof globalThis & {
+      Bun?: { sleep?: (ms: number) => Promise<void> };
+    };
+    const originalBun = runtime.Bun;
+    const originalSleep = runtime.Bun?.sleep;
+    const sleepMock = vi.fn(async () => { });
+    runtime.Bun = {
+      ...(runtime.Bun ?? {}),
+      sleep: sleepMock,
+    };
+    try {
+      const redacted = await sql.begin((tx) =>
+        redactSatelliteTable(tx, `${schema}.orders`, "user_ref", "yield-user", "hmac-user", 1)
+      );
+
+      expect(redacted).toBe(3);
+      expect(sleepMock).toHaveBeenCalledTimes(3);
+      expect(sleepMock).toHaveBeenCalledWith(0);
+    } finally {
+      if (originalBun) {
+        runtime.Bun = { ...originalBun, sleep: originalSleep };
+      } else {
+        Reflect.deleteProperty(runtime, "Bun");
+      }
+    }
+  });
+});

package/tests/schema-compatibility.test.ts

@@ -0,0 +1,237 @@
+import { afterAll, describe, expect, it } from "vitest";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { tmpdir } from "node:os";
+import { assertConfigSchemaCompatibility, readWorkerConfig } from "@modules/config";
+import { createTestSql, dropSchemas, uniqueSchema } from "./helpers";
+import type { Sql } from "@/types";
+
+const masterKeyHex = "42".repeat(32);
+const hmacKeyBase64 = Buffer.from(new Uint8Array(32).fill(0x24)).toString("base64");
+
+async function writeYaml(contents: string): Promise<string> {
+  const directory = await mkdtemp(join(tmpdir(), "worker-compat-"));
+  const path = join(directory, "compliance.worker.yml");
+  await writeFile(path, contents, "utf8");
+  return path;
+}
+
+async function removeYaml(path: string) {
+  await rm(path, { force: true });
+  await rm(dirname(path), { recursive: true, force: true });
+}
+
+describe("Worker config schema compatibility", () => {
+  const sql: Sql = createTestSql();
+  const schemasToDrop: string[] = [];
+  const pathsToDelete: string[] = [];
+
+  afterAll(async () => {
+    for (const path of pathsToDelete.splice(0, pathsToDelete.length)) {
+      await removeYaml(path);
+    }
+    await dropSchemas(sql, ...schemasToDrop);
+    await sql.end();
+  });
+
+  async function createCompatibleSchema(schema: string, includeRootLookupColumn: boolean) {
+    await dropSchemas(sql, schema);
+    await sql`CREATE SCHEMA ${sql(schema)}`;
+    await sql`
+      CREATE TABLE ${sql(schema)}.users (
+        id TEXT PRIMARY KEY,
+        ${includeRootLookupColumn ? sql`user_identifier TEXT NOT NULL,` : sql``}
+        email TEXT NOT NULL,
+        full_name TEXT NOT NULL
+      )
+    `;
+    await sql`
+      CREATE TABLE ${sql(schema)}.marketing_leads (
+        id BIGSERIAL PRIMARY KEY,
+        email TEXT NOT NULL,
+        name TEXT NOT NULL
+      )
+    `;
+    await sql`
+      CREATE TABLE ${sql(schema)}.system_audit_logs (
+        id BIGSERIAL PRIMARY KEY,
+        user_identifier TEXT NOT NULL,
+        message TEXT NOT NULL
+      )
+    `;
+    await sql`
+      CREATE TABLE ${sql(schema)}.transactions (
+        id TEXT NOT NULL,
+        transaction_ref TEXT PRIMARY KEY,
+        amount NUMERIC(18,2) NOT NULL
+      )
+    `;
+    await sql`
+      CREATE TABLE ${sql(schema)}.invoices (
+        id TEXT NOT NULL,
+        invoice_ref TEXT PRIMARY KEY,
+        total NUMERIC(18,2) NOT NULL
+      )
+    `;
+    await sql`
+      CREATE TABLE ${sql(schema)}.kyc_documents (
+        id TEXT NOT NULL,
+        document_ref TEXT PRIMARY KEY
+      )
+    `;
+  }
+
+  async function loadConfig(appSchema: string, engineSchema: string) {
+    const path = await writeYaml(`
+version: "1.0"
+database:
+  app_schema: ${appSchema}
+  engine_schema: ${engineSchema}
+compliance_policy:
+  default_retention_years: 0
+  notice_window_hours: 48
+  retention_rules:
+    - rule_name: PMLA_FINANCIAL
+      legal_citation: "Prevention of Money Laundering Act, 2002, Sec 12"
+      if_has_data_in:
+        - transactions
+        - invoices
+      retention_years: 10
+    - rule_name: RBI_KYC
+      legal_citation: "RBI KYC Directions, 2016, Sec 38"
+      if_has_data_in:
+        - kyc_documents
+      retention_years: 5
+graph:
+  root_table: users
+  root_id_column: id
+  max_depth: 32
+  notice_email_column: email
+  notice_name_column: full_name
+  root_pii_columns:
+    email: HMAC
+    full_name: STATIC_MASK
+satellite_targets:
+  - table: marketing_leads
+    lookup_column: email
+    action: redact
+    masking_rules:
+      email: HMAC
+      name: STATIC_MASK
+  - table: system_audit_logs
+    lookup_column: user_identifier
+    action: hard_delete
+outbox:
+  batch_size: 10
+  lease_seconds: 60
+  max_attempts: 10
+  base_backoff_ms: 1000
+security:
+  notification_lease_seconds: 120
+  master_key_env: DPDP_MASTER_KEY
+  hmac_key_env: DPDP_HMAC_KEY
+integrity:
+  expected_schema_hash: "${"1".repeat(64)}"
+legal_attestation:
+  dpo_identifier: "dpo-name@client.com"
+  configuration_version: "v1.2.0"
+  legal_review_date: "2026-04-20"
+  acknowledgment: "I confirm this configuration accurately reflects our obligations."
+`);
+    pathsToDelete.push(path);
+
+    return await readWorkerConfig(
+      {
+        DPDP_MASTER_KEY: masterKeyHex,
+        DPDP_HMAC_KEY: `base64:${hmacKeyBase64}`,
+      },
+      path
+    );
+  }
+
+  it("accepts a schema that satisfies every configured root, satellite, and evidence reference", async () => {
+    const appSchema = uniqueSchema("compat_ok_app");
+    const engineSchema = uniqueSchema("compat_ok_engine");
+    schemasToDrop.push(appSchema, engineSchema);
+    await createCompatibleSchema(appSchema, true);
+
+    const config = await loadConfig(appSchema, engineSchema);
+    await expect(assertConfigSchemaCompatibility(sql, config)).resolves.toBeUndefined();
+  });
+
+  it("fails closed when a satellite lookup column is missing from the root table", async () => {
+    const appSchema = uniqueSchema("compat_bad_app");
+    const engineSchema = uniqueSchema("compat_bad_engine");
+    schemasToDrop.push(appSchema, engineSchema);
+    await createCompatibleSchema(appSchema, false);
+
+    const config = await loadConfig(appSchema, engineSchema);
+    await expect(assertConfigSchemaCompatibility(sql, config)).rejects.toThrow(
+      new RegExp(`missing root column ${appSchema}\\.users\\.user_identifier`, "i")
+    );
+  });
+
+  it("fails closed when enabled purge policy references a missing root column", async () => {
+    const appSchema = uniqueSchema("compat_purge_bad_app");
+    const engineSchema = uniqueSchema("compat_purge_bad_engine");
+    schemasToDrop.push(appSchema, engineSchema);
+    await createCompatibleSchema(appSchema, true);
+
+    const path = await writeYaml(`
+version: "1.0"
+database:
+  app_schema: ${appSchema}
+  engine_schema: ${engineSchema}
+compliance_policy:
+  default_retention_years: 0
+  notice_window_hours: 48
+  retention_rules:
+    - rule_name: RBI_KYC
+      legal_citation: "RBI KYC Directions, 2016, Sec 38"
+      if_has_data_in:
+        - kyc_documents
+      retention_years: 5
+graph:
+  root_table: users
+  root_id_column: id
+  max_depth: 32
+  root_pii_columns:
+    email: HMAC
+    full_name: STATIC_MASK
+purge_policy:
+  enabled: true
+  selector:
+    kind: boolean_column
+    column: purge_eligible
+    value: true
+outbox:
+  batch_size: 10
+  lease_seconds: 60
+  max_attempts: 10
+  base_backoff_ms: 1000
+security:
+  notification_lease_seconds: 120
+  master_key_env: DPDP_MASTER_KEY
+  hmac_key_env: DPDP_HMAC_KEY
+integrity:
+  expected_schema_hash: "${"1".repeat(64)}"
+legal_attestation:
+  dpo_identifier: "dpo-name@client.com"
+  configuration_version: "v1.2.0"
+  legal_review_date: "2026-04-20"
+  acknowledgment: "I confirm this configuration accurately reflects our obligations."
+`);
+    pathsToDelete.push(path);
+
+    const config = await readWorkerConfig(
+      {
+        DPDP_MASTER_KEY: masterKeyHex,
+        DPDP_HMAC_KEY: `base64:${hmacKeyBase64}`,
+      },
+      path
+    );
+    await expect(assertConfigSchemaCompatibility(sql, config)).rejects.toThrow(
+      new RegExp(`missing root column ${appSchema}\\.users\\.purge_eligible`, "i")
+    );
+  });
+});

package/tests/schema-integrity.test.ts

@@ -0,0 +1,64 @@
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import type { Sql } from "@/types";
+import { createTestSql, dropSchemas, uniqueSchema } from "./helpers";
+import { detectSchemaDrift } from "@modules/db";
+import { assertSchemaIntegrity } from "@modules/bootstrap";
+
+describe("Schema Drift Detection", () => {
+  let sql: Sql;
+  const schemasToDrop: string[] = [];
+
+  beforeAll(() => {
+    sql = createTestSql();
+  });
+
+  afterAll(async () => {
+    await dropSchemas(sql, ...schemasToDrop);
+    await sql.end();
+  });
+
+  async function createSchema() {
+    const schema = uniqueSchema("drift_app");
+    schemasToDrop.push(schema);
+
+    await dropSchemas(sql, schema);
+    await sql`CREATE SCHEMA ${sql(schema)}`;
+    await sql`
+      CREATE TABLE ${sql(schema)}.users (
+        id SERIAL PRIMARY KEY,
+        email TEXT NOT NULL,
+        full_name TEXT NOT NULL
+      )
+    `;
+    await sql`
+      CREATE TABLE ${sql(schema)}.orders (
+        id SERIAL PRIMARY KEY,
+        user_id INTEGER NOT NULL REFERENCES ${sql(schema)}.users(id),
+        amount NUMERIC NOT NULL
+      )
+    `;
+
+    return schema;
+  }
+
+  it("returns a deterministic digest and changes when the schema changes", async () => {
+    const schema = await createSchema();
+
+    const first = await detectSchemaDrift(sql, schema);
+    const second = await detectSchemaDrift(sql, schema);
+    expect(second).toBe(first);
+
+    await sql`ALTER TABLE ${sql(schema)}.users ADD COLUMN phone TEXT`;
+
+    const third = await detectSchemaDrift(sql, schema);
+    expect(third).not.toBe(first);
+  });
+
+  it("fails closed when the manifest hash does not match the live schema digest", async () => {
+    const schema = await createSchema();
+    const liveHash = await detectSchemaDrift(sql, schema);
+
+    await expect(assertSchemaIntegrity(sql, schema, liveHash)).resolves.toBe(liveHash);
+    await expect(assertSchemaIntegrity(sql, schema, "0".repeat(64))).rejects.toThrow(/schema drift detected/i);
+  });
+});

package/tests/shredder.test.ts

@@ -0,0 +1,163 @@
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { dispatchPreErasureNotice, shredUser, vaultUser } from "@modules/engine";
+import {
+  TEST_SECRETS,
+  createTestSql,
+  dropSchemas,
+  insertUser,
+  prepareWorkerSchemas,
+  uniqueSchema,
+} from "./helpers";
+import type { Sql } from "@/types";
+
+describe("Crypto-Shredder Engine", () => {
+  let sql: Sql;
+  const schemasToDrop: string[] = [];
+
+  beforeAll(() => {
+    sql = createTestSql();
+  });
+
+  afterAll(async () => {
+    await dropSchemas(sql, ...schemasToDrop);
+    await sql.end();
+  });
+
+  async function prepare() {
+    const appSchema = uniqueSchema("shred_app");
+    const engineSchema = uniqueSchema("shred_engine");
+    schemasToDrop.push(appSchema, engineSchema);
+    await prepareWorkerSchemas(sql, appSchema, engineSchema, { withDependencies: true });
+    return { appSchema, engineSchema };
+  }
+
+  async function seedUser(appSchema: string, engineSchema: string) {
+    const userId = await insertUser(sql, appSchema, "shred.me@example.com", "Shred Me");
+    await vaultUser(sql, userId, TEST_SECRETS, {
+      appSchema,
+      engineSchema,
+      now: new Date("2020-01-01T00:00:00.000Z"),
+      defaultRetentionYears: 1,
+      noticeWindowHours: 48,
+      rootTable: "users",
+      rootIdColumn: "id",
+      rootPiiColumns: {
+        email: "HMAC",
+        full_name: "STATIC_MASK",
+      },
+      satelliteTargets: [],
+      compiledTargets: [
+        { table: `${appSchema}.users`, pii_columns: ["email", "full_name"] },
+        {
+          table: `${appSchema}.orders`,
+          parent: `${appSchema}.users`,
+          join: `${appSchema}.users.id = ${appSchema}.orders.user_id`,
+          pii_columns: [],
+        },
+      ],
+    });
+    return userId;
+  }
+
+  async function sendNotice(appSchema: string, engineSchema: string, userId: number) {
+    await dispatchPreErasureNotice(
+      sql,
+      userId,
+      TEST_SECRETS,
+      {
+        sendEmail: vi.fn().mockResolvedValue(undefined),
+      },
+      {
+        appSchema,
+        engineSchema,
+        now: new Date("2020-12-30T00:00:00.000Z"),
+      }
+    );
+  }
+
+  it("shreds the key after retention expiry and replaces the vault payload with a destroyed sentinel", async () => {
+    const { appSchema, engineSchema } = await prepare();
+    const userId = await seedUser(appSchema, engineSchema);
+    await sendNotice(appSchema, engineSchema, userId);
+
+    const result = await shredUser(sql, userId, {
+      appSchema,
+      engineSchema,
+      now: new Date("2021-01-02T00:00:00.000Z"),
+    });
+
+    expect(result.action).toBe("shredded");
+
+    const keysAfter = await sql`
+      SELECT *
+      FROM ${sql(engineSchema)}.user_keys
+      WHERE user_uuid_hash = ${result.userHash}
+    `;
+    const [vaultAfter] = await sql`
+      SELECT encrypted_pii, shredded_at
+      FROM ${sql(engineSchema)}.pii_vault
+      WHERE root_schema = ${appSchema}
+        AND root_table = 'users'
+        AND root_id = ${userId.toString()}
+    `;
+    const outboxRows = await sql`
+      SELECT *
+      FROM ${sql(engineSchema)}.outbox
+      WHERE idempotency_key = ${`shred:${appSchema}:users:${userId}`}
+    `;
+
+    expect(keysAfter).toHaveLength(0);
+    expect(vaultAfter?.encrypted_pii).toEqual({ v: 1, destroyed: true });
+    expect(vaultAfter?.shredded_at).toBeDefined();
+    expect(outboxRows).toHaveLength(1);
+    expect(outboxRows[0]?.event_type).toBe("SHRED_SUCCESS");
+  });
+
+  it("refuses to shred before the retention expiry", async () => {
+    const { appSchema, engineSchema } = await prepare();
+    const userId = await seedUser(appSchema, engineSchema);
+    await sendNotice(appSchema, engineSchema, userId);
+
+    await expect(
+      shredUser(sql, userId, {
+        appSchema,
+        engineSchema,
+        now: new Date("2020-12-31T00:00:00.000Z"),
+      })
+    ).rejects.toThrow(/before retention expiry/i);
+  });
+
+  it("refuses to shred if the pre-erasure notice has not been sent", async () => {
+    const { appSchema, engineSchema } = await prepare();
+    const userId = await seedUser(appSchema, engineSchema);
+
+    await expect(
+      shredUser(sql, userId, {
+        appSchema,
+        engineSchema,
+        now: new Date("2021-01-02T00:00:00.000Z"),
+      })
+    ).rejects.toThrow(/notice has been sent/i);
+  });
+
+  it("is idempotent when the same shred request is replayed", async () => {
+    const { appSchema, engineSchema } = await prepare();
+    const userId = await seedUser(appSchema, engineSchema);
+    await sendNotice(appSchema, engineSchema, userId);
+
+    const first = await shredUser(sql, userId, {
+      appSchema,
+      engineSchema,
+      now: new Date("2021-01-02T00:00:00.000Z"),
+    });
+    const second = await shredUser(sql, userId, {
+      appSchema,
+      engineSchema,
+      now: new Date("2021-01-03T00:00:00.000Z"),
+    });
+
+    expect(first.action).toBe("shredded");
+    expect(second.action).toBe("already_shredded");
+    expect(second.userHash).toBe(first.userHash);
+  });
+});