dpdp-erasure-cli 1.0.0

package/src/modules/cli/sign.ts

@@ -0,0 +1,50 @@
+import pc from "picocolors";
+import path from "node:path";
+import { UI, exitWithError } from "./ui";
+import { base64ToBytes, bytesToBase64 } from "@/lib";
+
+/**
+ * Signs the compliance manifest with a private key.
+ */
+export async function signAction(options: { config: string; key?: string }) {
+  UI.header("Manifest Signing");
+
+  const configPath = path.resolve(options.config);
+  const keyPath = options.key ? path.resolve(options.key) : "worker.pkcs8.key";
+
+  UI.info(`Manifest : ${pc.bold(options.config)}`);
+  UI.info(`Key      : ${pc.bold(keyPath)}`);
+
+  const spinner = UI.spinner("Signing manifest data...");
+
+  try {
+    const keyData = (await Bun.file(keyPath).text()).trim();
+    const manifestData = new Uint8Array(await Bun.file(configPath).arrayBuffer());
+
+    const privateKey = await globalThis.crypto.subtle.importKey(
+      "pkcs8",
+      base64ToBytes(keyData).buffer as ArrayBuffer,
+      { name: "Ed25519" },
+      false,
+      ["sign"]
+    );
+
+    const signature = await globalThis.crypto.subtle.sign(
+      "Ed25519",
+      privateKey,
+      manifestData.buffer as ArrayBuffer
+    );
+
+    const signatureBase64 = bytesToBase64(new Uint8Array(signature));
+    const sigPath = `${configPath}.sig`;
+
+    await Bun.write(sigPath, signatureBase64);
+    spinner.stop();
+
+    UI.success(`Detached signature generated: ${pc.bold(sigPath)}`);
+    UI.info("Ensure this file is present alongside your manifest in the worker environment.");
+  } catch (err) {
+    spinner.fail("Signing failed");
+    exitWithError("Process failed", err instanceof Error ? err.message : String(err));
+  }
+}

package/src/modules/cli/ui.ts

@@ -0,0 +1,61 @@
+import pc from "picocolors";
+import ora, { type Ora } from "ora";
+import Table from "cli-table3";
+import boxen from "boxen";
+
+/**
+ * Standardized UI components for the Compliance Worker CLI.
+ * Designed for professional output in diverse terminal environments.
+ */
+export const UI = {
+  header: (title: string) => {
+    console.log(
+      boxen(pc.bold(pc.blue(`COMPLIANCE WORKER — ${title.toUpperCase()}`)), {
+        padding: { top: 0, bottom: 0, left: 2, right: 2 },
+        margin: { top: 1, bottom: 1 },
+        borderStyle: "round",
+        borderColor: "blue",
+      })
+    );
+  },
+
+  divider: () => console.log(pc.gray("─".repeat(process.stdout.columns || 60))),
+
+  spinner: (text: string): Ora => ora({ text, color: "blue" }).start(),
+
+  success: (msg: string) => console.log(`\n${pc.green("✔")} ${pc.bold(msg)}`),
+  error: (msg: string) => console.error(`\n${pc.red("✖")} ${pc.bold(msg)}`),
+  warn: (msg: string) => console.log(`\n${pc.yellow("⚠")} ${pc.bold(msg)}`),
+  info: (msg: string) => console.log(`\n${pc.cyan("ℹ")} ${msg}`),
+
+  step: (n: number, msg: string) => {
+    console.log(`\n${pc.bold(pc.white(`${n}. ${msg}`))}`);
+  },
+
+  subStep: (msg: string, indent = 3) => {
+    console.log(`${" ".repeat(indent)}${pc.gray("└─")} ${pc.white(msg)}`);
+  },
+
+  keyValue: (key: string, val: string) => {
+    console.log(`   ${pc.gray(key.padEnd(20))}: ${pc.bold(val)}`);
+  },
+
+  table: (head: string[]) => {
+    return new Table({
+      head: head.map((h) => pc.bold(pc.white(h))),
+      style: { head: [], border: ["gray"] },
+    });
+  },
+
+  hint: (msg: string) => {
+    console.log(pc.italic(pc.gray(`\nPRO-TIP: ${msg}`)));
+  },
+};
+
+export function exitWithError(msg: string, detail?: string): never {
+  UI.error(msg);
+  if (detail) {
+    console.error(pc.gray(`\n${pc.bold("Detail:")}\n${detail}\n`));
+  }
+  process.exit(1);
+}

package/src/modules/cli/verify-schema.ts

@@ -0,0 +1,31 @@
+import postgres from "postgres";
+import { verifySchemaIntegrity } from "@modules/introspector";
+import { UI, exitWithError } from "./ui";
+
+/**
+ * Executes the CI/CD Privacy-as-Code schema gate.
+ *
+ * @param options - Manifest path and database URL.
+ * @returns Resolves only when the live schema hash matches legal attestation.
+ */
+export async function verifySchemaAction(options: { config: string; url?: string }): Promise<void> {
+  UI.header("Privacy-as-Code Schema Verification");
+
+  const dbUrl = options.url ?? process.env.DATABASE_URL;
+  if (!dbUrl) {
+    exitWithError("Database URL required.", "Pass --url or set DATABASE_URL.");
+  }
+
+  const sql = postgres(dbUrl, { max: 1 });
+  const spinner = UI.spinner(`Comparing live schema to ${options.config}...`);
+  try {
+    const liveHash = await verifySchemaIntegrity({ sql, configPath: options.config });
+    spinner.succeed("Schema hash matches legal attestation");
+    UI.keyValue("Live Schema Hash", liveHash);
+  } catch (error) {
+    spinner.fail("Schema verification failed");
+    exitWithError("Privacy-as-Code gate failed", error instanceof Error ? error.message : String(error));
+  } finally {
+    await sql.end();
+  }
+}

package/src/modules/cli/verify.ts

@@ -0,0 +1,85 @@
+import postgres from "postgres";
+import pc from "picocolors";
+import { UI, exitWithError } from "./ui";
+import path from "node:path";
+import { assertConfigSchemaCompatibility, readWorkerConfig } from "../config";
+import { detectSchemaDrift } from "../db";
+import { verifySignedWorkerConfig } from "@/secrets/signature";
+
+/**
+ * Validates the local manifest against the database and signing keys.
+ */
+export async function verifyAction(options: { config: string; url?: string }) {
+  UI.header("Integrity Verification");
+
+  const dbUrl = options.url || process.env.DATABASE_URL;
+  if (!dbUrl) exitWithError("Database URL required.", "Provide --url or set DATABASE_URL env.");
+
+  const configPath = path.resolve(options.config);
+  UI.info(`Verifying manifest: ${pc.bold(options.config)}`);
+
+  // Use dummy keys if not provided for verification path
+  const mockEnv = {
+    ...process.env,
+    DPDP_MASTER_KEY: process.env.DPDP_MASTER_KEY || "0".repeat(64),
+    DPDP_HMAC_KEY: process.env.DPDP_HMAC_KEY || "0".repeat(64),
+  };
+
+  let config;
+  try {
+    config = await readWorkerConfig(mockEnv, configPath);
+  } catch (err) {
+    exitWithError("Validation failed", err instanceof Error ? err.message : String(err));
+  }
+
+  const sql = postgres(dbUrl);
+
+  try {
+    UI.step(1, "Database Schema Compatibility");
+    const compatSpinner = UI.spinner("Checking existence of configured objects...");
+    try {
+      await assertConfigSchemaCompatibility(sql, config);
+      compatSpinner.succeed("All configured tables and columns verified.");
+    } catch (err) {
+      compatSpinner.fail("Schema mismatch detected.");
+      console.log(pc.red(`   ${err instanceof Error ? err.message : String(err)}`));
+    }
+
+    UI.step(2, "Schema Drift Detection");
+    const hashSpinner = UI.spinner("Computing SHA-256 fingerprint...");
+    const liveHash = await detectSchemaDrift(sql, config.database.app_schema);
+    const expectedHash = config.integrity.expected_schema_hash;
+    hashSpinner.stop();
+
+    UI.keyValue("Manifest Hash", expectedHash);
+    UI.keyValue("Live Schema Hash", liveHash);
+
+    if (liveHash === expectedHash) {
+      UI.success("Fingerprints match. Configuration is current.");
+    } else {
+      UI.error("DRIFT DETECTED: Database structure has changed.");
+      UI.hint(`If this was intended, update 'integrity.expected_schema_hash' to: ${pc.bold(liveHash)}`);
+    }
+
+    UI.step(3, "Cryptographic Signatures");
+    if (process.env.DPDP_CONFIG_SIGNING_PUBLIC_KEY_SPKI_BASE64) {
+      const sigSpinner = UI.spinner("Verifying detached Ed25519 signature...");
+      try {
+        await verifySignedWorkerConfig(process.env, configPath);
+        sigSpinner.succeed("Signature valid. Manifest is untampered.");
+      } catch (err) {
+        sigSpinner.fail("Signature verification failed.");
+        console.log(pc.red(`   ${err instanceof Error ? err.message : String(err)}`));
+      }
+    } else {
+      UI.info("Skipped: DPDP_CONFIG_SIGNING_PUBLIC_KEY_SPKI_BASE64 not set.");
+    }
+
+    UI.divider();
+    UI.success("Verification pipeline concluded.");
+  } catch (err) {
+    exitWithError("System failure", err instanceof Error ? err.message : String(err));
+  } finally {
+    await sql.end();
+  }
+}

package/src/modules/config/compatibility.ts

@@ -0,0 +1,271 @@
+import type { WorkerConfig } from "./validation";
+import { fail } from "@/errors";
+import type { Sql } from "@/types";
+
+interface SchemaColumnRow {
+  table_name: string;
+  column_name: string;
+}
+
+interface SchemaTableRow {
+  table_name: string;
+}
+
+function formatColumn(tableName: string, columnName: string): string {
+  return `${tableName}.${columnName}`;
+}
+
+function tableNameFromReference(reference: string, appSchema: string): string {
+  const parts = reference.split(".");
+  if (parts.length === 1) {
+    return parts[0]!;
+  }
+
+  if (parts[0] !== appSchema) {
+    return "";
+  }
+
+  return parts[1]!;
+}
+
+/**
+ * Verifies that the live application schema satisfies every column/table reference in the
+ * worker configuration before any task execution begins.
+ *
+ * This catches configuration drift such as missing root columns, satellite lookup columns,
+ * masking-rule targets, and retention evidence tables at boot time instead of failing after
+ * the worker has already leased work.
+ *
+ * @param sql - Postgres pool used for metadata inspection.
+ * @param config - Parsed worker configuration.
+ * @throws {WorkerError} When the application schema does not satisfy the worker configuration.
+ */
+export async function assertConfigSchemaCompatibility(
+  sql: Sql,
+  config: WorkerConfig
+): Promise<void> {
+  const rows = await sql<SchemaColumnRow[]>`
+    SELECT table_name, column_name
+    FROM information_schema.columns
+    WHERE table_schema = ${config.database.app_schema}
+    ORDER BY table_name ASC, ordinal_position ASC
+  `;
+  const tableRows = await sql<SchemaTableRow[]>`
+    SELECT table_name
+    FROM information_schema.tables
+    WHERE table_schema = ${config.database.app_schema}
+      AND table_type = 'BASE TABLE'
+    ORDER BY table_name ASC
+  `;
+
+  const columnsByTable = new Map<string, Set<string>>();
+  for (const row of rows) {
+    const existing = columnsByTable.get(row.table_name) ?? new Set<string>();
+    existing.add(row.column_name);
+    columnsByTable.set(row.table_name, existing);
+  }
+
+  const violations: string[] = [];
+  const rootTable = config.graph.root_table;
+  const rootColumns = columnsByTable.get(rootTable);
+
+  if (!rootColumns) {
+    violations.push(`missing root table ${config.database.app_schema}.${rootTable}`);
+  } else {
+    const requiredRootColumns = new Set<string>([
+      config.graph.root_id_column,
+      ...Object.keys(config.graph.root_pii_columns),
+      ...config.satellite_targets.map((target) => target.lookup_column),
+    ]);
+
+    if (config.graph.notice_email_column) {
+      requiredRootColumns.add(config.graph.notice_email_column);
+    }
+
+    if (config.graph.notice_name_column) {
+      requiredRootColumns.add(config.graph.notice_name_column);
+    }
+
+    if (config.purge_policy?.enabled && config.purge_policy.selector?.kind == "boolean_column") {
+      requiredRootColumns.add(config.purge_policy.selector.column);
+    }
+
+    for (const column of requiredRootColumns) {
+      if (!rootColumns.has(column)) {
+        violations.push(
+          `missing root column ${formatColumn(`${config.database.app_schema}.${rootTable}`, column)}`
+        );
+      }
+    }
+  }
+
+  for (const target of config.satellite_targets) {
+    const targetColumns = columnsByTable.get(target.table);
+    if (!targetColumns) {
+      violations.push(`missing satellite table ${config.database.app_schema}.${target.table}`);
+      continue;
+    }
+
+    if (!targetColumns.has(target.lookup_column)) {
+      violations.push(
+        `missing satellite lookup column ${formatColumn(
+          `${config.database.app_schema}.${target.table}`,
+          target.lookup_column
+        )}`
+      );
+    }
+
+    for (const column of Object.keys(target.masking_rules ?? {})) {
+      if (!targetColumns.has(column)) {
+        violations.push(
+          `missing satellite masking column ${formatColumn(
+            `${config.database.app_schema}.${target.table}`,
+            column
+          )}`
+        );
+      }
+    }
+  }
+
+  for (const rule of config.compliance_policy.retention_rules) {
+    for (const tableName of rule.if_has_data_in) {
+      if (!columnsByTable.has(tableName)) {
+        violations.push(`missing retention evidence table ${config.database.app_schema}.${tableName}`);
+      }
+    }
+  }
+
+  const compiledRules = config.rules ?? [];
+  if (compiledRules.length > 0) {
+    const dagTables = new Set<string>();
+    for (const rule of compiledRules) {
+      for (const target of rule.targets) {
+        const tableName = tableNameFromReference(target.table, config.database.app_schema);
+        if (!tableName) {
+          violations.push(`compiled DAG target ${target.table} is outside ${config.database.app_schema}`);
+          continue;
+        }
+
+        dagTables.add(tableName);
+        const targetColumns = columnsByTable.get(tableName);
+        if (!targetColumns) {
+          violations.push(`compiled DAG references missing table ${config.database.app_schema}.${tableName}`);
+          continue;
+        }
+
+        for (const column of target.pii_columns) {
+          if (!targetColumns.has(column)) {
+            violations.push(
+              `compiled DAG references missing PII column ${formatColumn(
+                `${config.database.app_schema}.${tableName}`,
+                column
+              )}`
+            );
+          }
+        }
+
+        for (const column of Object.keys(target.mutation_rules ?? {})) {
+          if (!targetColumns.has(column)) {
+            violations.push(
+              `compiled DAG references missing mutation column ${formatColumn(
+                `${config.database.app_schema}.${tableName}`,
+                column
+              )}`
+            );
+          }
+        }
+
+        for (const column of target.child_columns) {
+          if (!targetColumns.has(column)) {
+            violations.push(
+              `compiled DAG references missing child join column ${formatColumn(
+                `${config.database.app_schema}.${tableName}`,
+                column
+              )}`
+            );
+          }
+        }
+
+        for (const column of target.primary_key_columns) {
+          if (!targetColumns.has(column)) {
+            violations.push(
+              `compiled DAG references missing primary key column ${formatColumn(
+                `${config.database.app_schema}.${tableName}`,
+                column
+              )}`
+            );
+          }
+        }
+
+        if (tableName !== config.graph.root_table && target.pii_columns.length > 0 && !target.action) {
+          violations.push(
+            `compiled DAG target ${config.database.app_schema}.${tableName} contains PII columns but has no mutation action`
+          );
+        }
+
+        if (target.action === "redact" && (!target.mutation_rules || Object.keys(target.mutation_rules).length === 0)) {
+          violations.push(
+            `compiled DAG target ${config.database.app_schema}.${tableName} has redact action but no mutation_rules`
+          );
+        }
+
+        if (tableName !== config.graph.root_table) {
+          if (!target.parent) {
+            violations.push(`compiled DAG target ${config.database.app_schema}.${tableName} is missing parent`);
+          } else {
+            const parentTableName = tableNameFromReference(target.parent, config.database.app_schema);
+            const parentColumns = parentTableName ? columnsByTable.get(parentTableName) : undefined;
+            if (!parentTableName) {
+              violations.push(`compiled DAG parent ${target.parent} is outside ${config.database.app_schema}`);
+            } else if (!parentColumns) {
+              violations.push(`compiled DAG references missing parent table ${config.database.app_schema}.${parentTableName}`);
+            } else {
+              for (const column of target.parent_columns) {
+                if (!parentColumns.has(column)) {
+                  violations.push(
+                    `compiled DAG references missing parent join column ${formatColumn(
+                      `${config.database.app_schema}.${parentTableName}`,
+                      column
+                    )}`
+                  );
+                }
+              }
+            }
+
+            if (
+              target.parent_columns.length !== target.child_columns.length ||
+              (target.parent_columns.length === 0 && !target.join && !target.fk_condition)
+            ) {
+              violations.push(
+                `compiled DAG target ${config.database.app_schema}.${tableName} has incomplete join columns`
+              );
+            }
+          }
+        }
+      }
+    }
+
+    for (const table of tableRows) {
+      if (!dagTables.has(table.table_name)) {
+        violations.push(
+          `live table ${config.database.app_schema}.${table.table_name} is missing from compiled DAG`
+        );
+      }
+    }
+  }
+
+  if (violations.length > 0) {
+    fail({
+      code: "CONFIG_SCHEMA_MISMATCH",
+      title: "Worker config does not match the application schema",
+      detail: `Detected ${violations.length} schema compatibility violation(s): ${violations.join("; ")}`,
+      category: "configuration",
+      retryable: false,
+      fatal: true,
+      context: {
+        appSchema: config.database.app_schema,
+        violations,
+      },
+    });
+  }
+}

package/src/modules/config/index.ts

@@ -0,0 +1,4 @@
+export * from "./signature";
+export * from "./validation";
+export * from "./reader";
+export * from "./compatibility";

package/src/modules/config/reader.ts

@@ -0,0 +1,149 @@
+import { asWorkerError } from "@/errors";
+import {
+  type WorkerConfig,
+  type WorkerYamlConfig,
+  workerYamlSchema,
+  normalizeWorkerYaml,
+} from "./validation";
+import yaml from "js-yaml";
+import type { EnvType } from "@/types";
+import { resolveConfiguredKey, resolveConfiguredKeyAsync } from "@/secrets";
+
+/**
+* Reads a configuration file by detecting the current JavaScript runtime.
+* 
+* It prioritizes `Bun.file` for performance if available, otherwise it
+* dynamically imports `node:fs/promises` to read the file in Node.js.
+* 
+* @param configPath - The path or URL to the configuration file.
+* @returns A promise that resolves to the file's content as a UTF-8 string.
+* @throws Error if the file cannot be found or read.
+*/
+async function readConfigText(configPath: string | URL): Promise<string> {
+  /**
+   * Augments @module `globalThis` with Bun to satisfy type definition.
+   */
+  const bunRuntime = (globalThis as typeof globalThis & {
+    Bun?: { file: (path: string | URL) => { text: () => Promise<string> } };
+  }).Bun
+
+  if (bunRuntime) {
+    return bunRuntime.file(configPath).text();
+  }
+
+  const { readFile } = await import("node:fs/promises");
+  return readFile(configPath, "utf-8");
+}
+
+/**
+ * 
+ * @param configPath - Path to config.
+ * @returns {WorkerYamlConfig}
+ */
+async function readAndValidateWorkerYaml(configPath: string | URL): Promise<WorkerYamlConfig> {
+  const yamlText = await readConfigText(configPath);
+  let parsedYaml: unknown;
+  try {
+    parsedYaml = yaml.load(yamlText);
+  } catch (error) {
+    throw asWorkerError(error, {
+      code: "CONFIG_YAML_INVALID",
+      title: "Invalid worker YAML",
+      detail: `Failed to parse ${String(configPath)} as YAML.`,
+      category: "configuration",
+      retryable: false,
+      fatal: true,
+      context: { configPath: String(configPath) },
+    })
+  }
+
+  let parsedConfig: WorkerYamlConfig;
+  try {
+    parsedConfig = normalizeWorkerYaml(workerYamlSchema.parse(parsedYaml))
+  } catch (error) {
+    throw asWorkerError(error, {
+      code: "CONFIG_SCHEMA_INVALID",
+      title: "Invalid worker configuration",
+      category: "configuration",
+      retryable: false,
+      fatal: true,
+      context: { configPath: String(configPath) },
+    });
+  }
+
+  return parsedConfig;
+}
+
+/**
+ * Reads `compliance.worker.yml`, validates it strictly, and resolves local cryptographic secrets.
+ *
+ * This synchronous path is intended for tests and local env/file sources. Production boot should
+ * use `readWorkerConfigFromRuntime` so remote KMS/Vault providers can be resolved without blocking
+ * or silently falling back to process env.
+ *
+ * @param env - Environment map used to resolve key material.
+ * @param configPath - Worker YAML path.
+ * @returns Fully validated worker configuration with decoded binary keys.
+ * @throws {WorkerError} When YAML parsing, schema validation, or local secret decoding fails.
+ */
+export async function readWorkerConfig(
+  env: Record<string, string | undefined> = process.env,
+  configPath: string | URL = new URL("../../compliance.worker.yml", import.meta.url)
+): Promise<WorkerConfig> {
+  const parsedConfig = await readAndValidateWorkerYaml(configPath);
+  const masterKey = await resolveConfiguredKeyAsync({
+    env,
+    keyName: parsedConfig.security.master_key_env,
+    legacyEnvName: parsedConfig.security.master_key_env,
+    source: parsedConfig.security.master_key_source,
+  });
+  const hmacKey = await resolveConfiguredKeyAsync({
+    env,
+    keyName: parsedConfig.security.hmac_key_env,
+    legacyEnvName: parsedConfig.security.hmac_key_env,
+    fallbackLegacyEnvName: parsedConfig.security.master_key_env,
+    source: parsedConfig.security.hmac_key_source,
+  });
+
+  return {
+    ...parsedConfig,
+    masterKey,
+    hmacKey,
+  };
+}
+
+/**
+ * Reads `compliance.worker.yml` and resolves env, file, AWS KMS, GCP Secret Manager, or Vault keys.
+ * 
+ * @param env - Environment map used for provider credentials and legacy env key fallback.
+ * @param configPath - Worker YAML path.
+ * @returns Fully validated worker configuration with runtime-resolved binary keys.
+ * @throws {WorkerError} When YAML, schema validation, provider access, or key decoding fails.
+ */
+export const readWorkerConfigFromRuntime = async (
+  env: EnvType = process.env,
+  configPath: string | URL = new URL("../../compliance.worker.yml", import.meta.url)
+): Promise<WorkerConfig> => {
+  const parsedConfig = await readAndValidateWorkerYaml(configPath);
+  const [masterKey, hmacKey] = await Promise.all([
+    resolveConfiguredKey({
+      env,
+      keyName: parsedConfig.security.hmac_key_env,
+      legacyEnvName: parsedConfig.security.master_key_env,
+      source: parsedConfig.security.master_key_source
+    }),
+    resolveConfiguredKey({
+      env,
+      keyName: parsedConfig.security.hmac_key_env,
+      legacyEnvName: parsedConfig.security.hmac_key_env,
+      fallbackLegacyEnvName: parsedConfig.security.master_key_env,
+      source: parsedConfig.security.hmac_key_source,
+    }),
+  ]);
+
+  return {
+    ...parsedConfig,
+    masterKey,
+    hmacKey,
+  }
+};