dpdp-erasure-cli 1.0.0

package/src/modules/bootstrap/preflight.ts

@@ -0,0 +1,296 @@
+import type { Sql } from "@/types";
+import { assertIdentifier } from "@/utils";
+import type { CompiledExecutionTarget, WorkerConfig } from "@modules/config/validation";
+import { fail } from "@/errors";
+
+export interface IndexRequirement {
+  schema: string;
+  table: string;
+  columns: string[];
+  reason: string;
+}
+
+export interface IndexPreflightResult {
+  checked: number,
+  missing: IndexRequirement[]
+}
+
+interface IndexPrefixRow {
+  exists: boolean;
+}
+
+interface ColumnExistsRow {
+  exists: boolean;
+}
+
+function qualifiedTable(value: string, defaultSchema: string): { schema: string; table: string } {
+  const parts = value.split(".");
+  if (parts.length === 1) {
+    return {
+      schema: defaultSchema,
+      table: assertIdentifier(parts[0]!, "table name"),
+    };
+  }
+
+  if (parts.length === 2) {
+    return {
+      schema: assertIdentifier(parts[0]!, "table schema"),
+      table: assertIdentifier(parts[1]!, "table name"),
+    };
+  }
+
+  fail({
+    code: "INDEX_PREFLIGHT_TABLE_INVALID",
+    title: "Invalid index preflight table reference",
+    detail: `Expected table or schema.table, received "${value}".`,
+    category: "configuration",
+    retryable: false,
+    fatal: true,
+  });
+}
+
+function addRequirement(
+  requirements: Map<string, IndexRequirement>,
+  requirement: IndexRequirement
+): void {
+  const safeRequirement: IndexRequirement = {
+    schema: assertIdentifier(requirement.schema, "index requirement schema"),
+    table: assertIdentifier(requirement.table, "index requirement schema"),
+    columns: requirement.columns.map((column) => assertIdentifier(column, "index requirement schema")),
+    reason: requirement.reason,
+  }
+  const key = `${safeRequirement.schema}.${safeRequirement.table}:${safeRequirement.columns.join(",")}`;
+  requirements.set(key, safeRequirement);
+}
+
+function addCompiledTargetRequirements(
+  requirements: Map<string, IndexRequirement>,
+  target: CompiledExecutionTarget,
+  defaultSchema: string
+): void {
+  const child = qualifiedTable(target.table, defaultSchema);
+  if (target.child_columns.length > 0) {
+    addRequirement(requirements, {
+      ...child,
+      columns: target.child_columns,
+      reason: `compiled DAG child join for ${target.table}`,
+    });
+  }
+
+  const primaryKeyColumns = target.primary_key_columns.length > 0 ? target.primary_key_columns : ["id"];
+  addRequirement(requirements, {
+    ...child,
+    columns: primaryKeyColumns,
+    reason: `compiled DAG bounded mutation identity for ${target.table}`,
+  });
+
+  if (!target.parent || target.parent_columns.length === 0) {
+    return;
+  }
+
+  const parent = qualifiedTable(target.parent, defaultSchema);
+  addRequirement(requirements, {
+    ...parent,
+    columns: target.parent_columns,
+    reason: `compiled DAG parent join for ${target.table}`,
+  });
+}
+
+/**
+ * Builds the exact lookup/index contract required for safe runtime execution.
+ *
+ * @param config - DPO-attested worker configuration.
+ * @returns Deduplicated index requirements for root, evidence, satellite, blob, and compiled-DAG lookups.
+ */
+export function collectIndexRequirements(config: WorkerConfig): IndexRequirement[] {
+  const requirements = new Map<string, IndexRequirement>();
+  const appSchema = config.database.app_schema;
+
+  addRequirement(requirements, {
+    schema: appSchema,
+    table: config.graph.root_table,
+    columns: [config.graph.root_id_column],
+    reason: "root row lock and task subject lookup",
+  });
+
+  for (const rule of config.compliance_policy.retention_rules) {
+    for (const table of rule.if_has_data_in) {
+      addRequirement(requirements, {
+        schema: appSchema,
+        table,
+        columns: [config.graph.root_id_column],
+        reason: `retention evidence rule ${rule.rule_name}`,
+      });
+    }
+  }
+
+  for (const target of config.satellite_targets) {
+    addRequirement(requirements, {
+      schema: appSchema,
+      table: target.table,
+      columns: [target.lookup_column],
+      reason: `satellite ${target.action} lookup`,
+    });
+  }
+
+  for (const target of config.blob_targets) {
+    if (target.lookup_column) {
+      addRequirement(requirements, {
+        schema: appSchema,
+        table: target.table,
+        columns: [target.lookup_column],
+        reason: `blob target lookup for ${target.table}.${target.column}`,
+      });
+    }
+  }
+
+  for (const rule of config.rules ?? []) {
+    for (const target of rule.targets) {
+      addCompiledTargetRequirements(requirements, target, appSchema);
+    }
+  }
+
+  return Array.from(requirements.values()).sort((left, right) =>
+    `${left.schema}.${left.table}.${left.columns.join(".")}`.localeCompare(
+      `${right.schema}.${right.table}.${right.columns.join(".")}`
+    )
+  );
+}
+
+async function hasIndexPrefix(
+  sql: Sql,
+  requirement: IndexRequirement
+): Promise<boolean> {
+  const [row] = await sql<IndexPrefixRow[]>`
+    SELECT EXISTS (
+      SELECT 1
+      FROM pg_index AS idx
+      JOIN pg_class AS table_class
+        ON table_class.oid = idx.indrelid
+      JOIN pg_namespace AS table_namespace
+        ON table_namespace.oid = table_class.relnamespace
+      JOIN LATERAL unnest(idx.indkey) WITH ORDINALITY AS indexed(attnum, ordinality)
+        ON indexed.ordinality <= ${requirement.columns.length}
+      JOIN pg_attribute AS attribute
+        ON attribute.attrelid = table_class.oid
+       AND attribute.attnum = indexed.attnum
+      WHERE table_namespace.nspname = ${requirement.schema}
+        AND table_class.relname = ${requirement.table}
+        AND idx.indisvalid
+        AND idx.indisready
+        AND idx.indpred IS NULL
+      GROUP BY idx.indexrelid
+      HAVING array_agg(attribute.attname ORDER BY indexed.ordinality) = ${requirement.columns}
+         OR array_agg(attribute.attname ORDER BY attribute.attname) = ${[...requirement.columns].sort()}
+    ) AS exists
+  `;
+
+  return row?.exists ?? false;
+}
+
+async function tableHasColumn(
+  sql: Sql,
+  schema: string,
+  table: string,
+  column: string
+): Promise<boolean> {
+  const [row] = await sql<ColumnExistsRow[]>`
+    SELECT EXISTS (
+      SELECT 1
+      FROM information_schema.columns
+      WHERE table_schema = ${schema}
+        AND table_name = ${table}
+        AND column_name = ${column}
+    ) AS exists
+  `;
+  return row?.exists ?? false;
+}
+
+async function collectTenantScopedRequirements(
+  sql: Sql,
+  requirements: IndexRequirement[]
+): Promise<IndexRequirement[]> {
+  const tenantRequirements: IndexRequirement[] = [];
+  const seenTables = new Map<string, boolean>();
+
+  for (const requirement of requirements) {
+    if (requirement.columns.includes("tenant_id")) {
+      continue;
+    }
+    if (
+      requirement.reason.startsWith("root row lock") ||
+      requirement.reason.startsWith("compiled DAG parent join")
+    ) {
+      continue;
+    }
+
+    const key = `${requirement.schema}.${requirement.table}`;
+    let hasTenantId = seenTables.get(key);
+    if (hasTenantId === undefined) {
+      hasTenantId = await tableHasColumn(sql, requirement.schema, requirement.table, "tenant_id");
+      seenTables.set(key, hasTenantId);
+    }
+
+    if (hasTenantId) {
+      tenantRequirements.push({
+        ...requirement,
+        columns: [...requirement.columns, "tenant_id"],
+        reason: `${requirement.reason} with tenant isolation`,
+      });
+    }
+  }
+
+  return tenantRequirements;
+}
+
+/**
+ * Verifies that configured runtime lookups are backed by usable non-partial indexes.
+ *
+ * The worker can mutate safely only when root locks, retention evidence probes, satellite
+ * batches, and compiled-DAG joins are index-backed. This preflight prevents a bad YAML from
+ * creating table scans or broad lock pressure against very large tenant databases.
+ *
+ * @param sql - Postgres pool used for catalog inspection.
+ * @param config - Parsed worker configuration.
+ * @returns Requirement summary when every lookup is index-backed.
+ * @throws {WorkerError} When one or more required indexes are missing.
+ */
+export async function assertIndexPreflight(
+  sql: Sql,
+  config: WorkerConfig
+): Promise<IndexPreflightResult> {
+  const baseRequirements = collectIndexRequirements(config);
+  const requirements = [
+    ...baseRequirements,
+    ...await collectTenantScopedRequirements(sql, baseRequirements),
+  ];
+  const missing: IndexRequirement[] = [];
+
+  for (const requirement of requirements) {
+    if (!await hasIndexPrefix(sql, requirement)) {
+      missing.push(requirement);
+    }
+  }
+
+  if (missing.length > 0) {
+    fail({
+      code: "INDEX_PREFLIGHT_FAILED",
+      title: "Required lookup indexes are missing",
+      detail: `Detected ${missing.length} missing index requirement(s): ${missing
+        .map((item) => `${item.schema}.${item.table}(${item.columns.join(", ")}) for ${item.reason}`)
+        .join("; ")}`,
+      category: "configuration",
+      retryable: false,
+      fatal: true,
+      context: {
+        checked: requirements.length,
+        missing,
+      },
+    });
+  }
+
+  return {
+    checked: requirements.length,
+    missing,
+  };
+}

package/src/modules/cli/check-integrity.ts

@@ -0,0 +1,48 @@
+import postgres from "postgres";
+import path from "node:path";
+import { UI, exitWithError } from "./ui";
+import { assertSchemaIntegrity } from "@modules/bootstrap";
+import { assertConfigSchemaCompatibility, readWorkerConfig } from "@modules/config";
+
+/**
+ * Runs fail-closed integrity validation for CI/CD and worker boot gates.
+ *
+ * @param options - Manifest path and database URL.
+ * @returns Resolves only when schema hash and compiled DAG checks pass.
+ */
+export async function checkIntegrityAction(options: { config: string; url?: string }) {
+  UI.header("Fail-Closed Integrity Check");
+
+  const dbUrl = options.url || process.env.DATABASE_URL;
+  if (!dbUrl) {
+    exitWithError("Database URL required.", "Provide --url or set DATABASE_URL env.");
+  }
+
+  const configPath = path.resolve(options.config);
+  const mockEnv = {
+    ...process.env,
+    DPDP_MASTER_KEY: process.env.DPDP_MASTER_KEY || "0".repeat(64),
+    DPDP_HMAC_KEY: process.env.DPDP_HMAC_KEY || "0".repeat(64),
+  };
+
+  const config = await readWorkerConfig(mockEnv, configPath);
+  const expectedSchemaHash =
+    config.legal_attestation.schema_hash ?? config.integrity.expected_schema_hash;
+  const sql = postgres(dbUrl);
+
+  try {
+    const detectedHash = await assertSchemaIntegrity(
+      sql,
+      config.database.app_schema,
+      expectedSchemaHash
+    );
+    await assertConfigSchemaCompatibility(sql, config);
+
+    UI.keyValue("Live Schema Hash", detectedHash);
+    UI.success("Schema hash, legal attestation, and compiled DAG are current.");
+  } catch (error) {
+    exitWithError("Integrity check failed", error instanceof Error ? error.message : String(error));
+  } finally {
+    await sql.end();
+  }
+}

package/src/modules/cli/dry-run.ts

@@ -0,0 +1,90 @@
+import postgres from "postgres";
+import pc from "picocolors";
+import { UI, exitWithError } from "./ui";
+import path from "node:path";
+import { readWorkerConfig } from "../config";
+import { vaultUser } from "../engine";
+
+/**
+ * Preview the results of a vault operation for a specific subject ID.
+ */
+export async function dryRunAction(options: {
+  id: string,
+  config: string,
+  url?: string,
+}) {
+  UI.header("Vault Simulation");
+
+  const dbUrl = options.url || process.env.DATABASE_URL;
+  if (!dbUrl) exitWithError("Database URL required.", "Provider --url or set DATABASE_URL env.");
+
+  const configPath = path.resolve(options.config);
+  UI.info(`Subject  : ${pc.bold(options.id)}`);
+  UI.info(`Manifest : ${pc.bold(options.config)}`);
+
+  const mockEnv = {
+    ...process.env,
+    DPDP_MASTER_KEY: process.env.DPDP_MASTER_KEY || "0".repeat(64),
+    DPDP_HMAC_KEY: process.env.DPDP_HMAC_KEY || "0".repeat(64),
+  };
+
+  let config;
+  try {
+    config = await readWorkerConfig(mockEnv, configPath);
+  } catch (err) {
+    exitWithError("Config load failed", err instanceof Error ? err.message : String(err));
+  }
+
+  const spinner = UI.spinner(`Executing dry-run pipeline for ${options.id}`);
+  const sql = postgres(dbUrl);
+
+  try {
+    const result = await vaultUser(sql, options.id, {
+      kek: config.masterKey,
+      hmacKey: config.hmacKey,
+    }, {
+      appSchema: config.database.app_schema,
+      engineSchema: config.database.engine_schema,
+      defaultRetentionYears: config.compliance_policy.default_retention_years,
+      noticeWindowHours: config.compliance_policy.notice_window_hours,
+      graphMaxDepth: config.graph.max_depth,
+      rootTable: config.graph.root_table,
+      rootIdColumn: config.graph.root_id_column,
+      rootPiiColumns: config.graph.root_pii_columns,
+      satelliteTargets: config.satellite_targets,
+      blobTargets: config.blob_targets,
+      retentionRules: config.compliance_policy.retention_rules,
+      dryRun: true,
+      now: new Date(),
+    });
+    spinner.stop();
+
+    if (!result.plan) exitWithError("Internal Error", "Vault engine failed to return a plan.");
+
+    UI.step(1, "Executive Summary");
+    UI.subStep(result.plan.summary);
+
+    UI.step(2, "Integrity Checks");
+    result.plan.checks.forEach(c => UI.subStep(c));
+
+    UI.step(3, "Cryptographic Pipeline");
+    result.plan.cryptoSteps.forEach(s => UI.subStep(pc.yellow(s)));
+
+    UI.step(4, "Proposed SQL Transactions");
+    result.plan.sqlSteps.forEach(s => UI.subStep(pc.magenta(s)));
+
+    UI.divider();
+    UI.info("FINAL PREVIEW:");
+    UI.keyValue("Planned Action", result.action);
+    UI.keyValue("Worker Hash", result.userHash || "N/A");
+    UI.keyValue("Dependencies", result.dependencyCount.toString());
+    UI.keyValue("Retention", `${result.retentionYears ?? 0} years (${result.appliedRuleName})`);
+
+    UI.success("Dry-run complete. No production data was modified.");
+  } catch (err) {
+    spinner.fail("Simulation failed");
+    exitWithError("Dry-run error", err instanceof Error ? err.message : String(err));
+  } finally {
+    await sql.end();
+  }
+}

package/src/modules/cli/graph.ts

@@ -0,0 +1,87 @@
+import { type DependencyNode, getDependencyGraph } from "@modules/db";
+import { readWorkerConfig } from "@modules/config";
+import pc from "picocolors";
+import { UI, exitWithError } from "./ui";
+import postgres from "postgres";
+
+function renderTree(rootName: string, nodes: DependencyNode[]) {
+  const nodesByParent: Record<string, DependencyNode[]> = {};
+  nodes.forEach(n => {
+    if (!nodesByParent[n.parent_table]) nodesByParent[n.parent_table] = [];
+    nodesByParent[n.parent_table]?.push(n);
+  });
+
+  function print(tableName: string, prefix: string, isLast: boolean) {
+    const children = nodesByParent[tableName] || [];
+    children.forEach((child, i) => {
+      const isChildLast = i === children.length - 1;
+      const marker = isChildLast ? "└── " : "├── ";
+      const unsafe = ["CASCADE", "SET_NULL", "SET_DEFAULT"].includes(child.delete_action);
+      const color = unsafe ? pc.red : pc.cyan;
+
+      console.log(`${prefix}${marker}${color(child.table_name)} ${pc.gray(`(${child.column_name})`)}`);
+      print(child.table_name, prefix + (isChildLast ? "    " : "│   "), isChildLast);
+    });
+  }
+
+  print(rootName, "", true);
+}
+
+/**
+ * Visualizes the recursive foreign-key graph.
+ */
+export async function graphAction(options: {
+  table: string,
+  url?: string,
+  schema?: string,
+  maxDepth: string,
+}): Promise<void> {
+  UI.header("Dependecy Graph Audit");
+
+  const dbUrl = options.url || process.env.DATABASE_URL
+  if (!dbUrl) exitWithError("Database URL required.", "Pass --url or set DATABASE_URL env.");
+
+  let appSchema = options.schema;
+  if (!appSchema) {
+    try {
+      const config = await readWorkerConfig(process.env);
+      appSchema = config.database.app_schema;
+    } catch (err) {
+      exitWithError("Schema missing.", "Provide --schema or ensure compliance.worker.yml exists.");
+    }
+  }
+
+  const maxDepth = parseInt(options.maxDepth, 10);
+  const spinner = UI.spinner(`Crawling relationships for ${appSchema}.${options.table}...`);
+  const sql = postgres(dbUrl);
+
+  try {
+    const nodes = await getDependencyGraph(sql, appSchema, options.table, {
+      maxDepth,
+      failOnUnsafeDeleteAction: false,
+    });
+    spinner.stop();
+
+    if (nodes.length === 0) {
+      UI.success("Leaf table detected: 0 downstream dependencies found.");
+      return;
+    }
+
+    console.log(`\n${pc.bold(pc.white(options.table))}`);
+    renderTree(options.table, nodes);
+
+    const unsafe = nodes.filter((n) => ["CASCADE", "SET_NULL", "SET_DEFAULT"].includes(n.delete_action));
+    if (unsafe.length > 0) {
+      UI.error(`${unsafe.length} UNSAFE CONSTRAINTS FOUND`);
+      unsafe.forEach(n => console.log(`   ${pc.red("✖")} ${n.table_name}.${n.column_name} — ON DELETE ${n.delete_action}`));
+      UI.warn("Live execution will fail until these are converted to NO ACTION or RESTRICT.");
+    } else {
+      UI.success("Graph validation passed: All constraints are safe for anonymization.");
+    }
+  } catch (err) {
+    spinner.fail("Graph crawl failed");
+    exitWithError("Analysis error", err instanceof Error ? err.message : String(err));
+  } finally {
+    sql.end();
+  }
+}