dpdp-erasure-cli 1.0.0

package/src/modules/introspector/dag.ts

@@ -0,0 +1,276 @@
+import { MAX_DEPTH } from "@/constants";
+import type { CompileDagOptions, DagTarget, PotentialLogicalLink, QualifiedTable } from "./types";
+import { fail } from "@/errors";
+import { formatJoinCondition, parseQualifiedTable } from "./naming";
+import { parseBuildCommand } from "typescript";
+
+interface DagRow {
+  constraint_schema: string;
+  constraint_name: string;
+  child_schema: string;
+  child_table: string;
+  parent_schema: string;
+  parent_table: string;
+  child_columns: string[];
+  parent_columns: string[];
+  depth: number;
+}
+
+interface ColumnRow {
+  table_schema: string;
+  table_name: string;
+  column_name: string;
+}
+
+function toQualifiedTable(schema: string, table: string): QualifiedTable {
+  return { schema, table };
+}
+
+function rootTarget(root: QualifiedTable): DagTarget {
+  return {
+    table: root,
+    parentTable: null,
+    constraintName: null,
+    childColumns: [],
+    parentColumns: [],
+    depth: 0,
+    fkCondition: "ROOT",
+  };
+}
+
+/**
+ * Compiles foreign-key dependencies from `information_schema` into a static DAG target list.
+ *
+ * The query is read-only, bounded by `maxDepth`, and scoped to the root table schema so
+ * unrelated tenant/test schemas cannot slow or block compilation. Composite foreign keys are
+ * preserved by aligning `key_column_usage.position_in_unique_constraint` with the referenced key columns.
+ *
+ * @param options - Database handle, root table, default schema, and recursion breaker.
+ * @returns Root target plus dependent satellite tables with explicit join predicates.
+ * @throws {WorkerError} If the depth breaker is invalid.
+ */
+export async function compileStaticDag(options: CompileDagOptions): Promise<DagTarget[]> {
+  const maxDepth = options.maxDepth ?? MAX_DEPTH;
+  if (!Number.isInteger(maxDepth) || maxDepth < 1 || maxDepth > MAX_DEPTH) {
+    fail({
+      code: "INTROSPECTOR_DEPTH_INVALID",
+      title: "Invalid introspector depth",
+      detail: "Static DAG maxDepth must be an integer between 1 and 32.",
+      category: "validation",
+      retryable: false,
+      fatal: true,
+      context: { maxDepth }
+    });
+  }
+
+  const root = parseQualifiedTable(options.rootTable, options.defaultSchema);
+  const rows = await options.sql<DagRow[]>`
+    WITH RECURSIVE fk_columns AS (
+      SELECT
+        tc.constraint_schema,
+        tc.constraint_name,
+        kcu.table_schema AS child_schema,
+        kcu.table_name AS child_table,
+        pk.table_schema AS parent_schema,
+        pk.table_name AS parent_table,
+        kcu.column_name AS child_column,
+        pk.column_name AS parent_column,
+        kcu.ordinal_position
+      FROM information_schema.table_constraints AS tc
+      JOIN information_schema.key_column_usage AS kcu
+        ON kcu.constraint_schema = tc.constraint_schema
+       AND kcu.constraint_name = tc.constraint_name
+       AND kcu.table_schema = tc.table_schema
+       AND kcu.table_name = tc.table_name
+      JOIN information_schema.referential_constraints AS rc
+        ON rc.constraint_schema = tc.constraint_schema
+       AND rc.constraint_name = tc.constraint_name
+      JOIN information_schema.key_column_usage AS pk
+        ON pk.constraint_schema = rc.unique_constraint_schema
+       AND pk.constraint_name = rc.unique_constraint_name
+       AND pk.ordinal_position = kcu.position_in_unique_constraint
+      JOIN information_schema.constraint_column_usage AS ccu
+        ON ccu.constraint_schema = tc.constraint_schema
+       AND ccu.constraint_name = tc.constraint_name
+      WHERE tc.constraint_type = 'FOREIGN KEY'
+        AND kcu.table_schema = ${root.schema}
+        AND ccu.table_schema = pk.table_schema
+        AND ccu.table_name = pk.table_name
+    ),
+    fk_edges AS (
+      SELECT
+        constraint_schema,
+        constraint_name,
+        child_schema,
+        child_table,
+        parent_schema,
+        parent_table,
+        array_agg(child_column ORDER BY ordinal_position) AS child_columns,
+        array_agg(parent_column ORDER BY ordinal_position) AS parent_columns
+      FROM fk_columns
+      GROUP BY
+        constraint_schema,
+        constraint_name,
+        child_schema,
+        child_table,
+        parent_schema,
+        parent_table
+    ),
+    graph AS (
+      SELECT
+        constraint_schema,
+        constraint_name,
+        child_schema,
+        child_table,
+        parent_schema,
+        parent_table,
+        child_columns,
+        parent_columns,
+        1 AS depth,
+        ARRAY[parent_schema || '.' || parent_table, child_schema || '.' || child_table] AS visited
+      FROM fk_edges
+      WHERE parent_schema = ${root.schema}
+        AND parent_table = ${root.table}
+
+      UNION ALL
+
+      SELECT
+        edge.constraint_schema,
+        edge.constraint_name,
+        edge.child_schema,
+        edge.child_table,
+        edge.parent_schema,
+        edge.parent_table,
+        edge.child_columns,
+        edge.parent_columns,
+        graph.depth + 1 AS depth,
+        graph.visited || (edge.child_schema || '.' || edge.child_table)
+      FROM fk_edges AS edge
+      JOIN graph
+        ON edge.parent_schema = graph.child_schema
+       AND edge.parent_table = graph.child_table
+      WHERE graph.depth < ${maxDepth}
+        AND NOT (edge.child_schema || '.' || edge.child_table) = ANY(graph.visited)
+    )
+    SELECT
+      constraint_schema,
+      constraint_name,
+      child_schema,
+      child_table,
+      parent_schema,
+      parent_table,
+      child_columns,
+      parent_columns,
+      depth
+    FROM graph
+    ORDER BY depth ASC, child_schema ASC, child_table ASC, constraint_name ASC
+  `;
+
+  const targets = new Map<string, DagTarget>();
+  targets.set(`${root.schema}.${root.table}`, rootTarget(root));
+
+  for (const row of rows) {
+    const table = toQualifiedTable(row.child_schema, row.child_table);
+    const parentTable = toQualifiedTable(row.parent_schema, row.parent_table);
+    const key = `${table.schema}.${table.table}:${row.constraint_schema}.${row.constraint_name}`;
+    targets.set(key, {
+      table,
+      parentTable,
+      constraintName: row.constraint_name,
+      childColumns: row.child_columns,
+      parentColumns: row.parent_columns,
+      depth: row.depth,
+      fkCondition: formatJoinCondition(parentTable, row.parent_columns, table, row.child_columns),
+    });
+  }
+
+  return Array.from(targets.values()).sort((left, right) => {
+    const byDepth = left.depth - right.depth;
+    if (byDepth !== 0) {
+      return byDepth;
+    }
+    return `${left.table.schema}.${left.table.table}`.localeCompare(`${right.table.schema}.${right.table.table}`);
+  });
+}
+
+function physicalLinkKey(left: QualifiedTable, right: QualifiedTable, column: string): string {
+  const [first, second] = [`${left.schema}.${left.table}`, `${right.schema}.${right.table}`].sort();
+  return `${first}|${second}|${column}`;
+}
+
+/**
+ * Finds likely unmodeled relationships by intersecting high-signal identifier column names.
+ *
+ * This pass is metadata-only. It does not mutate state and does not prove a relationship exists;
+ * it surfaces tables that commonly act as ORM-managed or microservice-managed satellites without
+ * physical foreign keys.
+ *
+ * @param options - Database handle, root table, and default schema.
+ * @param physicalDag - FK DAG used to suppress already-modeled relationships.
+ * @returns Potential logical links for human review in the generated YAML.
+ */
+export async function discoverPotentialLogicalLinks(
+  options: Pick<CompileDagOptions, "sql" | "rootTable" | "defaultSchema">,
+  physicalDag: readonly DagTarget[],
+): Promise<PotentialLogicalLink[]> {
+  const root = parseQualifiedTable(options.rootTable, options.defaultSchema);
+  const rows = await options.sql<ColumnRow[]>`
+    SELECT table_schema, table_name, column_name
+    FROM information_schema.columns
+    WHERE table_schema = ${root.schema}
+      AND table_schema NOT IN ('pg_catalog', 'information_schema')
+    ORDER BY table_name ASC, ordinal_position ASC
+  `;
+
+  const physicalLinks = new Set<string>();
+  for (const target of physicalDag) {
+    if (!target.parentTable) {
+      continue;
+    }
+
+    for (const column of [...target.childColumns, ...target.parentColumns]) {
+      physicalLinks.add(physicalLinkKey(target.parentTable, target.table, column))
+    }
+  }
+
+  const byColumn = new Map<string, QualifiedTable[]>();
+  for (const row of rows) {
+    const normalized = row.column_name.toLowerCase();
+    if (!/^(?:user_id|account_id|customer_id|member_id|subject_id|.*_user_id)$/.test(normalized)) {
+      continue;
+    }
+
+    const existing = byColumn.get(row.column_name) ?? [];
+    existing.push(toQualifiedTable(row.table_schema, row.table_name));
+    byColumn.set(row.column_name, existing);
+  }
+
+  const links: PotentialLogicalLink[] = [];
+  const emitted = new Set<string>();
+  for (const [column, tables] of byColumn.entries()) {
+    if (tables.length < 2) {
+      continue;
+    }
+
+    for (let leftIndex = 0; leftIndex < tables.length; leftIndex += 1) {
+      for (let rightIndex = leftIndex + 1; rightIndex < tables.length; rightIndex += 1) {
+        const left = tables[leftIndex]!;
+        const right = tables[rightIndex]!;
+        const key = physicalLinkKey(left, right, column);
+        if (physicalLinks.has(key) || emitted.has(key)) {
+          continue;
+        }
+
+        emitted.add(key);
+        links.push({
+          sourceTable: left,
+          targetTable: right,
+          column,
+          reason: `Both tables expose ${column} but no physical foreign key was found.`,
+        });
+      }
+    }
+  }
+
+  return links;
+}

package/src/modules/introspector/index.ts

@@ -0,0 +1,7 @@
+export * from "./run";
+export * from "./classifier";
+export * from "./report";
+export * from "./types";
+export * from "./naming";
+export * from "./dag";
+export * from "./s3-sampler";

package/src/modules/introspector/naming.ts

@@ -0,0 +1,75 @@
+import { assertIdentifier } from "@/utils";
+import type { QualifiedTable } from "./types";
+
+/**
+ * Parses a CLI table reference into validated schema and table identifiers.
+ * 
+ * @param value - `table` or `schema.table` reference.
+ * @param defaultSchema - Schema used when `value` omits one.
+ * @returns Validated qualified table reference.
+ */
+export function parseQualifiedTable(value: string, defaultSchema: string = "public"): QualifiedTable {
+  const parts = value.split(".");
+  if (parts.length > 2 || parts.some((part) => part.trim() == "")) {
+    assertIdentifier(value, "qualified table reference");
+  }
+
+  if (parts.length === 1) {
+    return {
+      schema: assertIdentifier(defaultSchema, "default schema"),
+      table: assertIdentifier(parts[0]!, "root table"),
+    }
+  }
+
+  return {
+    schema: assertIdentifier(parts[0]!, "root schema"),
+    table: assertIdentifier(parts[1]!, "root schema"),
+  };
+}
+
+/**
+ * Serializes a qualified table in audit-friendly `schema.table` form.
+ *
+ * @param table - Qualified table reference.
+ * @returns Dot-qualified table name.
+ */
+export function formatQualifiedTable(table: QualifiedTable): string {
+  return `${table.schema}.${table.table}`;
+}
+
+/**
+ * Serializes one join predicate without quoting so the YAML remains readable for DPO review.
+ *
+ * @param parent - Referenced parent table.
+ * @param parentColumns - Referenced parent columns.
+ * @param child - Dependent child table.
+ * @param childColumns - Foreign key columns on the child.
+ * @returns Stable join condition string.
+ */
+export function formatJoinCondition(
+  parent: QualifiedTable,
+  parentColumns: string[],
+  child: QualifiedTable,
+  childColumns: string[]
+): string {
+  return parentColumns
+    .map((parentColumn, index) => {
+      const childColumn = childColumns[index] ?? childColumns[0] ?? "UNKNOWN";
+      return `${formatQualifiedTable(parent)}.${parentColumn} = ${formatQualifiedTable(child)}.${childColumn}`;
+    })
+    .join(" AND ");
+}
+
+/**
+ * Quotes a YAML scalar only when needed.
+ *
+ * @param value - Scalar value.
+ * @returns YAML-safe scalar representation.
+ */
+export function yamlScalar(value: string): string {
+  if (/^[A-Za-z0-9_.:-]+$/.test(value)) {
+    return value;
+  }
+
+  return JSON.stringify(value);
+}

package/src/modules/introspector/report.ts

@@ -0,0 +1,153 @@
+import { formatQualifiedTable } from "./naming";
+import type {
+  IntrospectorDraft,
+  IntrospectorReport,
+  IntrospectorReportFinding,
+  IntrospectorReportSummary,
+} from "./types";
+
+const HIGH_CONFIDENCE_THRESHOLD = 0.9;
+
+function formatScore(value: number): string {
+  return value.toFixed(3);
+}
+
+function findingKey(finding: IntrospectorReportFinding): string {
+  return `${finding.table}.${finding.column}`;
+}
+
+function sortFindings(
+  left: IntrospectorReportFinding,
+  right: IntrospectorReportFinding
+): number {
+  const byConfidence = right.confidence - left.confidence;
+  if (byConfidence !== 0) {
+    return byConfidence;
+  }
+
+  return findingKey(left).localeCompare(findingKey(right));
+}
+
+/**
+ * Builds a stable developer review report from an introspector draft.
+ *
+ * @param draft - Static DAG, PII taxonomy, and schema metadata produced by the introspector.
+ * @returns A report object suitable for CLI output, JSON export, or Markdown review artifacts.
+ */
+export function buildIntrospectorReport(draft: IntrospectorDraft): IntrospectorReport {
+  const findings = draft.targets.flatMap((target) =>
+    target.piiColumns.map((column): IntrospectorReportFinding => ({
+      table: formatQualifiedTable(target.table),
+      column: column.column,
+      dataType: column.dataType,
+      confidence: column.confidence,
+      metadataScore: column.metadataScore,
+      contentMatchRatio: column.contentMatchRatio,
+      sampleSize: column.sampleSize,
+      matchedSignatures: column.matchedSignatures,
+    }))
+  ).sort(sortFindings);
+
+  const tablesWithPii = new Set(findings.map((finding) => finding.table));
+  const summary: IntrospectorReportSummary = {
+    rootTable: formatQualifiedTable(draft.root),
+    generatedAt: draft.generatedAt,
+    schemaHash: draft.schemaHash,
+    targetCount: draft.targets.length,
+    tablesWithPii: tablesWithPii.size,
+    piiColumnCount: findings.length,
+    highConfidenceCount: findings.filter((finding) => finding.confidence >= HIGH_CONFIDENCE_THRESHOLD).length,
+    reviewRequiredCount: findings.filter((finding) => finding.confidence < HIGH_CONFIDENCE_THRESHOLD).length,
+    potentialLogicalLinkCount: draft.potentialLogicalLinks.length,
+  };
+
+  return {
+    summary,
+    findings,
+    potentialLogicalLinks: draft.potentialLogicalLinks,
+    nextSteps: [
+      "Review every PII column and potential logical link with the application owner.",
+      "Copy reviewed targets into compliance.worker.yml and complete legal_attestation.",
+      "Run compliance-worker check-integrity before allowing live worker boot.",
+      "Sign the reviewed manifest with compliance-worker sign after DPO approval.",
+    ],
+  };
+}
+
+/**
+ * Renders a Markdown report for DPO/developer review.
+ *
+ * @param report - Report model created by `buildIntrospectorReport`.
+ * @returns Markdown content containing summary, findings, review warnings, and next steps.
+ */
+export function renderIntrospectorMarkdown(report: IntrospectorReport): string {
+  const lines = [
+    "# Compliance Introspector Report",
+    "",
+    "## Summary",
+    "",
+    `- Root table: \`${report.summary.rootTable}\``,
+    `- Generated at: \`${report.summary.generatedAt}\``,
+    `- Schema hash: \`${report.summary.schemaHash}\``,
+    `- DAG targets: ${report.summary.targetCount}`,
+    `- Tables with PII: ${report.summary.tablesWithPii}`,
+    `- PII columns: ${report.summary.piiColumnCount}`,
+    `- High-confidence findings: ${report.summary.highConfidenceCount}`,
+    `- Review-required findings: ${report.summary.reviewRequiredCount}`,
+    `- Potential logical links: ${report.summary.potentialLogicalLinkCount}`,
+    "",
+    "## PII Findings",
+    "",
+  ];
+
+  if (report.findings.length === 0) {
+    lines.push("No PII columns crossed the configured confidence threshold.", "");
+  } else {
+    lines.push("| Table | Column | Type | Confidence | Metadata | Content | Signatures |");
+    lines.push("| --- | --- | --- | ---: | ---: | ---: | --- |");
+    for (const finding of report.findings) {
+      lines.push([
+        `| \`${finding.table}\``,
+        `\`${finding.column}\``,
+        `\`${finding.dataType}\``,
+        formatScore(finding.confidence),
+        formatScore(finding.metadataScore),
+        formatScore(finding.contentMatchRatio),
+        finding.matchedSignatures.length > 0 ? finding.matchedSignatures.join(", ") : "metadata",
+        "|",
+      ].join(" "));
+    }
+    lines.push("");
+  }
+
+  lines.push("## Potential Logical Links", "");
+  if (report.potentialLogicalLinks.length === 0) {
+    lines.push("None detected.", "");
+  } else {
+    for (const link of report.potentialLogicalLinks) {
+      lines.push(
+        `- \`${formatQualifiedTable(link.sourceTable)}.${link.column}\` <-> ` +
+        `\`${formatQualifiedTable(link.targetTable)}.${link.column}\`: ${link.reason}`
+      );
+    }
+    lines.push("");
+  }
+
+  lines.push("## Next Steps", "");
+  for (const step of report.nextSteps) {
+    lines.push(`- ${step}`);
+  }
+  lines.push("");
+
+  return lines.join("\n");
+}
+
+/**
+ * Renders report JSON with deterministic indentation for CI artifacts.
+ *
+ * @param report - Report model created by `buildIntrospectorReport`.
+ * @returns Pretty-printed JSON report.
+ */
+export function renderIntrospectorJson(report: IntrospectorReport): string {
+  return `${JSON.stringify(report, null, 2)}\n`;
+}

package/src/modules/introspector/run.ts

@@ -0,0 +1,123 @@
+import { MAX_DEPTH } from "@/constants";
+import { formatQualifiedTable, parseQualifiedTable } from "./naming";
+import type {
+  IntrospectorDraft,
+  IntrospectorTargetDraft,
+  RunIntrospectorOptions,
+  VerifySchemaIntegrityOptions
+} from "./types";
+import { compileStaticDag, discoverPotentialLogicalLinks } from "./dag";
+import { classifyDagTargets } from "./classifier";
+import { detectSchemaDrift } from "@modules/db";
+import { renderIntrospectorYaml } from "./yaml";
+import { fail } from "@/errors";
+import { readWorkerConfig } from "../config";
+
+function targetKey(schema: string, table: string): string {
+  return `${schema}.${table}`;
+}
+
+/**
+ * Runs the offline Introspector pipeline: static DAG compilation, bounded PII
+ * classification, and deterministic YAML draft rendering.
+ *
+ * @param options - SQL handle, root table, sampling controls, and output timestamp.
+ * @returns Draft model and rendered YAML content.
+ */
+export async function runIntrospector(options: RunIntrospectorOptions): Promise<{
+  draft: IntrospectorDraft;
+  yaml: string;
+}> {
+  const maxDepth = options.maxDepth ?? MAX_DEPTH;
+  const root = parseQualifiedTable(options.rootTable, options.defaultSchema);
+
+  const dag = await compileStaticDag({
+    sql: options.sql,
+    rootTable: formatQualifiedTable(root),
+    defaultSchema: root.schema,
+    maxDepth,
+  });
+
+  const classifiedColumns = await classifyDagTargets({
+    sql: options.sql,
+    targets: dag,
+    samplePercent: options.samplePercent,
+    sampleLimit: options.sampleLimit,
+    threshold: options.threshold,
+  });
+
+  const [schemaHash, potentialLogicalLinks] = await Promise.all([
+    detectSchemaDrift(options.sql, root.schema),
+    discoverPotentialLogicalLinks(
+      {
+        sql: options.sql,
+        rootTable: formatQualifiedTable(root),
+        defaultSchema: root.schema
+      },
+      dag,
+    ),
+  ]);
+
+  const targets: IntrospectorTargetDraft[] = dag.map((target) => ({
+    table: target.table,
+    parentTable: target.parentTable,
+    fkCondition: target.fkCondition,
+    childColumns: target.childColumns,
+    parentColumns: target.parentColumns,
+    depth: target.depth,
+    piiColumns: classifiedColumns.get(targetKey(target.table.schema, target.table.table)) ?? [],
+  }));
+
+  const draft: IntrospectorDraft = {
+    root,
+    maxDepth,
+    generatedAt: (options.generatedAt ?? new Date()).toISOString(),
+    schemaHash,
+    targets,
+    potentialLogicalLinks,
+  };
+
+  return {
+    draft,
+    yaml: renderIntrospectorYaml(draft),
+  };
+}
+
+/**
+ * Verifies that the live schema still matches the DPO-attested manifest hash.
+ *
+ * @param options - SQL handle, manifest path, and optional env map for key placeholders.
+ * @returns Live schema hash when the manifest is current.
+ * @throws {WorkerError} When the legal attestation hash is absent or stale.
+ */
+export async function verifySchemaIntegrity(options: VerifySchemaIntegrityOptions): Promise<string> {
+  const config = await readWorkerConfig(
+    {
+      ...process.env,
+      ...options.env,
+      DPDP_MASTER_KEY: options.env?.DPDP_MASTER_KEY ?? process.env.DPDP_MASTER_KEY ?? "0".repeat(64),
+      DPDP_HMAC_KEY: options.env?.DPDP_HMAC_KEY ?? process.env.DPDP_HMAC_KEY ?? "0".repeat(64),
+    },
+    options.configPath
+  );
+  const expectedHash = config.legal_attestation.schema_hash ?? config.integrity.expected_schema_hash;
+  const liveHash = await detectSchemaDrift(options.sql, config.database.app_schema);
+
+  if (liveHash !== expectedHash) {
+    fail({
+      code: "INTROSPECTOR_SCHEMA_VERIFY_FAILED",
+      title: "Schema verification failed",
+      detail: `Live schema hash ${liveHash} does not match legal attestation hash ${expectedHash}.`,
+      category: "integrity",
+      retryable: false,
+      fatal: true,
+      context: {
+        appSchema: config.database.app_schema,
+        expectedHash,
+        liveHash,
+      },
+    });
+  }
+
+  return liveHash;
+}