@rehearsal-db/core 0.1.0-beta.1

package/scripts/lib/rehearsal/runtime_restore.mjs

@@ -0,0 +1,310 @@
+/**
+ * Purpose: Build the bounded PostgreSQL restore program for a verified sanitized
+ * Rehearsal baseline. Do not run directly; this module is reusable script infrastructure.
+ */
+
+import { createHash } from "node:crypto";
+
+const identifierPattern = /^[a-z][a-z0-9_]{0,62}$/u;
+const constraintFailurePattern =
+  /^ERROR:\s+(?:[A-Z0-9]{5}:\s+)?new row for relation "([a-z][a-z0-9_]*)" violates check constraint "([a-z][a-z0-9_]*)"$/mu;
+const sqlStatePattern = /^ERROR:\s+([A-Z0-9]{5}):/mu;
+const notNullFailurePattern =
+  /^ERROR:\s+(?:23502:\s+)?null value in column "([a-z][a-z0-9_]*)" of relation "([a-z][a-z0-9_]*)" violates not-null constraint$/mu;
+const quoteIdentifier = (value) => {
+  if (!identifierPattern.test(value)) {
+    throw new Error(`Unsafe Rehearsal restore identifier: ${value}`);
+  }
+  return `"${value}"`;
+};
+const quoteLiteral = (value) => `'${value.replaceAll("'", "''")}'`;
+
+const includedPolicies = (manifest) =>
+  manifest.tables
+    .filter((table) => table.sourceRows !== "EXCLUDE")
+    .sort((left, right) => left.name.localeCompare(right.name));
+
+export const summarizeRestoreError = (stderr) => {
+  const match = String(stderr).match(constraintFailurePattern);
+  if (match) return `Sanitized data violated ${match[1]}.${match[2]}.`;
+  const notNull = String(stderr).match(notNullFailurePattern);
+  if (notNull) {
+    return `Sanitized data omitted required column ${notNull[2]}.${notNull[1]}.`;
+  }
+  const sqlState = String(stderr).match(sqlStatePattern)?.[1];
+  return sqlState
+    ? `PostgreSQL rejected the sanitized baseline stream with SQLSTATE ${sqlState}; row details were withheld.`
+    : "PostgreSQL rejected the sanitized baseline stream; row details were withheld.";
+};
+
+export const encodeBaselineRecordForCopy = (line) => {
+  let record;
+  try {
+    record = JSON.parse(line.trimEnd());
+  } catch (error) {
+    throw new Error("A Rehearsal baseline record is not valid JSON.", {
+      cause: error,
+    });
+  }
+  if (
+    !identifierPattern.test(record?.table ?? "") ||
+    !record.row ||
+    typeof record.row !== "object" ||
+    Array.isArray(record.row)
+  ) {
+    throw new Error(
+      "A Rehearsal baseline record has an invalid restore shape.",
+    );
+  }
+  return `${record.table}\t${Buffer.from(JSON.stringify(record.row), "utf8").toString("base64")}`;
+};
+
+export const buildRestoreSqlPrefix = () => `\\set ON_ERROR_STOP on
+\\set VERBOSITY verbose
+begin;
+set local session_replication_role = replica;
+create temporary table rehearsal_restore_rows (
+	sequence bigint generated always as identity primary key,
+	table_name text not null,
+	encoded text not null
+) on commit drop;
+copy rehearsal_restore_rows (table_name, encoded) from stdin;
+`;
+
+const decodedPayloadSql =
+  "convert_from(decode(source.encoded, 'base64'), 'UTF8')::jsonb";
+
+const authReferenceSelects = (manifest) =>
+  includedPolicies(manifest).flatMap((table) =>
+    table.columns
+      .filter(
+        (column) =>
+          column.action !== "EXCLUDE" &&
+          column.foreignKey?.schema === "auth" &&
+          column.foreignKey?.table === "users" &&
+          column.foreignKey?.column === "id",
+      )
+      .map(
+        (
+          column,
+        ) => `select nullif(${decodedPayloadSql}->>${quoteLiteral(column.name)}, '')::uuid as id
+	from rehearsal_restore_rows source
+	where source.table_name = ${quoteLiteral(table.name)}`,
+      ),
+  );
+
+const buildAuthRestoreSql = (manifest) => {
+  const selects = authReferenceSelects(manifest);
+  if (selects.length === 0) return "";
+  return `with referenced_auth_users as (
+	${selects.join("\n\tunion\n\t")}
+)
+insert into auth.users (
+	instance_id,
+	id,
+	aud,
+	role,
+	email,
+	confirmation_token,
+	recovery_token,
+	email_change_token_new,
+	email_change,
+	email_change_token_current,
+	phone_change,
+	phone_change_token,
+	reauthentication_token,
+	raw_app_meta_data,
+	raw_user_meta_data,
+	is_sso_user,
+	is_anonymous,
+	created_at,
+	updated_at
+)
+select distinct
+	'00000000-0000-0000-0000-000000000000'::uuid,
+	id,
+	'authenticated',
+	'authenticated',
+	'rehearsal-' || replace(id::text, '-', '') || '@rehearsal.invalid',
+	'',
+	'',
+	'',
+	'',
+	'',
+	'',
+	'',
+	'',
+	'{"provider":"email","providers":["email"]}'::jsonb,
+	'{"rehearsal":true}'::jsonb,
+	false,
+	false,
+	now(),
+	now()
+from referenced_auth_users
+where id is not null
+on conflict (id) do nothing;
+`;
+};
+
+const buildTableRestoreSql = (table) => {
+  const tableName = quoteIdentifier(table.name);
+  const columns = table.columns.filter(
+    (column) => column.action !== "EXCLUDE" && column.generated !== "ALWAYS",
+  );
+  if (columns.length === 0) return "";
+  const columnList = columns
+    .map((column) => quoteIdentifier(column.name))
+    .join(", ");
+  const selectedColumns = columns
+    .map((column) => `restored.${quoteIdentifier(column.name)}`)
+    .join(", ");
+  const overridesIdentity = columns.some((column) => column.identity === "YES")
+    ? " overriding system value"
+    : "";
+  return `insert into public.${tableName} (${columnList})${overridesIdentity}
+select ${selectedColumns}
+from rehearsal_restore_rows source
+cross join lateral jsonb_populate_record(
+	null::public.${tableName},
+	${decodedPayloadSql}
+) restored
+where source.table_name = ${quoteLiteral(table.name)}
+order by source.sequence;
+`;
+};
+
+const buildIdentitySequenceSql = (table) =>
+  table.columns
+    .filter(
+      (column) =>
+        column.action !== "EXCLUDE" &&
+        column.identity === "YES" &&
+        column.generated !== "ALWAYS",
+    )
+    .map((column) => {
+      const relation = `public.${quoteIdentifier(table.name)}`;
+      const columnName = quoteIdentifier(column.name);
+      return `select setval(
+	pg_get_serial_sequence(${quoteLiteral(`public.${table.name}`)}, ${quoteLiteral(column.name)}),
+	coalesce((select max(${columnName}) from ${relation}), 1),
+	exists(select 1 from ${relation})
+);`;
+    })
+    .join("\n");
+
+const buildRowCountChecks = (tableCounts) =>
+  Object.entries(tableCounts)
+    .sort(([left], [right]) => left.localeCompare(right))
+    .map(([table, expected]) => {
+      quoteIdentifier(table);
+      if (!Number.isSafeInteger(expected) || expected < 0) {
+        throw new Error(`Invalid Rehearsal row count for ${table}.`);
+      }
+      return `if (select count(*) from public.${quoteIdentifier(table)}) <> ${expected} then
+		raise exception 'Rehearsal row-count mismatch for ${table}';
+	end if;`;
+    })
+    .join("\n\t");
+
+const buildForeignKeyChecks = (manifest) =>
+  includedPolicies(manifest)
+    .flatMap((table) =>
+      table.columns
+        .filter((column) => column.action !== "EXCLUDE" && column.foreignKey)
+        .map((column) => ({ table, column })),
+    )
+    .map(({ table, column }) => {
+      const target = column.foreignKey;
+      for (const value of [
+        table.name,
+        column.name,
+        target.schema,
+        target.table,
+        target.column,
+      ])
+        quoteIdentifier(value);
+      return `if exists (
+		select 1
+		from public.${quoteIdentifier(table.name)} child
+		left join ${quoteIdentifier(target.schema)}.${quoteIdentifier(target.table)} parent
+			on parent.${quoteIdentifier(target.column)} = child.${quoteIdentifier(column.name)}
+		where child.${quoteIdentifier(column.name)} is not null
+			and parent.${quoteIdentifier(target.column)} is null
+	) then
+		raise exception 'Rehearsal foreign-key mismatch for ${table.name}.${column.name}';
+	end if;`;
+    })
+    .join("\n\t");
+
+export const buildRestoreSqlSuffix = ({ manifest, tableCounts }) => {
+  if (!Array.isArray(manifest?.tables) || !tableCounts) {
+    throw new Error("A reviewed manifest and exact row counts are required.");
+  }
+  const tables = includedPolicies(manifest);
+  const expectedNames = tables.map((table) => table.name).sort();
+  if (expectedNames.join("\0") !== Object.keys(tableCounts).sort().join("\0")) {
+    throw new Error("Rehearsal restore table counts do not match the policy.");
+  }
+  return `\\.
+create index rehearsal_restore_rows_table_name_idx
+	on rehearsal_restore_rows (table_name, sequence);
+truncate table ${manifest.tables
+    .map((table) => `public.${quoteIdentifier(table.name)}`)
+    .join(", ")}
+restart identity cascade;
+${buildAuthRestoreSql(manifest)}
+${tables.map(buildTableRestoreSql).join("\n")}
+${tables.map(buildIdentitySequenceSql).filter(Boolean).join("\n")}
+set local session_replication_role = origin;
+do $rehearsal_checks$
+begin
+	${buildRowCountChecks(tableCounts)}
+	${buildForeignKeyChecks(manifest)}
+end
+$rehearsal_checks$;
+commit;
+`;
+};
+
+export const createCandidateMigrationReceipt = ({
+  baselineManifest,
+  currentFiles,
+}) => {
+  const baselineEntries = Object.entries(
+    baselineManifest?.migrations ?? {},
+  ).sort(([left], [right]) => left.localeCompare(right));
+  if (baselineEntries.length === 0) {
+    throw new Error("The active Rehearsal baseline has no migration bundle.");
+  }
+  if (baselineEntries.length > currentFiles.length) {
+    throw new Error(
+      "The active Rehearsal history is absent from local source.",
+    );
+  }
+  for (let index = 0; index < baselineEntries.length; index += 1) {
+    const [filename, receipt] = baselineEntries[index];
+    const current = currentFiles[index];
+    if (
+      current.filename !== filename ||
+      current.fileSha256 !== receipt.sha256
+    ) {
+      throw new Error(
+        `Local migration history diverges from the active baseline at ${filename}.`,
+      );
+    }
+  }
+  const candidates = currentFiles.slice(baselineEntries.length);
+  const candidateSha256 = createHash("sha256")
+    .update(
+      `${JSON.stringify(
+        candidates.map(({ filename, fileSha256 }) => ({
+          filename,
+          fileSha256,
+        })),
+        null,
+        "\t",
+      )}\n`,
+    )
+    .digest("hex");
+  return { candidates, candidateSha256 };
+};

package/scripts/lib/rehearsal/sanitization_policy.mjs

@@ -0,0 +1,169 @@
+/**
+ * Generic, fail-closed sanitization policy validation and value operations.
+ * Projects own the policy, schema inventory, pseudonym keys, and derivation logic.
+ */
+
+export const SANITIZATION_ACTIONS = Object.freeze({
+  KEEP: "KEEP",
+  PSEUDONYMIZE: "PSEUDONYMIZE",
+  REPLACE: "REPLACE",
+  EXCLUDE: "EXCLUDE",
+  DERIVE: "DERIVE",
+});
+
+export const EXCLUDED_VALUE = Symbol("REHEARSAL_EXCLUDED_VALUE");
+
+const actionAliases = new Map([
+  ["KEEP", SANITIZATION_ACTIONS.KEEP],
+  ["KEEP EXACTLY", SANITIZATION_ACTIONS.KEEP],
+  ["PSEUDONYMIZE", SANITIZATION_ACTIONS.PSEUDONYMIZE],
+  ["REPLACE", SANITIZATION_ACTIONS.REPLACE],
+  ["REPLACE WITH SYNTHETIC", SANITIZATION_ACTIONS.REPLACE],
+  ["EXCLUDE", SANITIZATION_ACTIONS.EXCLUDE],
+  ["DERIVE", SANITIZATION_ACTIONS.DERIVE],
+]);
+
+const identifierPattern = /^[a-z][a-z0-9_]{0,62}$/u;
+
+const assertIdentifier = (value, label) => {
+  if (!identifierPattern.test(value ?? "")) {
+    throw new Error(`${label} must be a lowercase PostgreSQL identifier.`);
+  }
+  return value;
+};
+
+export const normalizeSanitizationAction = (action) => {
+  const normalized = actionAliases.get(
+    String(action ?? "")
+      .trim()
+      .toUpperCase(),
+  );
+  if (!normalized) {
+    throw new Error(`Unknown Rehearsal sanitization action: ${action}.`);
+  }
+  return normalized;
+};
+
+const indexTables = (tables, label) => {
+  if (!Array.isArray(tables)) throw new Error(`${label} must be an array.`);
+  const index = new Map();
+  for (const table of tables) {
+    const name = assertIdentifier(table?.name, `${label} table name`);
+    if (index.has(name)) throw new Error(`${label} duplicates table ${name}.`);
+    if (!Array.isArray(table.columns)) {
+      throw new Error(`${label} table ${name} must list its columns.`);
+    }
+    const columns = new Map();
+    for (const column of table.columns) {
+      const columnName = assertIdentifier(
+        typeof column === "string" ? column : column?.name,
+        `${label} column name`,
+      );
+      if (columns.has(columnName)) {
+        throw new Error(`${label} duplicates column ${name}.${columnName}.`);
+      }
+      columns.set(columnName, column);
+    }
+    index.set(name, { table, columns });
+  }
+  return index;
+};
+
+export const validateSanitizationCoverage = ({ policy, schemaTables }) => {
+  if (!policy || typeof policy !== "object" || Array.isArray(policy)) {
+    throw new Error("A Rehearsal sanitization policy object is required.");
+  }
+  const schema = indexTables(schemaTables, "Schema inventory");
+  const classified = indexTables(policy.tables, "Sanitization policy");
+  const normalizedTables = [];
+
+  for (const [tableName, schemaEntry] of schema) {
+    const policyEntry = classified.get(tableName);
+    if (!policyEntry) {
+      throw new Error(
+        `Sanitization policy does not classify table ${tableName}.`,
+      );
+    }
+    const normalizedColumns = [];
+    for (const columnName of schemaEntry.columns.keys()) {
+      const column = policyEntry.columns.get(columnName);
+      if (!column) {
+        throw new Error(
+          `Sanitization policy does not classify column ${tableName}.${columnName}.`,
+        );
+      }
+      normalizedColumns.push({
+        ...column,
+        name: columnName,
+        action: normalizeSanitizationAction(column.action),
+      });
+    }
+    for (const columnName of policyEntry.columns.keys()) {
+      if (!schemaEntry.columns.has(columnName)) {
+        throw new Error(
+          `Sanitization policy references unknown column ${tableName}.${columnName}.`,
+        );
+      }
+    }
+    normalizedTables.push({
+      ...policyEntry.table,
+      name: tableName,
+      columns: normalizedColumns,
+    });
+  }
+
+  for (const tableName of classified.keys()) {
+    if (!schema.has(tableName)) {
+      throw new Error(
+        `Sanitization policy references unknown table ${tableName}.`,
+      );
+    }
+  }
+
+  return Object.freeze({
+    policyVersion: policy.policyVersion,
+    tableCount: normalizedTables.length,
+    columnCount: normalizedTables.reduce(
+      (total, table) => total + table.columns.length,
+      0,
+    ),
+    tables: Object.freeze(normalizedTables),
+  });
+};
+
+export const applySanitizationAction = ({
+  action,
+  value,
+  replace,
+  pseudonymize,
+  derive,
+  context = {},
+}) => {
+  switch (normalizeSanitizationAction(action)) {
+    case SANITIZATION_ACTIONS.KEEP:
+      return value;
+    case SANITIZATION_ACTIONS.EXCLUDE:
+      return EXCLUDED_VALUE;
+    case SANITIZATION_ACTIONS.REPLACE:
+      if (typeof replace !== "function") {
+        throw new Error(
+          "REPLACE requires a project-owned replacement function.",
+        );
+      }
+      return replace(value, context);
+    case SANITIZATION_ACTIONS.PSEUDONYMIZE:
+      if (typeof pseudonymize !== "function") {
+        throw new Error(
+          "PSEUDONYMIZE requires a project-owned pseudonymization function.",
+        );
+      }
+      return pseudonymize(value, context);
+    case SANITIZATION_ACTIONS.DERIVE:
+      if (typeof derive !== "function") {
+        throw new Error("DERIVE requires a project-owned derivation function.");
+      }
+      return derive(value, context);
+    default:
+      throw new Error("Unreachable Rehearsal sanitization action.");
+  }
+};

package/scripts/lib/rehearsal/schema_snapshot.mjs

@@ -0,0 +1,113 @@
+/**
+ * Purpose: Capture and compare bounded, data-free PostgreSQL public-schema
+ * snapshots for a local Rehearsal runtime. Do not run directly.
+ */
+
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { chmod, mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
+import { basename, join, resolve, sep } from "node:path";
+import { createCleanProcessEnvironment } from "./process_environment.mjs";
+
+const maximumSchemaBytes = 64 * 1024 * 1024;
+
+const assertArtifactRoot = (artifactRoot) => {
+  const resolved = resolve(artifactRoot);
+  if (basename(resolved) !== ".rehearsal" || resolved === sep) {
+    throw new Error(
+      `Refusing an unsafe Rehearsal artifact root: ${artifactRoot}`,
+    );
+  }
+  return resolved;
+};
+
+export const assertProductionSchemaDump = (schemaSql) => {
+  if (typeof schemaSql !== "string" || schemaSql.length === 0) {
+    throw new Error("The Rehearsal production schema snapshot is empty.");
+  }
+  if (Buffer.byteLength(schemaSql) > maximumSchemaBytes) {
+    throw new Error("The Rehearsal production schema snapshot is too large.");
+  }
+  if (!schemaSql.includes('CREATE SCHEMA IF NOT EXISTS "public";')) {
+    throw new Error(
+      "The Rehearsal production schema snapshot does not define public.",
+    );
+  }
+  if (/^COPY\s|^\\connect\s|^CREATE DATABASE\s/mu.test(schemaSql)) {
+    throw new Error(
+      "The Rehearsal production schema snapshot contains data or database switching commands.",
+    );
+  }
+  return schemaSql;
+};
+
+const comparableSchemaDump = (schemaSql) =>
+  assertProductionSchemaDump(schemaSql)
+    .split("\n")
+    .map((line) => line.trimEnd())
+    .filter(Boolean)
+    .map((line) => {
+      const checkConstraint = line.match(
+        /^(\s*CONSTRAINT "[^"]+" CHECK ).*(,?)$/u,
+      );
+      return checkConstraint
+        ? `${checkConstraint[1]}<database-normalized-expression>${checkConstraint[2]}`
+        : line;
+    })
+    .join("\n");
+
+export const compareProductionSchemaDumps = ({ expected, actual }) => {
+  const expectedComparable = comparableSchemaDump(expected);
+  const actualComparable = comparableSchemaDump(actual);
+  if (expectedComparable !== actualComparable) {
+    throw new Error(
+      "The local Rehearsal public schema does not structurally match the production snapshot.",
+    );
+  }
+  return createHash("sha256").update(actualComparable).digest("hex");
+};
+
+export const captureSupabasePublicSchema = async ({
+  repositoryRoot,
+  artifactRoot,
+  arguments_,
+}) => {
+  const root = assertArtifactRoot(artifactRoot);
+  await mkdir(root, { recursive: true, mode: 0o700 });
+  const temporaryDirectory = await mkdtemp(join(root, ".schema-capture-"));
+  const destination = join(temporaryDirectory, "production-schema.sql");
+  try {
+    const result = spawnSync(
+      "supabase",
+      [...arguments_, "--schema", "public", "--file", destination],
+      {
+        cwd: repositoryRoot,
+        env: createCleanProcessEnvironment(),
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "pipe"],
+        maxBuffer: 4 * 1024 * 1024,
+      },
+    );
+    if (result.error || result.status !== 0) {
+      throw new Error(
+        "The exact Rehearsal schema snapshot could not be captured.",
+        { cause: result.error },
+      );
+    }
+    await chmod(destination, 0o600);
+    return assertProductionSchemaDump(await readFile(destination, "utf8"));
+  } finally {
+    await rm(temporaryDirectory, { recursive: true, force: true });
+  }
+};
+
+export const captureLocalPublicSchema = ({
+  repositoryRoot,
+  artifactRoot,
+  workdir,
+}) =>
+  captureSupabasePublicSchema({
+    repositoryRoot,
+    artifactRoot,
+    arguments_: ["db", "dump", "--local", "--workdir", workdir],
+  });

package/scripts/lib/rehearsal/service_environment.mjs

@@ -0,0 +1,82 @@
+/**
+ * Purpose: Load an exact, project-owned allowlist of local service credentials
+ * without exposing values or inheriting ambient hosted credentials.
+ * Do not run directly; this module is reusable Rehearsal infrastructure.
+ */
+
+import { readFile, stat } from "node:fs/promises";
+import { parseEnv } from "node:util";
+
+const ENVIRONMENT_KEY_PATTERN = /^[A-Z][A-Z0-9_]*$/u;
+
+export const readRehearsalServiceEnvironment = async ({
+  path,
+  keys = [],
+  blockedKeys = [],
+}) => {
+  if (!path) {
+    if (keys.length) {
+      throw new Error(
+        "Rehearsal service environment keys require an owned environment file.",
+      );
+    }
+    return {};
+  }
+  if (!keys.length) {
+    throw new Error(
+      "A Rehearsal service environment file requires an exact variable allowlist.",
+    );
+  }
+  for (const key of keys) {
+    if (!ENVIRONMENT_KEY_PATTERN.test(key)) {
+      throw new Error(`Invalid Rehearsal service environment key: ${key}.`);
+    }
+    if (blockedKeys.includes(key)) {
+      throw new Error(
+        `Rehearsal service environment key ${key} is blocked by the safety policy.`,
+      );
+    }
+  }
+
+  let source;
+  let details;
+  try {
+    [details, source] = await Promise.all([stat(path), readFile(path, "utf8")]);
+  } catch (error) {
+    if (error?.code === "ENOENT") {
+      throw new Error(
+        `Rehearsal service environment file is missing: ${path}.`,
+        { cause: error },
+      );
+    }
+    throw error;
+  }
+  if (!details.isFile()) {
+    throw new Error(
+      `Rehearsal service environment path is not a file: ${path}.`,
+    );
+  }
+  if ((details.mode & 0o077) !== 0) {
+    throw new Error(
+      "Rehearsal service environment credentials must be owner-only (chmod 600).",
+    );
+  }
+
+  const parsed = parseEnv(source);
+  const unexpectedKeys = Object.keys(parsed).filter(
+    (key) => !keys.includes(key),
+  );
+  if (unexpectedKeys.length) {
+    throw new Error(
+      `Rehearsal service environment contains variables outside its allowlist: ${unexpectedKeys.join(", ")}.`,
+    );
+  }
+  const missingKeys = keys.filter((key) => !parsed[key]?.trim());
+  if (missingKeys.length) {
+    throw new Error(
+      `Rehearsal service environment is missing required variables: ${missingKeys.join(", ")}.`,
+    );
+  }
+
+  return Object.fromEntries(keys.map((key) => [key, parsed[key]]));
+};