@rehearsal-db/core 0.1.0-beta.1 → 0.1.0-beta.10

package/{scripts/lib/rehearsal/baseline_builder.mjs → src/baseline/builder.mjs}

@@ -12,13 +12,14 @@ import { pathToFileURL } from "node:url";
 import {
   createAndActivateBaseline,
   createBaselineGenerationId,
-} from "./baseline_artifact.mjs";
-import { loadRehearsalConfig } from "./configuration.mjs";
+} from "./artifact.mjs";
+import { loadRehearsalConfig } from "../project/configuration.mjs";
 import {
   buildMigrationLedgerInventory,
   createMigrationReplayReceipt,
   readMigrationSourceBundle,
-} from "./migration_history.mjs";
+} from "../runtime/migration_history.mjs";
+import { validateRuntimeSanitizationPolicy } from "./sanitization_policy.mjs";
 
 const resolveProjectInput = (projectRoot, value, label) => {
   if (typeof value !== "string" || value.trim() === "") {
@@ -103,13 +104,16 @@ export const createSyntheticBaselineFromFiles = async ({
         })),
       )
     : [];
-  const policy = JSON.parse(policyBytes.toString("utf8"));
-  if (!Array.isArray(policy.tables)) {
-    throw new Error("The configured sanitization policy does not list tables.");
-  }
+  const policy = validateRuntimeSanitizationPolicy(
+    JSON.parse(policyBytes.toString("utf8")),
+  );
   const expectedTables = policy.tables
     .filter((table) => table.sourceRows !== "EXCLUDE")
-    .map((table) => table.name);
+    .map((table) =>
+      (table.schema ?? "public") === "public"
+        ? table.name
+        : `${table.schema}.${table.name}`,
+    );
   if (expectedTables.length === 0) {
     throw new Error("The sanitization policy includes no restorable tables.");
   }
@@ -117,6 +121,11 @@ export const createSyntheticBaselineFromFiles = async ({
     JSON.parse(ledgerBytes.toString("utf8")),
     "Synthetic migration ledger",
   );
+  if (policy.migrationCutoff !== sourceMigrationHistory.at(-1).version) {
+    throw new Error(
+      `The sanitization policy cutoff ${policy.migrationCutoff} does not match migration ledger cutoff ${sourceMigrationHistory.at(-1).version}.`,
+    );
+  }
   const migrationFiles = await readMigrationSourceBundle({
     directory: new URL(
       "./",
@@ -148,8 +157,8 @@ export const createSyntheticBaselineFromFiles = async ({
   return {
     generationId,
     migrationCutoff: baseline.migrationCutoff,
-    migrationCount: baseline.migrationCount,
+    migrationCount: Object.keys(baseline.migrations).length,
     rowCount: baseline.rowCount,
-    tableCount: baseline.tableCount,
+    tableCount: Object.keys(baseline.tableCounts).length,
   };
 };

package/src/baseline/input_discovery.mjs

@@ -0,0 +1,155 @@
+/**
+ * Discover likely project-owned baseline inputs without exposing their values.
+ * Discovery is bounded, ignores generated/private directories, and only sniffs
+ * small JSON or NDJSON files before returning project-relative paths.
+ */
+
+import { open, readFile, readdir, stat } from "node:fs/promises";
+import { extname, join, relative, sep } from "node:path";
+
+const ignoredDirectories = new Set([
+  ".git",
+  ".next",
+  ".rehearsal",
+  ".turbo",
+  "build",
+  "coverage",
+  "dist",
+  "node_modules",
+]);
+const maximumDepth = 4;
+const maximumFiles = 2_000;
+const maximumJsonBytes = 2 * 1024 * 1024;
+const maximumNdjsonProbeBytes = 64 * 1024;
+
+const projectRelativePath = (projectRoot, path) =>
+  relative(projectRoot, path).split(sep).join("/");
+
+const preference = {
+  records: [
+    "rehearsal/sanitized-data.ndjson",
+    "rehearsal/synthetic-data.ndjson",
+  ],
+  ledgers: ["rehearsal/migration-ledger.json"],
+  assetManifests: ["rehearsal/assets.json"],
+};
+
+const sortCandidates = (kind, candidates) => {
+  const preferred = new Map(
+    preference[kind].map((path, index) => [path, index]),
+  );
+  return candidates.sort((left, right) => {
+    const leftRank = preferred.get(left) ?? Number.POSITIVE_INFINITY;
+    const rightRank = preferred.get(right) ?? Number.POSITIVE_INFINITY;
+    return leftRank - rightRank || left.localeCompare(right);
+  });
+};
+
+const readFirstRecord = async (path) => {
+  const handle = await open(path, "r");
+  try {
+    const buffer = Buffer.alloc(maximumNdjsonProbeBytes);
+    const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+    const probe = buffer.subarray(0, bytesRead).toString("utf8");
+    const lines = probe.split(/\r?\n/u);
+    for (const line of lines) {
+      if (!line.trim()) continue;
+      return JSON.parse(line);
+    }
+  } finally {
+    await handle.close();
+  }
+  return undefined;
+};
+
+const isRecord = (value) =>
+  Boolean(
+    value &&
+    typeof value === "object" &&
+    !Array.isArray(value) &&
+    typeof value.table === "string" &&
+    value.row &&
+    typeof value.row === "object" &&
+    !Array.isArray(value.row),
+  );
+
+const isMigrationLedger = (value) =>
+  Array.isArray(value) &&
+  value.length > 0 &&
+  value.every(
+    (entry) =>
+      entry &&
+      typeof entry === "object" &&
+      typeof entry.version === "string" &&
+      typeof entry.name === "string" &&
+      Array.isArray(entry.statements),
+  );
+
+const isAssetManifest = (value) =>
+  Array.isArray(value) &&
+  value.length > 0 &&
+  value.every(
+    (entry) =>
+      entry &&
+      typeof entry === "object" &&
+      typeof entry.bucket === "string" &&
+      typeof entry.objectPath === "string" &&
+      typeof entry.file === "string",
+  );
+
+const collectCandidateFiles = async (projectRoot) => {
+  const files = [];
+  const visit = async (directory, depth) => {
+    if (depth > maximumDepth || files.length >= maximumFiles) return;
+    const entries = await readdir(directory, { withFileTypes: true }).catch(
+      () => [],
+    );
+    entries.sort((left, right) => left.name.localeCompare(right.name));
+    for (const entry of entries) {
+      if (files.length >= maximumFiles) return;
+      if (entry.isSymbolicLink()) continue;
+      const path = join(directory, entry.name);
+      if (entry.isDirectory()) {
+        if (!ignoredDirectories.has(entry.name)) await visit(path, depth + 1);
+        continue;
+      }
+      if (!entry.isFile() || ![".json", ".ndjson"].includes(extname(path))) {
+        continue;
+      }
+      files.push(path);
+    }
+  };
+  await visit(projectRoot, 0);
+  return files;
+};
+
+export const discoverBaselineInputFiles = async ({
+  projectRoot = process.cwd(),
+} = {}) => {
+  const result = { records: [], ledgers: [], assetManifests: [] };
+  for (const path of await collectCandidateFiles(projectRoot)) {
+    const pathStats = await stat(path).catch(() => undefined);
+    if (!pathStats?.isFile()) continue;
+    const relativePath = projectRelativePath(projectRoot, path);
+    try {
+      if (extname(path) === ".ndjson") {
+        if (isRecord(await readFirstRecord(path))) {
+          result.records.push(relativePath);
+        }
+        continue;
+      }
+      if (pathStats.size > maximumJsonBytes) continue;
+      const value = JSON.parse(await readFile(path, "utf8"));
+      if (isMigrationLedger(value)) result.ledgers.push(relativePath);
+      else if (isAssetManifest(value)) result.assetManifests.push(relativePath);
+    } catch {
+      // Invalid or unreadable files are not candidates; explicit validation will
+      // provide actionable errors if the user enters one manually.
+    }
+  }
+  return {
+    records: sortCandidates("records", result.records),
+    ledgers: sortCandidates("ledgers", result.ledgers),
+    assetManifests: sortCandidates("assetManifests", result.assetManifests),
+  };
+};

package/src/baseline/policy_review.mjs

@@ -0,0 +1,92 @@
+/**
+ * Purpose: Complete and safely apply a generated REVIEW REQUIRED policy draft.
+ * The caller owns the interactive presentation; this module owns validation and
+ * concurrent-change refusal.
+ */
+
+import { readFile, writeFile } from "node:fs/promises";
+import { validateRuntimeSanitizationPolicy } from "./sanitization_policy.mjs";
+
+export const SAFE_COLUMN_PRESET = Object.freeze({
+  action: "REPLACE WITH SYNTHETIC",
+  generated: "NEVER",
+  identity: "NO",
+  foreignKey: null,
+});
+
+export const createSafeTablePreset = (columns) =>
+  Object.fromEntries(
+    columns.map((column) => [
+      typeof column === "string" ? column : column.name,
+      { ...SAFE_COLUMN_PRESET },
+    ]),
+  );
+
+export const suggestPolicyExceptionColumns = (columns) =>
+  columns
+    .map((column) => (typeof column === "string" ? column : column.name))
+    .filter(
+      (column) =>
+        column === "id" ||
+        column.endsWith("_id") ||
+        ["created_at", "updated_at", "deleted_at"].includes(column),
+    );
+
+export const readReviewablePolicyDraft = async (path) => {
+  const source = await readFile(path, "utf8");
+  const draft = JSON.parse(source);
+  if (draft?.draft !== true || !Array.isArray(draft.tables)) {
+    throw new Error(
+      "Interactive review requires a generated sanitization policy with draft: true.",
+    );
+  }
+  return { source, draft };
+};
+
+export const completePolicyDraft = async ({
+  draft,
+  reviewColumn,
+  reviewTable,
+}) => {
+  if (draft?.draft !== true || !Array.isArray(draft.tables)) {
+    throw new Error("A generated REVIEW REQUIRED policy draft is required.");
+  }
+  const tables = [];
+  for (const table of draft.tables) {
+    const columnNames = (table.columns ?? []).map((column) => column.name);
+    const tableDecisions = reviewTable
+      ? await reviewTable({ table: table.name, columns: columnNames })
+      : {};
+    const columns = [];
+    for (const column of table.columns ?? []) {
+      const reviewed =
+        tableDecisions?.[column.name] ??
+        (await reviewColumn?.({
+          table: table.name,
+          column: column.name,
+        }));
+      if (!reviewed) {
+        throw new Error(
+          `Policy review did not classify ${table.name}.${column.name}.`,
+        );
+      }
+      columns.push({ name: column.name, ...reviewed });
+    }
+    tables.push({ ...table, columns });
+  }
+  const reviewed = { ...draft, tables };
+  delete reviewed.draft;
+  return validateRuntimeSanitizationPolicy(reviewed);
+};
+
+export const applyReviewedPolicy = async ({ path, originalSource, policy }) => {
+  validateRuntimeSanitizationPolicy(policy);
+  if ((await readFile(path, "utf8")) !== originalSource) {
+    throw new Error(
+      "The sanitization policy changed during review; no reviewed policy was written.",
+    );
+  }
+  await writeFile(path, `${JSON.stringify(policy, null, 2)}\n`, {
+    mode: 0o600,
+  });
+};

package/src/baseline/preparation.mjs

@@ -0,0 +1,323 @@
+/**
+ * Purpose: Inspect explicit local synthetic records and migration evidence, then
+ * create a fail-closed sanitization-policy draft for human review.
+ */
+
+import { createReadStream } from "node:fs";
+import { access, lstat, mkdir, open, readFile } from "node:fs/promises";
+import { createInterface } from "node:readline";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
+import { loadRehearsalConfig } from "../project/configuration.mjs";
+import { buildMigrationLedgerInventory } from "../runtime/migration_history.mjs";
+
+const identifierPattern = /^[a-z][a-z0-9_]{0,62}$/u;
+const storageBucketPattern = /^[a-z0-9][a-z0-9.-]{0,99}$/u;
+
+const resolveProjectInput = (projectRoot, value, label) => {
+  if (typeof value !== "string" || value.trim() === "") {
+    throw new Error(`${label} is required.`);
+  }
+  const path = resolve(projectRoot, value);
+  const owned = relative(projectRoot, path);
+  if (
+    owned === "" ||
+    owned === ".." ||
+    owned.startsWith(`..${sep}`) ||
+    isAbsolute(owned)
+  ) {
+    throw new Error(`${label} must be a file inside the project root.`);
+  }
+  return path;
+};
+
+const pathExists = (path) =>
+  access(path)
+    .then(() => true)
+    .catch((error) => {
+      if (error?.code === "ENOENT") return false;
+      throw error;
+    });
+
+const assertRegularInput = async (path, label) => {
+  let stats;
+  try {
+    stats = await lstat(path);
+  } catch (error) {
+    if (error?.code === "ENOENT") throw new Error(`${label} does not exist.`);
+    throw error;
+  }
+  if (stats.isSymbolicLink() || !stats.isFile()) {
+    throw new Error(`${label} must be a regular project-local file.`);
+  }
+};
+
+const inspectSyntheticRecords = async (path) => {
+  const tables = new Map();
+  let rowCount = 0;
+  const lines = createInterface({
+    input: createReadStream(path, { encoding: "utf8" }),
+    crlfDelay: Infinity,
+  });
+  for await (const line of lines) {
+    if (!line.trim()) continue;
+    let record;
+    try {
+      record = JSON.parse(line);
+    } catch (error) {
+      throw new Error("Synthetic baseline input contains invalid NDJSON.", {
+        cause: error,
+      });
+    }
+    if (
+      !record ||
+      !identifierPattern.test(record.schema ?? "public") ||
+      !identifierPattern.test(record.table ?? "") ||
+      !record.row ||
+      typeof record.row !== "object" ||
+      Array.isArray(record.row)
+    ) {
+      throw new Error("A synthetic baseline record has an invalid shape.");
+    }
+    const columns = Object.keys(record.row);
+    if (
+      columns.length === 0 ||
+      columns.some((column) => !identifierPattern.test(column))
+    ) {
+      throw new Error(
+        `Synthetic baseline table ${record.table} has invalid or empty columns.`,
+      );
+    }
+    const schema = record.schema ?? "public";
+    const key =
+      schema === "public" ? record.table : `${schema}.${record.table}`;
+    const table = tables.get(key) ?? {
+      schema,
+      name: record.table,
+      rowCount: 0,
+      columns: new Set(),
+    };
+    table.rowCount += 1;
+    for (const column of columns) table.columns.add(column);
+    tables.set(key, table);
+    rowCount += 1;
+  }
+  if (rowCount === 0) {
+    throw new Error("Synthetic baseline input contains no records.");
+  }
+  return {
+    rowCount,
+    tables: [...tables.values()]
+      .sort((left, right) => left.name.localeCompare(right.name))
+      .map((table) => ({
+        schema: table.schema,
+        name: table.name,
+        rowCount: table.rowCount,
+        columns: [...table.columns].sort(),
+      })),
+  };
+};
+
+const inspectAssetManifest = async (path, projectRoot) => {
+  let manifest;
+  try {
+    manifest = JSON.parse(await readFile(path, "utf8"));
+  } catch (error) {
+    throw new Error("Storage asset manifest contains invalid JSON.", {
+      cause: error,
+    });
+  }
+  if (!Array.isArray(manifest)) {
+    throw new Error("Storage asset manifest must be a JSON array.");
+  }
+  const destinations = new Set();
+  for (const [index, asset] of manifest.entries()) {
+    if (
+      !asset ||
+      typeof asset !== "object" ||
+      Array.isArray(asset) ||
+      !storageBucketPattern.test(asset.bucket ?? "") ||
+      typeof asset.objectPath !== "string" ||
+      !asset.objectPath ||
+      asset.objectPath
+        .split("/")
+        .some((segment) => !segment || segment === "." || segment === "..") ||
+      typeof asset.file !== "string" ||
+      !asset.file.trim() ||
+      (asset.contentType !== undefined && typeof asset.contentType !== "string")
+    ) {
+      throw new Error(
+        `Storage asset manifest entry ${index + 1} has an invalid shape.`,
+      );
+    }
+    const destination = `${asset.bucket}/${asset.objectPath}`;
+    if (destinations.has(destination)) {
+      throw new Error(`Duplicate Storage asset destination: ${destination}.`);
+    }
+    destinations.add(destination);
+    const assetFile = resolveProjectInput(
+      projectRoot,
+      asset.file,
+      `Storage asset file in entry ${index + 1}`,
+    );
+    await assertRegularInput(
+      assetFile,
+      `Storage asset file in entry ${index + 1}`,
+    );
+  }
+  return { assetCount: manifest.length };
+};
+
+export const inspectBaselineInputFiles = async ({
+  projectRoot = process.cwd(),
+  configPath,
+  recordsPath,
+  ledgerPath,
+  assetsPath,
+}) => {
+  const loaded = await loadRehearsalConfig({ projectRoot, configPath });
+  const records = resolveProjectInput(
+    loaded.projectRoot,
+    recordsPath,
+    "Synthetic records path",
+  );
+  const ledger = resolveProjectInput(
+    loaded.projectRoot,
+    ledgerPath,
+    "Migration ledger path",
+  );
+  const assets = assetsPath
+    ? resolveProjectInput(
+        loaded.projectRoot,
+        assetsPath,
+        "Storage asset manifest path",
+      )
+    : undefined;
+  await Promise.all([
+    assertRegularInput(records, "Synthetic records path"),
+    assertRegularInput(ledger, "Migration ledger path"),
+    ...(assets
+      ? [assertRegularInput(assets, "Storage asset manifest path")]
+      : []),
+  ]);
+  let ledgerRows;
+  try {
+    ledgerRows = JSON.parse(await readFile(ledger, "utf8"));
+  } catch (error) {
+    throw new Error("Migration ledger contains invalid JSON.", {
+      cause: error,
+    });
+  }
+  const [inspection, assetInspection] = await Promise.all([
+    inspectSyntheticRecords(records),
+    assets
+      ? inspectAssetManifest(assets, loaded.projectRoot)
+      : Promise.resolve({ assetCount: 0 }),
+  ]);
+  const migrationHistory = buildMigrationLedgerInventory(
+    ledgerRows,
+    "Synthetic migration ledger",
+  );
+  return {
+    projectRoot: loaded.projectRoot,
+    recordsPath: relative(loaded.projectRoot, records),
+    ledgerPath: relative(loaded.projectRoot, ledger),
+    assetsPath: assets ? relative(loaded.projectRoot, assets) : undefined,
+    migrationHistory,
+    migrationCutoff: migrationHistory.at(-1).version,
+    migrationCount: migrationHistory.length,
+    rowCount: inspection.rowCount,
+    tables: inspection.tables,
+    assetCount: assetInspection.assetCount,
+  };
+};
+
+const renderPolicyDraft = ({ migrationCutoff, tables }) =>
+  `${JSON.stringify(
+    {
+      policyVersion: 1,
+      draft: true,
+      migrationCutoff,
+      tables: tables.map((table) => ({
+        ...(table.schema === "public" ? {} : { schema: table.schema }),
+        name: table.name,
+        group: "synthetic",
+        sourceRows: "STREAM AND SANITIZE",
+        columns: table.columns.map((name) => ({
+          name,
+          action: "REVIEW REQUIRED",
+          generated: "REVIEW REQUIRED",
+          identity: "REVIEW REQUIRED",
+          foreignKey: "REVIEW REQUIRED",
+        })),
+      })),
+    },
+    null,
+    2,
+  )}\n`;
+
+export const planBaselinePreparation = async ({
+  projectRoot = process.cwd(),
+  configPath,
+  recordsPath,
+  ledgerPath,
+}) => {
+  const loaded = await loadRehearsalConfig({ projectRoot, configPath });
+  const inspection = await inspectBaselineInputFiles({
+    projectRoot: loaded.projectRoot,
+    configPath,
+    recordsPath,
+    ledgerPath,
+  });
+  const destination = loaded.paths.sanitizationPolicy;
+  if (await pathExists(destination)) {
+    throw new Error(
+      `Rehearsal baseline preparation will not overwrite ${relative(loaded.projectRoot, destination)}.`,
+    );
+  }
+  return {
+    projectRoot: loaded.projectRoot,
+    destination,
+    destinationRelative: relative(loaded.projectRoot, destination),
+    recordsPath: inspection.recordsPath,
+    ledgerPath: inspection.ledgerPath,
+    migrationCutoff: inspection.migrationCutoff,
+    migrationCount: inspection.migrationCount,
+    rowCount: inspection.rowCount,
+    tables: inspection.tables,
+    content: renderPolicyDraft({
+      migrationCutoff: inspection.migrationCutoff,
+      tables: inspection.tables,
+    }),
+  };
+};
+
+export const applyBaselinePreparation = async (plan) => {
+  if (await pathExists(plan.destination)) {
+    throw new Error(
+      `Rehearsal baseline preparation will not overwrite ${plan.destinationRelative}.`,
+    );
+  }
+  await mkdir(dirname(plan.destination), { recursive: true, mode: 0o700 });
+  const handle = await open(plan.destination, "wx", 0o600);
+  try {
+    await handle.writeFile(plan.content);
+    await handle.sync();
+  } finally {
+    await handle.close();
+  }
+};
+
+export const summarizeBaselinePreparation = (plan, { mode }) => ({
+  mode,
+  destination: plan.destinationRelative,
+  recordsPath: plan.recordsPath,
+  ledgerPath: plan.ledgerPath,
+  migrationCutoff: plan.migrationCutoff,
+  migrationCount: plan.migrationCount,
+  rowCount: plan.rowCount,
+  tables: plan.tables,
+  nextAction:
+    mode === "written"
+      ? `Review every REVIEW REQUIRED decision in ${plan.destinationRelative}, remove draft only after review, then create the baseline.`
+      : "Review this schema-only summary, then rerun with --write to create the draft.",
+});