@rehearsal-db/core 0.1.0-beta.14 → 0.1.0-beta.16

package/src/baseline/builder.mjs

@@ -1,164 +0,0 @@
-/**
- * Build a synthetic baseline from explicit project-owned local files. This module does
- * not connect to a hosted source or decide which values are safe.
- */
-
-import { createHash } from "node:crypto";
-import { createReadStream } from "node:fs";
-import { readFile } from "node:fs/promises";
-import { isAbsolute, relative, resolve, sep } from "node:path";
-import { createInterface } from "node:readline";
-import { pathToFileURL } from "node:url";
-import {
-  createAndActivateBaseline,
-  createBaselineGenerationId,
-} from "./artifact.mjs";
-import { loadRehearsalConfig } from "../project/configuration.mjs";
-import {
-  buildMigrationLedgerInventory,
-  createMigrationReplayReceipt,
-  readMigrationSourceBundle,
-} from "../runtime/migration_history.mjs";
-import { validateRuntimeSanitizationPolicy } from "./sanitization_policy.mjs";
-
-const resolveProjectInput = (projectRoot, value, label) => {
-  if (typeof value !== "string" || value.trim() === "") {
-    throw new Error(`${label} is required.`);
-  }
-  const path = resolve(projectRoot, value);
-  const owned = relative(projectRoot, path);
-  if (
-    owned === "" ||
-    owned === ".." ||
-    owned.startsWith(`..${sep}`) ||
-    isAbsolute(owned)
-  ) {
-    throw new Error(`${label} must be a file inside the project root.`);
-  }
-  return path;
-};
-
-const readRecords = async function* (path) {
-  const lines = createInterface({
-    input: createReadStream(path, { encoding: "utf8" }),
-    crlfDelay: Infinity,
-  });
-  for await (const line of lines) {
-    if (line.trim() === "") continue;
-    let record;
-    try {
-      record = JSON.parse(line);
-    } catch (error) {
-      throw new Error("Synthetic baseline input contains invalid NDJSON.", {
-        cause: error,
-      });
-    }
-    yield record;
-  }
-};
-
-export const createSyntheticBaselineFromFiles = async ({
-  projectRoot = process.cwd(),
-  configPath,
-  recordsPath,
-  ledgerPath,
-  assetsPath,
-}) => {
-  const loaded = await loadRehearsalConfig({ projectRoot, configPath });
-  const records = resolveProjectInput(
-    loaded.projectRoot,
-    recordsPath,
-    "Synthetic records path",
-  );
-  const ledger = resolveProjectInput(
-    loaded.projectRoot,
-    ledgerPath,
-    "Migration ledger path",
-  );
-  const [policyBytes, ledgerBytes] = await Promise.all([
-    readFile(loaded.paths.sanitizationPolicy),
-    readFile(ledger),
-  ]);
-  const assets = assetsPath
-    ? await Promise.all(
-        JSON.parse(
-          await readFile(
-            resolveProjectInput(
-              loaded.projectRoot,
-              assetsPath,
-              "Synthetic assets manifest path",
-            ),
-            "utf8",
-          ),
-        ).map(async (asset) => ({
-          bucket: asset.bucket,
-          objectPath: asset.objectPath,
-          contentType: asset.contentType,
-          content: await readFile(
-            resolveProjectInput(
-              loaded.projectRoot,
-              asset.file,
-              "Synthetic asset file path",
-            ),
-          ),
-        })),
-      )
-    : [];
-  const policy = validateRuntimeSanitizationPolicy(
-    JSON.parse(policyBytes.toString("utf8")),
-  );
-  const expectedTables = policy.tables
-    .filter((table) => table.sourceRows !== "EXCLUDE")
-    .map((table) =>
-      (table.schema ?? "public") === "public"
-        ? table.name
-        : `${table.schema}.${table.name}`,
-    );
-  if (expectedTables.length === 0) {
-    throw new Error("The sanitization policy includes no restorable tables.");
-  }
-  const sourceMigrationHistory = buildMigrationLedgerInventory(
-    JSON.parse(ledgerBytes.toString("utf8")),
-    "Synthetic migration ledger",
-  );
-  if (policy.migrationCutoff !== sourceMigrationHistory.at(-1).version) {
-    throw new Error(
-      `The sanitization policy cutoff ${policy.migrationCutoff} does not match migration ledger cutoff ${sourceMigrationHistory.at(-1).version}.`,
-    );
-  }
-  const migrationFiles = await readMigrationSourceBundle({
-    directory: new URL(
-      "./",
-      pathToFileURL(`${loaded.paths.migrationDirectory}${sep}`),
-    ),
-    sourceLedger: sourceMigrationHistory,
-  });
-  const migrationReceipt = createMigrationReplayReceipt({
-    files: migrationFiles,
-    ledger: sourceMigrationHistory,
-  });
-  const generationId = createBaselineGenerationId();
-  const baseline = await createAndActivateBaseline({
-    artifactRoot: loaded.paths.artifactDirectory,
-    generationId,
-    records: readRecords(records),
-    metadata: {
-      migrationCutoff: sourceMigrationHistory.at(-1).version,
-      migrationHistorySha256: migrationReceipt.historySha256,
-      sanitizationPolicySha256: createHash("sha256")
-        .update(policyBytes)
-        .digest("hex"),
-      sourceMigrationHistory,
-    },
-    expectedTables,
-    migrationFiles,
-    assets,
-  });
-  return {
-    generationId,
-    migrationCutoff: baseline.migrationCutoff,
-    migrationCount: Object.keys(baseline.migrations).length,
-    rowCount: baseline.rowCount,
-    tableCount: Object.keys(baseline.tableCounts).length,
-  };
-};

package/src/baseline/input_discovery.mjs

@@ -1,155 +0,0 @@
-/**
- * Discover likely project-owned baseline inputs without exposing their values.
- * Discovery is bounded, ignores generated/private directories, and only sniffs
- * small JSON or NDJSON files before returning project-relative paths.
- */
-
-import { open, readFile, readdir, stat } from "node:fs/promises";
-import { extname, join, relative, sep } from "node:path";
-
-const ignoredDirectories = new Set([
-  ".git",
-  ".next",
-  ".rehearsal",
-  ".turbo",
-  "build",
-  "coverage",
-  "dist",
-  "node_modules",
-]);
-const maximumDepth = 4;
-const maximumFiles = 2_000;
-const maximumJsonBytes = 2 * 1024 * 1024;
-const maximumNdjsonProbeBytes = 64 * 1024;
-
-const projectRelativePath = (projectRoot, path) =>
-  relative(projectRoot, path).split(sep).join("/");
-
-const preference = {
-  records: [
-    "rehearsal/sanitized-data.ndjson",
-    "rehearsal/synthetic-data.ndjson",
-  ],
-  ledgers: ["rehearsal/migration-ledger.json"],
-  assetManifests: ["rehearsal/assets.json"],
-};
-
-const sortCandidates = (kind, candidates) => {
-  const preferred = new Map(
-    preference[kind].map((path, index) => [path, index]),
-  );
-  return candidates.sort((left, right) => {
-    const leftRank = preferred.get(left) ?? Number.POSITIVE_INFINITY;
-    const rightRank = preferred.get(right) ?? Number.POSITIVE_INFINITY;
-    return leftRank - rightRank || left.localeCompare(right);
-  });
-};
-
-const readFirstRecord = async (path) => {
-  const handle = await open(path, "r");
-  try {
-    const buffer = Buffer.alloc(maximumNdjsonProbeBytes);
-    const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
-    const probe = buffer.subarray(0, bytesRead).toString("utf8");
-    const lines = probe.split(/\r?\n/u);
-    for (const line of lines) {
-      if (!line.trim()) continue;
-      return JSON.parse(line);
-    }
-  } finally {
-    await handle.close();
-  }
-  return undefined;
-};
-
-const isRecord = (value) =>
-  Boolean(
-    value &&
-    typeof value === "object" &&
-    !Array.isArray(value) &&
-    typeof value.table === "string" &&
-    value.row &&
-    typeof value.row === "object" &&
-    !Array.isArray(value.row),
-  );
-
-const isMigrationLedger = (value) =>
-  Array.isArray(value) &&
-  value.length > 0 &&
-  value.every(
-    (entry) =>
-      entry &&
-      typeof entry === "object" &&
-      typeof entry.version === "string" &&
-      typeof entry.name === "string" &&
-      Array.isArray(entry.statements),
-  );
-
-const isAssetManifest = (value) =>
-  Array.isArray(value) &&
-  value.length > 0 &&
-  value.every(
-    (entry) =>
-      entry &&
-      typeof entry === "object" &&
-      typeof entry.bucket === "string" &&
-      typeof entry.objectPath === "string" &&
-      typeof entry.file === "string",
-  );
-
-const collectCandidateFiles = async (projectRoot) => {
-  const files = [];
-  const visit = async (directory, depth) => {
-    if (depth > maximumDepth || files.length >= maximumFiles) return;
-    const entries = await readdir(directory, { withFileTypes: true }).catch(
-      () => [],
-    );
-    entries.sort((left, right) => left.name.localeCompare(right.name));
-    for (const entry of entries) {
-      if (files.length >= maximumFiles) return;
-      if (entry.isSymbolicLink()) continue;
-      const path = join(directory, entry.name);
-      if (entry.isDirectory()) {
-        if (!ignoredDirectories.has(entry.name)) await visit(path, depth + 1);
-        continue;
-      }
-      if (!entry.isFile() || ![".json", ".ndjson"].includes(extname(path))) {
-        continue;
-      }
-      files.push(path);
-    }
-  };
-  await visit(projectRoot, 0);
-  return files;
-};
-
-export const discoverBaselineInputFiles = async ({
-  projectRoot = process.cwd(),
-} = {}) => {
-  const result = { records: [], ledgers: [], assetManifests: [] };
-  for (const path of await collectCandidateFiles(projectRoot)) {
-    const pathStats = await stat(path).catch(() => undefined);
-    if (!pathStats?.isFile()) continue;
-    const relativePath = projectRelativePath(projectRoot, path);
-    try {
-      if (extname(path) === ".ndjson") {
-        if (isRecord(await readFirstRecord(path))) {
-          result.records.push(relativePath);
-        }
-        continue;
-      }
-      if (pathStats.size > maximumJsonBytes) continue;
-      const value = JSON.parse(await readFile(path, "utf8"));
-      if (isMigrationLedger(value)) result.ledgers.push(relativePath);
-      else if (isAssetManifest(value)) result.assetManifests.push(relativePath);
-    } catch {
-      // Invalid or unreadable files are not candidates; explicit validation will
-      // provide actionable errors if the user enters one manually.
-    }
-  }
-  return {
-    records: sortCandidates("records", result.records),
-    ledgers: sortCandidates("ledgers", result.ledgers),
-    assetManifests: sortCandidates("assetManifests", result.assetManifests),
-  };
-};

package/src/baseline/policy_review.mjs

@@ -1,92 +0,0 @@
-/**
- * Purpose: Complete and safely apply a generated REVIEW REQUIRED policy draft.
- * The caller owns the interactive presentation; this module owns validation and
- * concurrent-change refusal.
- */
-
-import { readFile, writeFile } from "node:fs/promises";
-import { validateRuntimeSanitizationPolicy } from "./sanitization_policy.mjs";
-
-export const SAFE_COLUMN_PRESET = Object.freeze({
-  action: "REPLACE WITH SYNTHETIC",
-  generated: "NEVER",
-  identity: "NO",
-  foreignKey: null,
-});
-
-export const createSafeTablePreset = (columns) =>
-  Object.fromEntries(
-    columns.map((column) => [
-      typeof column === "string" ? column : column.name,
-      { ...SAFE_COLUMN_PRESET },
-    ]),
-  );
-
-export const suggestPolicyExceptionColumns = (columns) =>
-  columns
-    .map((column) => (typeof column === "string" ? column : column.name))
-    .filter(
-      (column) =>
-        column === "id" ||
-        column.endsWith("_id") ||
-        ["created_at", "updated_at", "deleted_at"].includes(column),
-    );
-
-export const readReviewablePolicyDraft = async (path) => {
-  const source = await readFile(path, "utf8");
-  const draft = JSON.parse(source);
-  if (draft?.draft !== true || !Array.isArray(draft.tables)) {
-    throw new Error(
-      "Interactive review requires a generated sanitization policy with draft: true.",
-    );
-  }
-  return { source, draft };
-};
-
-export const completePolicyDraft = async ({
-  draft,
-  reviewColumn,
-  reviewTable,
-}) => {
-  if (draft?.draft !== true || !Array.isArray(draft.tables)) {
-    throw new Error("A generated REVIEW REQUIRED policy draft is required.");
-  }
-  const tables = [];
-  for (const table of draft.tables) {
-    const columnNames = (table.columns ?? []).map((column) => column.name);
-    const tableDecisions = reviewTable
-      ? await reviewTable({ table: table.name, columns: columnNames })
-      : {};
-    const columns = [];
-    for (const column of table.columns ?? []) {
-      const reviewed =
-        tableDecisions?.[column.name] ??
-        (await reviewColumn?.({
-          table: table.name,
-          column: column.name,
-        }));
-      if (!reviewed) {
-        throw new Error(
-          `Policy review did not classify ${table.name}.${column.name}.`,
-        );
-      }
-      columns.push({ name: column.name, ...reviewed });
-    }
-    tables.push({ ...table, columns });
-  }
-  const reviewed = { ...draft, tables };
-  delete reviewed.draft;
-  return validateRuntimeSanitizationPolicy(reviewed);
-};
-
-export const applyReviewedPolicy = async ({ path, originalSource, policy }) => {
-  validateRuntimeSanitizationPolicy(policy);
-  if ((await readFile(path, "utf8")) !== originalSource) {
-    throw new Error(
-      "The sanitization policy changed during review; no reviewed policy was written.",
-    );
-  }
-  await writeFile(path, `${JSON.stringify(policy, null, 2)}\n`, {
-    mode: 0o600,
-  });
-};