@rehearsal-db/core 0.1.0-beta.9 → 0.1.0-rc.2

package/dist/src/baseline/builder.mjs

@@ -0,0 +1,142 @@
+/**
+ * Build a synthetic baseline from explicit project-owned local files. This module does
+ * not connect to a hosted source or decide which values are safe.
+ */
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+import { createInterface } from "node:readline";
+import { pathToFileURL } from "node:url";
+import { createAndActivateBaseline, createBaselineGenerationId, } from "./artifact.mjs";
+import { loadRehearsalConfig } from "../project/configuration.mjs";
+import { buildMigrationLedgerInventory, createMigrationReplayReceipt, readMigrationSourceBundle, } from "../runtime/migration_history.mjs";
+import { validateRuntimeSanitizationPolicy } from "./sanitization_policy.mjs";
+const isObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+const parseAssets = (value) => {
+    if (!Array.isArray(value)) {
+        throw new Error("Synthetic assets manifest must be an array.");
+    }
+    return value.map((entry, index) => {
+        if (!isObject(entry) ||
+            typeof entry.bucket !== "string" ||
+            typeof entry.objectPath !== "string" ||
+            typeof entry.file !== "string" ||
+            (entry.contentType !== undefined && typeof entry.contentType !== "string")) {
+            throw new Error(`Synthetic asset ${index} has an invalid shape.`);
+        }
+        return {
+            bucket: entry.bucket,
+            objectPath: entry.objectPath,
+            file: entry.file,
+            ...(entry.contentType === undefined
+                ? {}
+                : { contentType: entry.contentType }),
+        };
+    });
+};
+const resolveProjectInput = (projectRoot, value, label) => {
+    if (typeof value !== "string" || value.trim() === "") {
+        throw new Error(`${label} is required.`);
+    }
+    const path = resolve(projectRoot, value);
+    const owned = relative(projectRoot, path);
+    if (owned === "" ||
+        owned === ".." ||
+        owned.startsWith(`..${sep}`) ||
+        isAbsolute(owned)) {
+        throw new Error(`${label} must be a file inside the project root.`);
+    }
+    return path;
+};
+const readRecords = async function* (path) {
+    const lines = createInterface({
+        input: createReadStream(path, { encoding: "utf8" }),
+        crlfDelay: Infinity,
+    });
+    for await (const line of lines) {
+        if (line.trim() === "")
+            continue;
+        let record;
+        try {
+            record = JSON.parse(line);
+        }
+        catch (error) {
+            throw new Error("Synthetic baseline input contains invalid NDJSON.", {
+                cause: error,
+            });
+        }
+        yield record;
+    }
+};
+export const createSyntheticBaselineFromFiles = async ({ projectRoot = process.cwd(), configPath, recordsPath, ledgerPath, assetsPath, }) => {
+    const loaded = await loadRehearsalConfig({
+        projectRoot,
+        ...(configPath === undefined ? {} : { configPath }),
+    });
+    const records = resolveProjectInput(loaded.projectRoot, recordsPath, "Synthetic records path");
+    const ledger = resolveProjectInput(loaded.projectRoot, ledgerPath, "Migration ledger path");
+    const [policyBytes, ledgerBytes] = await Promise.all([
+        readFile(loaded.paths.sanitizationPolicy),
+        readFile(ledger),
+    ]);
+    const assets = assetsPath
+        ? await Promise.all(parseAssets(JSON.parse(await readFile(resolveProjectInput(loaded.projectRoot, assetsPath, "Synthetic assets manifest path"), "utf8"))).map(async (asset) => ({
+            bucket: asset.bucket,
+            objectPath: asset.objectPath,
+            ...(asset.contentType === undefined
+                ? {}
+                : { contentType: asset.contentType }),
+            content: await readFile(resolveProjectInput(loaded.projectRoot, asset.file, "Synthetic asset file path")),
+        })))
+        : [];
+    const policy = validateRuntimeSanitizationPolicy(JSON.parse(policyBytes.toString("utf8")));
+    const expectedTables = policy.tables
+        .filter((table) => table.sourceRows !== "EXCLUDE")
+        .map((table) => (table.schema ?? "public") === "public"
+        ? table.name
+        : `${table.schema}.${table.name}`);
+    if (expectedTables.length === 0) {
+        throw new Error("The sanitization policy includes no restorable tables.");
+    }
+    const sourceMigrationHistory = buildMigrationLedgerInventory(JSON.parse(ledgerBytes.toString("utf8")), "Synthetic migration ledger");
+    const migrationCutoff = sourceMigrationHistory.at(-1);
+    if (migrationCutoff === undefined) {
+        throw new Error("The synthetic migration ledger is empty.");
+    }
+    if (policy.migrationCutoff !== migrationCutoff.version) {
+        throw new Error(`The sanitization policy cutoff ${policy.migrationCutoff} does not match migration ledger cutoff ${migrationCutoff.version}.`);
+    }
+    const migrationFiles = await readMigrationSourceBundle({
+        directory: new URL("./", pathToFileURL(`${loaded.paths.migrationDirectory}${sep}`)),
+        sourceLedger: sourceMigrationHistory,
+    });
+    const migrationReceipt = createMigrationReplayReceipt({
+        files: migrationFiles,
+        ledger: sourceMigrationHistory,
+    });
+    const generationId = createBaselineGenerationId();
+    const baseline = await createAndActivateBaseline({
+        artifactRoot: loaded.paths.artifactDirectory,
+        generationId,
+        records: readRecords(records),
+        metadata: {
+            migrationCutoff: migrationCutoff.version,
+            migrationHistorySha256: migrationReceipt.historySha256,
+            sanitizationPolicySha256: createHash("sha256")
+                .update(policyBytes)
+                .digest("hex"),
+            sourceMigrationHistory,
+        },
+        expectedTables,
+        migrationFiles,
+        assets,
+    });
+    return {
+        generationId,
+        migrationCutoff: baseline.migrationCutoff,
+        migrationCount: Object.keys(baseline.migrations).length,
+        rowCount: baseline.rowCount,
+        tableCount: Object.keys(baseline.tableCounts).length,
+    };
+};

package/dist/src/baseline/input_discovery.d.mts

@@ -0,0 +1,13 @@
+/**
+ * Discover likely project-owned baseline inputs without exposing their values.
+ * Discovery is bounded, ignores generated/private directories, and only sniffs
+ * small JSON or NDJSON files before returning project-relative paths.
+ */
+export interface DiscoveredBaselineInputs {
+    records: string[];
+    ledgers: string[];
+    assetManifests: string[];
+}
+export declare const discoverBaselineInputFiles: ({ projectRoot, }?: {
+    projectRoot?: string;
+}) => Promise<DiscoveredBaselineInputs>;

package/dist/src/baseline/input_discovery.mjs

@@ -0,0 +1,149 @@
+/**
+ * Discover likely project-owned baseline inputs without exposing their values.
+ * Discovery is bounded, ignores generated/private directories, and only sniffs
+ * small JSON or NDJSON files before returning project-relative paths.
+ */
+import { open, readFile, readdir, stat } from "node:fs/promises";
+import { extname, join, relative, sep } from "node:path";
+const ignoredDirectories = new Set([
+    ".git",
+    ".next",
+    ".rehearsal",
+    ".turbo",
+    "build",
+    "coverage",
+    "dist",
+    "node_modules",
+]);
+const maximumDepth = 4;
+const maximumFiles = 2_000;
+const maximumJsonBytes = 2 * 1024 * 1024;
+const maximumNdjsonProbeBytes = 64 * 1024;
+const projectRelativePath = (projectRoot, path) => relative(projectRoot, path).split(sep).join("/");
+const preference = {
+    records: [
+        "rehearsal/sanitized-data.ndjson",
+        "rehearsal/synthetic-data.ndjson",
+    ],
+    ledgers: ["rehearsal/migration-ledger.json"],
+    assetManifests: ["rehearsal/assets.json"],
+};
+const sortCandidates = (kind, candidates) => {
+    const preferred = new Map(preference[kind].map((path, index) => [path, index]));
+    return candidates.sort((left, right) => {
+        const leftRank = preferred.get(left) ?? Number.POSITIVE_INFINITY;
+        const rightRank = preferred.get(right) ?? Number.POSITIVE_INFINITY;
+        return leftRank - rightRank || left.localeCompare(right);
+    });
+};
+const readFirstRecord = async (path) => {
+    const handle = await open(path, "r");
+    try {
+        const buffer = Buffer.alloc(maximumNdjsonProbeBytes);
+        const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+        const probe = buffer.subarray(0, bytesRead).toString("utf8");
+        const lines = probe.split(/\r?\n/u);
+        for (const line of lines) {
+            if (!line.trim())
+                continue;
+            return JSON.parse(line);
+        }
+    }
+    finally {
+        await handle.close();
+    }
+    return undefined;
+};
+const isRecord = (value) => Boolean(value &&
+    typeof value === "object" &&
+    !Array.isArray(value) &&
+    "table" in value &&
+    typeof value.table === "string" &&
+    "row" in value &&
+    value.row &&
+    typeof value.row === "object" &&
+    !Array.isArray(value.row));
+const isMigrationLedger = (value) => Array.isArray(value) &&
+    value.length > 0 &&
+    value.every((entry) => entry &&
+        typeof entry === "object" &&
+        "version" in entry &&
+        typeof entry.version === "string" &&
+        "name" in entry &&
+        typeof entry.name === "string" &&
+        "statements" in entry &&
+        Array.isArray(entry.statements));
+const isAssetManifest = (value) => Array.isArray(value) &&
+    value.length > 0 &&
+    value.every((entry) => entry &&
+        typeof entry === "object" &&
+        "bucket" in entry &&
+        typeof entry.bucket === "string" &&
+        "objectPath" in entry &&
+        typeof entry.objectPath === "string" &&
+        "file" in entry &&
+        typeof entry.file === "string");
+const collectCandidateFiles = async (projectRoot) => {
+    const files = [];
+    const visit = async (directory, depth) => {
+        if (depth > maximumDepth || files.length >= maximumFiles)
+            return;
+        const entries = await readdir(directory, { withFileTypes: true }).catch(() => []);
+        entries.sort((left, right) => left.name.localeCompare(right.name));
+        for (const entry of entries) {
+            if (files.length >= maximumFiles)
+                return;
+            if (entry.isSymbolicLink())
+                continue;
+            const path = join(directory, entry.name);
+            if (entry.isDirectory()) {
+                if (!ignoredDirectories.has(entry.name))
+                    await visit(path, depth + 1);
+                continue;
+            }
+            if (!entry.isFile() || ![".json", ".ndjson"].includes(extname(path))) {
+                continue;
+            }
+            files.push(path);
+        }
+    };
+    await visit(projectRoot, 0);
+    return files;
+};
+export const discoverBaselineInputFiles = async ({ projectRoot = process.cwd(), } = {}) => {
+    const result = {
+        records: [],
+        ledgers: [],
+        assetManifests: [],
+    };
+    for (const path of await collectCandidateFiles(projectRoot)) {
+        const pathStats = await stat(path).catch(() => undefined);
+        if (!pathStats?.isFile())
+            continue;
+        const relativePath = projectRelativePath(projectRoot, path);
+        try {
+            if (extname(path) === ".ndjson") {
+                if (isRecord(await readFirstRecord(path))) {
+                    result.records.push(relativePath);
+                }
+                continue;
+            }
+            if (pathStats.size > maximumJsonBytes)
+                continue;
+            const value = JSON.parse(await readFile(path, "utf8"));
+            if (isMigrationLedger(value))
+                result.ledgers.push(relativePath);
+            else if (isAssetManifest(value))
+                result.assetManifests.push(relativePath);
+        }
+        catch {
+            // Invalid or unreadable files are not candidates; explicit validation will
+            // provide actionable errors if the user enters one manually.
+        }
+    }
+    return {
+        records: sortCandidates("records", result.records),
+        ledgers: sortCandidates("ledgers", result.ledgers),
+        assetManifests: sortCandidates("assetManifests", result.assetManifests),
+    };
+};

package/dist/src/baseline/policy_review.d.mts

@@ -0,0 +1,56 @@
+/**
+ * Purpose: Complete and safely apply a generated REVIEW REQUIRED policy draft.
+ * The caller owns the interactive presentation; this module owns validation and
+ * concurrent-change refusal.
+ */
+export declare const SAFE_COLUMN_PRESET: Readonly<{
+    action: "REPLACE WITH SYNTHETIC";
+    generated: "NEVER";
+    identity: "NO";
+    foreignKey: null;
+}>;
+type ColumnReference = string | {
+    name: string;
+};
+type ReviewedColumn = Record<string, unknown>;
+interface PolicyColumn extends Record<string, unknown> {
+    name: string;
+}
+interface PolicyTable extends Record<string, unknown> {
+    name: string;
+    columns: PolicyColumn[];
+}
+interface PolicyDraft extends Record<string, unknown> {
+    draft: true;
+    tables: PolicyTable[];
+}
+export declare const createSafeTablePreset: (columns: readonly ColumnReference[]) => {
+    [k: string]: {
+        action: "REPLACE WITH SYNTHETIC";
+        generated: "NEVER";
+        identity: "NO";
+        foreignKey: null;
+    };
+};
+export declare const suggestPolicyExceptionColumns: (columns: readonly ColumnReference[]) => string[];
+export declare const readReviewablePolicyDraft: (path: string) => Promise<{
+    source: string;
+    draft: PolicyDraft;
+}>;
+export declare const completePolicyDraft: ({ draft, reviewColumn, reviewTable, }: {
+    draft: unknown;
+    reviewColumn?: (input: {
+        table: string;
+        column: string;
+    }) => ReviewedColumn | undefined | Promise<ReviewedColumn | undefined>;
+    reviewTable?: (input: {
+        table: string;
+        columns: string[];
+    }) => Record<string, ReviewedColumn> | undefined | Promise<Record<string, ReviewedColumn> | undefined>;
+}) => Promise<unknown>;
+export declare const applyReviewedPolicy: ({ path, originalSource, policy, }: {
+    path: string;
+    originalSource: string;
+    policy: unknown;
+}) => Promise<void>;
+export {};

package/dist/src/baseline/policy_review.mjs

@@ -0,0 +1,73 @@
+/**
+ * Purpose: Complete and safely apply a generated REVIEW REQUIRED policy draft.
+ * The caller owns the interactive presentation; this module owns validation and
+ * concurrent-change refusal.
+ */
+import { readFile, writeFile } from "node:fs/promises";
+import { validateRuntimeSanitizationPolicy } from "./sanitization_policy.mjs";
+export const SAFE_COLUMN_PRESET = Object.freeze({
+    action: "REPLACE WITH SYNTHETIC",
+    generated: "NEVER",
+    identity: "NO",
+    foreignKey: null,
+});
+const isPolicyDraft = (value) => typeof value === "object" &&
+    value !== null &&
+    "draft" in value &&
+    value.draft === true &&
+    "tables" in value &&
+    Array.isArray(value.tables);
+export const createSafeTablePreset = (columns) => Object.fromEntries(columns.map((column) => [
+    typeof column === "string" ? column : column.name,
+    { ...SAFE_COLUMN_PRESET },
+]));
+export const suggestPolicyExceptionColumns = (columns) => columns
+    .map((column) => (typeof column === "string" ? column : column.name))
+    .filter((column) => column === "id" ||
+    column.endsWith("_id") ||
+    ["created_at", "updated_at", "deleted_at"].includes(column));
+export const readReviewablePolicyDraft = async (path) => {
+    const source = await readFile(path, "utf8");
+    const draft = JSON.parse(source);
+    if (!isPolicyDraft(draft)) {
+        throw new Error("Interactive review requires a generated sanitization policy with draft: true.");
+    }
+    return { source, draft };
+};
+export const completePolicyDraft = async ({ draft, reviewColumn, reviewTable, }) => {
+    if (!isPolicyDraft(draft)) {
+        throw new Error("A generated REVIEW REQUIRED policy draft is required.");
+    }
+    const tables = [];
+    for (const table of draft.tables) {
+        const columnNames = (table.columns ?? []).map((column) => column.name);
+        const tableDecisions = reviewTable
+            ? await reviewTable({ table: table.name, columns: columnNames })
+            : {};
+        const columns = [];
+        for (const column of table.columns ?? []) {
+            const reviewed = tableDecisions?.[column.name] ??
+                (await reviewColumn?.({
+                    table: table.name,
+                    column: column.name,
+                }));
+            if (!reviewed) {
+                throw new Error(`Policy review did not classify ${table.name}.${column.name}.`);
+            }
+            columns.push({ name: column.name, ...reviewed });
+        }
+        tables.push({ ...table, columns });
+    }
+    const reviewed = { ...draft, tables };
+    delete reviewed.draft;
+    return validateRuntimeSanitizationPolicy(reviewed);
+};
+export const applyReviewedPolicy = async ({ path, originalSource, policy, }) => {
+    validateRuntimeSanitizationPolicy(policy);
+    if ((await readFile(path, "utf8")) !== originalSource) {
+        throw new Error("The sanitization policy changed during review; no reviewed policy was written.");
+    }
+    await writeFile(path, `${JSON.stringify(policy, null, 2)}\n`, {
+        mode: 0o600,
+    });
+};

package/dist/src/baseline/preparation.d.mts

@@ -0,0 +1,63 @@
+/**
+ * Purpose: Inspect explicit local synthetic records and migration evidence, then
+ * create a fail-closed sanitization-policy draft for human review.
+ */
+import { type MigrationLedgerEntry } from "../runtime/migration_history.mjs";
+export interface SyntheticTableInspection {
+    readonly schema: string;
+    readonly name: string;
+    readonly rowCount: number;
+    readonly columns: string[];
+}
+interface SyntheticRecordInspection {
+    readonly rowCount: number;
+    readonly tables: SyntheticTableInspection[];
+}
+export interface BaselineInputInspection extends SyntheticRecordInspection {
+    readonly projectRoot: string;
+    readonly recordsPath: string;
+    readonly ledgerPath: string;
+    readonly assetsPath?: string;
+    readonly migrationHistory: MigrationLedgerEntry[];
+    readonly migrationCutoff: string;
+    readonly migrationCount: number;
+    readonly assetCount: number;
+}
+export interface BaselinePreparationPlan extends SyntheticRecordInspection {
+    readonly projectRoot: string;
+    readonly destination: string;
+    readonly destinationRelative: string;
+    readonly recordsPath: string;
+    readonly ledgerPath: string;
+    readonly migrationCutoff: string;
+    readonly migrationCount: number;
+    readonly content: string;
+}
+export declare const inspectBaselineInputFiles: ({ projectRoot, configPath, recordsPath, ledgerPath, assetsPath, }: {
+    projectRoot?: string;
+    configPath?: string;
+    recordsPath: string;
+    ledgerPath: string;
+    assetsPath?: string;
+}) => Promise<BaselineInputInspection>;
+export declare const planBaselinePreparation: ({ projectRoot, configPath, recordsPath, ledgerPath, }: {
+    projectRoot?: string;
+    configPath?: string;
+    recordsPath: string;
+    ledgerPath: string;
+}) => Promise<BaselinePreparationPlan>;
+export declare const applyBaselinePreparation: (plan: BaselinePreparationPlan) => Promise<void>;
+export declare const summarizeBaselinePreparation: (plan: BaselinePreparationPlan, { mode }: {
+    mode: "preview" | "written";
+}) => {
+    mode: "preview" | "written";
+    destination: string;
+    recordsPath: string;
+    ledgerPath: string;
+    migrationCutoff: string;
+    migrationCount: number;
+    rowCount: number;
+    tables: readonly SyntheticTableInspection[];
+    nextAction: string;
+};
+export {};