@segmentstream/ontology 0.0.0-stage → 0.92.1

package/dist/migrate/apply.js

@@ -0,0 +1,526 @@
+import { requestShadowSupersession } from "../replication/shadow.js";
+import { replicationScanScopes } from "../replication/scans.js";
+import { enableShadowCapture, refreshShadowCapturePolicy } from "../replication/shadow.js";
+import { Duration, Effect, Schema } from "effect";
+import { SqlClient } from "effect/unstable/sql";
+import { canonicalJson, decodeManifestDocument, encodeManifest, manifestHash, } from "../manifest.js";
+import { CatalogInvalid, CatalogMismatch, MigrationFailed, MigrationRefused } from "./errors.js";
+import { isNoopMigration, NO_RETAINED_STORAGE, RetainedStorage, retainedStorageJson, } from "./model.js";
+import { planMigration } from "./plan.js";
+import { REPLICATION_DDL, REPLICATION_SCHEMA_VERSION, replicationEntityDdl, } from "../replication/schema.js";
+import { lockReplicationLifecycle, lockReplicationSchema } from "../replication/transaction.js";
+import { prepareReplicationBaseline } from "../replication/baseline.js";
+import { randomUUID } from "node:crypto";
+import { recordWorkflowCatalog } from "../runtime/workflow-admission.js";
+import { prepareReplicationEvolution, dropsServingReplicationStorage, refreshReplicationProjection, retainReplicationHistory, renameReplicationHistory, } from "../replication/evolution.js";
+import { migratedReplicationFills, migrationReplicationExclusions, effectiveReplicationExclusions, } from "../replication/migrate.js";
+import { AGENT_TASKS_TABLE, ATTACHMENT_SOURCE_CHECK, ATTACHMENT_SOURCES, ATTACHMENTS_TABLE, SIGN_IN_EMAILS_TABLE, WORKFLOW_STARTS_TABLE, } from "../storage/tables.js";
+import { principalsOntology, reconcileWorkflowUsers } from "../runtime/principals.js";
+import { FEDERATION_TABLES } from "../runtime/federation-tables.js";
+import { syncUserIdentities, USER_IDENTITY_ENTITY_ID } from "../runtime/user-identities.js";
+import { resolveReplicationSelection, } from "../replication/model.js";
+import { ACTION_ATTRIBUTION_TABLE, AGENT_TABLES, API_KEYS_TABLE, CATALOG_TABLE, OAUTH_TABLES, USER_ROLES_TABLE, quoteIdentifier, quoteLiteral, historyTableName, } from "./storage.js";
+const CATALOG = quoteIdentifier(CATALOG_TABLE);
+const IDENTITY_SYNC = "Copy members' sign-in identities the legacy identity stores hold.";
+/**
+ * Whether applying the plan copies the legacy identity stores into `user_identity`: its target has
+ * the entity. A target an earlier SDK built has none, and the copy waits for the next deploy that
+ * declares it again.
+ */
+function syncsIdentities(plan) {
+    return plan.target.entities.some((entity) => entity.id === USER_IDENTITY_ENTITY_ID && entity.source === undefined);
+}
+/** Serializes migrations of one database; transaction-scoped, so it ends with the transaction. */
+const MIGRATION_LOCK = "segmentstream.ontology.migration";
+const DEFAULT_STATEMENT_TIMEOUT = "5 minutes";
+const DEFAULT_LOCK_TIMEOUT = "5 seconds";
+const Presence = Schema.Struct({ present: Schema.Boolean });
+const CatalogRow = Schema.Struct({ hash: Schema.String, manifest: Schema.String });
+const RetainedRow = Schema.Struct({ retained: Schema.NullOr(Schema.String) });
+const decodeManifest = decodeManifestDocument;
+const readCatalogRow = (sql, options) => Effect.gen(function* () {
+    const presence = yield* sql
+        .unsafe(`SELECT to_regclass($1) IS NOT NULL AS present`, [CATALOG])
+        .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Presence))));
+    if (presence[0]?.present !== true) {
+        return undefined;
+    }
+    const lock = options.forUpdate ? "FOR UPDATE" : "";
+    const rows = yield* sql
+        .unsafe(`
+          SELECT "hash", "manifest"::text AS "manifest"
+          FROM ${CATALOG}
+          WHERE "id"
+          ${lock}
+        `)
+        .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(CatalogRow))));
+    return rows[0];
+}).pipe(Effect.catchTag("SchemaError", (cause) => Effect.fail(new CatalogInvalid({ message: "The _ontology catalog has an unexpected shape.", cause }))));
+/**
+ * The manifest recorded by the last applied migration, or undefined when none was applied. It is
+ * decoded with the manifest contract, so fields this SDK does not know are dropped.
+ */
+export const readCatalog = (input) => Effect.gen(function* () {
+    const row = yield* readCatalogRow(input.sql, { forUpdate: false });
+    if (row === undefined) {
+        return undefined;
+    }
+    const parsed = yield* Effect.try({
+        try: () => JSON.parse(row.manifest),
+        catch: (cause) => new CatalogInvalid({ message: "The _ontology manifest is not JSON.", cause }),
+    });
+    const manifest = yield* decodeManifest(parsed).pipe(Effect.mapError((cause) => new CatalogInvalid({
+        message: "The _ontology manifest does not match the contract.",
+        cause,
+    })));
+    if (manifest.hash !== row.hash ||
+        manifestHash(parsed) !== row.hash) {
+        return yield* new CatalogInvalid({
+            message: "The _ontology hash does not match the manifest it records.",
+            cause: { recordedHash: row.hash, manifestHash: manifest.hash },
+        });
+    }
+    return manifest;
+}).pipe(Effect.withSpan("ontology.migrate.readCatalog"));
+/**
+ * The storage the catalog records as kept: what removals retained, which declaring the entity or
+ * @legacy ontology.pre-upgrade-database-tolerance
+ * field again restores. An earlier SDK's catalog records none, and its `_ontology` table has no
+ * `retained` column; older SDKs neither read nor write it.
+ */
+export const readRetainedStorage = (input) => Effect.gen(function* () {
+    const presence = yield* input.sql
+        .unsafe(`SELECT to_regclass($1) IS NOT NULL AS present`, [CATALOG])
+        .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Presence))));
+    if (presence[0]?.present !== true) {
+        return NO_RETAINED_STORAGE;
+    }
+    const rows = yield* input.sql
+        .unsafe(`SELECT (to_jsonb(c) -> 'retained')::text AS "retained" FROM ${CATALOG} c WHERE c."id"`)
+        .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(RetainedRow))));
+    const text = rows[0]?.retained;
+    if (text === null || text === undefined || text === "null") {
+        return NO_RETAINED_STORAGE;
+    }
+    const parsed = yield* Effect.try({
+        try: () => JSON.parse(text),
+        catch: (cause) => new CatalogInvalid({ message: "The _ontology kept storage is not JSON.", cause }),
+    });
+    return yield* Schema.decodeUnknownEffect(RetainedStorage)(parsed).pipe(Effect.mapError((cause) => new CatalogInvalid({
+        message: "The _ontology kept storage does not match the contract.",
+        cause,
+    })));
+}).pipe(Effect.catchTag("SchemaError", (cause) => Effect.fail(new CatalogInvalid({ message: "The _ontology catalog has an unexpected shape.", cause }))), Effect.withSpan("ontology.migrate.readRetainedStorage"));
+/**
+ * The names in the current schema, for `planMigration`'s `existingStorage`: relations and types,
+ * which share one namespace, and the columns of each table. Names starting with `_`, which no
+ * entity's storage uses, are left out.
+ */
+export const readStorageNames = (input) => Effect.gen(function* () {
+    const names = yield* input.sql.unsafe(`SELECT c.relname AS "name" FROM pg_catalog.pg_class c
+       WHERE c.relnamespace = current_schema()::regnamespace AND left(c.relname, 1) <> '_'
+       UNION
+       SELECT t.typname FROM pg_catalog.pg_type t
+       WHERE t.typnamespace = current_schema()::regnamespace AND t.typcategory <> 'A'
+         AND left(t.typname, 1) <> '_'`);
+    const columns = yield* input.sql.unsafe(`SELECT c.relname AS "table", a.attname AS "column"
+       FROM pg_catalog.pg_attribute a JOIN pg_catalog.pg_class c ON c.oid = a.attrelid
+       WHERE c.relnamespace = current_schema()::regnamespace AND c.relkind IN ('r', 'p')
+         AND left(c.relname, 1) <> '_' AND a.attnum > 0 AND NOT a.attisdropped
+       ORDER BY 1, a.attnum`);
+    const byTable = {};
+    for (const { table, column } of columns) {
+        const list = Object.hasOwn(byTable, table) ? byTable[table] : undefined;
+        if (list === undefined) {
+            byTable[table] = [column];
+        }
+        else {
+            list.push(column);
+        }
+    }
+    return { names: [...new Set(names.map((row) => row.name))].sort(), columns: byTable };
+}).pipe(Effect.withSpan("ontology.migrate.readStorageNames"));
+const verifyCatalog = (sql, plan, options) => Effect.gen(function* () {
+    const row = yield* readCatalogRow(sql, options);
+    if (row?.hash !== plan.currentHash) {
+        return yield* new CatalogMismatch({
+            message: "The database's applied ontology is not the one the plan starts from; plan the migration again.",
+            ...(plan.currentHash === undefined ? {} : { expectedHash: plan.currentHash }),
+            ...(row === undefined ? {} : { actualHash: row.hash }),
+        });
+    }
+    // A plan made without the record leaves it as it is.
+    if (plan.retained === undefined) {
+        return;
+    }
+    const retained = yield* readRetainedStorage({ sql });
+    if (retainedStorageJson(retained) !== retainedStorageJson(plan.retained.current)) {
+        return yield* new CatalogMismatch({
+            message: "The storage the database keeps is not what the plan starts from; plan the migration again.",
+            ...(plan.currentHash === undefined ? {} : { expectedHash: plan.currentHash }),
+            ...(row === undefined ? {} : { actualHash: row.hash }),
+        });
+    }
+});
+/**
+ * Records the kept storage the plan leaves. The column is added the first time there is any to
+ * record, so that a database without kept storage keeps the catalog an earlier SDK created.
+ */
+const recordRetained = (sql, plan) => Effect.gen(function* () {
+    if (plan.retained === undefined) {
+        return;
+    }
+    const json = plan.retained.target.entities.length === 0 ? null : retainedStorageJson(plan.retained.target);
+    const [row] = yield* sql.unsafe(`SELECT to_jsonb(c) ? 'retained' AS "present" FROM ${CATALOG} c WHERE c."id"`);
+    if (row?.present !== true && json === null) {
+        return;
+    }
+    if (row?.present !== true) {
+        yield* sql.unsafe(`ALTER TABLE ${CATALOG} ADD COLUMN IF NOT EXISTS "retained" jsonb`);
+    }
+    yield* sql.unsafe(`UPDATE ${CATALOG} SET "retained" = $1::jsonb WHERE "id"`, [json]);
+});
+const runStep = (sql, step, stepIndex) => Effect.forEach(step.sql, (statement) => sql.unsafe(statement).unprepared, { discard: true }).pipe(Effect.mapError((cause) => new MigrationFailed({
+    message: `Migration step ${stepIndex + 1} failed: ${step.description}`,
+    stepIndex,
+    step: step.description,
+    cause,
+})));
+/**
+ * Reconciles the workflow users with the target's workflows, inside the migration's transaction so
+ * that only the deploy that recorded the target decides them.
+ */
+const reconcileWorkflows = (plan, sql, workspaceId) => workspaceId === undefined
+    ? Effect.void
+    : reconcileWorkflowUsers(principalsOntology(plan.target), { workspaceId }).pipe(Effect.provideService(SqlClient.SqlClient, sql));
+/**
+ * Applies a plan. Non-transactional steps run first, each statement on its own; they only add enum
+ * values and build indexes concurrently, and are safe to run again after a failure. Every other step
+ * then runs in one transaction with `statement_timeout` and `lock_timeout` set, which also verifies
+ * that the catalog still records the manifest the plan starts from and finally records the target. A
+ * failure rolls the transaction back and leaves the applied version in place. With `workspaceId`,
+ * the same transaction then reconciles the workflow users with the target's workflows. A no-op plan
+ * writes nothing else when the runtime schema is also current; with `workspaceId` it still
+ * reconciles the workflow users, under the migration lock while the catalog records its target.
+ */
+export const applyMigration = (plan, options) => Effect.gen(function* () {
+    const { sql } = options;
+    if (plan.refusals.length > 0) {
+        return yield* new MigrationRefused({
+            message: `The migration plan has ${plan.refusals.length} refusal(s); nothing was applied.`,
+            refusals: plan.refusals,
+        });
+    }
+    // @legacy ontology.workflow-cutover-artifacts
+    // Ontology SDKs 0.86.0 to 0.88.0 read the journal row before the catalog; lock in their order.
+    const lockWorkflowAdmission = Effect.gen(function* () {
+        const [presence] = yield* sql.unsafe("SELECT to_regclass('_workflow_admission') IS NOT NULL AS present");
+        if (presence?.present) {
+            yield* sql.unsafe("SELECT id FROM _workflow_admission WHERE id FOR UPDATE");
+        }
+    });
+    const statementTimeout = Duration.toMillis(options.statementTimeout ?? DEFAULT_STATEMENT_TIMEOUT);
+    const lockTimeout = Duration.toMillis(options.lockTimeout ?? DEFAULT_LOCK_TIMEOUT);
+    const configureTimeouts = Effect.gen(function* () {
+        yield* sql.unsafe(`SELECT set_config('statement_timeout', $1, true)`, [
+            String(statementTimeout),
+        ]);
+        yield* sql.unsafe(`SELECT set_config('lock_timeout', $1, true)`, [String(lockTimeout)]);
+    });
+    const [presence] = yield* sql.unsafe(`SELECT to_regclass('_ontology_replication') IS NOT NULL AS present,
+        to_regclass('_ontology_replication_targets') IS NOT NULL AS targets`);
+    if (presence?.present) {
+        const targetPresent = presence.targets && isNoopMigration(plan)
+            ? "EXISTS (SELECT 1 FROM _ontology_replication_targets WHERE generation = active_generation)"
+            : "false";
+        const [state] = yield* sql.unsafe(`SELECT version, enabled, ${targetPresent} AS target_present FROM _ontology_replication WHERE id`);
+        if (state?.enabled &&
+            state.version >= 6 &&
+            (yield* requestShadowSupersession(sql, plan.target.hash))) {
+            return yield* new MigrationRefused({
+                message: "replication_publication_pending",
+                refusals: [
+                    {
+                        code: "replication-publication-pending",
+                        path: "replication.publication",
+                        message: "replication_publication_pending",
+                    },
+                ],
+            });
+        }
+        if (isNoopMigration(plan) &&
+            state?.version === REPLICATION_SCHEMA_VERSION &&
+            (!state.enabled || state.target_present)) {
+            if (options.workspaceId !== undefined || syncsIdentities(plan)) {
+                // Under the migration lock, and only while the catalog still records this target.
+                yield* sql.withTransaction(Effect.gen(function* () {
+                    yield* configureTimeouts;
+                    yield* sql.unsafe(`SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, [
+                        MIGRATION_LOCK,
+                    ]);
+                    // Drain admitted writers before catalog, identity and workflow-user locks, even
+                    // without DDL.
+                    yield* lockReplicationLifecycle(sql);
+                    yield* lockWorkflowAdmission;
+                    yield* verifyCatalog(sql, plan, { forUpdate: true });
+                    // An earlier SDK may have served or migrated the database since: copy what it wrote.
+                    if (syncsIdentities(plan)) {
+                        // @legacy identities.legacy-stores-dual-write
+                        yield* syncUserIdentities(sql);
+                    }
+                    yield* reconcileWorkflows(plan, sql, options.workspaceId);
+                    yield* recordWorkflowCatalog(sql, plan.target);
+                }));
+            }
+            return { hash: plan.target.hash, appliedSteps: 0, changed: false };
+        }
+    }
+    yield* verifyCatalog(sql, plan, { forUpdate: false });
+    for (const [stepIndex, step] of plan.steps.entries()) {
+        if (!step.transactional) {
+            yield* runStep(sql, step, stepIndex);
+        }
+    }
+    yield* sql.withTransaction(Effect.gen(function* () {
+        yield* configureTimeouts;
+        yield* sql.unsafe(`SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, [
+            MIGRATION_LOCK,
+        ]);
+        // Recheck after serializing migrations: an earlier migration may have created the table.
+        const [replication] = yield* sql.unsafe("SELECT to_regclass('_ontology_replication') IS NOT NULL AS present");
+        if (replication?.present) {
+            yield* lockReplicationSchema(sql);
+        }
+        // Writers authorize workflows/files after lifecycle admission. Match that catalog order.
+        yield* lockWorkflowAdmission;
+        yield* verifyCatalog(sql, plan, { forUpdate: true });
+        const previousManifest = yield* readCatalog({ sql });
+        let refresh;
+        let active;
+        let exclusions;
+        if (replication?.present) {
+            const [state] = yield* sql.unsafe("SELECT * FROM _ontology_replication WHERE id FOR UPDATE");
+            if (state && previousManifest) {
+                exclusions = yield* Effect.try({
+                    try: () => migrationReplicationExclusions(previousManifest, plan.target, state.exclusions ?? []),
+                    catch: (cause) => new CatalogInvalid({ message: "The replication exclusions are invalid.", cause }),
+                });
+            }
+            if (state?.enabled) {
+                refresh = yield* Effect.try({
+                    try: () => resolveReplicationSelection(plan.target, effectiveReplicationExclusions(plan.target, exclusions ?? state.exclusions ?? [])),
+                    catch: (cause) => new CatalogInvalid({ message: "The replication exclusions are invalid.", cause }),
+                });
+                const [target] = presence.targets
+                    ? yield* sql.unsafe("SELECT * FROM _ontology_replication_targets WHERE generation = $1::uuid", [state.active_generation])
+                    : [];
+                if (target &&
+                    (yield* dropsServingReplicationStorage(sql, {
+                        current: previousManifest,
+                        target,
+                        version: state.version,
+                        retained: plan.retained?.current,
+                        steps: plan.steps,
+                    }))) {
+                    return yield* new MigrationRefused({
+                        message: "replication_source_compatibility_required",
+                        refusals: [
+                            {
+                                code: "drop-refused",
+                                path: "replication.compatibility",
+                                message: "replication_source_compatibility_required",
+                            },
+                        ],
+                    });
+                }
+                active = target;
+                // A runtime-only upgrade retains an existing explicit projection.
+                if (target?.manifest.hash === plan.target.hash) {
+                    refresh = target.selection;
+                }
+                // Both writer protocols are fenced. Capture and schema changes commit on one stream.
+                yield* sql.unsafe("UPDATE _ontology_replication SET enabled = false WHERE id");
+            }
+        }
+        // Install metadata before destructive steps so pending envelopes and stable import/scan
+        // cursors can survive storage changes. Capture stays fenced until everything commits.
+        for (const statement of REPLICATION_DDL) {
+            yield* sql.unsafe(statement).unprepared;
+        }
+        yield* sql.unsafe("UPDATE _ontology_replication SET version = $1 WHERE id", [
+            REPLICATION_SCHEMA_VERSION,
+        ]);
+        yield* prepareReplicationEvolution(sql);
+        if (active) {
+            yield* enableShadowCapture(sql, active, replicationScanScopes(active));
+        }
+        if (plan.steps.some((step) => step.kind === "dropEntities")) {
+            for (const entity of previousManifest?.entities ?? []) {
+                if (!entity.builtIn &&
+                    !entity.source &&
+                    !plan.target.entities.some((item) => item.id === entity.id)) {
+                    yield* retainReplicationHistory(sql, historyTableName(entity.name));
+                }
+            }
+        }
+        for (const [stepIndex, step] of plan.steps.entries()) {
+            if (step.transactional) {
+                yield* runStep(sql, step, stepIndex);
+            }
+        }
+        // Once the entity's table and the legacy stores exist: copy what an earlier SDK wrote.
+        if (syncsIdentities(plan)) {
+            yield* syncUserIdentities(sql).pipe(Effect.mapError((cause) => new MigrationFailed({
+                message: `Migration step ${plan.steps.length + 1} failed: ${IDENTITY_SYNC}`,
+                stepIndex: plan.steps.length,
+                step: IDENTITY_SYNC,
+                cause,
+            })));
+        }
+        for (const statement of replicationEntityDdl(plan.target, previousManifest)) {
+            yield* sql.unsafe(statement).unprepared;
+        }
+        // References survive entity/history table renames, including receipts for paused generations
+        // and the history of an entity restored under a new name.
+        for (const entity of plan.target.entities) {
+            const previous = previousManifest?.entities.find((item) => item.id === entity.id)?.name ??
+                plan.retained?.current.entities.find((item) => item.id === entity.id && item.removed)
+                    ?.name;
+            if (previous !== undefined && previous !== entity.name) {
+                yield* renameReplicationHistory(sql, historyTableName(previous), historyTableName(entity.name));
+            }
+        }
+        yield* sql.unsafe("UPDATE _ontology_replication SET version = $1 WHERE id", [
+            REPLICATION_SCHEMA_VERSION,
+        ]);
+        if (exclusions !== undefined) {
+            yield* sql.unsafe("UPDATE _ontology_replication SET exclusions = $1::jsonb WHERE id", [
+                JSON.stringify(exclusions),
+            ]);
+        }
+        yield* sql.unsafe("ALTER TABLE _ontology_replication VALIDATE CONSTRAINT _ontology_replication_protocol_floor");
+        yield* sql.unsafe("ALTER TABLE _ontology_replication VALIDATE CONSTRAINT _ontology_replication_schema_floor_v5");
+        yield* sql.unsafe("ALTER TABLE _ontology_replication VALIDATE CONSTRAINT _ontology_replication_schema_floor_v6");
+        yield* sql.unsafe(`
+            INSERT INTO ${CATALOG} ("id", "version", "hash", "manifest")
+            VALUES (true, $1, $2, $3::jsonb)
+            ON CONFLICT ("id") DO UPDATE SET
+              "version" = excluded."version",
+              "hash" = excluded."hash",
+              "manifest" = excluded."manifest",
+              "applied_at" = now()
+          `, [plan.target.version, plan.target.hash, canonicalJson(encodeManifest(plan.target))]);
+        yield* recordRetained(sql, plan);
+        if (refresh !== undefined) {
+            if (active) {
+                const fills = yield* migratedReplicationFills(sql, active, plan.target, refresh, plan.retained?.current);
+                yield* refreshReplicationProjection(sql, active, { ...active, manifest: plan.target, selection: refresh, fills }, plan.retained?.current).pipe(Effect.catchTag("ReplicationBaselineFailed", (cause) => Effect.fail(new MigrationRefused({
+                    message: "replication_publication_pending",
+                    refusals: [
+                        {
+                            code: "replication-publication-pending",
+                            path: "replication.publication",
+                            message: "replication_publication_pending",
+                        },
+                    ],
+                    cause,
+                }))));
+                yield* refreshShadowCapturePolicy(sql);
+                yield* sql.unsafe("UPDATE _ontology_replication SET enabled=true WHERE id");
+            }
+            else {
+                // Missing capture metadata is observable damage, so recover from authoritative rows
+                // under a fresh incarnation instead of asking the workspace to disable capture.
+                yield* prepareReplicationBaseline(sql, {
+                    generation: randomUUID(),
+                    manifest: plan.target,
+                    selection: refresh,
+                    exclusions: exclusions ?? [],
+                    newSource: true,
+                }).pipe(Effect.mapError((cause) => new CatalogInvalid({
+                    message: "Replication initialization could not be prepared.",
+                    cause,
+                })));
+            }
+        }
+        yield* sql.unsafe("UPDATE _ontology_replication SET requested_manifest_hash=NULL,requested_manifest_until=NULL WHERE id");
+        yield* reconcileWorkflows(plan, sql, options.workspaceId);
+        yield* recordWorkflowCatalog(sql, plan.target);
+    }));
+    return { hash: plan.target.hash, appliedSteps: plan.steps.length, changed: true };
+}).pipe(Effect.withSpan("ontology.migrate.apply", {
+    attributes: { "ontology.target_hash": plan.target.hash, "ontology.steps": plan.steps.length },
+}));
+/**
+ * Reads the catalog, plans the migration to the target and applies it. Run it on a Neon branch of the
+ * workspace database first as a dry run, then on production: the plan is made against each database's
+ * own catalog.
+ */
+/** The runtime's tables, beside the catalog and the entity tables. */
+export const RUNTIME_TABLES = [
+    ...FEDERATION_TABLES,
+    "_ontology_replication_scans",
+    "_ontology_replication_epochs",
+    "_ontology_replication_segments",
+    "_ontology_replication_control_jobs",
+    ACTION_ATTRIBUTION_TABLE,
+    USER_ROLES_TABLE,
+    API_KEYS_TABLE,
+    ...OAUTH_TABLES,
+    ...AGENT_TABLES,
+    ATTACHMENTS_TABLE,
+    WORKFLOW_STARTS_TABLE,
+    "_workflow_admission",
+    AGENT_TASKS_TABLE,
+    SIGN_IN_EMAILS_TABLE,
+];
+/**
+ * The runtime tables the database has, for `planMigration`'s `existingTables`: a database that an
+ * earlier SDK migrated lacks the tables added since.
+ * A grants table without app-session binding is reported incomplete so the idempotent OAuth DDL
+ * also upgrades an unchanged manifest, and so is an attachments table whose source check predates
+ * one of {@link ATTACHMENT_SOURCES}, which the idempotent attachment DDL widens.
+ */
+export const readRuntimeTables = (input) => input.sql
+    .unsafe(`SELECT "tablename" AS "name" FROM "pg_catalog"."pg_tables" ` +
+    `WHERE "schemaname" = current_schema() ` +
+    `AND "tablename" IN (${RUNTIME_TABLES.map(quoteLiteral).join(", ")}) ` +
+    `AND ("tablename" <> '_oauth_grants' OR EXISTS (` +
+    `SELECT 1 FROM information_schema.columns WHERE table_schema = current_schema() ` +
+    `AND table_name = '_oauth_grants' AND column_name = 'identity_id')) ` +
+    `AND ("tablename" <> ${quoteLiteral(ATTACHMENTS_TABLE)} OR EXISTS (` +
+    `SELECT 1 FROM pg_catalog.pg_constraint c ` +
+    `JOIN pg_catalog.pg_class t ON t.oid = c.conrelid ` +
+    `JOIN pg_catalog.pg_namespace n ON n.oid = t.relnamespace ` +
+    `WHERE n.nspname = current_schema() AND t.relname = ${quoteLiteral(ATTACHMENTS_TABLE)} ` +
+    `AND c.conname = ${quoteLiteral(ATTACHMENT_SOURCE_CHECK)} ` +
+    ATTACHMENT_SOURCES.map((source) => `AND pg_catalog.pg_get_constraintdef(c.oid) LIKE ${quoteLiteral(`%'${source}'%`)} `).join("") +
+    `))`)
+    .pipe(Effect.map((rows) => new Set(rows.map((row) => row.name))));
+/** Only valid, complete covering indexes satisfy the upgrade; failed builds are retried. */
+export const readHistoryActivityIndexes = (input) => input.sql
+    .unsafe(`SELECT c.relname AS name FROM pg_catalog.pg_index i
+     JOIN pg_catalog.pg_class c ON c.oid = i.indexrelid
+     WHERE i.indisvalid AND i.indisready AND i.indnatts = 2 AND i.indpred IS NULL
+       AND pg_catalog.pg_table_is_visible(c.oid)
+       AND pg_catalog.pg_get_indexdef(i.indexrelid, 1, true) = 'at'
+       AND pg_catalog.pg_get_indexdef(i.indexrelid, 2, true) = 'principal_id'`)
+    .pipe(Effect.map((rows) => new Set(rows.map((row) => row.name))));
+export const migrate = (input) => Effect.gen(function* () {
+    const current = yield* readCatalog({ sql: input.sql });
+    const retained = yield* readRetainedStorage({ sql: input.sql });
+    const existingTables = yield* readRuntimeTables({ sql: input.sql });
+    const existingHistoryActivityIndexes = yield* readHistoryActivityIndexes({ sql: input.sql });
+    const existingStorage = yield* readStorageNames({ sql: input.sql });
+    const plan = planMigration({
+        current,
+        retained,
+        existingHistoryActivityIndexes,
+        existingStorage,
+        target: input.target,
+        existingTables,
+        ...(input.options === undefined ? {} : { options: input.options }),
+    });
+    const result = yield* applyMigration(plan, input);
+    return { plan, result };
+});

package/dist/migrate/errors.d.ts

@@ -0,0 +1,46 @@
+import { Schema } from "effect";
+declare const MigrationRefused_base: Schema.Class<MigrationRefused, Schema.TaggedStruct<"MigrationRefused", {
+    readonly message: Schema.String;
+    readonly refusals: Schema.$Array<Schema.Struct<{
+        readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict", "replication-rebuild-required", "replication-publication-pending"]>;
+        readonly path: Schema.String;
+        readonly message: Schema.String;
+    }>>;
+    readonly cause: Schema.optionalKey<Schema.Unknown>;
+}>, import("effect/Cause").YieldableError>;
+/** The plan has refusals; nothing was applied. */
+export declare class MigrationRefused extends MigrationRefused_base {
+}
+declare const CatalogMismatch_base: Schema.Class<CatalogMismatch, Schema.TaggedStruct<"CatalogMismatch", {
+    readonly message: Schema.String;
+    readonly expectedHash: Schema.optionalKey<Schema.String>;
+    readonly actualHash: Schema.optionalKey<Schema.String>;
+}>, import("effect/Cause").YieldableError>;
+/**
+ * The applied manifest is not the one the plan was made from: another migration ran in between, or the
+ * plan was made against another database. Nothing was applied.
+ */
+export declare class CatalogMismatch extends CatalogMismatch_base {
+}
+declare const CatalogInvalid_base: Schema.Class<CatalogInvalid, Schema.TaggedStruct<"CatalogInvalid", {
+    readonly message: Schema.String;
+    readonly cause: Schema.Unknown;
+}>, import("effect/Cause").YieldableError>;
+/** The `_ontology` row cannot be read as a manifest. */
+export declare class CatalogInvalid extends CatalogInvalid_base {
+}
+declare const MigrationFailed_base: Schema.Class<MigrationFailed, Schema.TaggedStruct<"MigrationFailed", {
+    readonly message: Schema.String;
+    /** The position of the failed step in `plan.steps`. */
+    readonly stepIndex: Schema.Number;
+    readonly step: Schema.String;
+    readonly cause: Schema.Unknown;
+}>, import("effect/Cause").YieldableError>;
+/**
+ * A step failed. A failed transactional step rolls the whole transaction back; non-transactional steps
+ * that ran before it stay applied and are safe to run again. The message names the step only; the
+ * database's own explanation, which may quote stored values, is kept in `cause`.
+ */
+export declare class MigrationFailed extends MigrationFailed_base {
+}
+export {};

package/dist/migrate/errors.js

@@ -0,0 +1,38 @@
+import { Schema } from "effect";
+import { MigrationRefusal } from "./model.js";
+/** The plan has refusals; nothing was applied. */
+export class MigrationRefused extends Schema.TaggedError()("MigrationRefused", {
+    message: Schema.String,
+    refusals: Schema.Array(MigrationRefusal),
+    cause: Schema.optionalKey(Schema.Unknown),
+}) {
+}
+/**
+ * The applied manifest is not the one the plan was made from: another migration ran in between, or the
+ * plan was made against another database. Nothing was applied.
+ */
+export class CatalogMismatch extends Schema.TaggedError()("CatalogMismatch", {
+    message: Schema.String,
+    expectedHash: Schema.optionalKey(Schema.String),
+    actualHash: Schema.optionalKey(Schema.String),
+}) {
+}
+/** The `_ontology` row cannot be read as a manifest. */
+export class CatalogInvalid extends Schema.TaggedError()("CatalogInvalid", {
+    message: Schema.String,
+    cause: Schema.Unknown,
+}) {
+}
+/**
+ * A step failed. A failed transactional step rolls the whole transaction back; non-transactional steps
+ * that ran before it stay applied and are safe to run again. The message names the step only; the
+ * database's own explanation, which may quote stored values, is kept in `cause`.
+ */
+export class MigrationFailed extends Schema.TaggedError()("MigrationFailed", {
+    message: Schema.String,
+    /** The position of the failed step in `plan.steps`. */
+    stepIndex: Schema.Number,
+    step: Schema.String,
+    cause: Schema.Unknown,
+}) {
+}

package/dist/migrate/index.d.ts

@@ -0,0 +1,10 @@
+export { applyMigration, migrate, readCatalog, readRetainedStorage, readRuntimeTables, readHistoryActivityIndexes, readStorageNames, RUNTIME_TABLES, } from "./apply.js";
+export type { ApplyMigrationError, ApplyMigrationOptions, CatalogInput, MigrateInput, MigrationResult, } from "./apply.js";
+export { CatalogInvalid, CatalogMismatch, MigrationFailed, MigrationRefused } from "./errors.js";
+export { isNoopMigration, MIGRATION_REFUSAL_CODES, MIGRATION_STEP_CLASSIFICATIONS, MIGRATION_STEP_KINDS, MigrationPlan, MigrationRefusal, MigrationStep, RetainedEntity, RetainedStorage, } from "./model.js";
+export type { MigrationRefusalCode, MigrationStepClassification, MigrationStepKind, } from "./model.js";
+export { planMigration } from "./plan.js";
+export { preparePreviewDatabase } from "./preview.js";
+export type { MigrationOptions, PlanMigrationInput, StorageNames } from "./plan.js";
+/** The physical mapping: table, column, enum, index and constraint names, and the DDL builders. */
+export * as OntologyStorage from "./storage.js";

package/dist/migrate/index.js

@@ -0,0 +1,7 @@
+export { applyMigration, migrate, readCatalog, readRetainedStorage, readRuntimeTables, readHistoryActivityIndexes, readStorageNames, RUNTIME_TABLES, } from "./apply.js";
+export { CatalogInvalid, CatalogMismatch, MigrationFailed, MigrationRefused } from "./errors.js";
+export { isNoopMigration, MIGRATION_REFUSAL_CODES, MIGRATION_STEP_CLASSIFICATIONS, MIGRATION_STEP_KINDS, MigrationPlan, MigrationRefusal, MigrationStep, RetainedEntity, RetainedStorage, } from "./model.js";
+export { planMigration } from "./plan.js";
+export { preparePreviewDatabase } from "./preview.js";
+/** The physical mapping: table, column, enum, index and constraint names, and the DDL builders. */
+export * as OntologyStorage from "./storage.js";