@segmentstream-cloud/ontology 0.24.1 → 0.25.0

package/dist/migrate/apply.js

@@ -4,10 +4,11 @@ import { CatalogInvalid, CatalogMismatch, MigrationFailed, MigrationRefused } fr
 import { isNoopMigration } from "./model.js";
 import { planMigration } from "./plan.js";
 import { REPLICATION_DDL, REPLICATION_SCHEMA_VERSION, replicationEntityDdl, } from "../replication/schema.js";
+import { lockReplicationLifecycle } from "../replication/transaction.js";
 import { replicationMigrationRefusals } from "../replication/migrate.js";
 import { ATTACHMENTS_TABLE } from "../storage/tables.js";
 import { resolveReplicationSelection, } from "../replication/model.js";
-import { ACTION_ATTRIBUTION_TABLE, AGENT_TABLES, API_KEYS_TABLE, CATALOG_TABLE, OAUTH_TABLES, USER_ROLES_TABLE, quoteIdentifier, quoteLiteral, } from "./storage.js";
+import { ACTION_ATTRIBUTION_TABLE, AGENT_TABLES, API_KEYS_TABLE, CATALOG_TABLE, OAUTH_TABLES, USER_ROLES_TABLE, quoteIdentifier, quoteLiteral, historyTableName, } from "./storage.js";
 const CATALOG = quoteIdentifier(CATALOG_TABLE);
 /** Serializes migrations of one database; transaction-scoped, so it ends with the transaction. */
 const MIGRATION_LOCK = "segmentstream.ontology.migration";
@@ -117,8 +118,10 @@ export const applyMigration = (plan, options) => Effect.gen(function* () {
             MIGRATION_LOCK,
         ]);
         yield* verifyCatalog(sql, plan, { forUpdate: true });
+        const previousManifest = yield* readCatalog({ sql });
         let refresh;
         if (presence?.present) {
+            yield* lockReplicationLifecycle(sql);
             const [state] = yield* sql.unsafe("SELECT * FROM _ontology_replication WHERE id FOR UPDATE");
             if (state?.enabled) {
                 refresh = yield* Effect.try({
@@ -132,6 +135,28 @@ export const applyMigration = (plan, options) => Effect.gen(function* () {
                         cause: { generation: state.active_generation },
                     });
                 }
+                const incompleteScan = target.baseline_complete === false
+                    ? "baseline"
+                    : target.backfill_complete === false
+                        ? "backfill"
+                        : undefined;
+                if (incompleteScan && target.manifest.hash !== plan.target.hash) {
+                    return yield* new MigrationRefused({
+                        message: `Finish the replication ${incompleteScan} before changing its schema, ` +
+                            "or pause and rebuild replication.",
+                        refusals: [
+                            {
+                                code: "replication-rebuild-required",
+                                path: `replication.${incompleteScan}`,
+                                message: "An unfinished replication scan must retain its manifest and keyset cursors.",
+                            },
+                        ],
+                    });
+                }
+                // A runtime-only upgrade preserves an explicit projection as well as exclusions.
+                if (target.manifest.hash === plan.target.hash) {
+                    refresh = target.selection;
+                }
                 const refusals = replicationMigrationRefusals(target, plan.target, refresh);
                 if (refusals.length > 0) {
                     return yield* new MigrationRefused({
@@ -151,9 +176,17 @@ export const applyMigration = (plan, options) => Effect.gen(function* () {
         for (const statement of [...REPLICATION_DDL, ...replicationEntityDdl(plan.target)]) {
             yield* sql.unsafe(statement).unprepared;
         }
+        // References survive entity/history table renames, including receipts for paused generations.
+        for (const entity of plan.target.entities) {
+            const previous = previousManifest?.entities.find((item) => item.id === entity.id);
+            if (previous && previous.name !== entity.name) {
+                yield* sql.unsafe("UPDATE _ontology_replication_delivery SET history_table = $2 WHERE history_table = $1", [historyTableName(previous.name), historyTableName(entity.name)]);
+            }
+        }
         yield* sql.unsafe("UPDATE _ontology_replication SET version = $1 WHERE id", [
             REPLICATION_SCHEMA_VERSION,
         ]);
+        yield* sql.unsafe("ALTER TABLE _ontology_replication VALIDATE CONSTRAINT _ontology_replication_protocol_floor");
         yield* sql.unsafe(`
             INSERT INTO ${CATALOG} ("id", "version", "hash", "manifest")
             VALUES (true, $1, $2, $3::jsonb)

package/dist/replication/baseline.d.ts

@@ -8,20 +8,34 @@ export interface ReplicationTarget {
     readonly from_revision: string;
     readonly manifest: OntologyManifest;
     readonly selection: ReadonlyArray<ReplicationSelection>;
+    readonly protocol?: number;
+    readonly baseline_complete?: boolean;
+    readonly backfill_complete?: boolean;
+    readonly initial_revision?: string | null;
+    readonly replay_complete?: boolean;
 }
 declare const ReplicationBaselineFailed_base: Schema.Class<ReplicationBaselineFailed, Schema.TaggedStruct<"ReplicationBaselineFailed", {
     readonly message: Schema.String;
     readonly cause: Schema.optionalKey<Schema.Unknown>;
 }>, import("effect/Cause").YieldableError>;
-/** An initial snapshot cannot be created with the requested state or record projection. */
+/** An initial scan cannot be created with the requested state or record projection. */
 export declare class ReplicationBaselineFailed extends ReplicationBaselineFailed_base {
 }
-/** All baseline rows are committed with capture enablement; interrupted setup can safely resume. */
-export declare const createReplicationBaseline: (sql: SqlClient.SqlClient, input: {
+interface BaselineInput {
     readonly generation: string;
     readonly manifest: OntologyManifest;
     readonly exclusions?: ReadonlyArray<ReplicationExclusion>;
     readonly selection: ReadonlyArray<ReplicationSelection>;
     readonly newSource?: boolean;
-}) => Effect.Effect<ReplicationTarget, ReplicationBaselineFailed | import("effect/unstable/sql/SqlError").SqlError, never>;
+}
+/** Enable capture in a short fenced transaction before scanning any business records. */
+export declare const prepareReplicationBaseline: (sql: SqlClient.SqlClient, input: BaselineInput) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError | ReplicationBaselineFailed, never>;
+/**
+ * Commit one keyset page and its cursor together. Every concurrent mutation has a captured
+ * revision above zero, so it supersedes even a later scanned baseline row. A failed page retries
+ * without changing earlier pages, and the capture stream covers inserts/deletes across pages.
+ */
+export declare const advanceReplicationBaseline: (sql: SqlClient.SqlClient, generation: string) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError | ReplicationBaselineFailed, never>;
+/** Prepare once, then advance a bounded page; subsequent calls resume the same durable target. */
+export declare const createReplicationBaseline: (sql: SqlClient.SqlClient, input: BaselineInput) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError | ReplicationBaselineFailed, never>;
 export {};

package/dist/replication/baseline.js

@@ -3,35 +3,45 @@ import { Effect, Schema } from "effect";
 import { joinColumnNames, joinTableName, quoteIdentifier as qi, quoteLiteral as ql, tableName, } from "../migrate/storage.js";
 import { REPLICATION_RECORD_BYTES, validateReplicationSelection, } from "./model.js";
 import { snapshotExpression } from "./schema.js";
-/** An initial snapshot cannot be created with the requested state or record projection. */
+import { lockReplicationLifecycle, shareReplicationLifecycle } from "./transaction.js";
+/** An initial scan cannot be created with the requested state or record projection. */
 export class ReplicationBaselineFailed extends Schema.TaggedError()("ReplicationBaselineFailed", { message: Schema.String, cause: Schema.optionalKey(Schema.Unknown) }) {
 }
-/** All baseline rows are committed with capture enablement; interrupted setup can safely resume. */
-export const createReplicationBaseline = (sql, input) => sql.withTransaction(Effect.gen(function* () {
+const targetSql = `SELECT generation::text, source_id::text, from_revision::text, manifest,
+  selection, protocol, baseline_complete, backfill_complete, initial_revision::text,
+  replay_complete FROM _ontology_replication_targets WHERE generation = $1::uuid`;
+/** Enable capture in a short fenced transaction before scanning any business records. */
+export const prepareReplicationBaseline = (sql, input) => sql.withTransaction(Effect.gen(function* () {
     yield* sql.unsafe("SET LOCAL statement_timeout = '30s'");
     yield* sql.unsafe("SET LOCAL lock_timeout = '5s'");
-    const [state] = yield* sql.unsafe("SELECT source_id::text, revision::text, enabled FROM _ontology_replication WHERE id FOR UPDATE");
-    if (!state) {
+    yield* lockReplicationLifecycle(sql);
+    const [state] = yield* sql.unsafe("SELECT source_id::text, revision::text, enabled, version FROM _ontology_replication WHERE id");
+    if (!state || state.version < 4) {
         return yield* new ReplicationBaselineFailed({
             message: "Run the ontology SDK migration before enabling replication.",
         });
     }
-    const existing = yield* sql.unsafe("SELECT * FROM _ontology_replication_targets WHERE generation = $1::uuid", [input.generation]);
-    if (existing[0]) {
-        return existing[0];
+    const [existing] = yield* sql.unsafe(targetSql, [input.generation]);
+    if (existing) {
+        return existing;
+    }
+    const [catalog] = yield* sql.unsafe("SELECT hash FROM _ontology WHERE id");
+    if (catalog?.hash !== input.manifest.hash) {
+        return yield* new ReplicationBaselineFailed({
+            message: "The ontology changed before replication started; retry with the current manifest.",
+        });
     }
     if (state.enabled) {
         return yield* new ReplicationBaselineFailed({
             message: "Pause replication before starting a new generation.",
         });
     }
-    const manifest = input.manifest;
     yield* Effect.try({
-        try: () => validateReplicationSelection(manifest, input.selection),
+        try: () => validateReplicationSelection(input.manifest, input.selection),
         catch: (cause) => new ReplicationBaselineFailed({ message: "Invalid replication projection.", cause }),
     });
-    // Drain older SDK transactions that started before the compatibility fence was enabled.
-    const tables = manifest.entities
+    // Drain raw/pre-upgrade writers only at enablement; online pages never lock business tables.
+    const tables = input.manifest.entities
         .filter((e) => e.source === undefined && !e.builtIn)
         .flatMap((e) => [
         tableName(e.name),
@@ -43,64 +53,145 @@ export const createReplicationBaseline = (sql, input) => sql.withTransaction(Eff
         yield* sql.unsafe(`LOCK TABLE ${tables.map(qi).join(", ")} IN SHARE MODE`);
     }
     const source = input.newSource ? randomUUID() : state.source_id;
-    const selection = Object.fromEntries(input.selection.map((s) => [s.entityId, s]));
-    yield* sql.unsafe(`UPDATE _ontology_replication SET source_id = $1::uuid, selection = $2::jsonb, enabled = true,
-          exclusions = $3::jsonb, active_generation = $4::uuid WHERE id`, [
+    yield* sql.unsafe(`INSERT INTO _ontology_replication_targets
+          (generation, source_id, from_revision, manifest, selection, protocol, baseline_complete)
+         VALUES ($1::uuid, $2::uuid, 0, $3::jsonb, $4::jsonb, 4, false)`, [input.generation, source, JSON.stringify(input.manifest), JSON.stringify(input.selection)]);
+    yield* sql.unsafe(`UPDATE _ontology_replication SET source_id = $1::uuid, selection = $2::jsonb,
+          enabled = true, exclusions = $3::jsonb, active_generation = $4::uuid WHERE id`, [
         source,
-        JSON.stringify(selection),
+        JSON.stringify(Object.fromEntries(input.selection.map((s) => [s.entityId, s]))),
         JSON.stringify(input.exclusions ?? []),
         input.generation,
     ]);
-    const target = {
-        generation: input.generation,
-        source_id: source,
-        from_revision: state.revision,
-        manifest,
-        selection: input.selection,
-    };
-    yield* sql.unsafe(`INSERT INTO _ontology_replication_targets (generation, source_id, from_revision, manifest, selection)
-     VALUES ($1::uuid, $2::uuid, $3::bigint, $4::jsonb, $5::jsonb)`, [
-        target.generation,
-        source,
-        state.revision,
-        JSON.stringify(manifest),
-        JSON.stringify(input.selection),
-    ]);
-    const baseline = (entityId, kind, records) => Effect.gen(function* () {
-        // Keep the snapshot in PostgreSQL; large baselines must not fill a serverless process's heap.
-        const [size] = yield* sql.unsafe(`SELECT EXISTS (
-              SELECT 1 FROM (${records}) snapshot
-              WHERE octet_length(payload::text) > ${REPLICATION_RECORD_BYTES}
-            ) AS oversized`);
-        if (size.oversized) {
+    const [target] = yield* sql.unsafe(targetSql, [input.generation]);
+    return target;
+}));
+const BASELINE_PAGE_SIZE = 250;
+function baselineSources(target) {
+    return target.selection.flatMap((item) => {
+        const entity = target.manifest.entities.find((e) => e.id === item.entityId);
+        return [
+            {
+                entityId: entity.id,
+                kind: "entity",
+                records: `SELECT r.id::text AS record_key,
+          (SELECT jsonb_object_agg(key, value) FROM jsonb_each(${snapshotExpression(entity, "r")})
+           WHERE key = 'id' OR ${ql(JSON.stringify(item.fields))}::jsonb ? key) AS payload
+         FROM ${qi(tableName(entity.name))} r
+         WHERE r.id >= COALESCE(NULLIF($2::text, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid)
+           AND ($2::text = '' OR r.id > NULLIF($2::text, '')::uuid)
+         ORDER BY r.id LIMIT ${BASELINE_PAGE_SIZE}`,
+            },
+            ...entity.relationships
+                .filter((r) => item.relationships?.includes(r.name))
+                .map((relationship) => {
+                const columns = joinColumnNames({
+                    entity: entity.name,
+                    relationship: relationship.name,
+                    target: relationship.to,
+                });
+                return {
+                    entityId: entity.id,
+                    kind: `edge:${relationship.name}`,
+                    records: `SELECT ${qi(columns.source)}::text || ':' || ${qi(columns.target)}::text AS record_key,
+            jsonb_build_object('source_id', ${qi(columns.source)}::text,
+              'target_id', ${qi(columns.target)}::text) AS payload
+           FROM ${qi(joinTableName(entity.name, relationship.name))}
+           WHERE (${qi(columns.source)}, ${qi(columns.target)}) >= (
+             COALESCE(NULLIF(split_part($2::text, ':', 1), '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid),
+             COALESCE(NULLIF(split_part($2::text, ':', 2), '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid))
+             AND ($2::text = '' OR (${qi(columns.source)}, ${qi(columns.target)}) > (
+               NULLIF(split_part($2::text, ':', 1), '')::uuid, NULLIF(split_part($2::text, ':', 2), '')::uuid))
+           ORDER BY ${qi(columns.source)}, ${qi(columns.target)} LIMIT ${BASELINE_PAGE_SIZE}`,
+                };
+            }),
+        ];
+    });
+}
+/**
+ * Commit one keyset page and its cursor together. Every concurrent mutation has a captured
+ * revision above zero, so it supersedes even a later scanned baseline row. A failed page retries
+ * without changing earlier pages, and the capture stream covers inserts/deletes across pages.
+ */
+export const advanceReplicationBaseline = (sql, generation) => Effect.gen(function* () {
+    const progress = yield* sql.withTransaction(Effect.gen(function* () {
+        yield* sql.unsafe("SET LOCAL statement_timeout = '30s'");
+        yield* sql.unsafe("SET LOCAL lock_timeout = '5s'");
+        yield* shareReplicationLifecycle(sql);
+        const [target] = yield* sql.unsafe(`${targetSql.replace("replay_complete FROM", "replay_complete, baseline_cursor FROM")} FOR NO KEY UPDATE`, [generation]);
+        if (!target) {
+            return yield* new ReplicationBaselineFailed({
+                message: "The replication target is missing.",
+            });
+        }
+        if (target.baseline_complete !== false) {
+            return { target, finish: false };
+        }
+        const [state] = yield* sql.unsafe("SELECT enabled, active_generation::text FROM _ontology_replication WHERE id");
+        if (!state?.enabled || state.active_generation !== generation) {
+            return yield* new ReplicationBaselineFailed({
+                message: "Resume the active replication generation before scanning its baseline.",
+            });
+        }
+        const index = target.baseline_cursor.index ?? 0;
+        const source = baselineSources(target)[index];
+        if (!source) {
+            return { target, finish: true };
+        }
+        // Materialize once in PostgreSQL; only the small progress result crosses the transport.
+        const [page] = yield* sql.unsafe(`WITH page AS MATERIALIZED (
+          SELECT * FROM (${source.records}) records
+        ), valid AS (
+          SELECT NOT EXISTS(SELECT 1 FROM page WHERE octet_length(payload::text) > ${REPLICATION_RECORD_BYTES}) AS ok
+        ), inserted AS (
+          INSERT INTO _ontology_replication_baselines (generation, event)
+          SELECT $1::uuid, jsonb_build_object('source_id', $3::text, 'entity_id', $4::text,
+            'record_key', record_key, 'record_kind', $5::text, 'revision', '0',
+            'deleted', false, 'schema_hash', $6::text, 'payload', payload::text)
+          FROM page WHERE (SELECT ok FROM valid) RETURNING id
+        ), queued AS (
+          INSERT INTO _ontology_replication_delivery (generation, event_id, revision, is_baseline)
+          SELECT $1::uuid, id, 0, true FROM inserted RETURNING event_id
+        ) SELECT (SELECT count(*)::integer FROM queued) AS count,
+          (SELECT max(record_key) FROM page) AS key, NOT (SELECT ok FROM valid) AS oversized`, [
+            generation,
+            target.baseline_cursor.key ?? "",
+            target.source_id,
+            source.entityId,
+            source.kind,
+            target.manifest.hash,
+        ]);
+        if (page.oversized) {
             return yield* new ReplicationBaselineFailed({
                 message: `A projected baseline record exceeds ${REPLICATION_RECORD_BYTES} bytes; ` +
                     "exclude large fields before enabling replication.",
             });
         }
-        yield* sql.unsafe(`INSERT INTO _ontology_replication_baselines (generation, event)
-      SELECT $1::uuid, jsonb_build_object('source_id', $2::text, 'entity_id', $3::text,
-        'record_key', record_key, 'record_kind', $4::text, 'revision', $5::text,
-        'deleted', false, 'schema_hash', $6::text, 'payload', payload::text)
-      FROM (${records}) snapshot`, [target.generation, source, entityId, kind, state.revision, manifest.hash]);
-    });
-    for (const item of input.selection) {
-        const entity = manifest.entities.find((e) => e.id === item.entityId);
-        yield* baseline(entity.id, "entity", `SELECT r.id::text AS record_key,
-        (SELECT jsonb_object_agg(key, value) FROM jsonb_each(${snapshotExpression(entity, "r")})
-         WHERE key = 'id' OR ${ql(JSON.stringify(item.fields))}::jsonb ? key) AS payload
-       FROM ${qi(tableName(entity.name))} r`);
-        for (const relationship of entity.relationships.filter((r) => item.relationships?.includes(r.name))) {
-            const columns = joinColumnNames({
-                entity: entity.name,
-                relationship: relationship.name,
-                target: relationship.to,
+        const cursor = page.count < BASELINE_PAGE_SIZE ? { index: index + 1 } : { index, key: page.key };
+        yield* sql.unsafe("UPDATE _ontology_replication_targets SET baseline_cursor = $2::jsonb WHERE generation = $1::uuid", [generation, JSON.stringify(cursor)]);
+        return { target, finish: false };
+    }));
+    if (!progress.finish) {
+        return progress.target;
+    }
+    // Do not upgrade a shared lock: take the same legacy-row/exclusive-lock order as pause/migrate.
+    return yield* sql.withTransaction(Effect.gen(function* () {
+        yield* sql.unsafe("SET LOCAL lock_timeout = '5s'");
+        yield* lockReplicationLifecycle(sql);
+        const [state] = yield* sql.unsafe("SELECT enabled, active_generation::text FROM _ontology_replication WHERE id");
+        if (!state?.enabled || state.active_generation !== generation) {
+            return yield* new ReplicationBaselineFailed({
+                message: "Resume the active replication generation before completing its baseline.",
             });
-            yield* baseline(entity.id, `edge:${relationship.name}`, `SELECT ${qi(columns.source)}::text || ':' || ${qi(columns.target)}::text AS record_key,
-          jsonb_build_object('source_id', ${qi(columns.source)}::text,
-            'target_id', ${qi(columns.target)}::text) AS payload
-         FROM ${qi(joinTableName(entity.name, relationship.name))}`);
         }
-    }
-    return target;
-}));
+        yield* sql.unsafe(`UPDATE _ontology_replication_targets SET baseline_complete = true,
+        initial_revision = (SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM _ontology_replication_revision)
+        WHERE generation = $1::uuid AND NOT baseline_complete`, [generation]);
+        const [target] = yield* sql.unsafe(targetSql, [generation]);
+        return target;
+    }));
+});
+/** Prepare once, then advance a bounded page; subsequent calls resume the same durable target. */
+export const createReplicationBaseline = (sql, input) => prepareReplicationBaseline(sql, input).pipe(Effect.flatMap((target) => target.baseline_complete === false
+    ? advanceReplicationBaseline(sql, target.generation)
+    : Effect.succeed(target)));

package/dist/replication/legacy-baseline.d.ts

@@ -0,0 +1,27 @@
+import { Effect, Schema } from "effect";
+import type { SqlClient } from "effect/unstable/sql";
+import type { OntologyManifest } from "../manifest.js";
+import { type ReplicationSelection, type ReplicationExclusion } from "./model.js";
+export interface ReplicationTarget {
+    readonly generation: string;
+    readonly source_id: string;
+    readonly from_revision: string;
+    readonly manifest: OntologyManifest;
+    readonly selection: ReadonlyArray<ReplicationSelection>;
+}
+declare const ReplicationBaselineFailed_base: Schema.Class<ReplicationBaselineFailed, Schema.TaggedStruct<"ReplicationBaselineFailed", {
+    readonly message: Schema.String;
+    readonly cause: Schema.optionalKey<Schema.Unknown>;
+}>, import("effect/Cause").YieldableError>;
+/** An initial snapshot cannot be created with the requested state or record projection. */
+export declare class ReplicationBaselineFailed extends ReplicationBaselineFailed_base {
+}
+/** All baseline rows are committed with capture enablement; interrupted setup can safely resume. */
+export declare const createReplicationBaseline: (sql: SqlClient.SqlClient, input: {
+    readonly generation: string;
+    readonly manifest: OntologyManifest;
+    readonly exclusions?: ReadonlyArray<ReplicationExclusion>;
+    readonly selection: ReadonlyArray<ReplicationSelection>;
+    readonly newSource?: boolean;
+}) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError | ReplicationBaselineFailed, never>;
+export {};

package/dist/replication/legacy-baseline.js

@@ -0,0 +1,106 @@
+import { randomUUID } from "node:crypto";
+import { Effect, Schema } from "effect";
+import { joinColumnNames, joinTableName, quoteIdentifier as qi, quoteLiteral as ql, tableName, } from "../migrate/storage.js";
+import { REPLICATION_RECORD_BYTES, validateReplicationSelection, } from "./model.js";
+import { snapshotExpression } from "./schema.js";
+/** An initial snapshot cannot be created with the requested state or record projection. */
+export class ReplicationBaselineFailed extends Schema.TaggedError()("ReplicationBaselineFailed", { message: Schema.String, cause: Schema.optionalKey(Schema.Unknown) }) {
+}
+/** All baseline rows are committed with capture enablement; interrupted setup can safely resume. */
+export const createReplicationBaseline = (sql, input) => sql.withTransaction(Effect.gen(function* () {
+    yield* sql.unsafe("SET LOCAL statement_timeout = '30s'");
+    yield* sql.unsafe("SET LOCAL lock_timeout = '5s'");
+    const [state] = yield* sql.unsafe("SELECT source_id::text, revision::text, enabled FROM _ontology_replication WHERE id FOR UPDATE");
+    if (!state) {
+        return yield* new ReplicationBaselineFailed({
+            message: "Run the ontology SDK migration before enabling replication.",
+        });
+    }
+    const existing = yield* sql.unsafe("SELECT * FROM _ontology_replication_targets WHERE generation = $1::uuid", [input.generation]);
+    if (existing[0]) {
+        return existing[0];
+    }
+    if (state.enabled) {
+        return yield* new ReplicationBaselineFailed({
+            message: "Pause replication before starting a new generation.",
+        });
+    }
+    const manifest = input.manifest;
+    yield* Effect.try({
+        try: () => validateReplicationSelection(manifest, input.selection),
+        catch: (cause) => new ReplicationBaselineFailed({ message: "Invalid replication projection.", cause }),
+    });
+    // Drain older SDK transactions that started before the compatibility fence was enabled.
+    const tables = manifest.entities
+        .filter((e) => e.source === undefined && !e.builtIn)
+        .flatMap((e) => [
+        tableName(e.name),
+        ...e.relationships
+            .filter((r) => r.cardinality === "many")
+            .map((r) => joinTableName(e.name, r.name)),
+    ]);
+    if (tables.length > 0) {
+        yield* sql.unsafe(`LOCK TABLE ${tables.map(qi).join(", ")} IN SHARE MODE`);
+    }
+    const source = input.newSource ? randomUUID() : state.source_id;
+    const selection = Object.fromEntries(input.selection.map((s) => [s.entityId, s]));
+    yield* sql.unsafe(`UPDATE _ontology_replication SET source_id = $1::uuid, selection = $2::jsonb, enabled = true,
+          exclusions = $3::jsonb, active_generation = $4::uuid WHERE id`, [
+        source,
+        JSON.stringify(selection),
+        JSON.stringify(input.exclusions ?? []),
+        input.generation,
+    ]);
+    const target = {
+        generation: input.generation,
+        source_id: source,
+        from_revision: state.revision,
+        manifest,
+        selection: input.selection,
+    };
+    yield* sql.unsafe(`INSERT INTO _ontology_replication_targets (generation, source_id, from_revision, manifest, selection)
+     VALUES ($1::uuid, $2::uuid, $3::bigint, $4::jsonb, $5::jsonb)`, [
+        target.generation,
+        source,
+        state.revision,
+        JSON.stringify(manifest),
+        JSON.stringify(input.selection),
+    ]);
+    const baseline = (entityId, kind, records) => Effect.gen(function* () {
+        // Keep the snapshot in PostgreSQL; large baselines must not fill a serverless process's heap.
+        const [size] = yield* sql.unsafe(`SELECT EXISTS (
+              SELECT 1 FROM (${records}) snapshot
+              WHERE octet_length(payload::text) > ${REPLICATION_RECORD_BYTES}
+            ) AS oversized`);
+        if (size.oversized) {
+            return yield* new ReplicationBaselineFailed({
+                message: `A projected baseline record exceeds ${REPLICATION_RECORD_BYTES} bytes; ` +
+                    "exclude large fields before enabling replication.",
+            });
+        }
+        yield* sql.unsafe(`INSERT INTO _ontology_replication_baselines (generation, event)
+      SELECT $1::uuid, jsonb_build_object('source_id', $2::text, 'entity_id', $3::text,
+        'record_key', record_key, 'record_kind', $4::text, 'revision', $5::text,
+        'deleted', false, 'schema_hash', $6::text, 'payload', payload::text)
+      FROM (${records}) snapshot`, [target.generation, source, entityId, kind, state.revision, manifest.hash]);
+    });
+    for (const item of input.selection) {
+        const entity = manifest.entities.find((e) => e.id === item.entityId);
+        yield* baseline(entity.id, "entity", `SELECT r.id::text AS record_key,
+        (SELECT jsonb_object_agg(key, value) FROM jsonb_each(${snapshotExpression(entity, "r")})
+         WHERE key = 'id' OR ${ql(JSON.stringify(item.fields))}::jsonb ? key) AS payload
+       FROM ${qi(tableName(entity.name))} r`);
+        for (const relationship of entity.relationships.filter((r) => item.relationships?.includes(r.name))) {
+            const columns = joinColumnNames({
+                entity: entity.name,
+                relationship: relationship.name,
+                target: relationship.to,
+            });
+            yield* baseline(entity.id, `edge:${relationship.name}`, `SELECT ${qi(columns.source)}::text || ':' || ${qi(columns.target)}::text AS record_key,
+          jsonb_build_object('source_id', ${qi(columns.source)}::text,
+            'target_id', ${qi(columns.target)}::text) AS payload
+         FROM ${qi(joinTableName(entity.name, relationship.name))}`);
+        }
+    }
+    return target;
+}));

package/dist/replication/legacy-store.d.ts

@@ -0,0 +1,23 @@
+import { Effect } from "effect";
+import type { SqlClient } from "effect/unstable/sql";
+import { type ReplicationEvent } from "./model.js";
+import type { ReplicationTarget } from "./baseline.js";
+/** No timestamp/high-water cursor: every unacknowledged immutable event remains discoverable. */
+export declare function replicationEventsSql(target: ReplicationTarget): string;
+export declare const pendingReplicationEvents: (sql: SqlClient.SqlClient, target: ReplicationTarget) => Effect.Effect<{
+    readonly historyId: string;
+    readonly event: ReplicationEvent;
+}[], import("effect/unstable/sql/SqlError").SqlError, never>;
+/** Call only after Pub/Sub accepts the complete batch. Lost responses deliberately produce duplicates. */
+export declare const acknowledgeReplication: (sql: SqlClient.SqlClient, generation: string, ids: ReadonlyArray<string>) => import("effect/unstable/sql/Statement").Statement<object>;
+export declare const replayReplication: (sql: SqlClient.SqlClient, generation: string) => import("effect/unstable/sql/Statement").Statement<object>;
+export declare const replicationStatus: (sql: SqlClient.SqlClient, target: ReplicationTarget) => Effect.Effect<{
+    total: string;
+    accepted: string;
+    verified: string;
+}, import("effect/unstable/sql/SqlError").SqlError, never>;
+export declare const unverifiedReplicationEvents: (sql: SqlClient.SqlClient, target: ReplicationTarget) => Effect.Effect<{
+    historyId: string;
+    eventId: string;
+}[], import("effect/unstable/sql/SqlError").SqlError, never>;
+export declare const verifyReplicationEvents: (sql: SqlClient.SqlClient, generation: string, ids: ReadonlyArray<string>) => import("effect/unstable/sql/Statement").Statement<object>;

package/dist/replication/legacy-store.js

@@ -0,0 +1,61 @@
+import { Effect } from "effect";
+import { historyTableName, quoteIdentifier as qi } from "../migrate/storage.js";
+import { REPLICATION_BATCH_BYTES, REPLICATION_BATCH_SIZE, publishMessage, replicationEventId, } from "./model.js";
+/** No timestamp/high-water cursor: every unacknowledged immutable event remains discoverable. */
+export function replicationEventsSql(target) {
+    const histories = target.selection.map((item) => {
+        const entity = target.manifest.entities.find((e) => e.id === item.entityId);
+        return `SELECT id, replication AS event FROM ${qi(historyTableName(entity.name))}
+      WHERE replication IS NOT NULL AND replication->>'source_id' = $2
+      AND (replication->>'revision')::bigint > $3::bigint`;
+    });
+    const baseline = `SELECT id, event FROM _ontology_replication_baselines WHERE generation = $1::uuid
+    AND $2::text IS NOT NULL AND $3::bigint IS NOT NULL`;
+    return [baseline, ...histories].join(" UNION ALL ");
+}
+export const pendingReplicationEvents = (sql, target) => sql
+    .unsafe(`SELECT e.id::text, e.event FROM (${replicationEventsSql(target)}) e
+     WHERE NOT EXISTS (SELECT 1 FROM _ontology_replication_receipts r
+       WHERE r.generation = $1::uuid AND r.event_id = e.id)
+     ORDER BY (event->>'revision')::bigint, id LIMIT ${REPLICATION_BATCH_SIZE}`, [target.generation, target.source_id, target.from_revision])
+    .pipe(Effect.map((rows) => {
+    let size = 32;
+    const result = [];
+    for (const row of rows) {
+        const event = {
+            ...row.event,
+            generation: target.generation,
+            event_id: replicationEventId(target.source_id, row.id, target.generation),
+        };
+        const bytes = Buffer.byteLength(JSON.stringify(publishMessage(event))) + 1;
+        if (bytes + 32 > REPLICATION_BATCH_BYTES) {
+            throw new Error("A replication event exceeds the publish request limit.");
+        }
+        if (size + bytes > REPLICATION_BATCH_BYTES) {
+            break;
+        }
+        size += bytes;
+        result.push({ historyId: row.id, event });
+    }
+    return result;
+}));
+/** Call only after Pub/Sub accepts the complete batch. Lost responses deliberately produce duplicates. */
+export const acknowledgeReplication = (sql, generation, ids) => sql.unsafe(`INSERT INTO _ontology_replication_receipts (generation, event_id)
+     SELECT $1::uuid, value::uuid FROM jsonb_array_elements_text($2::jsonb) ON CONFLICT DO NOTHING`, [generation, JSON.stringify(ids)]);
+export const replayReplication = (sql, generation) => sql.unsafe("DELETE FROM _ontology_replication_receipts WHERE generation = $1::uuid", [
+    generation,
+]);
+export const replicationStatus = (sql, target) => sql
+    .unsafe(`SELECT count(*)::text AS total, count(r.event_id)::text AS accepted, count(r.verified_at)::text AS verified
+     FROM (${replicationEventsSql(target)}) e
+     LEFT JOIN _ontology_replication_receipts r ON r.generation = $1::uuid AND r.event_id = e.id`, [target.generation, target.source_id, target.from_revision])
+    .pipe(Effect.map((rows) => rows[0]));
+export const unverifiedReplicationEvents = (sql, target) => sql
+    .unsafe(`SELECT event_id::text AS id FROM _ontology_replication_receipts
+     WHERE generation = $1::uuid AND verified_at IS NULL ORDER BY accepted_at, event_id LIMIT 1000`, [target.generation])
+    .pipe(Effect.map((rows) => rows.map(({ id }) => ({
+    historyId: id,
+    eventId: replicationEventId(target.source_id, id, target.generation),
+}))));
+export const verifyReplicationEvents = (sql, generation, ids) => sql.unsafe(`UPDATE _ontology_replication_receipts SET verified_at = now()
+    WHERE generation = $1::uuid AND event_id IN (SELECT value::uuid FROM jsonb_array_elements_text($2::jsonb))`, [generation, JSON.stringify(ids)]);

package/dist/replication/schema.d.ts

@@ -1,5 +1,5 @@
 import type { EntityManifest, OntologyManifest } from "../manifest.js";
-export declare const REPLICATION_SCHEMA_VERSION = 3;
+export declare const REPLICATION_SCHEMA_VERSION = 4;
 /** Installed only by the SDK migration runner, never at service startup. */
 export declare const REPLICATION_DDL: string[];
 export declare function snapshotExpression(entity: EntityManifest, alias: string): string;