@segmentstream/ontology 0.0.0-stage → 0.92.1

package/dist/replication/source-replacement.js

@@ -0,0 +1,76 @@
+import { randomUUID } from "node:crypto";
+import { Effect } from "effect";
+import { ReplicationBaselineFailed } from "./baseline.js";
+import { lockReplicationLifecycle } from "./transaction.js";
+export const pendingSourceReplacement = (sql) => sql
+    .unsafe(`SELECT source_id::text,operation_id::text,expected,proof,
+      COALESCE(proof->>'published',attempts->0->'expected'->>'epoch') AS published
+      FROM _ontology_replication_source_replacements
+    WHERE state='pending' AND source_id=(SELECT source_id FROM _ontology_replication WHERE id)`)
+    .pipe(Effect.map((rows) => rows[0]));
+/** Only the trusted product publication can justify replacing an otherwise healthy old source. */
+export const prepareSourceReplacement = (sql, input) => sql.withTransaction(Effect.gen(function* () {
+    yield* lockReplicationLifecycle(sql);
+    const pending = yield* pendingSourceReplacement(sql);
+    if (pending) {
+        return pending;
+    }
+    const [state] = yield* sql.unsafe(`SELECT EXISTS(SELECT 1 FROM _ontology_replication r
+      WHERE r.id AND r.enabled AND r.source_id=$1::uuid
+        AND NOT EXISTS(SELECT 1 FROM _ontology_replication_targets WHERE generation=$2::uuid AND source_id=r.source_id)
+        AND NOT EXISTS(SELECT 1 FROM _ontology_replication_epochs
+          WHERE source_id=r.source_id AND (state='serving' OR installed))
+        AND NOT EXISTS(SELECT 1 FROM _ontology_replication_source_replacements WHERE source_id=r.source_id)) AS valid`, [input.sourceId, input.published]);
+    if (!state?.valid ||
+        input.observed.sourceId === input.sourceId ||
+        input.proof.published !== input.published ||
+        input.proof.sourceId !== input.observed.sourceId ||
+        input.proof.selectorIncarnation !== input.observed.incarnation ||
+        input.proof.workspaceId !== input.workspaceId ||
+        input.proof.projectId !== input.projectId) {
+        return yield* new ReplicationBaselineFailed({
+            message: "replication_source_replacement_not_proven",
+        });
+    }
+    const operation = randomUUID();
+    yield* sql.unsafe(`INSERT INTO _ontology_replication_source_replacements
+      (source_id,operation_id,expected,state,attempts,proof) VALUES($1::uuid,$2::uuid,$3::jsonb,'pending',
+        jsonb_build_array(jsonb_build_object('operation_id',$2::text,'expected',$3::jsonb)),$4::jsonb)`, [input.sourceId, operation, JSON.stringify(input.observed), JSON.stringify(input.proof)]);
+    yield* sql.unsafe(`UPDATE _ontology_replication SET destination_rebuilding=true,destination_repair_pending=true WHERE id`);
+    return {
+        source_id: input.sourceId,
+        operation_id: operation,
+        expected: input.observed,
+        proof: input.proof,
+        published: input.published,
+    };
+}));
+/** A confirmed no-op CAS may race an old-source handoff; retain its receipt before a new attempt. */
+export const retrySourceReplacement = (sql, intent, observed) => sql.withTransaction(Effect.gen(function* () {
+    yield* lockReplicationLifecycle(sql);
+    if (observed.sourceId !== intent.expected.sourceId ||
+        observed.incarnation !== intent.expected.incarnation) {
+        return yield* new ReplicationBaselineFailed({
+            message: "replication_source_transition_changed",
+        });
+    }
+    const operation = randomUUID();
+    yield* sql.unsafe(`UPDATE _ontology_replication_source_replacements SET expected=$3::jsonb,
+      operation_id=$4::uuid,attempts=attempts || jsonb_build_array(jsonb_build_object(
+        'operation_id',$4::text,'expected',$3::jsonb))
+      WHERE source_id=$1::uuid AND operation_id=$2::uuid AND state='pending'`, [intent.source_id, intent.operation_id, JSON.stringify(observed), operation]);
+}));
+/** Upgrade only an already admitted old journal; the provider evidence is read before this transaction. */
+export const recordSourcePublicationProof = (sql, intent, proof) => sql.withTransaction(Effect.gen(function* () {
+    yield* lockReplicationLifecycle(sql);
+    if (proof.published !== intent.published ||
+        proof.sourceId !== intent.expected.sourceId ||
+        proof.selectorIncarnation !== intent.expected.incarnation) {
+        return yield* new ReplicationBaselineFailed({
+            message: "replication_source_replacement_not_proven",
+        });
+    }
+    yield* sql.unsafe(`UPDATE _ontology_replication_source_replacements SET proof=$3::jsonb
+      WHERE source_id=$1::uuid AND operation_id=$2::uuid AND state='pending' AND proof IS NULL
+        AND source_id=(SELECT source_id FROM _ontology_replication WHERE id AND enabled)`, [intent.source_id, intent.operation_id, JSON.stringify(proof)]);
+}));

package/dist/replication/store.d.ts

@@ -0,0 +1,36 @@
+import { Effect } from "effect";
+import type { SqlClient } from "effect/unstable/sql";
+import { type ReplicationEvent, type ReplicationSelection, type ReplicationExclusion } from "./model.js";
+import { advanceReplicationBaseline, ReplicationBaselineFailed, type ReplicationTarget } from "./baseline.js";
+export type { ReplicationTarget } from "./baseline.js";
+export declare const startReplication: (sql: SqlClient.SqlClient, input: {
+    readonly generation: string;
+    readonly exclusions?: ReadonlyArray<ReplicationExclusion>;
+    readonly selection?: ReadonlyArray<ReplicationSelection>;
+    readonly newSource?: boolean;
+}) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError | ReplicationBaselineFailed | import("../index.js").CatalogInvalid | import("./legacy-baseline.js").ReplicationBaselineFailed, never>;
+export declare const activeReplicationTarget: (sql: SqlClient.SqlClient) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError, never>;
+export declare const pauseReplication: (sql: SqlClient.SqlClient) => Effect.Effect<void, import("effect/unstable/sql/SqlError").SqlError, never>;
+export declare const replicationTarget: (sql: SqlClient.SqlClient, generation: string) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError, never>;
+export { advanceReplicationBaseline };
+export { replicationInitiallyReady as legacyReplicationInitiallyReady } from "./legacy-store.js";
+/** Retained history is recovery storage; frequent delivery reads only indexed unaccepted references. */
+export declare const pendingReplicationEvents: (sql: SqlClient.SqlClient, target: ReplicationTarget, options?: {
+    readonly bulk?: boolean;
+    readonly through?: string;
+}) => Effect.Effect<{
+    readonly historyId: string;
+    readonly event: ReplicationEvent;
+}[], import("effect/unstable/sql/SqlError").SqlError, never>;
+/** Acceptance comes only from Pub/Sub receipts; historical verification columns are not used. */
+export declare const replicationStatus: (sql: SqlClient.SqlClient, target: ReplicationTarget) => Effect.Effect<{
+    total: string;
+    accepted: string;
+}, import("effect/unstable/sql/SqlError").SqlError, never>;
+/** Initialization is a fixed fence, so continuing writes cannot move the readiness goal. */
+export declare const replicationInitiallyReady: (sql: SqlClient.SqlClient, generation: string) => Effect.Effect<boolean, import("effect/unstable/sql/SqlError").SqlError, never>;
+export declare const advanceReplicationReplay: (sql: SqlClient.SqlClient, generation: string) => Effect.Effect<boolean, import("effect/unstable/sql/SqlError").SqlError, never>;
+/** Static pre-upgrade histories have no new inserts: v4 writers atomically enqueue new history. */
+export declare const advanceReplicationBackfill: (sql: SqlClient.SqlClient, generation: string) => Effect.Effect<boolean, import("effect/unstable/sql/SqlError").SqlError | import("../index.js").CatalogInvalid, never>;
+export declare const acknowledgeReplication: (sql: SqlClient.SqlClient, generation: string, ids: ReadonlyArray<string>, version?: number) => Effect.Effect<void, import("effect/unstable/sql/SqlError").SqlError, never>;
+export declare const replayReplication: (sql: SqlClient.SqlClient, generation: string) => Effect.Effect<void, import("effect/unstable/sql/SqlError").SqlError, never>;

package/dist/replication/store.js

@@ -0,0 +1,382 @@
+import { refreshShadowCapturePolicy, shadowWireEvent } from "./shadow.js";
+import { randomUUID } from "node:crypto";
+import { Effect } from "effect";
+import * as legacy from "./legacy-store.js";
+import { createReplicationBaseline as createLegacyBaseline } from "./legacy-baseline.js";
+import { lockReplicationLifecycle, shareReplicationLifecycle } from "./transaction.js";
+import { canonicalJson } from "../manifest.js";
+import { readCatalog } from "../migrate/apply.js";
+import { historyTableName, quoteIdentifier as qi } from "../migrate/storage.js";
+import { REPLICATION_BATCH_BYTES, REPLICATION_BATCH_SIZE, publishMessage, replicationEventId, resolveReplicationSelection, } from "./model.js";
+import { createReplicationBaseline, advanceReplicationBaseline, ReplicationBaselineFailed, } from "./baseline.js";
+import { effectiveReplicationExclusions, migratedReplicationFills } from "./migrate.js";
+import { refreshReplicationProjection } from "./evolution.js";
+import { REPLICATION_TARGET_FILLS, REPLICATION_TARGET_METADATA } from "./schema.js";
+const replicationPolicyKey = (policy) => JSON.stringify(policy
+    .map((item) => ({
+    entityId: item.entityId,
+    disabled: item.disabled ?? false,
+    fields: [...new Set(item.fields ?? [])].sort(),
+    relationships: [...new Set(item.relationships ?? [])].sort(),
+}))
+    .sort((a, b) => a.entityId.localeCompare(b.entityId)));
+export const startReplication = (sql, input) => Effect.gen(function* () {
+    const manifest = yield* readCatalog({ sql });
+    if (!manifest) {
+        return yield* new ReplicationBaselineFailed({
+            message: "The ontology has not been migrated.",
+        });
+    }
+    const version = yield* protocol(sql);
+    const [state] = yield* sql.unsafe("SELECT * FROM _ontology_replication WHERE id");
+    const requested = yield* replicationTarget(sql, input.generation);
+    const previous = version >= 6 && requested ? ((yield* activeReplicationTarget(sql)) ?? requested) : requested;
+    const policy = input.exclusions ?? state?.exclusions ?? [];
+    const persisted = replicationPolicyKey(policy) === replicationPolicyKey(state?.exclusions ?? []);
+    const paused = version >= 6 && state?.enabled === false;
+    if (previous &&
+        !paused &&
+        persisted &&
+        previous.manifest.hash === manifest.hash &&
+        input.selection === undefined) {
+        return previous;
+    }
+    const selection = input.selection ??
+        (yield* Effect.try({
+            try: () => resolveReplicationSelection(manifest, persisted ? effectiveReplicationExclusions(manifest, policy) : policy),
+            catch: (cause) => new ReplicationBaselineFailed({ message: "Invalid replication exclusions.", cause }),
+        }));
+    if (previous &&
+        (paused ||
+            !persisted ||
+            previous.manifest.hash !== manifest.hash ||
+            canonicalJson(previous.selection) !== canonicalJson(selection))) {
+        if (version >= 5) {
+            return yield* sql.withTransaction(Effect.gen(function* () {
+                yield* lockReplicationLifecycle(sql);
+                const [capture] = yield* sql.unsafe("SELECT enabled FROM _ontology_replication WHERE id");
+                const current = version >= 6
+                    ? (yield* activeReplicationTarget(sql))
+                    : (yield* replicationTarget(sql, input.generation));
+                const catalog = yield* readCatalog({ sql });
+                if (catalog?.hash !== manifest.hash) {
+                    return yield* new ReplicationBaselineFailed({
+                        message: "The ontology changed while configuring replication; retry the current projection.",
+                    });
+                }
+                const fills = yield* migratedReplicationFills(sql, current, manifest, selection);
+                yield* refreshReplicationProjection(sql, current, { ...current, manifest, selection, fills }, undefined, { resuming: version >= 6 && capture?.enabled === false });
+                yield* sql.unsafe("UPDATE _ontology_replication SET selection=$1::jsonb,exclusions=$2::jsonb,enabled=true WHERE id", [
+                    JSON.stringify(Object.fromEntries(selection.map((item) => [item.entityId, item]))),
+                    JSON.stringify(policy),
+                ]);
+                if (version >= 6) {
+                    yield* sql.unsafe(`UPDATE _ontology_replication_epochs SET policy_installed=false
+                WHERE id=(SELECT active_generation FROM _ontology_replication WHERE id)`);
+                    yield* refreshShadowCapturePolicy(sql);
+                    return (yield* activeReplicationTarget(sql));
+                }
+                return (yield* replicationTarget(sql, input.generation));
+            }));
+        }
+        // Older source schemas cannot hold scoped scan cursors. An automatic generation copy is
+        // @legacy ontology.replication-older-source-protocols
+        // the compatibility path until the next ordinary ontology migration installs schema 5.
+        return yield* version >= 4
+            ? createReplicationBaseline(sql, {
+                ...input,
+                manifest,
+                selection,
+                generation: randomUUID(),
+                newSource: true,
+            })
+            : createLegacyBaseline(sql, {
+                ...input,
+                manifest,
+                selection,
+                generation: randomUUID(),
+                newSource: true,
+            });
+    }
+    return yield* version >= 4
+        ? createReplicationBaseline(sql, { ...input, manifest, selection })
+        : createLegacyBaseline(sql, { ...input, manifest, selection }).pipe(Effect.map((target) => target));
+}).pipe(Effect.map((target) => target));
+const targetFields = `generation::text, source_id::text, from_revision::text, manifest, selection,
+  protocol, baseline_complete, backfill_complete, initial_revision::text, replay_complete,
+  ${REPLICATION_TARGET_FILLS}, ${REPLICATION_TARGET_METADATA}, 4 AS delivery_version`;
+const protocol = (sql) => sql
+    .unsafe("SELECT version FROM _ontology_replication WHERE id")
+    .pipe(Effect.map((rows) => rows[0]?.version ?? 3));
+export const activeReplicationTarget = (sql) => Effect.gen(function* () {
+    const version = yield* protocol(sql);
+    const fields = version >= 4
+        ? targetFields
+        : "generation::text, source_id::text, from_revision::text, manifest, selection";
+    const rows = yield* sql.unsafe(`SELECT ${fields} FROM _ontology_replication_targets
+     WHERE generation = (SELECT active_generation FROM _ontology_replication WHERE id)`);
+    return rows[0];
+});
+export const pauseReplication = (sql) => sql.withTransaction(Effect.gen(function* () {
+    yield* lockReplicationLifecycle(sql);
+    yield* sql.unsafe("UPDATE _ontology_replication SET enabled = false WHERE id");
+}));
+export const replicationTarget = (sql, generation) => Effect.gen(function* () {
+    const version = yield* protocol(sql);
+    const fields = version >= 4
+        ? targetFields
+        : "generation::text, source_id::text, from_revision::text, manifest, selection";
+    const rows = yield* sql.unsafe(`SELECT ${fields} FROM _ontology_replication_targets WHERE generation = $1::uuid`, [generation]);
+    return rows[0];
+});
+export { advanceReplicationBaseline };
+export { replicationInitiallyReady as legacyReplicationInitiallyReady } from "./legacy-store.js";
+/** Retained history is recovery storage; frequent delivery reads only indexed unaccepted references. */
+export const pendingReplicationEvents = (sql, target, options = {}) => Effect.gen(function* () {
+    if (target.delivery_version !== 4 &&
+        target.protocol !== 4) {
+        return yield* legacy.pendingReplicationEvents(sql, target);
+    }
+    const shadow = (target.schema_version ?? 0) >= 6;
+    const references = yield* sql.unsafe(shadow
+        ? `WITH event_refs AS MATERIALIZED (
+       SELECT event_id::text AS id, history_table, payload, revision, payload_bytes
+       FROM _ontology_replication_delivery d
+       WHERE generation = $1::uuid
+         AND suppressed_at IS NULL
+         AND (CASE WHEN $2::boolean THEN committed_at IS NULL AND segment_id IS NULL ELSE accepted_at IS NULL END)
+         AND ($3::bigint IS NULL OR revision <= $3::bigint)
+         AND ($2::boolean OR NOT EXISTS (SELECT 1 FROM _ontology_replication_epochs e
+           WHERE e.state='candidate' AND e.token IS NOT NULL AND d.revision>e.boundary))
+       ORDER BY revision, event_id LIMIT ${options.bulk ? 5000 : REPLICATION_BATCH_SIZE}
+       ), bounded AS (
+         SELECT *,sum(payload_bytes) OVER(ORDER BY revision,id) AS bytes FROM event_refs
+       ) SELECT id,history_table,payload FROM bounded
+         WHERE bytes<=${options.bulk ? 32_000_000 : REPLICATION_BATCH_BYTES}
+         ORDER BY revision,id`
+        : `SELECT event_id::text AS id,history_table,to_jsonb(d)->'payload' AS payload
+       FROM _ontology_replication_delivery d WHERE generation=$1::uuid AND accepted_at IS NULL
+       ORDER BY revision,event_id LIMIT ${REPLICATION_BATCH_SIZE}`, shadow
+        ? [target.generation, options.bulk ?? false, options.through ?? null]
+        : [target.generation]);
+    const groups = new Map();
+    for (const reference of references) {
+        if (reference.payload) {
+            continue;
+        }
+        const ids = groups.get(reference.history_table) ?? [];
+        ids.push(reference.id);
+        groups.set(reference.history_table, ids);
+    }
+    const events = new Map(references
+        .filter((reference) => reference.payload !== null)
+        .map((reference) => [reference.id, reference.payload]));
+    for (const [table, ids] of groups) {
+        const rows = yield* sql.unsafe(table === null
+            ? `SELECT id::text, event FROM _ontology_replication_baselines
+             WHERE generation = $1::uuid AND id IN (SELECT value::uuid FROM jsonb_array_elements_text($2::jsonb))`
+            : `SELECT id::text, replication AS event FROM ${qi(table)}
+             WHERE $1::uuid IS NOT NULL AND id IN (SELECT value::uuid FROM jsonb_array_elements_text($2::jsonb))`, [target.generation, JSON.stringify(ids)]);
+        for (const row of rows) {
+            events.set(row.id, row.event);
+        }
+    }
+    const [wire] = shadow
+        ? yield* sql.unsafe("SELECT capture_bindings,capture_versions FROM _ontology_replication_targets WHERE generation=$1::uuid", [target.generation])
+        : [];
+    const batchBytes = options.bulk ? 32_000_000 : REPLICATION_BATCH_BYTES;
+    const [policy] = shadow
+        ? yield* sql.unsafe("SELECT exclusions FROM _ontology_replication WHERE id")
+        : [];
+    const suppressed = [];
+    let size = 32;
+    const result = [];
+    for (const reference of references) {
+        const payload = events.get(reference.id);
+        if (!payload) {
+            return yield* Effect.die(new Error("A queued replication event is missing from retained history."));
+        }
+        let event = {
+            ...payload,
+            generation: target.generation,
+            event_id: replicationEventId(target.source_id, reference.id, target.generation),
+        };
+        if (wire?.capture_bindings) {
+            const bindings = wire.capture_versions?.find((version) => version.through === undefined || BigInt(event.revision) <= BigInt(version.through))?.bindings ?? wire.capture_bindings;
+            event = shadowWireEvent(event, bindings, target.selection);
+        }
+        const restriction = policy?.exclusions.find((item) => item.entityId === event.entity_id);
+        if (restriction?.disabled ||
+            (event.record_kind.startsWith("edge:") &&
+                restriction?.relationships?.includes(event.record_kind.slice(5)))) {
+            suppressed.push(reference.id);
+            continue;
+        }
+        if (restriction && event.record_kind === "entity" && !event.deleted) {
+            const fields = new Set([
+                ...(restriction.fields ?? []),
+                ...(restriction.relationships ?? []).map((name) => `${name}_id`),
+            ]);
+            event = {
+                ...event,
+                payload: JSON.stringify(Object.fromEntries(Object.entries(JSON.parse(event.payload)).filter(([key]) => !fields.has(wire?.capture_bindings?.[event.entity_id]?.[key] ?? key)))),
+            };
+        }
+        const bytes = Buffer.byteLength(JSON.stringify(publishMessage(event))) + 1;
+        if (bytes + 32 > REPLICATION_BATCH_BYTES) {
+            return yield* Effect.die(new Error("A replication event exceeds the publish request limit."));
+        }
+        if (size + bytes > batchBytes) {
+            break;
+        }
+        size += bytes;
+        result.push({ historyId: reference.id, event });
+    }
+    if (suppressed.length) {
+        yield* sql.unsafe(`UPDATE _ontology_replication_delivery SET suppressed_at=now()
+        WHERE generation=$1::uuid AND event_id IN (SELECT value::uuid FROM jsonb_array_elements_text($2::jsonb))`, [target.generation, JSON.stringify(suppressed)]);
+    }
+    return result;
+});
+/** Lost publish responses deliberately leave the same immutable IDs pending. */
+const acknowledgeReplicationV4 = (sql, generation, ids) => sql.withTransaction(Effect.gen(function* () {
+    // Receipt foreign keys touch targets after delivery rows; drain before migration table locks.
+    yield* shareReplicationLifecycle(sql);
+    yield* sql.unsafe(`WITH accepted AS (
+          UPDATE _ontology_replication_delivery SET accepted_at = now()
+          WHERE generation = $1::uuid AND accepted_at IS NULL
+            AND event_id IN (SELECT value::uuid FROM jsonb_array_elements_text($2::jsonb))
+          RETURNING event_id, accepted_at
+        ) INSERT INTO _ontology_replication_receipts (generation, event_id, accepted_at)
+          SELECT $1::uuid, event_id, accepted_at FROM accepted ON CONFLICT DO NOTHING`, [generation, JSON.stringify(ids)]);
+}));
+// @legacy ontology.replication-verification-remnants
+/** Acceptance comes only from Pub/Sub receipts; historical verification columns are not used. */
+export const replicationStatus = (sql, target) => target.delivery_version !== 4 &&
+    target.protocol !== 4
+    ? legacy.replicationStatus(sql, target)
+    : sql
+        .unsafe(`SELECT count(*)::text AS total, count(accepted_at)::text AS accepted
+           FROM _ontology_replication_delivery WHERE generation = $1::uuid`, [target.generation])
+        .pipe(Effect.map((rows) => rows[0]));
+/** Initialization is a fixed fence, so continuing writes cannot move the readiness goal. */
+export const replicationInitiallyReady = (sql, generation) => sql
+    .unsafe(`SELECT baseline_complete AND backfill_complete AND replay_complete AND NOT EXISTS (
+      SELECT 1 FROM _ontology_replication_delivery d WHERE d.generation = t.generation
+        AND d.accepted_at IS NULL AND d.revision <= t.initial_revision
+    ) AS ready FROM _ontology_replication_targets t WHERE generation = $1::uuid`, [generation])
+    .pipe(Effect.map((rows) => rows[0]?.ready ?? false));
+/** Replay initialization is constant time; reset work is resumed in bounded pages. */
+const replayReplicationV4 = (sql, generation) => sql.withTransaction(Effect.gen(function* () {
+    yield* lockReplicationLifecycle(sql);
+    yield* sql.unsafe(`UPDATE _ontology_replication_targets SET replay_complete = false, replay_cursor = NULL,
+        initial_revision = (SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM _ontology_replication_revision)
+       WHERE generation = $1::uuid`, [generation]);
+}));
+export const advanceReplicationReplay = (sql, generation) => sql.withTransaction(Effect.gen(function* () {
+    const [target] = yield* sql.unsafe(`SELECT replay_complete, backfill_complete,
+         COALESCE((to_jsonb(_ontology_replication_targets)->>'scans_complete')::boolean,true) AS scans_complete,
+         replay_cursor::text, initial_revision::text
+       FROM _ontology_replication_targets
+       WHERE generation = $1::uuid FOR NO KEY UPDATE`, [generation]);
+    if (!target || target.replay_complete) {
+        return true;
+    }
+    // A legacy import can add accepted rows behind any replay cursor. Wait until its full
+    // queue exists before resetting the fixed prefix, including when replay starts mid-import.
+    if (!target.backfill_complete || !target.scans_complete) {
+        return false;
+    }
+    const rows = yield* sql.unsafe(`SELECT event_id::text AS id, verified_at IS NOT NULL AS verified FROM _ontology_replication_delivery
+       WHERE generation = $1::uuid AND ($2::uuid IS NULL OR event_id > $2::uuid)
+         AND revision <= $3::bigint
+       ORDER BY event_id LIMIT 1000`, [generation, target.replay_cursor, target.initial_revision]);
+    const ids = JSON.stringify(rows.map((row) => row.id));
+    yield* sql.unsafe(`UPDATE _ontology_replication_delivery SET accepted_at = NULL, verified_at = NULL, next_verify_at = now()
+       WHERE generation = $1::uuid AND event_id IN (SELECT value::uuid FROM jsonb_array_elements_text($2::jsonb))`, [generation, ids]);
+    yield* sql.unsafe(`DELETE FROM _ontology_replication_receipts WHERE generation = $1::uuid
+       AND event_id IN (SELECT value::uuid FROM jsonb_array_elements_text($2::jsonb))`, [generation, ids]);
+    const complete = rows.length < 1000;
+    yield* sql.unsafe(`UPDATE _ontology_replication_targets SET replay_cursor = $2::uuid, replay_complete = $3,
+        verified_count = verified_count - $4::bigint WHERE generation = $1::uuid`, [
+        generation,
+        rows.at(-1)?.id ?? target.replay_cursor,
+        complete,
+        rows.filter((row) => row.verified).length,
+    ]);
+    return complete;
+}));
+/** Static pre-upgrade histories have no new inserts: v4 writers atomically enqueue new history. */
+export const advanceReplicationBackfill = (sql, generation) => sql.withTransaction(Effect.gen(function* () {
+    yield* shareReplicationLifecycle(sql);
+    const [target] = yield* sql.unsafe(`SELECT ${targetFields}, backfill_cursor, backfill_revision::text
+       FROM _ontology_replication_targets WHERE generation = $1::uuid FOR NO KEY UPDATE`, [generation]);
+    if (!target || target.backfill_complete) {
+        return true;
+    }
+    const catalog = yield* readCatalog({ sql });
+    const tables = target.backfill_sources?.map((source) => source.table) ?? [
+        null,
+        ...target.selection.map((item) => {
+            const entity = (catalog ?? target.manifest).entities.find((entity) => entity.id === item.entityId) ??
+                target.manifest.entities.find((entity) => entity.id === item.entityId);
+            return historyTableName(entity.name);
+        }),
+    ];
+    const position = target.backfill_cursor.table ?? 0;
+    const table = tables[position];
+    if (target.backfill_sources?.[position]?.skip || position >= tables.length) {
+        const complete = position + 1 >= tables.length;
+        yield* sql.unsafe(`UPDATE _ontology_replication_targets SET backfill_cursor=$2::jsonb,backfill_complete=$3
+           WHERE generation=$1::uuid`, [generation, JSON.stringify({ table: position + 1 }), complete]);
+        return complete;
+    }
+    const cursor = target.backfill_cursor.id ?? null;
+    const rows = yield* sql.unsafe(table === null
+        ? `SELECT id::text FROM _ontology_replication_baselines WHERE generation = $1::uuid
+           AND ($2::uuid IS NULL OR id > $2::uuid) ORDER BY id LIMIT 1000`
+        : `SELECT id::text FROM ${qi(table)} WHERE replication IS NOT NULL
+           AND replication->>'source_id' = $3 AND (replication->>'revision')::bigint > $4::bigint
+           AND (replication->>'revision')::bigint <= $5::bigint
+           AND ($2::uuid IS NULL OR id > $2::uuid) AND $1::uuid IS NOT NULL ORDER BY id LIMIT 1000`, table === null
+        ? [generation, cursor]
+        : [generation, cursor, target.source_id, target.from_revision, target.backfill_revision]);
+    const ids = JSON.stringify(rows.map((row) => row.id));
+    const eventSelect = table === null
+        ? `SELECT id, event FROM _ontology_replication_baselines WHERE generation = $1::uuid`
+        : `SELECT id, replication AS event FROM ${qi(table)}`;
+    yield* sql.unsafe(`WITH inserted AS (
+        INSERT INTO _ontology_replication_delivery
+          (generation, event_id, history_table, revision, is_baseline, accepted_at, verified_at, next_verify_at)
+        SELECT $1::uuid, e.id, $3::text, (e.event->>'revision')::bigint, $4::boolean,
+          r.accepted_at, r.verified_at, now() FROM (${eventSelect}) e
+        LEFT JOIN _ontology_replication_receipts r ON r.generation = $1::uuid AND r.event_id = e.id
+        WHERE e.id IN (SELECT value::uuid FROM jsonb_array_elements_text($2::jsonb))
+        ON CONFLICT DO NOTHING RETURNING verified_at
+      ) UPDATE _ontology_replication_targets SET verified_count = verified_count +
+        (SELECT count(verified_at) FROM inserted) WHERE generation = $1::uuid`, [generation, ids, table, table === null]);
+    const finishedTable = rows.length < 1000;
+    const complete = finishedTable && position + 1 >= tables.length;
+    const next = finishedTable
+        ? { table: position + 1 }
+        : { table: position, id: rows.at(-1).id };
+    yield* sql.unsafe(`UPDATE _ontology_replication_targets SET backfill_cursor = $2::jsonb, backfill_complete = $3
+       WHERE generation = $1::uuid`, [generation, JSON.stringify(next), complete]);
+    return complete;
+}));
+export const acknowledgeReplication = (sql, generation, ids, version) => Effect.gen(function* () {
+    const current = version ?? (yield* protocol(sql));
+    yield* current >= 4
+        ? acknowledgeReplicationV4(sql, generation, ids)
+        : legacy.acknowledgeReplication(sql, generation, ids);
+});
+export const replayReplication = (sql, generation) => Effect.gen(function* () {
+    const current = yield* protocol(sql);
+    if (current >= 4) {
+        yield* replayReplicationV4(sql, generation);
+        yield* advanceReplicationReplay(sql, generation);
+    }
+    else {
+        yield* legacy.replayReplication(sql, generation);
+    }
+});

package/dist/replication/transaction.d.ts

@@ -0,0 +1,23 @@
+import { Effect } from "effect";
+import type { SqlClient } from "effect/unstable/sql";
+import { SqlError } from "effect/unstable/sql/SqlError";
+export declare const REPLICATION_LIFECYCLE_LOCK = "segmentstream.ontology.replication.lifecycle";
+/** Legacy writers take the singleton first; lifecycle operations must keep that order. */
+export declare const lockReplicationLifecycle: (sql: SqlClient.SqlClient) => Effect.Effect<void, SqlError, never>;
+/** Acquire the relation before admission, like writers, even when the first page read is a target. */
+export declare const shareReplicationLifecycle: (sql: SqlClient.SqlClient) => Effect.Effect<readonly object[], SqlError, never>;
+/**
+ * Must run in the migration transaction, before catalog or business locks. Acquire the strongest
+ * relation lock up front: even ADD COLUMN IF NOT EXISTS needs it when the column already exists.
+ * Old writers read/lock the singleton before lifecycle admission, so waiting for DDL after taking
+ * lifecycle admission would prevent those readers from ever finishing.
+ *
+ * Older online replication pages can take shared lifecycle admission before the relation. Never
+ * wait for them while holding the relation: roll back just this acquisition savepoint, drain their
+ * lifecycle locks without any relation lock, then try again. No migration work is replayed.
+ * Bound contention to eight attempts; each blocking statement also retains the migration's SQL
+ * timeouts. Sustained old-worker traffic must fail the migration instead of holding its mutex forever.
+ */
+export declare const lockReplicationSchema: (sql: SqlClient.SqlClient) => Effect.Effect<undefined, SqlError, never>;
+/** Must run inside the write transaction, before any business row locks. */
+export declare const beginReplicationWrite: (sql: SqlClient.SqlClient) => Effect.Effect<void, SqlError, never>;

package/dist/replication/transaction.js

@@ -0,0 +1,88 @@
+import { Data, Effect } from "effect";
+import { LockTimeoutError, SqlError } from "effect/unstable/sql/SqlError";
+export const REPLICATION_LIFECYCLE_LOCK = "segmentstream.ontology.replication.lifecycle";
+// @legacy ontology.replication-older-source-protocols
+/** Legacy writers take the singleton first; lifecycle operations must keep that order. */
+export const lockReplicationLifecycle = (sql) => Effect.gen(function* () {
+    yield* sql.unsafe("SELECT id FROM _ontology_replication WHERE id FOR UPDATE");
+    yield* sql.unsafe("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [
+        REPLICATION_LIFECYCLE_LOCK,
+    ]);
+});
+const sharedLifecycleLock = (sql) => sql.unsafe("SELECT pg_advisory_xact_lock_shared(hashtextextended($1, 0))", [
+    REPLICATION_LIFECYCLE_LOCK,
+]);
+/** Acquire the relation before admission, like writers, even when the first page read is a target. */
+export const shareReplicationLifecycle = (sql) => sql
+    .unsafe("LOCK TABLE _ontology_replication IN ACCESS SHARE MODE")
+    .pipe(Effect.andThen(sharedLifecycleLock(sql)));
+class RetryReplicationSchemaLock extends Data.TaggedError("RetryReplicationSchemaLock") {
+}
+const MAX_SCHEMA_LOCK_ATTEMPTS = 8;
+/**
+ * Must run in the migration transaction, before catalog or business locks. Acquire the strongest
+ * relation lock up front: even ADD COLUMN IF NOT EXISTS needs it when the column already exists.
+ * Old writers read/lock the singleton before lifecycle admission, so waiting for DDL after taking
+ * lifecycle admission would prevent those readers from ever finishing.
+ *
+ * Older online replication pages can take shared lifecycle admission before the relation. Never
+ * wait for them while holding the relation: roll back just this acquisition savepoint, drain their
+ * lifecycle locks without any relation lock, then try again. No migration work is replayed.
+ * Bound contention to eight attempts; each blocking statement also retains the migration's SQL
+ * timeouts. Sustained old-worker traffic must fail the migration instead of holding its mutex forever.
+ */
+export const lockReplicationSchema = (sql) => Effect.gen(function* () {
+    for (let attempt = 0; attempt < MAX_SCHEMA_LOCK_ATTEMPTS; attempt += 1) {
+        const acquired = yield* sql
+            .withTransaction(Effect.gen(function* () {
+            yield* sql.unsafe("LOCK TABLE _ontology_replication IN ACCESS EXCLUSIVE MODE");
+            // Keep the legacy singleton fence as well as the v4 lifecycle fence until commit.
+            yield* sql.unsafe("SELECT id FROM _ontology_replication WHERE id FOR UPDATE");
+            const [lock] = yield* sql.unsafe("SELECT pg_try_advisory_xact_lock(hashtextextended($1, 0)) AS acquired", [REPLICATION_LIFECYCLE_LOCK]);
+            if (!lock?.acquired) {
+                return yield* Effect.fail(new RetryReplicationSchemaLock());
+            }
+        }))
+            .pipe(Effect.as(true), Effect.catchTag("RetryReplicationSchemaLock", () => Effect.succeed(false)));
+        if (acquired) {
+            return;
+        }
+        if (attempt === MAX_SCHEMA_LOCK_ATTEMPTS - 1) {
+            break;
+        }
+        // Roll back the successful wait too, so the next attempt starts without either lock.
+        yield* sql
+            .withTransaction(sql
+            .unsafe("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [
+            REPLICATION_LIFECYCLE_LOCK,
+        ])
+            .pipe(Effect.andThen(Effect.fail(new RetryReplicationSchemaLock()))))
+            .pipe(Effect.catchTag("RetryReplicationSchemaLock", () => Effect.void));
+    }
+    return yield* new SqlError({
+        reason: new LockTimeoutError({
+            message: "Replication schema admission remained contended after eight attempts.",
+            operation: "acquire replication schema locks",
+            cause: { attempts: MAX_SCHEMA_LOCK_ATTEMPTS, lock: REPLICATION_LIFECYCLE_LOCK },
+        }),
+    });
+});
+/** Must run inside the write transaction, before any business row locks. */
+export const beginReplicationWrite = (sql) => Effect.gen(function* () {
+    const [schema] = yield* sql.unsafe("SELECT to_regclass('_ontology_replication') IS NOT NULL AS present");
+    if (!schema?.present) {
+        return;
+    }
+    // Shared admission permits independent writes while lifecycle changes drain admitted writers.
+    const [state] = yield* sql.unsafe("SELECT version FROM _ontology_replication WHERE id");
+    if ((state?.version ?? 0) < 4) {
+        // A new service may be deployed before its owner applies the protocol upgrade.
+        yield* sql.unsafe("SELECT id FROM _ontology_replication WHERE id FOR UPDATE");
+        yield* sql.unsafe("SELECT set_config('segmentstream.writer', '1', true)");
+    }
+    else {
+        // The version read already holds the relation lock; do not acquire it after admission.
+        yield* sharedLifecycleLock(sql);
+        yield* sql.unsafe("SELECT set_config('segmentstream.writer', '2', true)");
+    }
+});