@segmentstream/ontology 0.0.0-stage → 0.92.0

package/dist/replication/shadow.js

@@ -0,0 +1,614 @@
+import { replicationIdentities } from "./identities.js";
+import { queryFields } from "../query.js";
+import { createHash, randomUUID } from "node:crypto";
+import { Effect } from "effect";
+import { canonicalJson } from "../manifest.js";
+import { ReplicationBaselineFailed } from "./baseline.js";
+import { initialReplicationLineage, renamedReplicationField } from "./lineage.js";
+import { replicationScanScopes, scheduleReplicationScans, } from "./scans.js";
+import { lockReplicationLifecycle } from "./transaction.js";
+export const SHADOW_HANDOFF_MS = 30_000;
+export const SHADOW_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+export const SHADOW_QUERY_LIFETIME_MS = 60 * 60 * 1000;
+const shadowJson = (value) => JSON.stringify(value, (_key, item) => (typeof item === "bigint" ? item.toString() : item));
+const epochFields = `id::text, source_id::text, desired_hash, target, routes, state,
+  predecessor::text, boundary::text, catchup_boundary::text, token::text, fence::text,
+  handoff_until::text, operation_id::text, installed, cancel_requested`;
+const segmentFields = `id::text, epoch::text, generation::text, source_id::text,
+  through_revision::text, events, event_ids, digest, object_uri, job_id, attempt, state, retry_after::text`;
+/** Names the target keys a stored route omits because its epoch target holds the same value. */
+const SHARED_FROM_EPOCH = "shared_from_epoch";
+/** SQL reads these keys from stored routes, for example retention checks on route generations. */
+const STORED_ROUTE_KEYS = new Set(["generation", "source_id"]);
+const sameSharedValue = (key, route, epoch) => {
+    if (route === undefined || epoch === undefined) {
+        return false;
+    }
+    if (route === epoch) {
+        return true;
+    }
+    if (key === "manifest") {
+        const routeHash = route?.hash;
+        return typeof routeHash === "string" && routeHash === epoch?.hash;
+    }
+    try {
+        return canonicalJson(route) === canonicalJson(epoch);
+    }
+    catch {
+        return false;
+    }
+};
+/**
+ * Every route carries a full target, and most routes repeat the epoch target's manifest and
+ * projection. One copy per scope grows with entities times manifest size, past what a single
+ * PostgreSQL message may carry, so a stored route names the keys it takes from its epoch target.
+ */
+export function compactShadowRoutes(routes, target) {
+    const epoch = target;
+    return routes.map((route) => {
+        const record = route.target;
+        const shared = Object.keys(record).filter((key) => !STORED_ROUTE_KEYS.has(key) && sameSharedValue(key, record[key], epoch[key]));
+        if (shared.length === 0) {
+            return route;
+        }
+        const stored = { [SHARED_FROM_EPOCH]: shared };
+        for (const [key, value] of Object.entries(record)) {
+            if (!shared.includes(key)) {
+                stored[key] = value;
+            }
+        }
+        return { ...route, target: stored };
+    });
+}
+/** Restores the keys a stored route took from its epoch target. Full stored routes pass through. */
+export function expandShadowRoutes(routes, target) {
+    const epoch = target;
+    return routes.map((stored) => {
+        const route = stored;
+        const record = route.target;
+        const shared = record[SHARED_FROM_EPOCH];
+        if (!Array.isArray(shared)) {
+            return route;
+        }
+        const expanded = {};
+        for (const key of shared) {
+            expanded[key] = epoch[key];
+        }
+        for (const [key, value] of Object.entries(record)) {
+            if (key !== SHARED_FROM_EPOCH) {
+                expanded[key] = value;
+            }
+        }
+        return { ...route, target: expanded };
+    });
+}
+const expandEpoch = (row) => row && { ...row, routes: expandShadowRoutes(row.routes, row.target) };
+export const shadowEpoch = (sql, id) => sql
+    .unsafe(`SELECT ${epochFields} FROM _ontology_replication_epochs WHERE id=$1::uuid`, [id])
+    .pipe(Effect.map((rows) => expandEpoch(rows[0])));
+export const servingShadowEpoch = (sql) => sql
+    .unsafe(`SELECT ${epochFields} FROM _ontology_replication_epochs
+    WHERE id=(SELECT serving_epoch FROM _ontology_replication WHERE id)`)
+    .pipe(Effect.map((rows) => expandEpoch(rows[0])));
+export const candidateShadowEpoch = (sql) => sql
+    .unsafe(`SELECT ${epochFields} FROM _ontology_replication_epochs WHERE state='candidate'`)
+    .pipe(Effect.map((rows) => expandEpoch(rows[0])));
+export const shadowProjectionHash = (target) => createHash("sha256")
+    .update(canonicalJson({
+    manifest: target.manifest.hash,
+    selection: target.selection,
+    fills: target.fills ?? {},
+    identities: replicationIdentities(target),
+}))
+    .digest("hex");
+/** Called under exclusive lifecycle admission; no provider I/O is permitted here. */
+export const adoptShadowServing = (sql, target) => Effect.gen(function* () {
+    const existing = yield* servingShadowEpoch(sql);
+    if (existing) {
+        return existing;
+    }
+    target = { ...target, identities: replicationIdentities(target) };
+    const routes = replicationScanScopes(target).map((scope) => ({ ...scope, target }));
+    yield* sql.unsafe(`INSERT INTO _ontology_replication_epochs
+      (id,source_id,desired_hash,target,routes,state) VALUES ($1::uuid,$2::uuid,$3,$4::jsonb,$5::jsonb,'serving')
+      ON CONFLICT (id) DO UPDATE SET state='serving'`, [
+        target.generation,
+        target.source_id,
+        shadowProjectionHash(target),
+        shadowJson(target),
+        shadowJson(compactShadowRoutes(routes, target)),
+    ]);
+    yield* sql.unsafe(`UPDATE _ontology_replication SET serving_epoch=$1::uuid WHERE id`, [
+        target.generation,
+    ]);
+    yield* enableShadowCapture(sql, target, replicationScanScopes(target));
+    return (yield* shadowEpoch(sql, target.generation));
+});
+/** Capture bindings name wire fields, not public columns. Renames never rewrite historical JSON. */
+export const enableShadowCapture = (sql, target, scopes) => sql
+    .unsafe(`UPDATE _ontology_replication_targets SET capturing=true,
+    identities=COALESCE(identities,$4::jsonb),
+    capture_scopes=$2::jsonb, capture_bindings=COALESCE(capture_bindings,$3::jsonb),
+    capture_versions=COALESCE(capture_versions,jsonb_build_array(jsonb_build_object('bindings',$3::jsonb)))
+    WHERE generation=$1::uuid`, [
+    target.generation,
+    JSON.stringify(scopes),
+    JSON.stringify(Object.fromEntries(target.selection.map((item) => [
+        item.entityId,
+        Object.fromEntries([...new Set(["id", ...item.fields])].map((field) => [field, field])),
+    ]))),
+    JSON.stringify(replicationIdentities(target)),
+])
+    .pipe(Effect.andThen(refreshShadowCapturePolicy(sql)));
+/**
+ * Immutable epochs may share unaffected generations. Only complete affected entity scopes get a
+ * fresh incarnation. Generation-zero snapshots cannot overwrite mutations in another incarnation.
+ */
+export const prepareShadowEpoch = (sql, input) => Effect.gen(function* () {
+    if (input.predecessor) {
+        const previous = input.predecessor;
+        yield* sql.unsafe(`INSERT INTO _ontology_replication_epochs
+        (id,source_id,desired_hash,target,routes,state)
+        VALUES ($1::uuid,$2::uuid,$3,$4::jsonb,$5::jsonb,'retired') ON CONFLICT DO NOTHING`, [
+            previous.id,
+            previous.source_id,
+            previous.desired_hash,
+            shadowJson(previous.target),
+            shadowJson(compactShadowRoutes(previous.routes, previous.target)),
+        ]);
+    }
+    const desiredHash = shadowProjectionHash(input.target);
+    const pending = yield* candidateShadowEpoch(sql);
+    if (!pending?.cancel_requested &&
+        pending?.desired_hash === desiredHash &&
+        pending.predecessor === (input.predecessor?.id ?? null) &&
+        (!input.force || pending.routes.every((route) => route.target.generation === pending.id))) {
+        const [capture] = yield* sql.unsafe("SELECT enabled FROM _ontology_replication WHERE id");
+        if (capture?.enabled) {
+            return pending;
+        }
+        // Pause interrupted capture admission. A fresh incarnation scans authoritative rows and
+        // fences late old loads; reusing this candidate would leave its source disabled forever.
+    }
+    // Supersession with an installed handoff must first reconcile its destination fence. A migration
+    // can commit desired metadata, but cannot make another candidate publication-eligible yet.
+    if (pending?.token) {
+        return yield* new ReplicationBaselineFailed({
+            message: "replication_publication_reconciliation_pending",
+        });
+    }
+    if (pending) {
+        yield* sql.unsafe(`UPDATE _ontology_replication_scans SET complete=true,required=false
+        WHERE generation=$1::uuid`, [pending.id]);
+        yield* sql.unsafe(`UPDATE _ontology_replication_epochs SET state='cancelled',owner=NULL,owner_until=NULL
+      WHERE id=$1::uuid`, [pending.id]);
+    }
+    const id = input.target.generation === input.predecessor?.id || input.target.generation === pending?.id
+        ? randomUUID()
+        : input.target.generation;
+    const target = {
+        ...input.target,
+        generation: id,
+        schema_version: 6,
+        identities: replicationIdentities(input.target),
+        protocol: 4,
+        baseline_complete: input.affected.length === 0,
+        scans_complete: input.affected.length === 0,
+        lineage: initialReplicationLineage(input.target),
+    };
+    yield* sql.unsafe(`INSERT INTO _ontology_replication_targets
+    (generation,source_id,from_revision,manifest,selection,protocol,baseline_complete,
+      scans_complete,lineage,fills,identities)
+    VALUES ($1::uuid,$2::uuid,0,$3::jsonb,$4::jsonb,4,$5,$5,$6::jsonb,$7::jsonb,$8::jsonb) ON CONFLICT DO NOTHING`, [
+        id,
+        target.source_id,
+        JSON.stringify(target.manifest),
+        JSON.stringify(target.selection),
+        input.affected.length === 0,
+        JSON.stringify(target.lineage),
+        JSON.stringify(target.fills ?? {}),
+        JSON.stringify(target.identities),
+    ]);
+    const affectedEntities = new Set(input.affected.map((scope) => scope.entityId));
+    const scopes = replicationScanScopes(target).filter((scope) => affectedEntities.has(scope.entityId));
+    const captures = yield* sql.unsafe("SELECT generation::text,capture_bindings FROM _ontology_replication_targets WHERE capturing");
+    const routes = replicationScanScopes(target).map((scope) => {
+        const old = input.predecessor?.routes.find((route) => route.entityId === scope.entityId && route.kind === scope.kind);
+        if (!old || affectedEntities.has(scope.entityId)) {
+            return { ...scope, target };
+        }
+        // Reuse retained events through a schema-aware projection, without changing old epoch metadata.
+        const projected = {
+            ...old.target,
+            manifest: target.manifest,
+            identities: target.identities,
+            selection: target.selection,
+            fills: target.fills,
+        };
+        const bindings = captures.find((capture) => capture.generation === old.target.generation)
+            ?.capture_bindings ?? {};
+        const previousLineage = old.target.lineage ?? initialReplicationLineage(old.target);
+        const last = previousLineage.epochs.at(-1);
+        const lineage = {
+            ...previousLineage,
+            epochs: previousLineage.epochs.map((epoch) => ({
+                ...epoch,
+                fields: Object.fromEntries(target.selection.map((selected) => [
+                    selected.entityId,
+                    Object.fromEntries([...new Set(["id", ...selected.fields])].map((field) => {
+                        const slot = Object.entries(bindings[selected.entityId] ?? {}).find(([, current]) => current === field)?.[0];
+                        const previousField = Object.entries(last.fields[selected.entityId] ?? {}).find(([, binding]) => binding.key === slot)?.[0];
+                        const binding = previousField
+                            ? epoch.fields[selected.entityId]?.[previousField]
+                            : undefined;
+                        const fills = target.fills?.[selected.entityId] ?? {};
+                        return [
+                            field,
+                            binding ?? {
+                                key: slot ?? null,
+                                ...(Object.hasOwn(fills, field) ? { fill: fills[field] } : {}),
+                            },
+                        ];
+                    })),
+                ])),
+            })),
+        };
+        return {
+            ...scope,
+            target: {
+                ...projected,
+                lineage: {
+                    ...lineage,
+                    epochs: lineage.epochs.map((epoch, index, epochs) => index === epochs.length - 1 ? { ...epoch, throughRevision: undefined } : epoch),
+                },
+            },
+        };
+    });
+    yield* sql.unsafe(`INSERT INTO _ontology_replication_epochs
+    (id,source_id,desired_hash,target,routes,state,predecessor)
+    VALUES ($1::uuid,$2::uuid,$3,$4::jsonb,$5::jsonb,'candidate',$6::uuid)`, [
+        id,
+        target.source_id,
+        desiredHash,
+        shadowJson(target),
+        shadowJson(compactShadowRoutes(routes, target)),
+        input.predecessor?.id ?? null,
+    ]);
+    yield* enableShadowCapture(sql, target, scopes);
+    yield* scheduleReplicationScans(sql, id, scopes, { initial: true });
+    yield* sql.unsafe(`UPDATE _ontology_replication SET active_generation=$1::uuid, enabled=true WHERE id`, [id]);
+    return (yield* shadowEpoch(sql, id));
+});
+/** Remap destination wire bindings before installing the next source snapshot trigger. */
+export const migrateShadowCapture = (sql, current, next, selection, identities) => Effect.gen(function* () {
+    const targets = yield* sql.unsafe(`SELECT generation::text,capture_bindings FROM _ontology_replication_targets WHERE capturing`);
+    const [boundary] = yield* sql.unsafe("SELECT (CASE WHEN is_called THEN last_value ELSE 0 END)::text AS revision FROM _ontology_replication_revision");
+    for (const target of targets) {
+        const bindings = Object.fromEntries(Object.entries(target.capture_bindings ?? {}).map(([entity, fields]) => [
+            entity,
+            Object.fromEntries(Object.entries(fields).map(([wire, key]) => {
+                const declaration = next.entities.find((item) => item.id === entity);
+                const renamed = declaration?.properties.find((property) => renamedReplicationField(current, next, entity, property.name) === key)?.name;
+                return [wire, renamed ?? key];
+            })),
+        ]));
+        for (const selected of selection) {
+            const fields = bindings[selected.entityId];
+            if (!fields) {
+                continue;
+            }
+            const entity = next.entities.find((item) => item.id === selected.entityId);
+            for (const field of queryFields(entity).filter((field) => selected.fields.includes(field.name))) {
+                if (!Object.values(fields).includes(field.name)) {
+                    fields[`__ss_${createHash("sha256")
+                        .update(next.hash + entity.id + field.name + identities[entity.id]?.[field.name])
+                        .digest("hex")
+                        .slice(0, 24)}`] = field.name;
+                }
+            }
+        }
+        yield* sql.unsafe(`UPDATE _ontology_replication_targets SET capture_bindings=$2::jsonb,
+          capture_versions=(SELECT jsonb_agg(CASE WHEN NOT item ? 'through'
+            THEN item || jsonb_build_object('through',$3::text) ELSE item END ORDER BY n)
+            FROM jsonb_array_elements(COALESCE(capture_versions,
+              jsonb_build_array(jsonb_build_object('bindings',capture_bindings)))) WITH ORDINALITY AS versions(item,n))
+            || jsonb_build_array(jsonb_build_object('bindings',$2::jsonb))
+          WHERE generation=$1::uuid`, [target.generation, JSON.stringify(bindings), boundary.revision]);
+    }
+    yield* refreshShadowCapturePolicy(sql);
+});
+/** Every pass has a source-side lease as well as the product lease; stale completions are rejected. */
+export const claimShadowEpoch = (sql, id, owner) => sql
+    .unsafe(`UPDATE _ontology_replication_epochs SET owner=$2::uuid,owner_until=now()+interval '2 minutes'
+    WHERE id=$1::uuid AND state='candidate' AND NOT cancel_requested
+      AND EXISTS(SELECT 1 FROM _ontology_replication WHERE id AND enabled AND NOT destination_repair_pending)
+      AND (owner=$2::uuid OR owner_until IS NULL OR owner_until<now())
+    RETURNING id`, [id, owner])
+    .pipe(Effect.map((rows) => rows.length === 1));
+export const releaseShadowEpoch = (sql, id, owner) => sql.unsafe(`UPDATE _ontology_replication_epochs SET owner=NULL,owner_until=NULL WHERE id=$1::uuid AND owner=$2::uuid`, [id, owner]);
+/** A closed source boundary is selected only after all admitted writers and scans have drained. */
+export const beginShadowHandoff = (sql, id, owner, minimumFence = "0") => sql.withTransaction(Effect.gen(function* () {
+    yield* sql.unsafe("SET LOCAL lock_timeout='5s'");
+    yield* lockReplicationLifecycle(sql);
+    yield* sql.unsafe(`SELECT setval('_ontology_replication_cutover_fence',GREATEST(
+        (SELECT last_value FROM _ontology_replication_cutover_fence),$1::bigint),true)`, [minimumFence]);
+    const [epoch] = yield* sql.unsafe(`UPDATE _ontology_replication_epochs
+      SET boundary=(SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM _ontology_replication_revision),
+        token=gen_random_uuid(),fence=nextval('_ontology_replication_cutover_fence'),
+        operation_id=gen_random_uuid(),handoff_until=now()+interval '30 seconds'
+      WHERE id=$1::uuid AND owner=$2::uuid AND owner_until>now() AND state='candidate'
+        AND NOT cancel_requested AND token IS NULL AND installed
+        AND id=(SELECT active_generation FROM _ontology_replication WHERE id AND enabled)
+        AND NOT EXISTS(SELECT 1 FROM _ontology_replication_scans WHERE generation=$1::uuid AND NOT complete)
+      RETURNING ${epochFields}`, [id, owner]);
+    return expandEpoch(epoch);
+}));
+/** Destination invalidation must have succeeded (or switched) before this transaction runs. */
+export const abandonShadowHandoff = (sql, epoch) => sql.withTransaction(Effect.gen(function* () {
+    yield* lockReplicationLifecycle(sql);
+    yield* sql.unsafe(`UPDATE _ontology_replication_epochs SET catchup_boundary=GREATEST(COALESCE(catchup_boundary,0),boundary),
+      token=NULL,boundary=NULL,
+      handoff_until=NULL,operation_id=NULL WHERE id=$1::uuid AND token=$2::uuid AND state='candidate'`, [epoch.id, epoch.token]);
+}));
+export const completeShadowEpoch = (sql, epoch) => sql.withTransaction(Effect.gen(function* () {
+    yield* lockReplicationLifecycle(sql);
+    const [current] = yield* sql.unsafe(`SELECT e.state,e.operation_id::text,r.active_generation::text,r.destination_repair_pending
+          FROM _ontology_replication_epochs e
+          CROSS JOIN _ontology_replication r WHERE e.id=$1::uuid AND r.id FOR UPDATE OF e`, [epoch.id]);
+    if (current?.state === "serving") {
+        return;
+    }
+    if (!current ||
+        current.destination_repair_pending ||
+        current.state !== "candidate" ||
+        current.operation_id !== epoch.operation_id ||
+        current.active_generation !== epoch.id) {
+        return yield* new ReplicationBaselineFailed({
+            message: "replication_publication_superseded",
+        });
+    }
+    // An external switch has already committed. Adopt its immutable epoch even after a lost local
+    // completion, before considering any newer desired state. Never revert the warehouse here.
+    yield* sql.unsafe(`UPDATE _ontology_replication_epochs SET state='retired',retired_at=now()
+      WHERE state='serving' AND id<>$1::uuid`, [epoch.id]);
+    yield* sql.unsafe("UPDATE _ontology_replication_epochs SET retired_at=now() WHERE state='retired' AND retired_at IS NULL");
+    yield* sql.unsafe(`UPDATE _ontology_replication_epochs SET state='serving',owner=NULL,owner_until=NULL
+      WHERE id=$1::uuid AND operation_id=$2::uuid`, [epoch.id, epoch.operation_id]);
+    yield* sql.unsafe(`UPDATE _ontology_replication SET serving_epoch=$1::uuid,destination_rebuilding=false,
+          destination_repair_pending=false
+          WHERE id AND source_id=$2::uuid`, [epoch.id, epoch.source_id]);
+    const generations = [...new Set(epoch.routes.map((route) => route.target.generation))];
+    for (const generation of generations) {
+        yield* sql.unsafe(`UPDATE _ontology_replication_targets SET capture_scopes=$2::jsonb
+        WHERE generation=$1::uuid`, [
+            generation,
+            JSON.stringify(epoch.routes
+                .filter((route) => route.target.generation === generation)
+                .map(({ entityId, kind }) => ({ entityId, kind }))),
+        ]);
+    }
+    yield* sql.unsafe(`UPDATE _ontology_replication_targets SET capturing=false
+      WHERE NOT (generation IN (SELECT value::uuid FROM jsonb_array_elements_text($1::jsonb)))`, [JSON.stringify(generations)]);
+}));
+export const pendingShadowSegment = (sql, epoch) => sql
+    .unsafe(`SELECT ${segmentFields} FROM _ontology_replication_segments
+    WHERE epoch=$1::uuid AND state<>'committed' ORDER BY created_at,id LIMIT 1`, [epoch])
+    .pipe(Effect.map((rows) => rows[0]));
+/** Source events, immutable object names and job identities commit before the first upload. */
+export const journalShadowSegment = (sql, input) => sql.withTransaction(Effect.gen(function* () {
+    const id = randomUUID();
+    const events = input.events.map((item) => item.event);
+    if (!events.length) {
+        return undefined;
+    }
+    const digest = createHash("sha256").update(JSON.stringify(events)).digest("hex");
+    const through = input.events
+        .reduce((maximum, item) => BigInt(item.event.revision) > maximum ? BigInt(item.event.revision) : maximum, 0n)
+        .toString();
+    const [segment] = yield* sql.unsafe(`INSERT INTO _ontology_replication_segments
+    (id,epoch,generation,source_id,through_revision,events,event_ids,digest,object_uri,job_id)
+    SELECT $1::uuid,$2::uuid,$3::uuid,$4::uuid,$5::bigint,$6::jsonb,$7::jsonb,$8,$9,$10
+    WHERE EXISTS(SELECT 1 FROM _ontology_replication_epochs WHERE id=$2::uuid
+      AND owner=$11::uuid AND owner_until>now() AND state='candidate') RETURNING ${segmentFields}`, [
+        id,
+        input.epoch.id,
+        events[0].generation,
+        input.epoch.source_id,
+        through,
+        JSON.stringify(events),
+        JSON.stringify(input.events.map((item) => item.historyId)),
+        digest,
+        `gs://${input.projectId}-staging/ontology-replication/${input.epoch.source_id}/${id}-${digest}.ndjson`,
+        `ontology_load_${id.replaceAll("-", "")}_0`,
+        input.owner,
+    ]);
+    if (!segment) {
+        return undefined;
+    }
+    yield* sql.unsafe(`UPDATE _ontology_replication_delivery SET segment_id=$3::uuid
+    WHERE generation=$1::uuid AND event_id IN (SELECT value::uuid FROM jsonb_array_elements_text($2::jsonb))
+      AND committed_at IS NULL`, [segment.generation, JSON.stringify(segment.event_ids), id]);
+    return segment;
+}));
+export const finishShadowSegment = (sql, segment, owner) => sql.withTransaction(Effect.gen(function* () {
+    const [completed] = yield* sql.unsafe(`UPDATE _ontology_replication_segments SET state='committed',committed_at=now()
+      WHERE id=$1::uuid AND job_id=$2 AND EXISTS(SELECT 1 FROM _ontology_replication_epochs
+        WHERE id=$3::uuid AND owner=$4::uuid AND owner_until>now() AND state='candidate') RETURNING id`, [segment.id, segment.job_id, segment.epoch, owner]);
+    if (!completed) {
+        return false;
+    }
+    yield* sql.unsafe(`UPDATE _ontology_replication_delivery SET committed_at=now()
+      WHERE segment_id=$1::uuid`, [segment.id]);
+    return true;
+}));
+export const retryShadowSegment = (sql, segment, owner, failure) => sql.unsafe(`UPDATE _ontology_replication_segments SET attempt=attempt+1,
+    retry_after=now()+make_interval(secs=>least(3600,30*power(2,least(attempt,7)))::integer),
+    job_id='ontology_load_' || replace(id::text,'-','') || '_' || (attempt+1)::text,failure=$5::jsonb
+    WHERE id=$1::uuid AND job_id=$2 AND EXISTS(SELECT 1 FROM _ontology_replication_epochs
+      WHERE id=$3::uuid AND owner=$4::uuid AND owner_until>now() AND state='candidate')`, [segment.id, segment.job_id, segment.epoch, owner, JSON.stringify(failure)]);
+/** Recheck references under the lifecycle fence. Audit journals and immutable epoch routes survive. */
+export const retireShadowCapture = (sql) => sql.withTransaction(Effect.gen(function* () {
+    yield* lockReplicationLifecycle(sql);
+    yield* sql.unsafe(`UPDATE _ontology_replication_targets t SET capturing=false
+    WHERE NOT EXISTS(SELECT 1 FROM _ontology_replication_epochs e,
+      jsonb_array_elements(e.routes) r WHERE e.state IN ('serving','candidate')
+      AND r->'target'->>'generation'=t.generation::text)`);
+}));
+/** Filter by the destination's immutable wire schema, including missing keys and explicit nulls. */
+export function shadowWireEvent(event, bindings, selection) {
+    const allowed = selection.find((item) => item.entityId === event.entity_id);
+    if (!allowed || event.deleted || event.record_kind !== "entity") {
+        return event;
+    }
+    const source = JSON.parse(event.payload);
+    const fields = bindings[event.entity_id] ??
+        Object.fromEntries(["id", ...allowed.fields].map((field) => [field, field]));
+    return {
+        ...event,
+        payload: JSON.stringify(Object.fromEntries(Object.entries(fields)
+            .filter(([, key]) => Object.hasOwn(source, key))
+            .map(([wire, key]) => [wire, source[key]]))),
+    };
+}
+/** Capture is the union needed by live destinations; export shaping remains destination-specific. */
+export const refreshShadowCapturePolicy = (sql) => sql.unsafe(`UPDATE _ontology_replication SET selection=COALESCE((
+    SELECT jsonb_object_agg(entity_id,
+      jsonb_build_object('entityId',entity_id,'fields',fields,'relationships',relationships))
+    FROM (
+      SELECT entity_id,
+        (SELECT COALESCE(jsonb_agg(DISTINCT value),'[]'::jsonb) FROM jsonb_array_elements(all_fields) value) AS fields,
+        (SELECT COALESCE(jsonb_agg(DISTINCT value),'[]'::jsonb)
+          FROM jsonb_array_elements(all_relationships) value) AS relationships
+      FROM (
+        SELECT binding.key AS entity_id,jsonb_agg(DISTINCT field.value) AS all_fields,
+          COALESCE((SELECT jsonb_agg(DISTINCT substring(scope->>'kind' FROM 6))
+            FILTER (WHERE scope->>'kind' LIKE 'edge:%')
+            FROM _ontology_replication_targets other,jsonb_array_elements(other.capture_scopes) scope
+            WHERE other.capturing AND scope->>'entityId'=binding.key),'[]'::jsonb) AS all_relationships
+        FROM _ontology_replication_targets t,jsonb_each(t.capture_bindings) binding,
+          jsonb_each_text(binding.value) field WHERE t.capturing
+          AND t.source_id=(SELECT source_id FROM _ontology_replication WHERE id)
+          AND NOT EXISTS (
+          SELECT 1 FROM _ontology_replication state,jsonb_array_elements(state.exclusions) exclusion
+          WHERE exclusion->>'entityId'=binding.key AND
+            (COALESCE((exclusion->>'disabled')::boolean,false) OR exclusion->'fields' ? field.value))
+          GROUP BY binding.key
+      ) union_fields
+    ) selections),'{}'::jsonb) WHERE id`);
+/** Only generations retired for seven days qualify; creation time is never a query-safety fence. */
+export const retiredShadowGenerations = (sql) => sql.withTransaction(Effect.gen(function* () {
+    yield* lockReplicationLifecycle(sql);
+    return yield* sql.unsafe(`SELECT t.generation::text,t.source_id::text
+    FROM _ontology_replication_targets t WHERE NOT t.capturing
+      AND t.source_id=(SELECT source_id FROM _ontology_replication WHERE id)
+      AND t.created_at<now()-interval '31 days'
+      AND NOT EXISTS (SELECT 1 FROM _ontology_replication_epochs e,jsonb_array_elements(e.routes) route
+        WHERE route->'target'->>'generation'=t.generation::text AND
+          (e.state IN ('serving','candidate') OR e.retired_at>=now()-interval '7 days'))
+      AND NOT EXISTS(SELECT 1 FROM _ontology_replication_segments segment
+        JOIN _ontology_replication_epochs epoch ON epoch.id=segment.epoch
+        WHERE segment.generation=t.generation AND (epoch.state='candidate' OR segment.state='pending'))
+    ORDER BY t.generation LIMIT 20`);
+}));
+/** The destination has already fenced and deselected an observably destroyed event table. */
+export const replaceDestroyedShadowDestination = (sql, input) => sql.withTransaction(Effect.gen(function* () {
+    yield* lockReplicationLifecycle(sql);
+    const [current] = yield* sql.unsafe("SELECT active_generation::text,source_id::text FROM _ontology_replication WHERE id AND enabled");
+    if (current?.active_generation !== input.target.generation ||
+        current.source_id !== input.target.source_id) {
+        return yield* new ReplicationBaselineFailed({
+            message: "replication_destination_repair_superseded",
+        });
+    }
+    yield* sql.unsafe(`SELECT setval('_ontology_replication_cutover_fence',GREATEST(
+    (SELECT last_value FROM _ontology_replication_cutover_fence),$1::bigint),true)`, [input.fence]);
+    yield* sql.unsafe(`UPDATE _ontology_replication_epochs SET state=CASE WHEN state='serving'
+    THEN 'retired' ELSE 'cancelled' END,retired_at=now(),token=NULL,owner=NULL,owner_until=NULL
+    WHERE state IN ('serving','candidate')`);
+    yield* sql.unsafe(`UPDATE _ontology_replication SET serving_epoch=NULL,destination_rebuilding=true,
+        destination_repair_pending=false WHERE id`);
+    yield* sql.unsafe("UPDATE _ontology_replication_targets SET capturing=false");
+    return yield* prepareShadowEpoch(sql, {
+        target: { ...input.target, generation: randomUUID() },
+        affected: replicationScanScopes(input.target),
+    });
+}));
+export const installShadowEpoch = (sql, id, owner) => sql
+    .unsafe(`UPDATE _ontology_replication_epochs SET installed=true
+    WHERE id=$1::uuid AND owner=$2::uuid AND owner_until>now() AND state='candidate'
+      AND id=(SELECT active_generation FROM _ontology_replication WHERE id AND enabled) RETURNING id`, [id, owner])
+    .pipe(Effect.map((rows) => rows.length === 1));
+/** Strict warehouse job completion, not Pub/Sub acceptance, is the publication prerequisite. */
+export const shadowPublicationReady = (sql, epoch, owner) => sql
+    .unsafe(`SELECT EXISTS(SELECT 1 FROM _ontology_replication_epochs e
+    WHERE id=$1::uuid AND owner=$2::uuid AND owner_until>now() AND state='candidate' AND installed
+      AND NOT cancel_requested
+      AND token=$3::uuid AND boundary=$4::bigint AND handoff_until>now()
+      AND id=(SELECT active_generation FROM _ontology_replication
+        WHERE id AND enabled AND NOT destination_repair_pending)
+      AND NOT EXISTS(SELECT 1 FROM _ontology_replication_scans WHERE generation=e.id AND NOT complete)
+      AND NOT EXISTS(SELECT 1 FROM _ontology_replication_delivery WHERE generation=e.id
+        AND revision<=e.boundary AND committed_at IS NULL AND suppressed_at IS NULL)) AS ready`, [epoch.id, owner, epoch.token, epoch.boundary])
+    .pipe(Effect.map((rows) => rows[0].ready));
+export const unfinishedRetiredSegments = (sql) => sql.unsafe(`SELECT ${segmentFields} FROM _ontology_replication_segments
+    WHERE state='pending' AND created_at<now()-interval '7 days' AND epoch IN (
+      SELECT id FROM _ontology_replication_epochs WHERE state IN ('retired','cancelled'))
+    ORDER BY created_at,id LIMIT 10`);
+export const settleRetiredSegment = (sql, segment, committed) => sql.unsafe(`UPDATE _ontology_replication_segments SET state=$3,
+    committed_at=CASE WHEN $3='committed' THEN now() ELSE committed_at END
+    WHERE id=$1::uuid AND job_id=$2 AND state='pending' AND epoch IN (
+      SELECT id FROM _ontology_replication_epochs WHERE state IN ('retired','cancelled'))`, [segment.id, segment.job_id, committed ? "committed" : "failed"]);
+/** Restriction-only refresh is independent of build leases and cannot confer publication eligibility. */
+export const shadowPolicyViewOwnership = (sql, generation, source, exclusions) => sql
+    .unsafe(`SELECT EXISTS(SELECT 1 FROM _ontology_replication
+    WHERE id AND enabled AND NOT destination_repair_pending AND active_generation=$1::uuid
+      AND source_id=$2::uuid AND exclusions=$3::jsonb) AS valid`, [generation, source, JSON.stringify(exclusions)])
+    .pipe(Effect.map((rows) => rows[0].valid));
+export const recordShadowPolicyViews = (sql, generation, source, exclusions) => sql.unsafe(`UPDATE _ontology_replication_epochs SET policy_installed=true WHERE id=$1::uuid
+    AND EXISTS(SELECT 1 FROM _ontology_replication WHERE id AND enabled AND NOT destination_repair_pending
+      AND active_generation=$1::uuid AND source_id=$2::uuid AND exclusions=$3::jsonb)`, [generation, source, JSON.stringify(exclusions)]);
+/** Used after reading a destination view's etag, before its conditional DDL update. */
+export const shadowViewOwnership = (sql, id) => sql
+    .unsafe(`SELECT EXISTS(SELECT 1 FROM _ontology_replication_epochs e
+    WHERE e.id=$1::uuid
+      AND EXISTS(SELECT 1 FROM _ontology_replication WHERE id AND NOT destination_repair_pending)
+      AND (
+      (e.state='candidate' AND NOT e.cancel_requested
+        AND e.id=(SELECT active_generation FROM _ontology_replication WHERE id AND enabled))
+      OR (e.state='serving' AND e.id=(SELECT serving_epoch FROM _ontology_replication WHERE id AND enabled)
+        AND NOT EXISTS(SELECT 1 FROM _ontology_replication_epochs WHERE state='candidate')))) AS valid`, [id])
+    .pipe(Effect.map((rows) => rows[0].valid));
+/** A finite background goal prevents continuous writes from moving initialization's finish line. */
+export const closeShadowCatchup = (sql, id, owner) => sql.withTransaction(Effect.gen(function* () {
+    yield* sql.unsafe("SET LOCAL lock_timeout='5s'");
+    yield* lockReplicationLifecycle(sql);
+    yield* sql.unsafe(`UPDATE _ontology_replication_epochs SET catchup_boundary=(
+      SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM _ontology_replication_revision)
+      WHERE id=$1::uuid AND owner=$2::uuid AND owner_until>now() AND state='candidate'
+        AND catchup_boundary IS NULL AND installed=false
+        AND id=(SELECT active_generation FROM _ontology_replication WHERE id AND enabled)
+        AND NOT EXISTS(SELECT 1 FROM _ontology_replication_scans WHERE generation=$1::uuid AND NOT complete)`, [id, owner]);
+}));
+/** Persist cancellation before returning a retryable migration decision; provider fencing follows. */
+export const requestShadowSupersession = (sql, manifestHash) => sql.withTransaction(Effect.gen(function* () {
+    yield* sql.unsafe("SET LOCAL lock_timeout='5s'");
+    yield* sql.unsafe("SET LOCAL statement_timeout='30s'");
+    yield* lockReplicationLifecycle(sql);
+    const rows = yield* sql.unsafe(`UPDATE _ontology_replication_epochs SET cancel_requested=true
+      WHERE state='candidate' AND (token IS NOT NULL OR cancel_requested)
+        AND target->'manifest'->>'hash'<>$1 RETURNING id`, [manifestHash]);
+    if (rows.length) {
+        yield* sql.unsafe(`UPDATE _ontology_replication SET requested_manifest_hash=$1,
+        requested_manifest_until=now()+interval '5 minutes' WHERE id`, [manifestHash]);
+    }
+    return rows.length > 0;
+}));
+/** The destination token has been invalidated, or no token ever existed. */
+export const cancelShadowEpoch = (sql, epoch) => sql.withTransaction(Effect.gen(function* () {
+    yield* lockReplicationLifecycle(sql);
+    yield* sql.unsafe(`UPDATE _ontology_replication_epochs SET state='cancelled',owner=NULL,
+      owner_until=NULL,token=NULL,boundary=NULL,handoff_until=NULL
+      WHERE id=$1::uuid AND state='candidate' AND cancel_requested`, [epoch.id]);
+    yield* sql.unsafe(`UPDATE _ontology_replication_scans SET complete=true,required=false
+      WHERE generation=$1::uuid`, [epoch.id]);
+}));

package/dist/replication/source-replacement.d.ts

@@ -0,0 +1,33 @@
+import type { SourcePublicationProof } from "./model.js";
+import { Effect } from "effect";
+import type { SqlClient } from "effect/unstable/sql";
+import { ReplicationBaselineFailed } from "./baseline.js";
+export interface SourceReplacementSelector {
+    readonly sourceId: string;
+    readonly operationId: string | null;
+    readonly epoch: string | null;
+    readonly token: string | null;
+    readonly fence: string;
+    readonly incarnation: string;
+}
+export interface ShadowSourceReplacement {
+    readonly source_id: string;
+    readonly operation_id: string;
+    readonly expected: SourceReplacementSelector;
+    readonly proof: SourcePublicationProof | null;
+    readonly published: string;
+}
+export declare const pendingSourceReplacement: (sql: SqlClient.SqlClient) => Effect.Effect<ShadowSourceReplacement, import("effect/unstable/sql/SqlError").SqlError, never>;
+/** Only the trusted product publication can justify replacing an otherwise healthy old source. */
+export declare const prepareSourceReplacement: (sql: SqlClient.SqlClient, input: {
+    readonly sourceId: string;
+    readonly published: string;
+    readonly observed: SourceReplacementSelector;
+    readonly proof: SourcePublicationProof;
+    readonly projectId: string;
+    readonly workspaceId: string;
+}) => Effect.Effect<ShadowSourceReplacement, import("effect/unstable/sql/SqlError").SqlError | ReplicationBaselineFailed, never>;
+/** A confirmed no-op CAS may race an old-source handoff; retain its receipt before a new attempt. */
+export declare const retrySourceReplacement: (sql: SqlClient.SqlClient, intent: ShadowSourceReplacement, observed: SourceReplacementSelector) => Effect.Effect<undefined, import("effect/unstable/sql/SqlError").SqlError | ReplicationBaselineFailed, never>;
+/** Upgrade only an already admitted old journal; the provider evidence is read before this transaction. */
+export declare const recordSourcePublicationProof: (sql: SqlClient.SqlClient, intent: ShadowSourceReplacement, proof: SourcePublicationProof) => Effect.Effect<undefined, import("effect/unstable/sql/SqlError").SqlError | ReplicationBaselineFailed, never>;