@rikalabs/akter 0.1.0-alpha.0

package/dist/runtime/connections/owner.js

@@ -0,0 +1,896 @@
+import { Cause, Clock, Deferred, Effect, Exit, Option, Queue, Schema, Semaphore, Stream, } from "effect";
+import { inTenant } from "../database/tenancy.js";
+import { Entity } from "effect/cluster";
+import { SqlClient, SqlError } from "effect/sql";
+import { ActorError, ActorUnavailable, NotCreated, RunnerAtCapacity, SessionEnded, } from "../../errors/actor.js";
+import { ConnectionPhase, } from "../members.js";
+import { Caller, principal } from "../../identity/caller.js";
+import { replayEvents } from "../events/replay.js";
+import { compress, decompress } from "../storage/codec.js";
+import { FrameworkClock } from "../turn/admission.js";
+import { actorRow, emptyActivationCache, forget as forgetGeneration, heldGeneration, } from "../storage/generation.js";
+import { Committed, FEED_MEMBER, FeedFrame, HolderItem, isWatchMember, watchedQuery, watchMember, } from "./protocol.js";
+import { streamSubscriptions } from "./streams.js";
+import { HolderUnreachable } from "./transport.js";
+import { WatchTap } from "./watch.js";
+/** Encoded bytes one connection's session may hold. */
+export const MAX_SESSION_BYTES = 16_384;
+/** Open connections one actor may have per connection member. */
+const MAX_MEMBER_CONNECTIONS = 10_000;
+/** Events one `read.follow` page reads at most. */
+const FOLLOW_PAGE = 1_000;
+/** Progress frames one actor accepts per second, with a burst of as many. */
+const PROGRESS_PER_SECOND = 20;
+/** How long a job check's answer stands before a later frame reads the row again. */
+const JOB_CHECK_MS = 5_000;
+/** How long after a failed job check the next one waits. */
+const JOB_CHECK_RETRY_MS = 1_000;
+/** Clock skew allowed between the executor's lease deadline and the owner's clock. */
+const PROGRESS_SKEW_MS = 1_000;
+/** Progress entries one `read.progress` subscription buffers; the oldest goes first. */
+const STREAM_PROGRESS_BUFFER = 16;
+/** How long a progress record stays after its last frame, check, or close. */
+const PROGRESS_RECORD_MS = 60_000;
+const closedJob = (at) => ({
+    open: false,
+    checkedAt: at,
+    touchedAt: at,
+    attempt: 0,
+    seq: 0,
+    principal: Option.none(),
+    payload: "",
+    holders: new Set(),
+});
+const samePrincipal = (a, b) => Option.isSome(a) && Option.isSome(b) && a.value.subject === b.value.subject;
+const utf8 = new TextEncoder();
+/** A session is stored inside its codec's `{"value":…}` envelope, which the limit does not count. */
+const SESSION_ENVELOPE_BYTES = utf8.encode(JSON.stringify({ value: null })).byteLength - utf8.encode("null").byteLength;
+const encodeCaller = Schema.encodeEffect(Schema.fromJsonString(Caller));
+const decodeCaller = Schema.decodeEffect(Schema.fromJsonString(Caller));
+const emptyResult = {
+    session: undefined,
+    changed: false,
+    sends: [],
+    broadcasts: [],
+    close: false,
+};
+const ended = (cause, resync) => SessionEnded.make({ cause, resync });
+/** Answers a database failure as `ActorUnavailable`, so the caller retries. */
+const sqlUnavailable = (cause) => Effect.fail(ActorError.make({ reason: ActorUnavailable.make({ cause }) }));
+const unavailable = (message) => ActorError.make({ reason: ActorUnavailable.make({ cause: new Error(message) }) });
+/**
+ * The owner side of connections for one actor type: the activation registry
+ * shared with commands, the fenced `actor_connections` writes, and ordered,
+ * post-commit delivery of frames to every holder.
+ *
+ * Ordering and safety: every holder learns this owner before any of its
+ * broadcasts, so it can resync if the owner dies, and a holder that cannot be
+ * reached is excluded, its rows dropped, so it ends those connections itself.
+ * Dropping a generation's state means whatever runs next acquires a new one,
+ * reloads the rows, and numbers each holder's messages from 1 again, which a
+ * holder requires of every new generation; a clean end tells holders not to
+ * resync. A flush that read an older head than a concurrent one never moves the
+ * cursor back. Ends run on their own fibers so one slow handler never holds up the
+ * rest. A handler defect closes only its connection.
+ *
+ * Connections: feeds are framework connections, so an actor type with feeds loads
+ * its rows like one with members, and a feed has no handler, so its open only
+ * inserts the row and fixes its baseline and its close only deletes the row. A
+ * feed's holder rereads events itself once the new owner answers, and a feed
+ * serves the current shape even under a `writeVersion`. A connection's lock lives
+ * only as long as its row. A retried open cannot know whether its opening frames
+ * reached the holder. The owner runs frames only from the connection's stored
+ * holder, and a frame or subscription that reaches it past its authorization bound
+ * never runs.
+ *
+ * Subscriptions: followers wait on `advanced`, and a new head wakes every one. A
+ * replay reads pages from where the previous stopped up to the head, which only
+ * moves after a commit, so nothing is skipped or read twice. The subscription
+ * limit is checked and taken in one step so concurrent subscriptions cannot all
+ * pass it. A hook that hangs is retried at the next tick while the bound still
+ * ends the session, and an answer past the bound never extends it. A stream
+ * subscriber's live progress is a sliding buffer, so a slow reader loses the
+ * oldest.
+ *
+ * Progress frames: a route or settle that committed while the job check read
+ * keeps the job closed. A row that has not counted an attempt yet does not
+ * prove a frame open. Every frame of a job shares one activation record so its
+ * order and holders stay whole, and a record decides after a permit wait, never a
+ * copy taken before it. A record nothing touched for a while goes, and a late
+ * frame runs the job check again.
+ */
+export const activationOwner = ({ registration, transport, authorize, clock, role, }) => {
+    const activations = new Map();
+    const hasConnections = registration.connections.size > 0 ||
+        registration.feeds.size > 0 ||
+        registration.watches.size > 0;
+    const hasStreams = registration.streams.size > 0;
+    const wanted = new Set([
+        ...[...registration.connections.values()].flatMap((member) => [
+            ...(member.progress?.jobs ?? []),
+        ]),
+        ...[...registration.streams.values()].flatMap((member) => [...member.progress]),
+    ]);
+    const hasProgress = wanted.size > 0;
+    const now = Effect.map(Clock.currentTimeMillis, (millis) => millis + clock.offsetMillis());
+    const advance = (activation, head) => {
+        if (BigInt(head) <= BigInt(activation.head))
+            return;
+        activation.head = head;
+        const previous = activation.advanced;
+        activation.advanced = Deferred.makeUnsafe();
+        Deferred.doneUnsafe(previous, Exit.void);
+    };
+    const encodeFeedFrame = Schema.encodeEffect(Schema.fromJsonString(FeedFrame));
+    const encodeCommitted = Schema.encodeEffect(Schema.fromJsonString(Committed));
+    /**
+     * The frame every watch of the actor receives after a commit, one broadcast
+     * per watchable query so each reaches its own connections. A commit that
+     * wrote nothing a query could have read sends none.
+     */
+    const watchBroadcasts = (activation, writes, version) => Effect.gen(function* () {
+        const nothing = !writes.state &&
+            writes.events.length === 0 &&
+            writes.tables.length === 0 &&
+            writes.blobs.length === 0;
+        if (nothing || registration.watches.size === 0 || (yield* WatchTap).dropsCommitted())
+            return [];
+        if (![...(activation.rows?.values() ?? [])].some((row) => isWatchMember(row.member)))
+            return [];
+        const frame = yield* encodeCommitted({ version, writes }).pipe(Effect.orDie);
+        return [...registration.watches].map((query) => ({
+            member: watchMember(query),
+            frame,
+        }));
+    });
+    /** A committed turn's feed events, broadcast to every open feed of the actor. */
+    const feedBroadcasts = (committed) => Effect.forEach(committed.events.flatMap((event, index) => registration.feeds.has(event.tag)
+        ? [{ event, cursor: String(BigInt(committed.after) + BigInt(index) + 1n) }]
+        : []), ({ event, cursor }) => registration.upcastEvent(event.tag, event.version, event.value).pipe(Effect.flatMap((value) => encodeFeedFrame({
+        tag: event.tag,
+        value,
+        commandId: committed.commandId,
+        timestampMs: committed.emittedAtMs,
+    }).pipe(Effect.orDie)), Effect.map((frame) => ({ member: FEED_MEMBER, frame, event: cursor }))));
+    const enter = (entityId, ref, key) => Effect.acquireRelease(Effect.sync(() => {
+        const found = activations.get(entityId);
+        if (found !== undefined) {
+            found.presence += 1;
+            return found;
+        }
+        const created = {
+            ref,
+            key,
+            cache: emptyActivationCache(),
+            presence: 1,
+            rows: undefined,
+            head: "0",
+            advanced: Deferred.makeUnsafe(),
+            streams: new Set(),
+            progress: new Map(),
+            checking: new Set(),
+            checkFailed: new Map(),
+            listeners: new Set(),
+            progressTokens: PROGRESS_PER_SECOND,
+            progressRefilledAt: 0,
+            progressSweptAt: 0,
+            through: "0",
+            channels: new Map(),
+            flush: Semaphore.makeUnsafe(1),
+            loading: Semaphore.makeUnsafe(1),
+            locks: new Map(),
+            opened: new Set(),
+            keptAwake: undefined,
+            acquiring: Semaphore.makeUnsafe(1),
+        };
+        activations.set(entityId, created);
+        return created;
+    }), (activation) => Effect.gen(function* () {
+        activation.presence -= 1;
+        if (activation.presence > 0)
+            return;
+        activations.delete(entityId);
+        yield* seal(activation);
+    }));
+    const channelOf = (activation, holder, epoch) => {
+        const name = `${holder}|${epoch}`;
+        const found = activation.channels.get(name);
+        if (found !== undefined)
+            return found;
+        const created = { holder, epoch, seq: 0 };
+        activation.channels.set(name, created);
+        return created;
+    };
+    const dropRows = (activation, ids) => Effect.gen(function* () {
+        if (ids.length === 0)
+            return;
+        const sql = yield* SqlClient.SqlClient;
+        const actor = actorRow({ sql, actor: activation });
+        yield* sql `DELETE FROM actor_connections WHERE ${actor} AND connection_id IN ${sql.in(ids)}`;
+        for (const id of ids)
+            activation.rows?.delete(id);
+        const holder = activation.keptAwake;
+        if (holder !== undefined && (activation.rows?.size ?? 0) === 0) {
+            activation.keptAwake = undefined;
+            yield* Entity.keepAlive(false).pipe(Effect.provideContext(holder));
+        }
+    });
+    const send = (activation, channel, items) => Effect.gen(function* () {
+        channel.seq += 1;
+        const message = {
+            epoch: channel.epoch,
+            owner: transport.holder,
+            ownerEpoch: transport.epoch,
+            ref: activation.ref,
+            generation: activation.cache.generation ?? "0",
+            seq: channel.seq,
+            through: activation.through,
+            items,
+        };
+        const held = [...(activation.rows?.values() ?? [])].flatMap((row) => row.holder === channel.holder && row.holderEpoch === channel.epoch
+            ? [row.connectionId]
+            : []);
+        const answer = yield* transport
+            .deliver(channel.holder, channel.epoch, message)
+            .pipe(Effect.exit);
+        if (Exit.isFailure(answer)) {
+            if (Option.isSome(Cause.findErrorOption(answer.cause)))
+                return yield* dropRows(activation, held);
+            return yield* Effect.die(HolderUnreachable.make({ message: "Holder delivery failed" }));
+        }
+        if (answer.value.wrongEpoch)
+            return yield* dropRows(activation, held);
+        const mine = new Set(held);
+        yield* dropRows(activation, answer.value.unknown.filter((id) => mine.has(id)));
+    }).pipe(Effect.catchIf(SqlError.isSqlError, () => Effect.void));
+    const seal = (activation) => activation.flush
+        .withPermit(Effect.forEach(new Set([
+        ...activation.channels.values(),
+        ...[...(activation.rows?.values() ?? [])].map((row) => channelOf(activation, row.holder, row.holderEpoch)),
+    ]), (channel) => send(activation, channel, [HolderItem.cases.Seal.make({})]), { discard: true }))
+        .pipe(Effect.timeout("2 seconds"), Effect.ignore);
+    const forget = (activation) => {
+        forgetGeneration(activation.cache);
+        activation.rows = undefined;
+        activation.channels.clear();
+    };
+    /** Fences this activation's generation and loads committed state, as a command turn would. */
+    const acquire = (activation) => activation.cache.generation !== undefined && activation.cache.state !== undefined
+        ? Effect.void
+        : activation.acquiring.withPermit(acquireOnce(activation));
+    const acquireOnce = (activation) => Effect.gen(function* () {
+        if (activation.cache.generation !== undefined && activation.cache.state !== undefined)
+            return;
+        const sql = yield* SqlClient.SqlClient;
+        const actor = actorRow({ sql, actor: activation });
+        const { ref, key } = activation;
+        const acquired = yield* sql.withTransaction(Effect.gen(function* () {
+            yield* sql `INSERT INTO actor_generations (routing_key, tenant_id, actor_type, actor_id)
+            VALUES (${key}, ${ref.tenant}, ${ref.actor}, ${ref.id}) ON CONFLICT DO NOTHING`;
+            const [row] = activation.cache.generation === undefined
+                ? yield* sql `
+                  UPDATE actor_generations SET generation = generation + 1 WHERE ${actor}
+                  RETURNING generation::text AS generation, event_sequence::text AS head`
+                : yield* sql `
+                  SELECT generation::text AS generation, event_sequence::text AS head
+                  FROM actor_generations WHERE ${actor}`;
+            if (activation.cache.generation !== undefined &&
+                activation.cache.generation !== row.generation) {
+                forget(activation);
+                return yield* unavailable("Stale actor generation");
+            }
+            const state = yield* sql `
+            SELECT key, value FROM actor_state WHERE ${actor}`;
+            return { row: row, state };
+        }));
+        activation.cache.generation = acquired.row.generation;
+        activation.cache.state = new Map(acquired.state.map(({ key, value }) => [key, decompress(value)]));
+        advance(activation, acquired.row.head);
+        activation.through = acquired.row.head;
+    });
+    /** Loads the actor's connection rows once per activation, excluding holders that are gone. */
+    const load = (activation) => activation.rows !== undefined || !hasConnections
+        ? Effect.void
+        : activation.loading.withPermit(loadOnce(activation));
+    const loadOnce = (activation) => Effect.gen(function* () {
+        if (activation.rows !== undefined)
+            return;
+        const sql = yield* SqlClient.SqlClient;
+        const actor = actorRow({ sql, actor: activation });
+        const stored = yield* sql `SELECT connection_id, member, holder, holder_epoch, caller, session, frame_seq::text AS frame_seq,
+          opened_through::text AS opened_through
+        FROM actor_connections WHERE ${actor}`;
+        const rows = new Map();
+        for (const row of stored)
+            rows.set(row.connection_id, {
+                connectionId: row.connection_id,
+                member: row.member,
+                holder: row.holder,
+                holderEpoch: row.holder_epoch,
+                caller: yield* decodeCaller(row.caller).pipe(Effect.orDie),
+                baseline: row.opened_through,
+                session: row.session === null ? undefined : decompress(row.session),
+                frameSeq: Number(row.frame_seq),
+            });
+        activation.rows = rows;
+        yield* setKeepAwake(activation);
+        yield* activation.flush.withPermit(Effect.forEach(new Set([...rows.values()].map((row) => `${row.holder}|${row.holderEpoch}`)), (name) => {
+            const [holder, epoch] = name.split("|");
+            return send(activation, channelOf(activation, holder, epoch), []);
+        }, { discard: true }));
+    }).pipe(Effect.catchIf(SqlError.isSqlError, Effect.die));
+    /** Readies an activation that is about to run a turn: fenced, with its connection rows. */
+    const prepare = (activation) => hasConnections
+        ? Effect.andThen(acquire(activation), load(activation)).pipe(Effect.orDie)
+        : Effect.void;
+    const list = (activation) => (member) => Effect.sync(() => [...(activation.rows?.values() ?? [])]
+        .filter((row) => row.member === member && row.buffered === undefined)
+        .slice(0, 1_000)
+        .map((row) => ({
+        connectionId: row.connectionId,
+        caller: row.caller,
+        session: row.session,
+    })));
+    /** Sends committed frames to their holders in order and advances the flushed-through cursor. */
+    const flush = (activation, broadcasts, head, own) => activation.flush.withPermit(Effect.gen(function* () {
+        if (activation.rows === undefined || (broadcasts.length === 0 && own === undefined)) {
+            advance(activation, head);
+            if (BigInt(head) > BigInt(activation.through))
+                activation.through = head;
+            return;
+        }
+        const perChannel = new Map();
+        const add = (row, item) => {
+            const channel = channelOf(activation, row.holder, row.holderEpoch);
+            const items = perChannel.get(channel) ?? [];
+            const last = items.at(-1);
+            if (last?._tag === "Frame" &&
+                last.frame === item.frame &&
+                last.member === item.member &&
+                last.event === item.event &&
+                !last.to.includes(item.to[0]))
+                perChannel.set(channel, [
+                    ...items.slice(0, -1),
+                    { ...last, to: [...last.to, ...item.to] },
+                ]);
+            else
+                perChannel.set(channel, [...items, item]);
+        };
+        const self = own === undefined ? undefined : activation.rows.get(own.connectionId);
+        if (own !== undefined && self !== undefined) {
+            const stamp = registration.connections.get(own.member)?.stampCursor ?? true;
+            for (const frame of own.frames)
+                add(self, HolderItem.cases.Frame.make({
+                    member: own.member,
+                    to: [own.connectionId],
+                    frame: frame.frame,
+                    event: frame.event,
+                    stamp,
+                    replay: own.replay === true,
+                }));
+        }
+        for (const broadcast of broadcasts) {
+            const stamp = registration.connections.get(broadcast.member)?.stampCursor ?? true;
+            const to = broadcast.to === undefined ? undefined : new Set(broadcast.to);
+            const except = new Set(broadcast.except ?? []);
+            for (const row of activation.rows.values())
+                if (row.member === broadcast.member &&
+                    (to === undefined || to.has(row.connectionId)) &&
+                    !except.has(row.connectionId))
+                    if (row.buffered !== undefined)
+                        row.buffered.push({ frame: broadcast.frame, event: broadcast.event });
+                    else
+                        add(row, HolderItem.cases.Frame.make({
+                            member: row.member,
+                            to: [row.connectionId],
+                            frame: broadcast.frame,
+                            event: broadcast.event,
+                            stamp,
+                        }));
+        }
+        const advanced = BigInt(head) > BigInt(activation.through);
+        for (const [channel, items] of perChannel)
+            yield* send(activation, channel, advanced ? [...items, HolderItem.cases.Flushed.make({ through: head })] : items);
+        advance(activation, head);
+        if (advanced)
+            activation.through = head;
+    }));
+    const lockOf = (activation, connectionId) => {
+        const found = activation.locks.get(connectionId);
+        if (found !== undefined)
+            return found;
+        const created = Semaphore.makeUnsafe(1);
+        activation.locks.set(connectionId, created);
+        return created;
+    };
+    const withLock = (activation, connectionId, effect) => lockOf(activation, connectionId)
+        .withPermit(effect)
+        .pipe(Effect.ensuring(Effect.sync(() => {
+        if (activation.rows?.has(connectionId) !== true)
+            activation.locks.delete(connectionId);
+    })));
+    const events = (activation, sql) => (tag, after, limit) => replayEvents(activation.ref, activation.key, [tag], after, BigInt(activation.head), limit).pipe(inTenant({ sql, role, tenant: activation.ref.tenant }), Effect.catchIf(SqlError.isSqlError, Effect.die), Effect.provideService(SqlClient.SqlClient, sql));
+    const run = (activation, row, phase, commands) => Effect.gen(function* () {
+        const connection = registration.connections.get(row.member);
+        const sql = yield* SqlClient.SqlClient;
+        return yield* connection.run({
+            ref: activation.ref,
+            connectionId: row.connectionId,
+            member: row.member,
+            caller: row.caller,
+            resumed: !ConnectionPhase.guards.Open(phase) && !activation.opened.has(row.connectionId),
+            cursor: activation.through,
+            state: [...(activation.cache.state ?? new Map())],
+            session: row.session,
+            connections: list(activation),
+            events: events(activation, sql),
+            commands,
+        }, phase);
+    });
+    const checkSession = (result) => result.session !== undefined &&
+        utf8.encode(result.session).byteLength - SESSION_ENVELOPE_BYTES > MAX_SESSION_BYTES
+        ? Effect.die(new Error("Connection session exceeds 16 KiB"))
+        : Effect.void;
+    const setKeepAwake = (activation) => Effect.gen(function* () {
+        if (registration.policy.connections !== "keepAwake")
+            return;
+        const open = (activation.rows?.size ?? 0) > 0;
+        if (open === (activation.keptAwake !== undefined))
+            return;
+        if (open) {
+            activation.keptAwake = yield* Effect.context();
+            yield* Entity.keepAlive(true);
+            return;
+        }
+        const holder = activation.keptAwake;
+        activation.keptAwake = undefined;
+        yield* Entity.keepAlive(false).pipe(Effect.provideContext(holder));
+    });
+    const closeOnDefect = (activation, connectionId) => (cause) => Effect.gen(function* () {
+        yield* Effect.logError("Connection handler defect", Cause.die(cause));
+        yield* dropRows(activation, [connectionId]);
+        return { _tag: "Closed", ended: ended("Defect", false) };
+    });
+    const identity = (activation) => ({
+        generation: activation.cache.generation,
+        owner: transport.holder,
+        ownerEpoch: transport.epoch,
+    });
+    const open = (activation, request) => withLock(activation, request.connectionId, Effect.gen(function* () {
+        const feed = request.member === FEED_MEMBER;
+        const watch = isWatchMember(request.member);
+        if (feed
+            ? registration.feeds.size === 0
+            : watch
+                ? !registration.watches.has(watchedQuery(request.member))
+                : !registration.connections.has(request.member))
+            return yield* Effect.die(new Error(`Unregistered connection ${request.member}`));
+        yield* acquire(activation);
+        yield* load(activation);
+        const existing = activation.rows.get(request.connectionId);
+        if (existing !== undefined && existing.holderEpoch === request.holderEpoch)
+            return {
+                _tag: "Opened",
+                ...identity(activation),
+                baseline: existing.baseline,
+                recovered: true,
+            };
+        const sql = yield* SqlClient.SqlClient;
+        const actor = actorRow({ sql, actor: activation });
+        if (registration.policy.createdBy !== undefined) {
+            const [created] = yield* sql `
+            SELECT created FROM actor_generations WHERE ${actor}`;
+            if (created?.created !== true)
+                return yield* ActorError.make({ reason: NotCreated.make({}) });
+        }
+        if (watch &&
+            [...activation.rows.values()].filter((row) => isWatchMember(row.member)).length >=
+                registration.policy.watch.maxPerActor)
+            return yield* ActorError.make({ reason: RunnerAtCapacity.make({}) });
+        if ([...activation.rows.values()].filter((row) => row.member === request.member).length >=
+            MAX_MEMBER_CONNECTIONS)
+            return yield* feed
+                ? ActorError.make({ reason: RunnerAtCapacity.make({}) })
+                : unavailable("Actor is at its connection limit for this member");
+        const baseline = activation.through;
+        const row = {
+            connectionId: request.connectionId,
+            member: request.member,
+            holder: request.holder,
+            holderEpoch: request.holderEpoch,
+            caller: request.caller,
+            baseline,
+            session: undefined,
+        };
+        activation.rows.set(request.connectionId, { ...row, frameSeq: 0, buffered: [] });
+        const result = yield* (feed || watch
+            ? Effect.succeed(emptyResult)
+            : run(activation, row, ConnectionPhase.cases.Open.make({ params: request.params }), request.commands)).pipe(Effect.catchDefect((cause) => Effect.gen(function* () {
+            yield* Effect.logError("Connection open defect", Cause.die(cause));
+            return yield* ActorError.make({ reason: ended("Defect", false) });
+        })), Effect.catchIf((error) => "failure" in error, (error) => Effect.succeed({ failure: error.failure })));
+        if ("failure" in result)
+            return { _tag: "Failed", value: result.failure };
+        yield* checkSession(result).pipe(Effect.catchDefect(() => ActorError.make({ reason: ended("Defect", false) })));
+        const caller = yield* encodeCaller(request.caller).pipe(Effect.orDie);
+        const inserted = yield* sql `
+          INSERT INTO actor_connections (routing_key, connection_id, bucket, tenant_id, actor_type, actor_id,
+            member, holder, holder_epoch, caller, session, opened_at_ms, opened_through)
+          SELECT routing_key, ${request.connectionId}, (routing_key >> 56)::integer, tenant_id, actor_type, actor_id,
+            ${request.member}, ${request.holder}, ${request.holderEpoch}, ${caller},
+            ${result.session === undefined ? null : compress(result.session)},
+            floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint, ${baseline}::bigint
+          FROM (${heldGeneration({ sql, actor: activation, generation: activation.cache.generation, lock: "SHARE" })}) g
+          RETURNING connection_id`;
+        if (inserted.length === 0) {
+            forget(activation);
+            return yield* unavailable("Stale actor generation");
+        }
+        const buffered = activation.rows.get(request.connectionId)?.buffered ?? [];
+        activation.rows.set(request.connectionId, { ...row, session: result.session, frameSeq: 0 });
+        activation.opened.add(request.connectionId);
+        yield* setKeepAwake(activation);
+        yield* flush(activation, result.broadcasts, activation.head, {
+            connectionId: request.connectionId,
+            member: request.member,
+            frames: [...result.sends, ...buffered],
+        });
+        if (result.close)
+            yield* closeRow(activation, request.connectionId, ended("ServerClosed", false));
+        return { _tag: "Opened", ...identity(activation), baseline };
+    }).pipe(Effect.ensuring(Effect.sync(() => {
+        if (activation.rows?.get(request.connectionId)?.buffered !== undefined)
+            activation.rows.delete(request.connectionId);
+    })), Effect.catchIf(SqlError.isSqlError, sqlUnavailable)));
+    const closeRow = (activation, connectionId, cause) => Effect.gen(function* () {
+        const row = activation.rows?.get(connectionId);
+        if (row === undefined)
+            return;
+        yield* activation.flush.withPermit(Effect.gen(function* () {
+            yield* dropRows(activation, [connectionId]);
+            yield* send(activation, channelOf(activation, row.holder, row.holderEpoch), [
+                HolderItem.cases.End.make({ connectionId, ended: cause }),
+            ]);
+        }));
+        yield* setKeepAwake(activation);
+    });
+    /**
+     * Runs `body` under the connection's lock with the activation fenced and
+     * its rows loaded, on the connection's row when the requesting holder still
+     * owns it. A database failure answers `ActorUnavailable`, so the holder retries.
+     */
+    const onOwnedRow = (activation, request, body) => withLock(activation, request.connectionId, Effect.gen(function* () {
+        yield* acquire(activation);
+        yield* load(activation);
+        const row = activation.rows?.get(request.connectionId);
+        return yield* body(row !== undefined &&
+            row.holder === request.holder &&
+            row.holderEpoch === request.holderEpoch
+            ? row
+            : undefined);
+    }).pipe(Effect.catchIf(SqlError.isSqlError, sqlUnavailable)));
+    /** Whether a request's authorization has lapsed on the framework clock. */
+    const lapsed = (authorizedUntil) => Effect.gen(function* () {
+        const clock = yield* FrameworkClock;
+        return (yield* Clock.currentTimeMillis) + clock.offsetMillis() >= authorizedUntil;
+    });
+    /** Drops a connection the server ended and answers it `ServerClosed`. */
+    const serverClosed = (activation, connectionId) => Effect.gen(function* () {
+        yield* dropRows(activation, [connectionId]);
+        yield* setKeepAwake(activation);
+        return { _tag: "Closed", ended: ended("ServerClosed", false) };
+    });
+    const frame = (activation, request) => onOwnedRow(activation, request, (row) => Effect.gen(function* () {
+        if (row === undefined)
+            return { _tag: "Closed", ended: ended("ServerClosed", true) };
+        if (request.seq <= row.frameSeq)
+            return { _tag: "Acked", ...identity(activation) };
+        if (yield* lapsed(request.authorizedUntil))
+            return yield* serverClosed(activation, request.connectionId);
+        return yield* Effect.gen(function* () {
+            const result = yield* run(activation, row, ConnectionPhase.cases.Frame.make({ frame: request.frame }), request.commands).pipe(Effect.catch(() => Effect.die(new Error("A frame handler cannot fail"))));
+            yield* checkSession(result);
+            if (result.changed) {
+                const sql = yield* SqlClient.SqlClient;
+                const written = yield* sql `
+              UPDATE actor_connections c SET session = ${result.session === undefined ? null : compress(result.session)},
+                frame_seq = ${request.seq}
+              FROM (${heldGeneration({ sql, actor: activation, generation: activation.cache.generation, lock: "SHARE" })}) g
+              WHERE c.routing_key = g.routing_key AND c.tenant_id = g.tenant_id
+                AND c.actor_type = g.actor_type AND c.actor_id = g.actor_id
+                AND c.connection_id = ${request.connectionId} AND c.frame_seq < ${request.seq}
+              RETURNING c.connection_id`;
+                if (written.length === 0) {
+                    forget(activation);
+                    return yield* unavailable("Stale actor generation");
+                }
+                row.session = result.session;
+            }
+            row.frameSeq = request.seq;
+            yield* flush(activation, result.broadcasts, activation.head, {
+                connectionId: request.connectionId,
+                member: row.member,
+                frames: result.sends,
+            });
+            if (result.close)
+                return yield* serverClosed(activation, request.connectionId);
+            return { _tag: "Acked", ...identity(activation) };
+        }).pipe(Effect.catchDefect(closeOnDefect(activation, request.connectionId)));
+    }));
+    const close = (activation, request) => onOwnedRow(activation, request, (row) => Effect.gen(function* () {
+        if (row === undefined)
+            return;
+        const result = row.member === FEED_MEMBER || isWatchMember(row.member)
+            ? undefined
+            : yield* run(activation, row, ConnectionPhase.cases.Close.make({ reason: request.cause.cause })).pipe(Effect.catchDefect((cause) => Effect.as(Effect.logError("Connection close defect", Cause.die(cause)), undefined)), Effect.orElseSucceed(() => undefined));
+        yield* dropRows(activation, [request.connectionId]);
+        activation.opened.delete(request.connectionId);
+        activation.locks.delete(request.connectionId);
+        yield* setKeepAwake(activation);
+        if (result !== undefined)
+            yield* flush(activation, result.broadcasts, activation.head);
+    }));
+    const resync = (activation, request) => onOwnedRow(activation, request, (row) => Effect.gen(function* () {
+        if (row === undefined)
+            return { _tag: "Closed", ended: ended("OwnerLost", true) };
+        if (yield* lapsed(request.authorizedUntil))
+            return yield* serverClosed(activation, request.connectionId);
+        if (row.member === FEED_MEMBER ||
+            isWatchMember(row.member) ||
+            !registration.connections.get(row.member).hasResync)
+            return { _tag: "Replayed", ...identity(activation) };
+        return yield* Effect.gen(function* () {
+            const result = yield* run(activation, row, ConnectionPhase.cases.Resync.make({ after: request.after })).pipe(Effect.catch(() => Effect.die(new Error("A resync handler cannot fail"))));
+            yield* flush(activation, result.broadcasts, activation.head, {
+                connectionId: request.connectionId,
+                member: row.member,
+                frames: result.sends,
+                replay: true,
+            });
+            if (result.close)
+                return yield* serverClosed(activation, request.connectionId);
+            return { _tag: "Replayed", ...identity(activation) };
+        }).pipe(Effect.catchDefect(closeOnDefect(activation, request.connectionId)));
+    }));
+    const follow = (activation, sql) => (tag, after) => Stream.paginate({ cursor: after, first: true }, ({ cursor, first }) => Effect.gen(function* () {
+        const signal = activation.advanced;
+        const head = activation.head;
+        if (!first && BigInt(head) <= BigInt(cursor ?? "0")) {
+            yield* Deferred.await(signal);
+            return [[], Option.some({ cursor, first: false })];
+        }
+        const page = yield* replayEvents(activation.ref, activation.key, [tag], cursor, BigInt(head), FOLLOW_PAGE).pipe(inTenant({ sql, role, tenant: activation.ref.tenant }), Effect.catchIf(SqlError.isSqlError, Effect.die), Effect.provideService(SqlClient.SqlClient, sql));
+        const next = page.length === FOLLOW_PAGE ? page.at(-1).cursor : head;
+        return [page, Option.some({ cursor: next, first: false })];
+    }));
+    const streams = streamSubscriptions({
+        registration,
+        activations,
+        transport,
+        authorize,
+        now,
+        prepare: Effect.fnUntraced(function* (activation) {
+            yield* acquire(activation).pipe(Effect.catchIf(SqlError.isSqlError, sqlUnavailable));
+            const sql = yield* SqlClient.SqlClient;
+            return {
+                ref: activation.ref,
+                cursor: activation.head,
+                state: [...(activation.cache.state ?? new Map())],
+                events: events(activation, sql),
+                follow: follow(activation, sql),
+                progress: (tag, jobId) => progressFeed(activation, tag, jobId),
+            };
+        }),
+    });
+    const progressFeed = (activation, tag, jobId) => Stream.unwrap(Effect.gen(function* () {
+        const queue = yield* Queue.sliding(STREAM_PROGRESS_BUFFER);
+        const listener = { tag, jobId, queue };
+        yield* Effect.acquireRelease(Effect.sync(() => activation.listeners.add(listener)), () => Effect.sync(() => activation.listeners.delete(listener)));
+        return Stream.fromQueue(queue);
+    }));
+    const takeProgressToken = (activation, at) => {
+        activation.progressTokens = Math.min(PROGRESS_PER_SECOND, activation.progressTokens +
+            ((at - activation.progressRefilledAt) * PROGRESS_PER_SECOND) / 1000);
+        activation.progressRefilledAt = at;
+        if (activation.progressTokens < 1)
+            return false;
+        activation.progressTokens -= 1;
+        return true;
+    };
+    /**
+     * Whether the job is still open: one indexed read of its outbox row per
+     * job id per `JOB_CHECK_MS`. A row that is gone or no longer a job
+     * closes it for this activation.
+     */
+    const checkJob = (activation, message, at) => Effect.gen(function* () {
+        const current = activation.progress.get(message.jobId);
+        if (current !== undefined && (!current.open || at - current.checkedAt < JOB_CHECK_MS))
+            return current;
+        const failed = activation.checkFailed.get(message.jobId);
+        if (failed !== undefined && at - failed < JOB_CHECK_RETRY_MS)
+            return undefined;
+        const sql = yield* SqlClient.SqlClient;
+        const actor = actorRow({ sql, actor: activation });
+        activation.checking.add(message.jobId);
+        const read = yield* sql `SELECT kind, attempts, cancelled_at_ms IS NOT NULL AS cancelled, caller, payload
+          FROM actor_outbox
+          WHERE ${actor} AND intent_id = ${message.jobId}`.pipe(Effect.ensuring(Effect.sync(() => activation.checking.delete(message.jobId))), Effect.exit);
+        if (Exit.isFailure(read)) {
+            activation.checkFailed.set(message.jobId, at);
+            return undefined;
+        }
+        activation.checkFailed.delete(message.jobId);
+        const closedMeanwhile = activation.progress.get(message.jobId);
+        if (closedMeanwhile !== undefined && !closedMeanwhile.open)
+            return closedMeanwhile;
+        const [row] = read.value;
+        if (row === undefined || row.kind !== "job" || row.cancelled) {
+            const closed = closedJob(at);
+            activation.progress.set(message.jobId, closed);
+            return closed;
+        }
+        if (row.attempts < message.attempt)
+            return undefined;
+        const stored = activation.progress.get(message.jobId);
+        if (stored !== undefined && stored.open) {
+            stored.checkedAt = at;
+            stored.touchedAt = at;
+            return stored;
+        }
+        const checked = {
+            open: true,
+            checkedAt: at,
+            touchedAt: at,
+            attempt: 0,
+            seq: 0,
+            principal: principal(yield* decodeCaller(row.caller).pipe(Effect.orDie)),
+            payload: row.payload,
+            holders: new Set(),
+        };
+        activation.progress.set(message.jobId, checked);
+        return checked;
+    });
+    const after = (message, job) => message.attempt > job.attempt || (message.attempt === job.attempt && message.seq > job.seq);
+    /**
+     * Admits one executor progress frame and forwards it through this
+     * activation's ordered channels to the member connections that want it,
+     * and to stream subscribers. Every refusal drops the frame; nothing here
+     * fails, writes, or closes a session.
+     */
+    const progress = (activation, message) => Effect.gen(function* () {
+        const { ref } = activation;
+        if (message.ref.tenant !== ref.tenant ||
+            message.ref.actor !== ref.actor ||
+            message.ref.id !== ref.id ||
+            !wanted.has(message.job))
+            return;
+        const at = yield* now;
+        if (at - activation.progressSweptAt >= PROGRESS_RECORD_MS) {
+            activation.progressSweptAt = at;
+            for (const [jobId, record] of activation.progress)
+                if (at - record.touchedAt >= PROGRESS_RECORD_MS)
+                    activation.progress.delete(jobId);
+        }
+        if (at > message.leaseUntil + PROGRESS_SKEW_MS)
+            return;
+        const known = activation.progress.get(message.jobId);
+        if (known !== undefined && (!known.open || !after(message, known)))
+            return;
+        if (!takeProgressToken(activation, at))
+            return;
+        yield* acquire(activation);
+        yield* load(activation);
+        const job = yield* checkJob(activation, message, at);
+        if (job === undefined)
+            return;
+        yield* activation.flush.withPermit(Effect.gen(function* () {
+            if (activation.progress.get(message.jobId) !== job)
+                return;
+            if (!job.open || !after(message, job))
+                return;
+            job.attempt = message.attempt;
+            job.seq = message.seq;
+            job.touchedAt = at;
+            const perChannel = new Map();
+            for (const row of activation.rows?.values() ?? []) {
+                const wants = registration.connections.get(row.member)?.progress;
+                if (wants === undefined ||
+                    !wants.jobs.has(message.job) ||
+                    row.buffered !== undefined ||
+                    (wants.to === "principal" && !samePrincipal(principal(row.caller), job.principal)))
+                    continue;
+                const channel = channelOf(activation, row.holder, row.holderEpoch);
+                const members = perChannel.get(channel) ?? new Map();
+                members.set(row.member, [...(members.get(row.member) ?? []), row.connectionId]);
+                perChannel.set(channel, members);
+            }
+            for (const [channel, members] of perChannel) {
+                job.holders.add(`${channel.holder}|${channel.epoch}`);
+                yield* send(activation, channel, [...members].map(([member, to]) => HolderItem.cases.Progress.make({
+                    member,
+                    to,
+                    job: message.job,
+                    jobId: message.jobId,
+                    attempt: message.attempt,
+                    seq: message.seq,
+                    frame: message.frame,
+                })));
+            }
+            for (const listener of activation.listeners)
+                if (listener.tag === message.job &&
+                    (listener.jobId === undefined || listener.jobId === message.jobId))
+                    Queue.offerUnsafe(listener.queue, {
+                        jobId: message.jobId,
+                        job: job.payload,
+                        attempt: message.attempt,
+                        seq: message.seq,
+                        frame: message.frame,
+                    });
+        }));
+    }).pipe(Effect.catchCause((cause) => Effect.logDebug("Progress frame dropped", cause)));
+    /**
+     * Closes a job's progress on this activation, once its route turn (whose
+     * command id is the job id) commits or its settle is reported: holders
+     * that received its progress discard what they still buffer, before the
+     * route's own broadcasts.
+     */
+    const closeProgress = (activation, jobId) => Effect.suspend(() => {
+        const job = activation.progress.get(jobId);
+        if (job === undefined && !activation.checking.has(jobId))
+            return Effect.void;
+        return activation.flush.withPermit(Effect.gen(function* () {
+            const current = activation.progress.get(jobId);
+            activation.progress.set(jobId, closedJob(yield* now));
+            if (current === undefined || !current.open)
+                return;
+            current.open = false;
+            for (const name of current.holders) {
+                const channel = activation.channels.get(name);
+                if (channel !== undefined)
+                    yield* send(activation, channel, [HolderItem.cases.ProgressEnd.make({ jobId })]);
+            }
+        }));
+    });
+    const progressClosed = (activation, message) => {
+        const job = activation.progress.get(message.jobId);
+        return job !== undefined && message.attempt < job.attempt
+            ? Effect.void
+            : closeProgress(activation, message.jobId);
+    };
+    const hibernate = (entityId) => Effect.gen(function* () {
+        const activation = activations.get(entityId);
+        if (activation === undefined)
+            return;
+        yield* streams.end(activation);
+        yield* seal(activation);
+        forget(activation);
+        activation.opened.clear();
+        activation.progress.clear();
+        activation.checkFailed.clear();
+        activation.head = "0";
+        activation.through = "0";
+    });
+    return {
+        activations,
+        hasConnections,
+        hasStreams,
+        hasProgress,
+        progress,
+        closeProgress,
+        progressClosed,
+        feedBroadcasts,
+        watchBroadcasts,
+        hibernate,
+        subscribe: streams.subscribe,
+        endStreams: streams.end,
+        watchStreams: streams.watch,
+        enter,
+        prepare,
+        list,
+        flush,
+        open,
+        frame,
+        close,
+        resync,
+    };
+};