@rikalabs/akter 0.1.0-alpha.0

package/dist/runtime/subscriptions/relay.js

@@ -0,0 +1,763 @@
+import { DateTime, Effect, Match, Result, Schema } from "effect";
+import { SqlClient, SqlError } from "effect/sql";
+import {} from "../database/shards.js";
+import { SubscriptionFailure } from "../../errors/subscription.js";
+import { Outcome, Request } from "../request.js";
+import {} from "../members.js";
+import { ActorRef, System } from "../../identity/caller.js";
+import { decompress, routingKey } from "../storage/codec.js";
+import { databaseTime, FrameworkClock } from "../turn/admission.js";
+import { TurnHooks } from "../turn/hooks.js";
+import { ControlPayload, StringsJson, textArray } from "../turn/outbox.js";
+import { candidates, outboxNow } from "../turn/relay.js";
+import { changeRows, rowColumns } from "./storage.js";
+import { deliveryCommandId } from "./identity.js";
+import { count, Metrics } from "../telemetry/metrics.js";
+import { SpanNames } from "../telemetry/spans.js";
+/** How long a row whose declaration was removed stays due before cleanup deletes it. */
+const REMOVED_AFTER_MS = 86_400_000;
+/** Source-side subscription rows one expansion statement marks at most. */
+const EXPANSION_PAGE = 1000;
+/** A claimed feed row to expand, or control row to register at its source. */
+const OutboxWork = Schema.Struct({
+    kind: Schema.Literals(["feed", "control"]),
+    routing_key: Schema.String,
+    intent_id: Schema.String,
+    attempts: Schema.Int,
+    tenant_id: Schema.String,
+    actor_type: Schema.String,
+    actor_id: Schema.String,
+    target_type: Schema.String,
+    target_id: Schema.String,
+    command: Schema.String,
+    payload: Schema.String,
+    scheduled_at_ms: Schema.String,
+    claimed_until: Schema.String,
+});
+/** A claimed subscription row to deliver. */
+const SubscriptionRow = Schema.Struct({
+    kind: Schema.Literal("subscription"),
+    routing_key: Schema.String,
+    tenant_id: Schema.String,
+    source_type: Schema.String,
+    source_id: Schema.String,
+    subscriber_type: Schema.String,
+    subscription: Schema.String,
+    subscriber_id: Schema.String,
+    events: Schema.Array(Schema.String),
+    epoch: Schema.String,
+    delivered: Schema.String,
+    attempts: Schema.Int,
+    gap_through: Schema.NullOr(Schema.String),
+    claimed_until: Schema.String,
+});
+/** A unit of relay work, as a claim returns it in its `work` column: an outbox row to expand or a subscription row to deliver. */
+const SubscriptionWork = Schema.fromJsonString(Schema.Union([OutboxWork, SubscriptionRow]));
+const JsonText = Schema.fromJsonString(Schema.Unknown);
+/**
+ * A delivery as the relay encodes it for any subscriber's handler: the event
+ * stays the source's stored JSON, which the handler's own schema decodes.
+ */
+const WireEvent = Schema.TaggedStruct("Event", {
+    subscription: Schema.String,
+    source: ActorRef,
+    cursor: Schema.String,
+    event: Schema.Unknown,
+    commandId: Schema.String,
+    timestamp: Schema.String,
+});
+const WireRejected = Schema.TaggedStruct("Rejected", {
+    subscription: Schema.String,
+    source: ActorRef,
+    reason: Schema.Literal("UnknownCursor"),
+    cursor: Schema.String,
+});
+const WireGap = Schema.TaggedStruct("RetentionGap", {
+    subscription: Schema.String,
+    source: ActorRef,
+    after: Schema.String,
+    resumeAfter: Schema.String,
+});
+const WireDelivery = Schema.Union([WireEvent, WireGap, WireRejected]);
+const encodePayload = Schema.encodeEffect(Schema.fromJsonString(Schema.Struct({ value: WireDelivery })));
+const decodeStrings = Schema.decodeEffect(StringsJson);
+const decodeJson = Schema.decodeEffect(JsonText);
+const decodeWork = Schema.decodeEffect(SubscriptionWork);
+const backoff = (attempts, settings) => Math.max(settings.claimLeaseMs(), Math.min(1000 * 2 ** Math.max(attempts - 1, 0), settings.maxBackoffMs));
+/** A request-shaped value naming the work at a fault point, for `TurnHooks`. */
+const hookRequest = (ref, command, commandId) => Request.make({
+    ref,
+    caller: System.make({ source: "subscription" }),
+    command,
+    commandId,
+    payload: "",
+});
+/**
+ * The relay's subscription work on one runner: expanding source feeds,
+ * registering dynamic subscriptions at their sources, and delivering due
+ * subscription rows of the subscriber types this runner registers. A runner
+ * that registers none claims none of it.
+ *
+ * Delivery: rows this runner can deliver that an expansion makes due are leased at
+ * once, up to its free delivery slots, and delivered without a claim pass; the
+ * rest become due for any runner. Updates bound their rows by the page's key
+ * range, read from the key index, rather than joining the page, so a planner on
+ * stale row counts cannot rescan the page for every stored row. A renewal never
+ * shortens a claim, so it cannot undo a lease a test clock moved. Shutdown
+ * releases a claim at once; the receipt and cursor make redelivery safe.
+ *
+ * Subscribing: a source that has never been created can be subscribed to. Only a
+ * refused subscribe leaves a tombstone at its own epoch. A subscription is due at
+ * once when history after its start already matches, so one to a quiet source
+ * still delivers it. Widening only adds this declaration's classes to the row's
+ * list. Every write that changes a row's `active` or `events` goes through
+ * `changeRows`, which moves the tag summary in the same transaction.
+ *
+ * Gaps: pruning removes a prefix, so history after `delivered` is missing when the
+ * oldest retained event is past its successor or nothing is retained. An
+ * operator's skip plants the position it skips through as the gap's end. The
+ * detection time fixes the gap's id and its range is kept, so a redelivery reports
+ * the same gap even after pruning advances. An id route needs the pruned event to
+ * name a subscriber. A short batch scanned every matching event through the head
+ * it read, and after a gap older than the pruning it stops at the gap to find the
+ * next one.
+ *
+ * Delivery outcomes: the subscriber reads the current shape, and a value its chain
+ * cannot read backs the row off at this event rather than skipping it. A failure
+ * backs the row off, and a stale or unsubscribed acknowledgement removes this
+ * epoch's row. The marked position is read before the settle in its own snapshot:
+ * a commit after it raised `marked` through its expansion, which the settle reads
+ * from the row.
+ */
+export const subscriptionRelay = Effect.fnUntraced(function* (options) {
+    const sql = yield* SqlClient.SqlClient;
+    const hooks = yield* TurnHooks;
+    const clock = yield* FrameworkClock;
+    const { settings } = options;
+    const now = () => outboxNow({ sql, offsetMillis: clock.offsetMillis() });
+    const sourceWhere = (alias, key, source) => sql `${sql(alias)}.routing_key = ${key} AND ${sql(alias)}.tenant_id = ${source.tenant}
+      AND ${sql(alias)}.source_type = ${source.actor} AND ${sql(alias)}.source_id = ${source.id}`;
+    const eventsOf = (alias, key, source) => sql `${sql(alias)}.routing_key = ${key} AND ${sql(alias)}.tenant_id = ${source.tenant}
+      AND ${sql(alias)}.actor_type = ${source.actor} AND ${sql(alias)}.actor_id = ${source.id}`;
+    /** When a claim of the row aliased `alias` ends: one lease, or its backoff once that is longer. */
+    const leaseEnd = (alias) => sql `${now()} + greatest(${settings.claimLeaseMs()}::bigint,
+      least(1000 * power(2, least(${sql(alias)}.attempts, 31)), ${settings.maxBackoffMs}::bigint))::bigint`;
+    const outboxClaim = (kind, now, limit, range) => {
+        const probe = 4 * limit;
+        const found = sql.literal(`${kind}_candidates`);
+        const locked = sql.literal(`${kind}_locked`);
+        const claimed = sql.literal(`${kind}_claimed`);
+        return sql `${found} AS (
+        ${candidates({ sql, kind, now, limit: probe, range })}
+        ORDER BY o.due_at_ms LIMIT ${probe}
+      ),
+      ${locked} AS (
+        SELECT o.routing_key, o.intent_id FROM actor_outbox o
+        JOIN ${found} USING (routing_key, intent_id)
+        WHERE o.kind = ${kind} AND o.due_at_ms <= ${now}
+        ORDER BY o.due_at_ms LIMIT ${limit}
+        FOR UPDATE OF o SKIP LOCKED
+      ),
+      ${claimed} AS (
+        UPDATE actor_outbox o SET attempts = o.attempts + 1, due_at_ms = ${leaseEnd("o")}
+        FROM ${locked} c
+        WHERE o.routing_key = c.routing_key AND o.intent_id = c.intent_id
+        RETURNING jsonb_build_object('kind', o.kind, 'routing_key', o.routing_key::text,
+          'intent_id', o.intent_id, 'attempts', o.attempts, 'tenant_id', o.tenant_id,
+          'actor_type', o.actor_type, 'actor_id', o.actor_id, 'target_type', o.target_type,
+          'target_id', o.target_id, 'command', o.command, 'payload', o.payload,
+          'scheduled_at_ms', o.scheduled_at_ms::text,
+          'claimed_until', o.due_at_ms::text)::text AS work
+      )`;
+    };
+    const subscribed = (local) => sql `subscribed (subscriber_type, subscription, known) AS (VALUES ${sql.csv(local.map(({ subscriberType, subscription }) => sql `(${subscriberType}::text, ${subscription.tag}::text, ${textArray({
+        sql,
+        values: [...subscription.events, ...subscription.retired],
+    })})`))})`;
+    const claimedRow = sql `jsonb_build_object('kind', 'subscription', 'routing_key', s.routing_key::text,
+    'tenant_id', s.tenant_id, 'source_type', s.source_type, 'source_id', s.source_id,
+    'subscriber_type', s.subscriber_type, 'subscription', s.subscription,
+    'subscriber_id', s.subscriber_id, 'events', to_jsonb(s.events),
+    'epoch', s.epoch::text, 'delivered', s.delivered::text, 'attempts', s.attempts,
+    'gap_through', s.gap_through::text, 'claimed_until', s.due_at_ms::text)`;
+    /**
+     * Claims due feed and control rows and due subscription rows, each up to
+     * its free slots, as parts of the relay's claim statement. Subscription
+     * rows are probed per bucket and subscriber type through
+     * `actor_subscriptions_due`, so caught-up rows and other types' rows are
+     * never read, and only rows whose event tags this runner's declaration
+     * knows are taken: a runner never settles past a class it can't decode.
+     */
+    const claim = (slots, range) => {
+        const local = options.local();
+        if (local.length === 0)
+            return undefined;
+        const parts = [];
+        const results = [];
+        for (const kind of ["feed", "control"])
+            if (slots[kind] > 0) {
+                parts.push(outboxClaim(kind, now(), slots[kind], range));
+                results.push(sql `SELECT work FROM ${sql.literal(`${kind}_claimed`)}`);
+            }
+        if (slots.subscription > 0) {
+            const probe = 4 * slots.subscription;
+            parts.push(sql `${subscribed(local)},
+          subscription_candidates AS (
+            SELECT s.* FROM generate_series(${range.first}::int, ${range.last}::int) AS b(bucket)
+            CROSS JOIN (SELECT DISTINCT subscriber_type FROM subscribed) AS t
+            CROSS JOIN LATERAL (
+              SELECT routing_key, tenant_id, source_type, source_id, subscriber_type, subscription,
+                subscriber_id, due_at_ms
+              FROM actor_subscriptions
+              WHERE actor_subscriptions.bucket = b.bucket
+                AND actor_subscriptions.subscriber_type = t.subscriber_type
+                AND actor_subscriptions.due_at_ms <= ${now()}
+                AND EXISTS (SELECT 1 FROM subscribed m WHERE m.subscriber_type = actor_subscriptions.subscriber_type
+                  AND m.subscription = actor_subscriptions.subscription
+                  AND m.known @> actor_subscriptions.events)
+              ORDER BY actor_subscriptions.due_at_ms LIMIT ${probe}
+            ) s
+            ORDER BY s.due_at_ms LIMIT ${probe}
+          ),
+          subscription_locked AS (
+            SELECT s.routing_key, s.tenant_id, s.source_type, s.source_id, s.subscriber_type,
+              s.subscription, s.subscriber_id
+            FROM actor_subscriptions s
+            JOIN subscription_candidates c USING (routing_key, tenant_id, source_type, source_id,
+              subscriber_type, subscription, subscriber_id)
+            WHERE s.due_at_ms <= ${now()} AND s.active
+            ORDER BY s.due_at_ms LIMIT ${slots.subscription}
+            FOR UPDATE OF s SKIP LOCKED
+          ),
+          subscription_claimed AS (
+            UPDATE actor_subscriptions s SET attempts = s.attempts + 1, due_at_ms = ${leaseEnd("s")}
+            FROM subscription_locked c
+            WHERE s.routing_key = c.routing_key AND s.tenant_id = c.tenant_id
+              AND s.source_type = c.source_type AND s.source_id = c.source_id
+              AND s.subscriber_type = c.subscriber_type AND s.subscription = c.subscription
+              AND s.subscriber_id = c.subscriber_id
+            RETURNING ${claimedRow}::text AS work
+          )`);
+            results.push(sql `SELECT work FROM subscription_claimed`);
+        }
+        return results.length === 0 ? undefined : { parts, results };
+    };
+    const decode = (work) => Effect.orDie(decodeWork(work));
+    const placementKey = Effect.fnUntraced(function* (ref) {
+        const placement = yield* options.placementOf(ref.actor);
+        if (placement === undefined)
+            return yield* SubscriptionFailure.make({
+                message: `Source type ${ref.actor} has no recorded placement`,
+            });
+        return routingKey({ ref, placement });
+    });
+    const rowKey = (alias, row, key, source) => sql `${sourceWhere(alias, key, source)} AND ${sql(alias)}.subscriber_type = ${row.subscriber_type}
+      AND ${sql(alias)}.subscription = ${row.subscription} AND ${sql(alias)}.subscriber_id = ${row.subscriber_id}`;
+    /** Deletes the rows `old_rows` locked, returning each as no longer active. */
+    const deleteLocked = sql `DELETE FROM actor_subscriptions s USING old_rows o
+    WHERE s.routing_key = o.routing_key AND s.tenant_id = o.tenant_id
+      AND s.source_type = o.source_type AND s.source_id = o.source_id
+      AND s.subscriber_type = o.subscriber_type AND s.subscription = o.subscription
+      AND s.subscriber_id = o.subscriber_id
+    RETURNING s.routing_key, s.tenant_id, s.source_type, s.source_id, s.subscriber_type,
+      s.subscription, s.subscriber_id, false AS active, s.events`;
+    /**
+     * Deletes a dynamic row the subscriber acknowledged as stale or
+     * unsubscribed, but only at the delivery's epoch, so an old acknowledgement
+     * never removes a newer subscription.
+     */
+    const deleteRow = (row, key, source) => changeRows(sql `SELECT ${rowColumns({ sql, alias: "s" })} FROM actor_subscriptions s
+        WHERE ${rowKey("s", row, key, source)} AND s.epoch = ${row.epoch} FOR UPDATE`, deleteLocked);
+    /**
+     * Expands one source's feed: a claimed or backing-off row gets `marked`
+     * raised to the source's head, and a caught-up row with a matching event
+     * after its `delivered` becomes due. A row already due and unclaimed is left
+     * alone, so a burst of commits beside many waiting rows writes nothing. A claimed or backing-off row keeps its due
+     * time, so a commit can't cut a retry short, and `marked` alone keeps it
+     * due after its settle. The feed row goes only if no commit reset its
+     * `attempts` meanwhile; otherwise it stays due and is expanded again.
+     */
+    const expand = Effect.fnUntraced(function* (row, handoff) {
+        const key = BigInt(row.routing_key);
+        const source = { tenant: row.tenant_id, actor: row.actor_type, id: row.actor_id };
+        yield* hooks.at("afterClaim", hookRequest(source, "$feed", row.intent_id));
+        const local = options.local();
+        let after = ["", "", ""];
+        for (;;) {
+            const free = local.length === 0 ? 0 : Math.max(0, yield* handoff.free);
+            const leasable = free === 0
+                ? sql `leasable AS MATERIALIZED (
+              SELECT NULL::text AS subscriber_type, NULL::text AS subscription,
+                NULL::text AS subscriber_id WHERE false)`
+                : sql `${subscribed(local)},
+            leasable AS MATERIALIZED (
+              SELECT p.subscriber_type, p.subscription, p.subscriber_id
+              FROM page p JOIN subscribed m ON m.subscriber_type = p.subscriber_type
+                AND m.subscription = p.subscription AND m.known @> p.events, head
+              WHERE p.due_at_ms IS NULL AND p.marked < head.h
+                -- OFFSET 0 keeps this a per-row probe bounded by the row's
+                -- cursor in the index, rather than a join that reads the
+                -- source's whole event log for every row of the page.
+                AND EXISTS (
+                  SELECT 1 FROM actor_events e
+                  WHERE ${eventsOf("e", key, source)}
+                    AND e.sequence > p.delivered AND e.sequence <= head.h
+                    AND e.event = ANY(p.events) OFFSET 0)
+              LIMIT ${free})`;
+            const inPage = sql `(s.subscriber_type, s.subscription, s.subscriber_id) > (${after[0]}, ${after[1]}, ${after[2]})
+        AND (s.subscriber_type, s.subscription, s.subscriber_id)
+          <= (SELECT subscriber_type, subscription, subscriber_id FROM last)`;
+            const [page] = yield* sql `
+        WITH head AS (
+          SELECT event_sequence AS h FROM actor_generations g
+          WHERE ${eventsOf("g", key, source)}),
+        page AS MATERIALIZED (
+          SELECT s.subscriber_type, s.subscription, s.subscriber_id, s.due_at_ms, s.marked,
+            s.delivered, s.events
+          FROM actor_subscriptions s
+          WHERE ${sourceWhere("s", key, source)} AND s.active
+            AND (s.subscriber_type, s.subscription, s.subscriber_id) > (${after[0]}, ${after[1]}, ${after[2]})
+          ORDER BY s.subscriber_type, s.subscription, s.subscriber_id
+          LIMIT ${EXPANSION_PAGE}),
+        last AS (
+          SELECT subscriber_type, subscription, subscriber_id FROM page
+          ORDER BY subscriber_type DESC, subscription DESC, subscriber_id DESC LIMIT 1),
+        ${leasable},
+        -- Leased only while still unclaimed in the version this updates, so a
+        -- row another runner claimed meanwhile is left to the marking below.
+        leased AS (
+          UPDATE actor_subscriptions s SET marked = greatest(s.marked, head.h),
+            due_at_ms = ${leaseEnd("s")}, attempts = s.attempts + 1
+          FROM leasable l, head
+          WHERE ${sourceWhere("s", key, source)} AND ${inPage}
+            AND s.subscriber_type = l.subscriber_type AND s.subscription = l.subscription
+            AND s.subscriber_id = l.subscriber_id AND s.due_at_ms IS NULL
+          RETURNING s.subscriber_type, s.subscription, s.subscriber_id, ${claimedRow}::text AS work),
+        marked AS (
+          UPDATE actor_subscriptions s SET marked = greatest(s.marked, head.h),
+            due_at_ms = coalesce(s.due_at_ms, ${now()})
+          FROM head
+          WHERE NOT EXISTS (SELECT 1 FROM leased x WHERE x.subscriber_type = s.subscriber_type
+              AND x.subscription = s.subscription AND x.subscriber_id = s.subscriber_id)
+            AND ${sourceWhere("s", key, source)} AND s.active
+            AND ${inPage} AND s.marked < head.h
+            -- A row due and unclaimed since its last settle reads through the
+            -- head when it is claimed, so it needs no write.
+            AND NOT (s.due_at_ms IS NOT NULL AND s.due_at_ms <= ${now()} AND s.attempts = 0)
+            AND (s.due_at_ms IS NOT NULL OR EXISTS (
+                SELECT 1 FROM actor_events e
+                WHERE ${eventsOf("e", key, source)}
+                  AND e.sequence > s.delivered AND e.sequence <= head.h AND e.event = ANY(s.events)))
+          RETURNING 1)
+        SELECT (SELECT count(*) FROM page)::int AS rows,
+          (SELECT json_agg(work)::text FROM leased) AS leased,
+          (SELECT json_build_array(subscriber_type, subscription, subscriber_id)::text FROM last) AS last`;
+            if (page.leased !== null)
+                for (const work of yield* decodeStrings(page.leased))
+                    yield* handoff.start(yield* decode(work));
+            if (page.rows < EXPANSION_PAGE || page.last === null)
+                break;
+            after = yield* decodeStrings(page.last);
+        }
+        yield* hooks.at("afterExpand", hookRequest(source, "$feed", row.intent_id));
+        yield* sql `DELETE FROM actor_outbox WHERE routing_key = ${key} AND intent_id = ${row.intent_id}
+      AND kind = 'feed' AND attempts = ${row.attempts}`;
+        yield* options.wake;
+    });
+    /** Delivers one subscription delivery from `source` and answers how it settled. */
+    const deliverOne = Effect.fnUntraced(function* (subscriber, handler, source, delivery, issuedAt, value) {
+        const envelope = { ...delivery, sourceType: source.actor, sourceId: source.id };
+        const commandId = yield* deliveryCommandId({
+            subscriber,
+            envelope,
+            issuedAt,
+            retryWindowMs: settings.retryWindowMs,
+        });
+        const request = Request.make({
+            ref: subscriber,
+            caller: System.make({ source: "subscription", ref: source }),
+            command: handler,
+            commandId,
+            payload: yield* encodePayload({ value }),
+            delivery: envelope,
+        });
+        return yield* options.deliver(request).pipe(Effect.result);
+    });
+    const failure = (outcome) => {
+        if (Result.isFailure(outcome))
+            return outcome.failure.reason._tag;
+        return Outcome.guards.Defect(outcome.success) ? String(outcome.success.cause) : undefined;
+    };
+    /**
+     * Registers one dynamic subscription change at its source with a framework
+     * statement, never a turn, so the source isn't activated. Holding the
+     * source's generation row `FOR SHARE` orders it against the emit path's
+     * `FOR UPDATE`, and a row changes only for a strictly newer epoch, so a
+     * rerun after a crash, or an older change delivered late, changes nothing.
+     * A cursor past the source's head is refused: any older row becomes a
+     * tombstone at this epoch and the subscriber receives `Rejected`.
+     */
+    const register = Effect.fnUntraced(function* (row) {
+        const change = yield* Schema.decodeEffect(ControlPayload)(row.payload);
+        const subscriber = { tenant: row.tenant_id, actor: row.actor_type, id: row.actor_id };
+        const source = { tenant: row.tenant_id, actor: row.target_type, id: row.target_id };
+        const epoch = BigInt(change.epoch);
+        yield* hooks.at("afterClaim", hookRequest(source, "$control", row.intent_id));
+        const key = yield* placementKey(source);
+        const pk = sourceWhere("s", key, source);
+        const target = sql `${pk} AND s.subscriber_type = ${subscriber.actor}
+      AND s.subscription = ${row.command} AND s.subscriber_id = ${subscriber.id}`;
+        const rejected = yield* sql.withTransaction(Effect.gen(function* () {
+            yield* sql `INSERT INTO actor_generations (routing_key, tenant_id, actor_type, actor_id)
+          VALUES (${key}, ${source.tenant}, ${source.actor}, ${source.id}) ON CONFLICT DO NOTHING`;
+            const [generation] = yield* sql `
+          SELECT event_sequence::text AS head FROM actor_generations g
+          WHERE ${eventsOf("g", key, source)} FOR SHARE`;
+            const head = BigInt(generation.head);
+            const [existing] = yield* sql `
+          SELECT s.epoch::text AS epoch, s.active FROM actor_subscriptions s
+          WHERE ${target} FOR UPDATE`;
+            const refused = change.op === "subscribe" &&
+                change.start !== "now" &&
+                change.start !== "start" &&
+                BigInt(change.start) > head;
+            if (existing !== undefined && BigInt(existing.epoch) >= epoch)
+                return refused && BigInt(existing.epoch) === epoch && !existing.active;
+            const old = sql `SELECT ${rowColumns({ sql, alias: "s" })} FROM actor_subscriptions s
+          WHERE ${target} FOR UPDATE`;
+            if (change.op === "remove" || refused) {
+                yield* changeRows(old, sql `INSERT INTO actor_subscriptions AS s (routing_key, tenant_id, source_type, source_id,
+                subscriber_type, subscription, subscriber_id, events, epoch, active, delivered, bucket)
+              VALUES (${key}, ${source.tenant}, ${source.actor}, ${source.id}, ${subscriber.actor},
+                ${row.command}, ${subscriber.id}, ${textArray({ sql, values: change.events })}, ${epoch}, false,
+                ${head}, ${Number(key >> 56n)})
+              ON CONFLICT (routing_key, tenant_id, source_type, source_id, subscriber_type, subscription, subscriber_id)
+              DO UPDATE SET epoch = EXCLUDED.epoch, active = false, due_at_ms = NULL, attempts = 0,
+                last_error = NULL, gap_at_ms = NULL, gap_through = NULL
+              RETURNING ${rowColumns({ sql, alias: "s" })}`);
+                return refused;
+            }
+            const delivered = Match.value(change.start).pipe(Match.when("now", () => head), Match.when("start", () => 0n), Match.orElse((cursor) => BigInt(cursor)));
+            yield* changeRows(old, sql `INSERT INTO actor_subscriptions AS s (routing_key, tenant_id, source_type, source_id,
+              subscriber_type, subscription, subscriber_id, events, epoch, active, delivered, bucket,
+              due_at_ms)
+            SELECT ${key}, ${source.tenant}, ${source.actor}, ${source.id}, ${subscriber.actor},
+              ${row.command}, ${subscriber.id}, x.events, ${epoch}, true, ${delivered},
+              ${Number(key >> 56n)},
+              CASE WHEN EXISTS (SELECT 1 FROM actor_events e
+                  WHERE ${eventsOf("e", key, source)}
+                    AND e.sequence > ${delivered} AND e.sequence <= ${head} AND e.event = ANY(x.events))
+                THEN ${now()} END
+            FROM (SELECT ${textArray({ sql, values: change.events })} AS events) AS x
+            ON CONFLICT (routing_key, tenant_id, source_type, source_id, subscriber_type, subscription, subscriber_id)
+            DO UPDATE SET events = EXCLUDED.events, epoch = EXCLUDED.epoch, active = true,
+              delivered = EXCLUDED.delivered, marked = 0, due_at_ms = EXCLUDED.due_at_ms, attempts = 0,
+              last_error = NULL, gap_at_ms = NULL, gap_through = NULL
+            RETURNING ${rowColumns({ sql, alias: "s" })}`);
+            return false;
+        }));
+        const claim = sql `routing_key = ${BigInt(row.routing_key)} AND intent_id = ${row.intent_id}
+      AND kind = 'control' AND due_at_ms = ${BigInt(row.claimed_until)}`;
+        if (rejected) {
+            const outcome = yield* deliverOne(subscriber, yield* handlerOf(subscriber.actor, row.command), source, {
+                subscription: row.command,
+                epoch: change.epoch,
+                kind: "rejected",
+                position: change.start,
+            }, Number(row.scheduled_at_ms), WireRejected.make({
+                subscription: row.command,
+                source,
+                reason: "UnknownCursor",
+                cursor: change.start,
+            }));
+            const failed = failure(outcome);
+            if (failed !== undefined) {
+                yield* sql `UPDATE actor_outbox SET last_error = ${failed},
+            due_at_ms = ${(yield* databaseTime) + backoff(row.attempts, settings)}
+          WHERE ${claim}`;
+                return;
+            }
+        }
+        yield* hooks.at("beforeOutboxDelete", hookRequest(source, "$control", row.intent_id));
+        yield* sql `DELETE FROM actor_outbox WHERE ${claim}`;
+        yield* options.wake;
+    });
+    const handlerOf = Effect.fnUntraced(function* (subscriberType, tag) {
+        const found = options
+            .local()
+            .find((local) => local.subscriberType === subscriberType && local.subscription.tag === tag);
+        if (found === undefined)
+            return yield* SubscriptionFailure.make({
+                message: `Subscription ${subscriberType}.${tag} is not declared on this runner`,
+            });
+        return found.subscription.handler;
+    });
+    /**
+     * Delivers one claimed subscription row: up to `batch` matching events
+     * after `delivered`, one command at a time in cursor order, each waiting
+     * for the previous one's outcome, renewing the claim after each. It then
+     * settles `delivered` to the position it scanned through, fenced on the
+     * claim and the epoch. A failed delivery settles the progress before it and
+     * backs the row off; later events wait behind it.
+     */
+    const deliverRow = Effect.fnUntraced(function* (row, claim) {
+        const local = options
+            .local()
+            .find(({ subscriberType, subscription }) => subscriberType === row.subscriber_type && subscription.tag === row.subscription);
+        const source = { tenant: row.tenant_id, actor: row.source_type, id: row.source_id };
+        const key = BigInt(row.routing_key);
+        const pk = sql `${sourceWhere("s", key, source)}
+      AND s.subscriber_type = ${row.subscriber_type} AND s.subscription = ${row.subscription}
+      AND s.subscriber_id = ${row.subscriber_id}`;
+        const held = () => sql `${pk} AND s.epoch = ${row.epoch} AND s.due_at_ms = ${claim.lease}`;
+        const name = `${row.subscriber_type}.${row.subscription}`;
+        const lostClaim = (step) => Effect.logWarning(`Subscription ${step} lost its claim`).pipe(Effect.annotateLogs({ subscription: name }));
+        const at = (point) => hooks.at(point, hookRequest(source, row.subscription, row.subscriber_id));
+        yield* at("afterClaim");
+        const backOff = (progress, cause) => Effect.gen(function* () {
+            yield* Effect.logWarning("Subscription delivery failed; retrying with backoff").pipe(Effect.annotateLogs({
+                subscription: name,
+                source: `${source.actor}/${source.id}`,
+                tenant: source.tenant,
+                cause,
+            }));
+            yield* sql `UPDATE actor_subscriptions s SET delivered = greatest(s.delivered, ${progress}),
+            last_error = ${cause},
+            due_at_ms = ${(yield* databaseTime) + backoff(row.attempts, settings)}
+          WHERE ${held()}`;
+            yield* count(Metrics.relayRetried, { kind: "subscription" }, 1);
+        });
+        if (local === undefined)
+            return yield* backOff(BigInt(row.delivered), "Undeclared subscription");
+        const { subscription } = local;
+        const batch = yield* sql `SELECT (SELECT event_sequence::text FROM actor_generations g
+          WHERE ${eventsOf("g", key, source)}) AS head,
+        (SELECT min(o.sequence)::text FROM actor_events o
+          WHERE ${eventsOf("o", key, source)}) AS oldest,
+        e.sequence::text AS sequence, e.event, e.command_id, e.value, e.payload_version,
+        e.emitted_at_ms::text AS emitted_at_ms
+      FROM (VALUES (1)) AS one (x)
+      LEFT JOIN LATERAL (
+        SELECT sequence, event, command_id, value, payload_version, emitted_at_ms FROM actor_events e
+        WHERE ${eventsOf("e", key, source)}
+          AND e.sequence > ${BigInt(row.delivered)}
+          AND e.event = ANY(${textArray({ sql, values: subscription.events })})
+        ORDER BY e.sequence LIMIT ${settings.batch}) e ON true`;
+        const head = BigInt(batch[0].head);
+        const oldest = batch[0].oldest;
+        const events = batch.filter((event) => event.sequence !== null);
+        let progress = BigInt(row.delivered);
+        let continuous = true;
+        let uncountedGaps = 0;
+        const settleOutcome = Effect.fnUntraced(function* (outcome, position) {
+            const failed = failure(outcome);
+            if (failed !== undefined) {
+                yield* backOff(progress, failed);
+                return false;
+            }
+            const settled = outcome.success;
+            if (Outcome.guards.Acknowledged(settled) &&
+                (settled.reason === "Stale" || settled.reason === "Unsubscribed")) {
+                yield* deleteRow(row, key, source);
+                return false;
+            }
+            if (Outcome.guards.Acknowledged(settled) && settled.reason === "NotCreated")
+                yield* Effect.logInfo("Subscription event skipped: the subscriber is not created").pipe(Effect.annotateLogs({
+                    subscription: name,
+                    cursor: String(position),
+                }));
+            progress = position;
+            return yield* renew;
+        });
+        const renew = Effect.gen(function* () {
+            const renewed = yield* sql `UPDATE actor_subscriptions s
+          SET due_at_ms = greatest(s.due_at_ms, ${(yield* databaseTime) + settings.claimLeaseMs()})
+          WHERE ${held()} RETURNING s.due_at_ms::text AS due_at_ms`;
+            if (renewed.length === 0)
+                return yield* lostClaim("delivery").pipe(Effect.as(false));
+            claim.lease = BigInt(renewed[0].due_at_ms);
+            return true;
+        });
+        const planted = row.gap_through === null ? undefined : BigInt(row.gap_through);
+        if (progress < head &&
+            ((planted !== undefined && planted > progress) ||
+                oldest === null ||
+                BigInt(oldest) > progress + 1n)) {
+            const resumeAfter = planted !== undefined && planted > progress
+                ? planted
+                : oldest === null
+                    ? head
+                    : BigInt(oldest) - 1n;
+            const [gap] = yield* sql `UPDATE actor_subscriptions s
+          SET gap_at_ms = coalesce(s.gap_at_ms, ${yield* databaseTime}),
+            gap_through = coalesce(s.gap_through, ${resumeAfter})
+          WHERE ${held()}
+          RETURNING s.gap_at_ms::text AS at, s.gap_through::text AS through`;
+            if (gap === undefined)
+                return yield* lostClaim("delivery");
+            const through = BigInt(gap.through);
+            continuous = through === resumeAfter;
+            if (subscription.routed === "id") {
+                yield* Effect.logWarning("Subscription gap without a recipient").pipe(Effect.annotateLogs({
+                    subscription: name,
+                    source: `${source.actor}/${source.id}`,
+                    after: String(progress),
+                    resumeAfter: String(through),
+                }));
+                uncountedGaps = 1;
+                progress = through;
+            }
+            else {
+                const outcome = yield* deliverOne({
+                    tenant: source.tenant,
+                    actor: row.subscriber_type,
+                    id: subscription.routed === "singleton" ? "singleton" : row.subscriber_id,
+                }, subscription.handler, source, {
+                    subscription: row.subscription,
+                    epoch: row.epoch,
+                    kind: "gap",
+                    position: String(through),
+                }, Number(gap.at), WireGap.make({
+                    subscription: row.subscription,
+                    source,
+                    after: String(progress),
+                    resumeAfter: String(through),
+                }));
+                if (!(yield* settleOutcome(outcome, through)))
+                    return;
+            }
+        }
+        for (const event of continuous ? events.filter((e) => BigInt(e.sequence) > progress) : []) {
+            const upcast = yield* subscription
+                .upcast(event.event, event.payload_version, decompress(event.value))
+                .pipe(Effect.result);
+            if (Result.isFailure(upcast))
+                return yield* backOff(progress, `Event does not decode: ${upcast.failure.message}`);
+            const value = upcast.success;
+            const subscriberId = subscription.routed === undefined
+                ? Result.succeed(row.subscriber_id)
+                : yield* subscription.route(event.event, value, source).pipe(Effect.result);
+            if (Result.isFailure(subscriberId))
+                return yield* backOff(progress, `Route failed: ${String(subscriberId.failure)}`);
+            if (subscriberId.success.length === 0)
+                return yield* backOff(progress, "Route failed: route returned an empty id");
+            const subscriber = {
+                tenant: source.tenant,
+                actor: row.subscriber_type,
+                id: subscriberId.success,
+            };
+            const outcome = yield* deliverOne(subscriber, subscription.handler, source, {
+                subscription: row.subscription,
+                epoch: row.epoch,
+                kind: "event",
+                position: event.sequence,
+            }, Number(event.emitted_at_ms), WireEvent.make({
+                subscription: row.subscription,
+                source,
+                cursor: event.sequence,
+                event: yield* decodeJson(value),
+                commandId: event.command_id,
+                timestamp: DateTime.formatIso(DateTime.makeUnsafe(Number(event.emitted_at_ms))),
+            }));
+            if (!(yield* settleOutcome(outcome, BigInt(event.sequence))))
+                return;
+        }
+        const delivered = continuous && events.length < settings.batch ? (head > progress ? head : progress) : progress;
+        yield* at("beforeSettle");
+        const [pending] = yield* sql `SELECT EXISTS (
+        SELECT 1 FROM actor_events e
+        WHERE ${eventsOf("e", key, source)}
+          AND e.sequence > ${delivered} AND e.event = ANY(${textArray({ sql, values: subscription.events })})
+      ) AS due`;
+        yield* at("afterSettleSnapshot");
+        const declared = textArray({ sql, values: subscription.events });
+        const settled = yield* changeRows(sql `SELECT ${rowColumns({ sql, alias: "s" })} FROM actor_subscriptions s WHERE ${held()} FOR UPDATE`, sql `UPDATE actor_subscriptions s SET delivered = ${delivered}, attempts = 0, last_error = NULL,
+          gaps = s.gaps + ${uncountedGaps}, gap_at_ms = NULL, gap_through = NULL,
+          due_at_ms = CASE WHEN ${pending.due} OR s.marked > ${delivered} THEN ${now()} END,
+          events = CASE WHEN s.events @> ${declared} THEN s.events
+            ELSE ARRAY(SELECT DISTINCT x FROM unnest(s.events || ${declared}) AS u(x) ORDER BY x) END
+        WHERE ${held()}
+        RETURNING ${rowColumns({ sql, alias: "s" })}`);
+        if (settled === 0)
+            return yield* lostClaim("settle");
+        yield* count(Metrics.relayDelivered, { kind: "subscription" }, 1);
+        yield* count(Metrics.undeliverableGaps, { subscriber_type: row.subscriber_type, subscription: row.subscription }, uncountedGaps);
+    });
+    const release = (work, claim) => Effect.gen(function* () {
+        const at = yield* databaseTime;
+        if (work.kind !== "subscription")
+            return yield* sql `UPDATE actor_outbox SET due_at_ms = ${at}
+          WHERE routing_key = ${BigInt(work.routing_key)} AND intent_id = ${work.intent_id}
+            AND kind = ${work.kind} AND due_at_ms = ${claim.lease}`;
+        return yield* sql `UPDATE actor_subscriptions SET due_at_ms = ${at}
+        WHERE routing_key = ${BigInt(work.routing_key)} AND tenant_id = ${work.tenant_id}
+          AND source_type = ${work.source_type} AND source_id = ${work.source_id}
+          AND subscriber_type = ${work.subscriber_type} AND subscription = ${work.subscription}
+          AND subscriber_id = ${work.subscriber_id} AND epoch = ${work.epoch}
+          AND due_at_ms = ${claim.lease}`;
+    }).pipe(Effect.ignore);
+    /**
+     * Widens the rows of one dynamic subscription to its declaration's event
+     * classes, 1,000 rows per transaction, with their tag summary. It keeps every cursor and epoch, never
+     * narrows a row, and is idempotent, so every runner of a rolling deploy may
+     * run it. A widened row with an event of an added class after its
+     * `delivered` becomes due. Rows claimed right now are skipped; their settle
+     * widens them.
+     */
+    const widen = Effect.fnUntraced(function* (subscriberType, declared) {
+        const tags = textArray({ sql, values: declared.events });
+        for (;;) {
+            const widened = yield* changeRows(sql `SELECT ${rowColumns({ sql, alias: "s" })} FROM actor_subscriptions s
+          WHERE s.subscriber_type = ${subscriberType} AND s.subscription = ${declared.tag}
+            AND s.active AND NOT s.events @> ${tags}
+          LIMIT ${EXPANSION_PAGE}
+          FOR UPDATE SKIP LOCKED`, sql `UPDATE actor_subscriptions s
+          SET events = ARRAY(SELECT DISTINCT u.x FROM unnest(s.events || ${tags}) AS u(x) ORDER BY u.x),
+            due_at_ms = CASE WHEN s.due_at_ms IS NULL AND EXISTS (
+                SELECT 1 FROM actor_events e
+                WHERE e.routing_key = s.routing_key AND e.tenant_id = s.tenant_id
+                  AND e.actor_type = s.source_type AND e.actor_id = s.source_id
+                  AND e.sequence > s.delivered AND e.event = ANY(${tags})
+                  AND NOT e.event = ANY(o.events))
+              THEN ${now()} ELSE s.due_at_ms END
+          FROM old_rows o
+          WHERE s.routing_key = o.routing_key AND s.tenant_id = o.tenant_id
+            AND s.source_type = o.source_type AND s.source_id = o.source_id
+            AND s.subscriber_type = o.subscriber_type AND s.subscription = o.subscription
+            AND s.subscriber_id = o.subscriber_id
+          RETURNING ${rowColumns({ sql, alias: "s" })}`);
+            if (widened < EXPANSION_PAGE)
+                break;
+        }
+    });
+    /**
+     * Deletes rows of `subscriberType` whose subscription this runner doesn't
+     * declare once they have been due for a day. Such a row is never claimed
+     * here, so a rolling deploy's newer runners get a day to take it first.
+     */
+    const cleanupRemoved = Effect.fnUntraced(function* (subscriberType, declared) {
+        const cutoff = (yield* databaseTime) - REMOVED_AFTER_MS;
+        return yield* changeRows(sql `SELECT ${rowColumns({ sql, alias: "s" })} FROM actor_subscriptions s
+        WHERE s.subscriber_type = ${subscriberType}
+          AND NOT s.subscription = ANY(${textArray({ sql, values: declared })})
+          AND s.due_at_ms < ${cutoff}
+        FOR UPDATE`, deleteLocked);
+    });
+    const run = (work, handoff) => {
+        const claim = { lease: BigInt(work.claimed_until) };
+        const running = work.kind === "subscription"
+            ? deliverRow(work, claim).pipe(Effect.withSpan(SpanNames.relaySubscription, {
+                kind: "producer",
+                attributes: {
+                    "actor.type": work.subscriber_type,
+                    "actor.tenant": work.tenant_id,
+                    "subscription.name": work.subscription,
+                    "subscription.source_type": work.source_type,
+                    "relay.attempt": work.attempts,
+                },
+            }, { captureStackTrace: false }))
+            : Match.value(work.kind).pipe(Match.when("feed", () => expand(work, handoff)), Match.orElse(() => register(work)));
+        return running.pipe(Effect.onInterrupt(() => release(work, claim)));
+    };
+    return { claim, decode, run, widen, cleanupRemoved };
+});