@rikalabs/akter 0.1.0-alpha.0 → 0.1.0-alpha.3

package/dist/runtime/registration.js

@@ -22,7 +22,7 @@ const ROUTED_SUBSCRIBER_WAIT_MS = 5000;
  * connection holder, and the sweeps, and forgotten when its layer's scope
  * closes.
  */
-export const actorRegistration = ({ registrations, queryRegistrations, jobRegistrations, residency, diagnostics, owners, heldTypes, heldType, sweepsWorkflows, checked, services, frameworkClock, content, retryWindowMs, tableRole, adoptionRole, enforcedTypes, writerDeclarations, refreshPayloadWriters, subscriptions, transport, authorize, gate, writable, defectLog, outbox, }) => {
+export const actorRegistration = ({ registrations, queryRegistrations, jobRegistrations, residency, diagnostics, owners, heldTypes, heldType, sweepsWorkflows, checked, services, frameworkClock, content, retryWindowMs, tableRole, adoptionRole, enforcedTypes, writerDeclarations, refreshPayloadWriters, subscriptions, transport, authorize, gate, writable, defectLog, outbox, recorder, }) => {
     /** Concurrent layers can exceed checkout capacity; only a refused checkout is safe to retry here. */
     const startupSql = (effect) => retryPoolRefusal(effect).pipe(Effect.orDie);
     /**
@@ -150,9 +150,9 @@ export const actorRegistration = ({ registrations, queryRegistrations, jobRegist
                 heldTypes.delete(registration.name);
                 sweepsWorkflows.delete(registration.name);
             }));
-            const { isResident, owner, diagnose, restarting } = yield* registerActor(registration, transport, authorize, gate, writable).pipe(Effect.provideService(DefectLog, defectLog), Effect.provideContext(services), Effect.provideService(OutboxRuntime, outbox));
+            const { isResident, owner, diagnose, restarting, resident } = yield* registerActor(registration, transport, authorize, gate, writable, recorder).pipe(Effect.provideService(DefectLog, defectLog), Effect.provideContext(services), Effect.provideService(OutboxRuntime, outbox));
             residency.set(registration.name, isResident);
-            diagnostics.set(registration.name, { diagnose, restarting });
+            diagnostics.set(registration.name, { diagnose, restarting, resident });
             owners.set(registration.name, owner);
         }),
         registerQueries: Effect.fnUntraced(function* (registration) {

package/dist/runtime/runner.d.ts

@@ -29,6 +29,18 @@ declare const RunnerReadiness_base: Context.ServiceClass<RunnerReadiness, "@rika
 /** The public runner's readiness checks acquired shards, not just a reachable database. */
 export declare class RunnerReadiness extends RunnerReadiness_base {
 }
+declare const RunnerPeers_base: Context.ServiceClass<RunnerPeers, "@rikalabs/akter/runtime/runner/RunnerPeers", {
+    /** Healthy or not; undefined when the storage could not be read. */
+    readonly peers: Effect.Effect<number | undefined>;
+}>;
+/** How many runners other than this one the cluster's runner storage lists. */
+export declare class RunnerPeers extends RunnerPeers_base {
+}
+/** Counts the runners the storage lists under any address but this runner's own. */
+export declare const runnerPeers: Effect.Effect<{
+    /** Healthy or not; undefined when the storage could not be read. */
+    readonly peers: Effect.Effect<number | undefined>;
+}, never, RunnerStorage.RunnerStorage | ShardingConfig.ShardingConfig>;
 /** Builds assignment expectations independently of the runner's local acquisition state. */
 export declare const acquiredShards: Effect.Effect<{
     readonly acquired: (sharding: Pick<Sharding.Sharding["Service"], "hasShardId">) => Effect.Effect<boolean>;

package/dist/runtime/runner.js

@@ -20,6 +20,18 @@ export class RunnerWiring extends Context.Service()("@rikalabs/akter/runtime/run
 /** The public runner's readiness checks acquired shards, not just a reachable database. */
 export class RunnerReadiness extends Context.Service()("@rikalabs/akter/runtime/runner/RunnerReadiness") {
 }
+/** How many runners other than this one the cluster's runner storage lists. */
+export class RunnerPeers extends Context.Service()("@rikalabs/akter/runtime/runner/RunnerPeers") {
+}
+/** Counts the runners the storage lists under any address but this runner's own. */
+export const runnerPeers = Effect.gen(function* () {
+    const config = yield* ShardingConfig.ShardingConfig;
+    const storage = yield* RunnerStorage.RunnerStorage;
+    const self = Option.map(config.runnerAddress, PrimaryKey.value);
+    return RunnerPeers.of({
+        peers: storage.getRunners.pipe(Effect.map((runners) => runners.filter(([runner]) => Option.isNone(self) || PrimaryKey.value(runner.address) !== self.value).length), Effect.orElseSucceed(() => undefined)),
+    });
+});
 /** Builds assignment expectations independently of the runner's local acquisition state. */
 export const acquiredShards = Effect.gen(function* () {
     const config = yield* ShardingConfig.ShardingConfig;

package/dist/runtime/storage/codec.d.ts

@@ -5,24 +5,36 @@ export declare const PLACEMENT_ENCODING = 1;
  * An actor type's placement. A parent-placed type names its parent type and
  * that type's own placement, down to an actor-placed root.
  */
-export type Placement = "tenant" | "actor" | {
+export type Placement = "tenant" | "actor" | "authority" | {
     readonly parent: string;
     readonly placement: Placement;
 };
+/**
+ * The bucket every authority-placed key falls in. A topology that splits a
+ * database keeps it on the authoritative shard, beside the tables it does not
+ * route, so a turn of an authority-placed actor reads and writes the real
+ * control tables in a transaction that reaches one shard.
+ */
+export declare const AUTHORITY_BUCKET = -128;
+/** `key` moved into `AUTHORITY_BUCKET`, keeping its low 56 bits, so distinct keys stay distinct. */
+export declare const authorityKey: (key: bigint) => bigint;
 /** A parent-placed type's parent and the parent's placement; `undefined` for any other. */
 export declare const parentPlacement: (placement: Placement) => {
     readonly parent: string;
     readonly placement: Placement;
 } | undefined;
 /** The value `actor_placements.placement` records for `placement`. */
-export declare const placementKind: (placement: Placement) => "actor" | "parent" | "tenant";
+export declare const placementKind: (placement: Placement) => "actor" | "authority" | "parent" | "tenant";
+/** The placement at the root of `placement`'s parent chain. */
+export declare const rootPlacement: (placement: Placement) => "tenant" | "actor" | "authority";
 /**
  * The 64-bit shard key shared by every row an actor owns. Tenant
- * placement colocates a tenant's actors; actor placement spreads them; a
- * parent-placed actor takes its root's key, read from the parent id its own
- * id carries, so a family shares one shard. A parent-placed id without a
- * parent id throws: ids are decoded against their actor's key before any
- * routing, so that is a defect, not an input error.
+ * placement colocates a tenant's actors; actor placement spreads them;
+ * authority placement colocates a tenant's authority-placed actors in
+ * `AUTHORITY_BUCKET`; a parent-placed actor takes its root's key, read from
+ * the parent id its own id carries, so a family shares one shard. A
+ * parent-placed id without a parent id throws: ids are decoded against their
+ * actor's key before any routing, so that is a defect, not an input error.
  */
 export declare const routingKey: ({ ref, placement, }: {
     readonly ref: ActorRef;

package/dist/runtime/storage/codec.js

@@ -2,17 +2,31 @@ import { parseChildId } from "../../identity/child.js";
 import { compressBytes, decompressBytes, hash64 } from "./platform.js";
 /** Increment only with a migration: every stored `routing_key` depends on this encoding. */
 export const PLACEMENT_ENCODING = 1;
+/**
+ * The bucket every authority-placed key falls in. A topology that splits a
+ * database keeps it on the authoritative shard, beside the tables it does not
+ * route, so a turn of an authority-placed actor reads and writes the real
+ * control tables in a transaction that reaches one shard.
+ */
+export const AUTHORITY_BUCKET = -128;
+const LOW_56 = (1n << 56n) - 1n;
+/** `key` moved into `AUTHORITY_BUCKET`, keeping its low 56 bits, so distinct keys stay distinct. */
+export const authorityKey = (key) => BigInt.asIntN(64, (key & LOW_56) | (BigInt(AUTHORITY_BUCKET + 256) << 56n));
+const isRoot = (placement) => placement === "tenant" || placement === "actor" || placement === "authority";
 /** A parent-placed type's parent and the parent's placement; `undefined` for any other. */
-export const parentPlacement = (placement) => placement === "tenant" || placement === "actor" ? undefined : placement;
+export const parentPlacement = (placement) => (isRoot(placement) ? undefined : placement);
 /** The value `actor_placements.placement` records for `placement`. */
-export const placementKind = (placement) => placement === "tenant" || placement === "actor" ? placement : "parent";
+export const placementKind = (placement) => (isRoot(placement) ? placement : "parent");
+/** The placement at the root of `placement`'s parent chain. */
+export const rootPlacement = (placement) => isRoot(placement) ? placement : rootPlacement(placement.placement);
 /**
  * The 64-bit shard key shared by every row an actor owns. Tenant
- * placement colocates a tenant's actors; actor placement spreads them; a
- * parent-placed actor takes its root's key, read from the parent id its own
- * id carries, so a family shares one shard. A parent-placed id without a
- * parent id throws: ids are decoded against their actor's key before any
- * routing, so that is a defect, not an input error.
+ * placement colocates a tenant's actors; actor placement spreads them;
+ * authority placement colocates a tenant's authority-placed actors in
+ * `AUTHORITY_BUCKET`; a parent-placed actor takes its root's key, read from
+ * the parent id its own id carries, so a family shares one shard. A
+ * parent-placed id without a parent id throws: ids are decoded against their
+ * actor's key before any routing, so that is a defect, not an input error.
  */
 export const routingKey = ({ ref, placement, }) => {
     const above = parentPlacement(placement);
@@ -25,6 +39,8 @@ export const routingKey = ({ ref, placement, }) => {
             placement: above.placement,
         });
     }
+    if (placement === "authority")
+        return authorityKey(tenantRoutingKey(ref.tenant));
     return placement === "tenant"
         ? tenantRoutingKey(ref.tenant)
         : placementHash(["actor", ref.tenant, ref.actor, ref.id]);

package/dist/runtime/storage/placements.d.ts

@@ -1,11 +1,16 @@
 import { Effect } from "effect";
 import { SqlClient, type SqlError } from "effect/sql";
 import { type Placement } from "./codec.ts";
+/** The per-actor tables `moveToAuthority` rewrites, for the catalog check that none is missed. */
+export declare const AUTHORITY_MOVED_TABLES: ReadonlyArray<string>;
 /**
  * Records `name`'s placement on first registration and refuses one that
  * differs from the record, for the type and every ancestor it routes through:
  * a changed placement, encoding, or parent would read and write under
- * different routing keys.
+ * different routing keys. The one change it makes instead of refusing is a
+ * tenant-placed type that now declares authority placement: its rows move to
+ * the new keys and the record changes in one transaction, under the record's
+ * row lock, so concurrent starts move them once.
  */
 export declare const checkPlacement: ({ name, placement, }: {
     readonly name: string;

package/dist/runtime/storage/placements.js

@@ -1,11 +1,89 @@
 import { Effect } from "effect";
 import { SqlClient } from "effect/sql";
-import { parentPlacement, PLACEMENT_ENCODING, placementKind } from "./codec.js";
+import { AUTHORITY_BUCKET, authorityKey, parentPlacement, PLACEMENT_ENCODING, placementKind, } from "./codec.js";
+/**
+ * The per-actor tables that keep an actor's rows under its routing key, each
+ * with the column that names the actor's type and whether it carries the
+ * key's `bucket`. `actor_generations` and `actor_workflow_executions` are
+ * referenced by the others' foreign keys, so their rows are copied to the new
+ * key and deleted afterwards instead of updated.
+ */
+const KEYED = [
+    { table: "actor_state", type: "actor_type" },
+    { table: "actor_receipts", type: "actor_type" },
+    { table: "actor_events", type: "actor_type" },
+    { table: "actor_outbox", type: "actor_type", bucket: true },
+    { table: "actor_dead_letters", type: "actor_type" },
+    { table: "actor_blobs", type: "actor_type" },
+    { table: "actor_connections", type: "actor_type", bucket: true },
+    { table: "actor_content_refs", type: "actor_type" },
+    { table: "actor_subscription_cursors", type: "actor_type" },
+    { table: "actor_subscriptions", type: "source_type", bucket: true },
+    { table: "actor_subscription_tags", type: "source_type" },
+    { table: "actor_operator_audit", type: "actor_type" },
+];
+/** The per-actor tables `moveToAuthority` rewrites, for the catalog check that none is missed. */
+export const AUTHORITY_MOVED_TABLES = [
+    "actor_generations",
+    "actor_workflow_executions",
+    "actor_workflow_step",
+    ...KEYED.map(({ table }) => table),
+];
+const LOW_56 = (1n << 56n) - 1n;
+/**
+ * Moves every row of `name`'s actors from its tenant-placed keys to its
+ * authority-placed keys, in the caller's transaction: the framework's
+ * per-actor rows and the rows of every table `actor_tables` records it as
+ * owning. Rows already in `AUTHORITY_BUCKET` keep their key, which is the
+ * same under both placements. It returns how many actors moved.
+ *
+ * A runner that still computes the old keys would recreate an actor under
+ * them, so this runs only at the start of a release that replaced every
+ * runner of the previous one.
+ */
+const moveToAuthority = Effect.fnUntraced(function* (name) {
+    const sql = yield* SqlClient.SqlClient;
+    const high = authorityKey(0n);
+    const moved = sql `(routing_key & ${LOW_56}) | ${high}`;
+    const stale = sql `(routing_key >> 56) <> ${AUTHORITY_BUCKET}`;
+    const copy = (table) => Effect.gen(function* () {
+        const columns = yield* sql `SELECT attname AS name FROM pg_attribute
+        WHERE attrelid = ${table}::regclass AND attnum > 0 AND NOT attisdropped ORDER BY attnum`;
+        return yield* sql `INSERT INTO ${sql(table)} (${sql.csv(columns.map(({ name }) => sql `${sql(name)}`))})
+        SELECT ${sql.csv(columns.map(({ name }) => (name === "routing_key" ? moved : sql `${sql(name)}`)))}
+        FROM ${sql(table)} WHERE actor_type = ${name} AND ${stale}
+        RETURNING actor_id`;
+    });
+    const generations = yield* copy("actor_generations");
+    yield* copy("actor_workflow_executions");
+    yield* sql `UPDATE actor_workflow_step SET routing_key = ${moved}
+    WHERE actor_type = ${name} AND ${stale}`;
+    yield* sql `DELETE FROM actor_workflow_executions WHERE actor_type = ${name} AND ${stale}`;
+    for (const keyed of KEYED)
+        yield* sql `UPDATE ${sql(keyed.table)} SET routing_key = ${moved}
+      ${keyed.bucket === true ? sql `, bucket = ${AUTHORITY_BUCKET}` : sql.literal("")}
+      WHERE ${sql(keyed.type)} = ${name} AND ${stale}`;
+    yield* sql `DELETE FROM actor_generations WHERE actor_type = ${name} AND ${stale}`;
+    const owned = yield* sql `
+    SELECT t.table_schema AS schema, t.table_name AS table FROM actor_tables t
+    WHERE t.actor_type = ${name} AND EXISTS (
+      SELECT 1 FROM pg_attribute a
+      WHERE a.attrelid = to_regclass(format('%I.%I', t.table_schema, t.table_name))
+        AND a.attname = 'routing_key' AND NOT a.attisdropped)
+    ORDER BY t.table_schema, t.table_name`;
+    for (const { schema, table } of owned)
+        yield* sql `UPDATE ${sql(schema)}.${sql(table)} SET routing_key = ${moved}
+      WHERE routing_key IS NOT NULL AND ${stale}`;
+    return generations.length;
+});
 /**
  * Records `name`'s placement on first registration and refuses one that
  * differs from the record, for the type and every ancestor it routes through:
  * a changed placement, encoding, or parent would read and write under
- * different routing keys.
+ * different routing keys. The one change it makes instead of refusing is a
+ * tenant-placed type that now declares authority placement: its rows move to
+ * the new keys and the record changes in one transaction, under the record's
+ * row lock, so concurrent starts move them once.
  */
 export const checkPlacement = ({ name, placement, }) => Effect.gen(function* () {
     const sql = yield* SqlClient.SqlClient;
@@ -15,10 +93,24 @@ export const checkPlacement = ({ name, placement, }) => Effect.gen(function* ()
     yield* sql `INSERT INTO actor_placements (actor_type, placement, encoding, parent_type)
     VALUES (${name}, ${kind}, ${PLACEMENT_ENCODING}, ${parent})
     ON CONFLICT DO NOTHING`;
-    const [recorded] = yield* sql `SELECT placement, encoding, parent_type FROM actor_placements WHERE actor_type = ${name}`;
-    if (recorded?.placement !== kind ||
-        recorded.encoding !== PLACEMENT_ENCODING ||
-        recorded.parent_type !== parent)
+    const read = (lock) => Effect.map(sql `SELECT placement, encoding, parent_type FROM actor_placements WHERE actor_type = ${name}
+        ${lock ? sql.literal("FOR UPDATE") : sql.literal("")}`, ([row]) => row);
+    const matches = (recorded) => recorded?.placement === kind &&
+        recorded.encoding === PLACEMENT_ENCODING &&
+        recorded.parent_type === parent;
+    const recorded = yield* read(false);
+    if (!matches(recorded) &&
+        kind === "authority" &&
+        recorded?.placement === "tenant" &&
+        recorded.encoding === PLACEMENT_ENCODING)
+        yield* sql.withTransaction(Effect.gen(function* () {
+            if (matches(yield* read(true)))
+                return;
+            const moved = yield* moveToAuthority(name);
+            yield* sql `UPDATE actor_placements SET placement = ${kind} WHERE actor_type = ${name}`;
+            yield* Effect.logInfo(`Moved ${moved} actors of ${name} to authority placement`);
+        }));
+    if (!matches(yield* read(false)))
         return yield* Effect.die(new Error(`Actor ${name} placement differs from the deployment; migrate explicitly`));
     if (above !== undefined)
         yield* checkPlacement({ name: above.parent, placement: above.placement });

package/dist/runtime/storage/retention.d.ts

@@ -64,6 +64,12 @@ export interface Swept {
  *
  * The sweep yields after each batch, so a turn waiting for PGlite's one
  * connection runs between batches instead of after the whole sweep.
+ *
+ * With data shards, every prune runs once per shard on a session targeted at
+ * it, under the same authoritative lock and that shard's local fence. A
+ * manifest is then pruned on the authority once no shard reports an open
+ * execution under it; an execution that starts under it after that read
+ * restores it, as a runner of an older deployment does.
  */
 export declare const sweep: (policies: Iterable<RetentionPolicy>, retryWindowMs: number) => Effect.Effect<{
     receipts: number;

package/dist/runtime/storage/retention.js

@@ -3,6 +3,7 @@ import { SqlClient } from "effect/sql";
 import { databaseTime } from "../turn/admission.js";
 import { CleanupHooks } from "../turn/hooks.js";
 import { count, Metrics } from "../telemetry/metrics.js";
+import { forEachRange, withinRange } from "../database/shards.js";
 import { coordinated } from "../database/coordination.js";
 /**
  * How long after its id expires a receipt stays. An id expires one retry
@@ -45,6 +46,12 @@ export const receiptMarginMs = (horizon) => Math.max(horizon.keepReceiptsMs - ho
  *
  * The sweep yields after each batch, so a turn waiting for PGlite's one
  * connection runs between batches instead of after the whole sweep.
+ *
+ * With data shards, every prune runs once per shard on a session targeted at
+ * it, under the same authoritative lock and that shard's local fence. A
+ * manifest is then pruned on the authority once no shard reports an open
+ * execution under it; an execution that starts under it after that read
+ * restores it, as a runner of an older deployment does.
  */
 export const sweep = Effect.fnUntraced(function* (policies, retryWindowMs) {
     const sql = yield* SqlClient.SqlClient;
@@ -74,11 +81,13 @@ export const sweep = Effect.fnUntraced(function* (policies, retryWindowMs) {
                 yield* Effect.yieldNow;
             }
         });
-        receipts += yield* pruneAll((from) => sql `
+        const everyRange = (prune) => Effect.map(forEachRange(prune), (counts) => counts.reduce((sum, one) => sum + one, 0));
+        receipts += yield* everyRange((range) => pruneAll((from) => sql `
         WITH doomed AS (
           SELECT r.routing_key, r.tenant_id, r.actor_type, r.actor_id, r.command_id, r.expires_at_ms
           FROM actor_receipts r
           WHERE r.actor_type = ${policy.actorType} AND r.expires_at_ms >= ${from}::bigint
+            ${withinRange({ sql, range, column: "r.routing_key" })}
             AND r.expires_at_ms <= ${receiptCutoff}
             AND NOT EXISTS (
               SELECT 1 FROM actor_outbox o
@@ -94,11 +103,12 @@ export const sweep = Effect.fnUntraced(function* (policies, retryWindowMs) {
             AND r.actor_type = d.actor_type AND r.actor_id = d.actor_id AND r.command_id = d.command_id
           RETURNING 1)
         SELECT count(*)::integer AS count, (SELECT max(expires_at_ms)::text FROM doomed) AS last
-        FROM gone`, Metrics.receiptsPruned);
-        events += yield* pruneAll((from) => sql `
+        FROM gone`, Metrics.receiptsPruned));
+        events += yield* everyRange((range) => pruneAll((from) => sql `
         WITH picked AS (
           SELECT routing_key, tenant_id, actor_type, actor_id, sequence, emitted_at_ms FROM actor_events
           WHERE actor_type = ${policy.actorType} AND emitted_at_ms >= ${from}::bigint
+            ${withinRange({ sql, range, column: "routing_key" })}
             AND emitted_at_ms <= ${eventCutoff}
           ORDER BY emitted_at_ms
           LIMIT ${hooks.batchSize}),
@@ -128,13 +138,14 @@ export const sweep = Effect.fnUntraced(function* (policies, retryWindowMs) {
             AND e.actor_type = u.actor_type AND e.actor_id = u.actor_id AND e.sequence <= u.last
           RETURNING 1)
         SELECT count(*)::integer AS count, (SELECT max(emitted_at_ms)::text FROM picked) AS last
-        FROM gone`, Metrics.eventsPruned);
+        FROM gone`, Metrics.eventsPruned));
         if (policy.workflows) {
             const workflowCutoff = now - policy.keepWorkflowsMs;
-            workflows += yield* pruneAll(() => sql `
+            workflows += yield* everyRange((range) => pruneAll(() => sql `
         WITH doomed AS (
           SELECT routing_key, execution_id FROM actor_workflow_executions
           WHERE actor_type = ${policy.actorType} AND status = 'finished'
+            ${withinRange({ sql, range, column: "routing_key" })}
             AND finished_at_ms <= ${workflowCutoff}
           ORDER BY finished_at_ms
           LIMIT ${hooks.batchSize}
@@ -143,15 +154,16 @@ export const sweep = Effect.fnUntraced(function* (policies, retryWindowMs) {
           DELETE FROM actor_workflow_executions x USING doomed d
           WHERE x.routing_key = d.routing_key AND x.execution_id = d.execution_id
           RETURNING 1)
-        SELECT count(*)::integer AS count FROM gone`);
+        SELECT count(*)::integer AS count FROM gone`));
+            const superseded = sql `m.actor_type = ${policy.actorType}
+        AND m.accepted_at_ms < (SELECT max(l.accepted_at_ms) FROM actor_workflow_manifests l
+          WHERE l.actor_type = m.actor_type AND l.workflow = m.workflow)`;
             yield* batch(sql `
-        DELETE FROM actor_workflow_manifests m
-        WHERE m.actor_type = ${policy.actorType}
-          AND m.accepted_at_ms < (SELECT max(l.accepted_at_ms) FROM actor_workflow_manifests l
-            WHERE l.actor_type = m.actor_type AND l.workflow = m.workflow)
-          AND NOT EXISTS (SELECT 1 FROM actor_workflow_executions x
-            WHERE x.actor_type = m.actor_type AND x.workflow = m.workflow
-              AND x.manifest_hash = m.manifest_hash AND x.status <> 'finished')`);
+          DELETE FROM actor_workflow_manifests m
+          WHERE ${superseded}
+            AND NOT EXISTS (SELECT 1 FROM actor_workflow_executions x
+              WHERE x.actor_type = m.actor_type AND x.workflow = m.workflow
+                AND x.manifest_hash = m.manifest_hash AND x.status <> 'finished')`);
         }
     }
     return { receipts, events, workflows };

package/dist/runtime/subscriptions/relay.js

@@ -1,6 +1,6 @@
 import { DateTime, Effect, Match, Result, Schema } from "effect";
 import { SqlClient, SqlError } from "effect/sql";
-import {} from "../database/shards.js";
+import { forEachRange, withinRange } from "../database/shards.js";
 import { SubscriptionFailure } from "../../errors/subscription.js";
 import { Outcome, Request } from "../request.js";
 import {} from "../members.js";
@@ -9,7 +9,7 @@ import { decompress, routingKey } from "../storage/codec.js";
 import { databaseTime, FrameworkClock } from "../turn/admission.js";
 import { TurnHooks } from "../turn/hooks.js";
 import { ControlPayload, StringsJson, textArray } from "../turn/outbox.js";
-import { candidates, outboxNow } from "../turn/relay.js";
+import { candidates, outboxClock, outboxNow } from "../turn/relay.js";
 import { changeRows, rowColumns } from "./storage.js";
 import { deliveryCommandId } from "./identity.js";
 import { count, Metrics } from "../telemetry/metrics.js";
@@ -135,6 +135,7 @@ export const subscriptionRelay = Effect.fnUntraced(function* (options) {
     const clock = yield* FrameworkClock;
     const { settings } = options;
     const now = () => outboxNow({ sql, offsetMillis: clock.offsetMillis() });
+    const statementClock = outboxClock({ sql });
     const sourceWhere = (alias, key, source) => sql `${sql(alias)}.routing_key = ${key} AND ${sql(alias)}.tenant_id = ${source.tenant}
       AND ${sql(alias)}.source_type = ${source.actor} AND ${sql(alias)}.source_id = ${source.id}`;
     const eventsOf = (alias, key, source) => sql `${sql(alias)}.routing_key = ${key} AND ${sql(alias)}.tenant_id = ${source.tenant}
@@ -308,7 +309,7 @@ export const subscriptionRelay = Effect.fnUntraced(function* (options) {
         AND (s.subscriber_type, s.subscription, s.subscriber_id)
           <= (SELECT subscriber_type, subscription, subscriber_id FROM last)`;
             const [page] = yield* sql `
-        WITH head AS (
+        WITH ${statementClock}, head AS (
           SELECT event_sequence AS h FROM actor_generations g
           WHERE ${eventsOf("g", key, source)}),
         page AS MATERIALIZED (
@@ -409,14 +410,14 @@ export const subscriptionRelay = Effect.fnUntraced(function* (options) {
       AND s.subscription = ${row.command} AND s.subscriber_id = ${subscriber.id}`;
         const rejected = yield* sql.withTransaction(Effect.gen(function* () {
             yield* sql `INSERT INTO actor_generations (routing_key, tenant_id, actor_type, actor_id)
-          VALUES (${key}, ${source.tenant}, ${source.actor}, ${source.id}) ON CONFLICT DO NOTHING`;
+        VALUES (${key}, ${source.tenant}, ${source.actor}, ${source.id}) ON CONFLICT DO NOTHING`;
             const [generation] = yield* sql `
-          SELECT event_sequence::text AS head FROM actor_generations g
-          WHERE ${eventsOf("g", key, source)} FOR SHARE`;
+        SELECT event_sequence::text AS head FROM actor_generations g
+        WHERE ${eventsOf("g", key, source)} FOR SHARE`;
             const head = BigInt(generation.head);
             const [existing] = yield* sql `
-          SELECT s.epoch::text AS epoch, s.active FROM actor_subscriptions s
-          WHERE ${target} FOR UPDATE`;
+        SELECT s.epoch::text AS epoch, s.active FROM actor_subscriptions s
+        WHERE ${target} FOR UPDATE`;
             const refused = change.op === "subscribe" &&
                 change.start !== "now" &&
                 change.start !== "start" &&
@@ -424,36 +425,36 @@ export const subscriptionRelay = Effect.fnUntraced(function* (options) {
             if (existing !== undefined && BigInt(existing.epoch) >= epoch)
                 return refused && BigInt(existing.epoch) === epoch && !existing.active;
             const old = sql `SELECT ${rowColumns({ sql, alias: "s" })} FROM actor_subscriptions s
-          WHERE ${target} FOR UPDATE`;
+        WHERE ${target} FOR UPDATE`;
             if (change.op === "remove" || refused) {
                 yield* changeRows(old, sql `INSERT INTO actor_subscriptions AS s (routing_key, tenant_id, source_type, source_id,
-                subscriber_type, subscription, subscriber_id, events, epoch, active, delivered, bucket)
-              VALUES (${key}, ${source.tenant}, ${source.actor}, ${source.id}, ${subscriber.actor},
-                ${row.command}, ${subscriber.id}, ${textArray({ sql, values: change.events })}, ${epoch}, false,
-                ${head}, ${Number(key >> 56n)})
-              ON CONFLICT (routing_key, tenant_id, source_type, source_id, subscriber_type, subscription, subscriber_id)
-              DO UPDATE SET epoch = EXCLUDED.epoch, active = false, due_at_ms = NULL, attempts = 0,
-                last_error = NULL, gap_at_ms = NULL, gap_through = NULL
-              RETURNING ${rowColumns({ sql, alias: "s" })}`);
+              subscriber_type, subscription, subscriber_id, events, epoch, active, delivered, bucket)
+            VALUES (${key}, ${source.tenant}, ${source.actor}, ${source.id}, ${subscriber.actor},
+              ${row.command}, ${subscriber.id}, ${textArray({ sql, values: change.events })}, ${epoch}, false,
+              ${head}, ${Number(key >> 56n)})
+            ON CONFLICT (routing_key, tenant_id, source_type, source_id, subscriber_type, subscription, subscriber_id)
+            DO UPDATE SET epoch = EXCLUDED.epoch, active = false, due_at_ms = NULL, attempts = 0,
+              last_error = NULL, gap_at_ms = NULL, gap_through = NULL
+            RETURNING ${rowColumns({ sql, alias: "s" })}`);
                 return refused;
             }
             const delivered = Match.value(change.start).pipe(Match.when("now", () => head), Match.when("start", () => 0n), Match.orElse((cursor) => BigInt(cursor)));
             yield* changeRows(old, sql `INSERT INTO actor_subscriptions AS s (routing_key, tenant_id, source_type, source_id,
-              subscriber_type, subscription, subscriber_id, events, epoch, active, delivered, bucket,
-              due_at_ms)
-            SELECT ${key}, ${source.tenant}, ${source.actor}, ${source.id}, ${subscriber.actor},
-              ${row.command}, ${subscriber.id}, x.events, ${epoch}, true, ${delivered},
-              ${Number(key >> 56n)},
-              CASE WHEN EXISTS (SELECT 1 FROM actor_events e
-                  WHERE ${eventsOf("e", key, source)}
-                    AND e.sequence > ${delivered} AND e.sequence <= ${head} AND e.event = ANY(x.events))
-                THEN ${now()} END
-            FROM (SELECT ${textArray({ sql, values: change.events })} AS events) AS x
-            ON CONFLICT (routing_key, tenant_id, source_type, source_id, subscriber_type, subscription, subscriber_id)
-            DO UPDATE SET events = EXCLUDED.events, epoch = EXCLUDED.epoch, active = true,
-              delivered = EXCLUDED.delivered, marked = 0, due_at_ms = EXCLUDED.due_at_ms, attempts = 0,
-              last_error = NULL, gap_at_ms = NULL, gap_through = NULL
-            RETURNING ${rowColumns({ sql, alias: "s" })}`);
+            subscriber_type, subscription, subscriber_id, events, epoch, active, delivered, bucket,
+            due_at_ms)
+          SELECT ${key}, ${source.tenant}, ${source.actor}, ${source.id}, ${subscriber.actor},
+            ${row.command}, ${subscriber.id}, x.events, ${epoch}, true, ${delivered},
+            ${Number(key >> 56n)},
+            CASE WHEN EXISTS (SELECT 1 FROM actor_events e
+                WHERE ${eventsOf("e", key, source)}
+                  AND e.sequence > ${delivered} AND e.sequence <= ${head} AND e.event = ANY(x.events))
+              THEN ${now()} END
+          FROM (SELECT ${textArray({ sql, values: change.events })} AS events) AS x
+          ON CONFLICT (routing_key, tenant_id, source_type, source_id, subscriber_type, subscription, subscriber_id)
+          DO UPDATE SET events = EXCLUDED.events, epoch = EXCLUDED.epoch, active = true,
+            delivered = EXCLUDED.delivered, marked = 0, due_at_ms = EXCLUDED.due_at_ms, attempts = 0,
+            last_error = NULL, gap_at_ms = NULL, gap_through = NULL
+          RETURNING ${rowColumns({ sql, alias: "s" })}`, statementClock);
             return false;
         }));
         const claim = sql `routing_key = ${BigInt(row.routing_key)} AND intent_id = ${row.intent_id}
@@ -677,7 +678,7 @@ export const subscriptionRelay = Effect.fnUntraced(function* (options) {
           events = CASE WHEN s.events @> ${declared} THEN s.events
             ELSE ARRAY(SELECT DISTINCT x FROM unnest(s.events || ${declared}) AS u(x) ORDER BY x) END
         WHERE ${held()}
-        RETURNING ${rowColumns({ sql, alias: "s" })}`);
+        RETURNING ${rowColumns({ sql, alias: "s" })}`, statementClock);
         if (settled === 0)
             return yield* lostClaim("settle");
         yield* count(Metrics.relayDelivered, { kind: "subscription" }, 1);
@@ -698,18 +699,18 @@ export const subscriptionRelay = Effect.fnUntraced(function* (options) {
     }).pipe(Effect.ignore);
     /**
      * Widens the rows of one dynamic subscription to its declaration's event
-     * classes, 1,000 rows per transaction, with their tag summary. It keeps every cursor and epoch, never
+     * classes on every range, 1,000 rows per transaction, with their tag summary. It keeps every cursor and epoch, never
      * narrows a row, and is idempotent, so every runner of a rolling deploy may
      * run it. A widened row with an event of an added class after its
      * `delivered` becomes due. Rows claimed right now are skipped; their settle
      * widens them.
      */
-    const widen = Effect.fnUntraced(function* (subscriberType, declared) {
+    const widenRange = Effect.fnUntraced(function* (subscriberType, declared, range) {
         const tags = textArray({ sql, values: declared.events });
         for (;;) {
             const widened = yield* changeRows(sql `SELECT ${rowColumns({ sql, alias: "s" })} FROM actor_subscriptions s
           WHERE s.subscriber_type = ${subscriberType} AND s.subscription = ${declared.tag}
-            AND s.active AND NOT s.events @> ${tags}
+            AND s.active AND NOT s.events @> ${tags} ${withinRange({ sql, range, column: "s.routing_key" })}
           LIMIT ${EXPANSION_PAGE}
           FOR UPDATE SKIP LOCKED`, sql `UPDATE actor_subscriptions s
           SET events = ARRAY(SELECT DISTINCT u.x FROM unnest(s.events || ${tags}) AS u(x) ORDER BY u.x),
@@ -725,23 +726,25 @@ export const subscriptionRelay = Effect.fnUntraced(function* (options) {
             AND s.source_type = o.source_type AND s.source_id = o.source_id
             AND s.subscriber_type = o.subscriber_type AND s.subscription = o.subscription
             AND s.subscriber_id = o.subscriber_id
-          RETURNING ${rowColumns({ sql, alias: "s" })}`);
+          RETURNING ${rowColumns({ sql, alias: "s" })}`, statementClock);
             if (widened < EXPANSION_PAGE)
                 break;
         }
     });
+    const widen = (subscriberType, declared) => Effect.asVoid(forEachRange((range) => widenRange(subscriberType, declared, range)));
     /**
-     * Deletes rows of `subscriberType` whose subscription this runner doesn't
-     * declare once they have been due for a day. Such a row is never claimed
+     * Deletes rows of `subscriberType` on every range whose subscription this
+     * runner doesn't declare once they have been due for a day. Such a row is never claimed
      * here, so a rolling deploy's newer runners get a day to take it first.
      */
     const cleanupRemoved = Effect.fnUntraced(function* (subscriberType, declared) {
         const cutoff = (yield* databaseTime) - REMOVED_AFTER_MS;
-        return yield* changeRows(sql `SELECT ${rowColumns({ sql, alias: "s" })} FROM actor_subscriptions s
-        WHERE s.subscriber_type = ${subscriberType}
-          AND NOT s.subscription = ANY(${textArray({ sql, values: declared })})
-          AND s.due_at_ms < ${cutoff}
-        FOR UPDATE`, deleteLocked);
+        const counts = yield* forEachRange((range) => changeRows(sql `SELECT ${rowColumns({ sql, alias: "s" })} FROM actor_subscriptions s
+          WHERE s.subscriber_type = ${subscriberType}
+            AND NOT s.subscription = ANY(${textArray({ sql, values: declared })})
+            AND s.due_at_ms < ${cutoff} ${withinRange({ sql, range, column: "s.routing_key" })}
+          FOR UPDATE`, deleteLocked));
+        return counts.reduce((sum, one) => sum + one, 0);
     });
     const run = (work, handoff) => {
         const claim = { lease: BigInt(work.claimed_until) };

package/dist/runtime/subscriptions/storage.d.ts

@@ -35,6 +35,7 @@ export declare const summarize: ({ sql, changed, }: {
  * version of each row, so a concurrent change that committed first is never
  * counted twice. A summary count the change lowered to 0 is deleted in the
  * same transaction, so no committed summary row counts 0. Returns how many
- * rows `change` wrote.
+ * rows `change` wrote. A `change` that reads the outbox clock passes its
+ * `clock` CTE, which leads the statement's `WITH`.
  */
-export declare const changeRows: (old: Statement.Fragment, change: Statement.Fragment) => Effect.Effect<number, import("effect/sql/SqlError").SqlError, SqlClient.SqlClient>;
+export declare const changeRows: (old: Statement.Fragment, change: Statement.Fragment, clock?: Statement.Fragment | undefined) => Effect.Effect<number, import("effect/sql/SqlError").SqlError, SqlClient.SqlClient>;

package/dist/runtime/subscriptions/storage.js

@@ -64,15 +64,16 @@ export const summarize = ({ sql, changed, }) => {
  * version of each row, so a concurrent change that committed first is never
  * counted twice. A summary count the change lowered to 0 is deleted in the
  * same transaction, so no committed summary row counts 0. Returns how many
- * rows `change` wrote.
+ * rows `change` wrote. A `change` that reads the outbox clock passes its
+ * `clock` CTE, which leads the statement's `WITH`.
  */
-export const changeRows = Effect.fnUntraced(function* (old, change) {
+export const changeRows = Effect.fnUntraced(function* (old, change, clock) {
     const sql = yield* SqlClient.SqlClient;
     return yield* sql.withTransaction(Effect.gen(function* () {
         const [locked] = yield* sql `
         SELECT coalesce(json_agg(o), '[]')::text AS rows FROM (${old}) o`;
         const [result] = yield* sql `
-        WITH old_rows AS (
+        WITH ${clock === undefined ? sql `` : sql `${clock},`} old_rows AS (
           SELECT * FROM json_to_recordset(${locked.rows}::json) AS o(routing_key bigint,
             tenant_id text, source_type text, source_id text, subscriber_type text,
             subscription text, subscriber_id text, active boolean, events text[])),
@@ -89,9 +90,11 @@ export const changeRows = Effect.fnUntraced(function* (old, change) {
             event)) FROM tags_moved WHERE rows = 0)::text AS emptied`;
         if (result.emptied !== null)
             yield* sql `DELETE FROM actor_subscription_tags t
-          USING json_array_elements(${result.emptied}::json) AS e
-          WHERE t.routing_key = (e->>0)::bigint AND t.tenant_id = e->>1
-            AND t.source_type = e->>2 AND t.source_id = e->>3 AND t.event = e->>4
+          USING (SELECT (e->>0)::bigint AS routing_key, e->>1 AS tenant_id, e->>2 AS source_type,
+              e->>3 AS source_id, e->>4 AS event
+            FROM json_array_elements(${result.emptied}::json) AS e) AS k
+          WHERE t.routing_key = k.routing_key AND t.tenant_id = k.tenant_id
+            AND t.source_type = k.source_type AND t.source_id = k.source_id AND t.event = k.event
             AND t.rows = 0`;
         return result.rows;
     }));