@rikalabs/akter 0.1.0-alpha.0 → 0.1.0-alpha.3

package/dist/runtime/layer.js

@@ -1,6 +1,6 @@
 import { PgClient, PgTypes } from "@effect/sql-pg";
 import { PgliteClient } from "@effect/sql-pglite";
-import { Cause, Clock, Semaphore, Context, Crypto, Duration, Deferred, Effect, Fiber, Layer, Option, Result, Schedule, Schema, Stream, } from "effect";
+import { Cause, Clock, Semaphore, Context, Crypto, Duration, Deferred, Effect, Exit, Fiber, Layer, Option, Predicate, Result, Schedule, Schema, Stream, } from "effect";
 import { ClusterError, Entity, EntityId, RunnerHealth, Runners, RunnerStorage, Sharding, ShardingConfig, } from "effect/cluster";
 import { SqlClient, SqlError } from "effect/sql";
 import { ActorError, ActorUnavailable, InvalidInput, Unauthorized, Timeout, MailboxFull, RunnerAtCapacity, } from "../errors/actor.js";
@@ -11,7 +11,7 @@ import { InternalActors } from "./actors.js";
 import { System } from "../identity/caller.js";
 import { deriveMintId } from "../identity/mint.js";
 import { migrate } from "./database/migrations.js";
-import { NekiTurnSessions } from "./database/neki/session.js";
+import { schemaChange } from "./database/schema.js";
 import { queryPoolLayer, ReadReplica, replicaLayer } from "./database/replica.js";
 import { Coordination, coordinationLayer } from "./database/coordination.js";
 import { withKeepalives } from "./database/keepalive.js";
@@ -30,6 +30,7 @@ import { recordedPlacement } from "./storage/placements.js";
 import { TurnHooks } from "./turn/hooks.js";
 import { requestAttributes, SpanNames } from "./telemetry/spans.js";
 import { DefectLog, boundedDefectLog } from "./telemetry/defects.js";
+import { liveRecorder, LiveRuntime, } from "./telemetry/live.js";
 import { OperatorRuntime, operatorRuntime } from "./operators/repair.js";
 import { seedRuntime } from "./operators/seed.js";
 import { TelemetrySampler } from "./telemetry/sampler.js";
@@ -56,7 +57,7 @@ import { RuntimeControl, runtimeControl, turnGate } from "./drain.js";
 import { eventFeeds } from "./feeds.js";
 import { committedReads } from "./queries.js";
 import { servingReadiness } from "./readiness.js";
-import { acquiredShards, checkRunnerConfiguration, RunnerReadiness, LocalRequestSerialization, RunnerWiring, } from "./runner.js";
+import { acquiredShards, checkRunnerConfiguration, RunnerPeers, runnerPeers, RunnerReadiness, LocalRequestSerialization, RunnerWiring, } from "./runner.js";
 import { actorRegistration } from "./registration.js";
 import { startSweeps } from "./sweeps.js";
 import { checkFleet } from "./fleet/checks.js";
@@ -74,6 +75,14 @@ const CLAIM_MARGIN_MS = 5000;
 const DEFAULT_CLAIM_LEASE_MS = 37_000;
 /** How long a progress send may take before it is given up as a lost frame. */
 const PROGRESS_SEND_TIMEOUT = "5 seconds";
+/**
+ * The reply to a command the closing runtime scope cut off or never started.
+ * The caller cannot interrupt an admitted command, so an interrupt there comes
+ * from shutdown, and the caller can retry on a runtime that is still serving.
+ */
+const shuttingDown = Exit.fail(ActorError.make({
+    reason: ActorUnavailable.make({ cause: new Error("Runtime is shutting down") }),
+}));
 /**
  * An entity type with no messages, registered before any actor. The cluster's
  * idle sweep takes its interval from the first entity type registered and
@@ -92,7 +101,10 @@ const IdleSweep = Entity.make("akter/IdleSweep", []);
  * Constraints the wiring keeps:
  * - Turns run through the drain gate, so a drain refuses new turns and
  *   interrupts the rest. The runtime scope, not the caller, owns an in-flight
- *   turn: interrupting a waiter never cancels it.
+ *   turn: interrupting a waiter never cancels it. Closing that scope does, and
+ *   a command it cuts off, or one sent after it starts closing, fails with a
+ *   retryable `ActorUnavailable` instead of an interrupt or a wait that never
+ *   ends; delivery stops retrying once the scope is closed.
  * - Commands are direct, so Cluster keeps no messages and the receipt is the
  *   only admission record; a restarted activation or lost runner drops the
  *   uncommitted attempt, the handle retries with the same command id, and the
@@ -210,6 +222,14 @@ export const layer = (options = {}) => {
         let refreshedAt;
         const services = yield* Effect.context();
         const defectLog = boundedDefectLog(defectCapacity);
+        let recorder;
+        const currentRecorder = () => recorder;
+        const enableRecorder = Effect.map(Clock.currentTimeMillis, (startedAtMs) => {
+            recorder ??= liveRecorder({ startedAtMs, epoch: runtimeId.slice(0, 8) });
+            return recorder;
+        });
+        const servedStreams = new Map();
+        const peers = yield* RunnerPeers;
         const database = yield* rowsDatabase;
         const clockOffset = yield* FrameworkClock;
         const content = options.content === undefined
@@ -341,7 +361,7 @@ export const layer = (options = {}) => {
         }).pipe(refreshing.withPermits(1), Effect.provideContext(services), Effect.provideService(FrameworkClock, frameworkClock));
         /**
          * A runtime that could not refresh its writer rows within the window
-         * may already count as gone to `durable payloads clear`, so it starts
+         * may already count as gone to `akter payloads clear`, so it starts
          * no turn until a refresh succeeds. The check is local.
          */
         const writable = Effect.gen(function* () {
@@ -466,8 +486,9 @@ export const layer = (options = {}) => {
                         reason: ActorUnavailable.make({ cause: Cause.squash(cause) }),
                     }));
                 }));
-                const retrying = (attempt) => deliver.pipe(Effect.catchIf((error) => (Schema.is(ActorUnavailable)(error.reason) && !isOverloaded(error)) ||
-                    Schema.is(RunnerAtCapacity)(error.reason), (error) => Effect.sleep(retryDelay(attempt)(error)).pipe(Effect.andThen(Effect.suspend(() => retrying(attempt + 1))))));
+                const retrying = (attempt) => deliver.pipe(Effect.catchIf((error) => !Predicate.isTagged(scope.state, "Closed") &&
+                    ((Schema.is(ActorUnavailable)(error.reason) && !isOverloaded(error)) ||
+                        Schema.is(RunnerAtCapacity)(error.reason)), (error) => Effect.sleep(retryDelay(attempt)(error)).pipe(Effect.andThen(Effect.suspend(() => retrying(attempt + 1))))));
                 const executed = yield* retrying(0);
                 if (external)
                     yield* authorize(request, executed.endedAtMs);
@@ -613,6 +634,51 @@ export const layer = (options = {}) => {
             wake: relay.wake,
             tenantScope,
         });
+        const live = LiveRuntime.of({
+            recorder: currentRecorder,
+            enable: enableRecorder,
+            resident: (tenant) => [...diagnostics].flatMap(([actorType, { resident }]) => resident().flatMap((activation) => activation.tenant === tenant
+                ? [{ actorType, actorId: activation.id, mailbox: activation.mailbox }]
+                : [])),
+            connections: (tenant) => [
+                ...(holder?.census(tenant) ?? []).map(({ ref, member, feed }) => ({
+                    actorType: ref.actor,
+                    actorId: ref.id,
+                    kind: member === FEED_MEMBER ? "feed" : isWatchMember(member) ? "watch" : "socket",
+                    events: member === FEED_MEMBER ? feed : [],
+                })),
+                ...[...servedStreams.values()].flatMap((served) => served.tenant === tenant
+                    ? Array.from({ length: served.count }, () => ({
+                        actorType: served.actorType,
+                        actorId: served.actorId,
+                        kind: "stream",
+                        events: [],
+                    }))
+                    : []),
+            ],
+            schedules: () => [...registrations.values()].flatMap((registration) => registration.cron.map((entry) => ({
+                actorType: registration.name,
+                key: entry.key,
+                command: entry.command,
+            }))),
+            peers: peers.peers,
+            streamOpened: (ref) => {
+                const key = JSON.stringify([ref.tenant, ref.actor, ref.id]);
+                const served = servedStreams.get(key) ?? {
+                    tenant: ref.tenant,
+                    actorType: ref.actor,
+                    actorId: ref.id,
+                    count: 0,
+                };
+                served.count += 1;
+                servedStreams.set(key, served);
+                return () => {
+                    served.count -= 1;
+                    if (served.count === 0)
+                        servedStreams.delete(key);
+                };
+            },
+        });
         const internalActors = InternalActors.of({
             fleet: (request) => fleetSubscribe === undefined
                 ? ActorError.make({ reason: InvalidInput.make({ code: "unknown_route" }) })
@@ -661,6 +727,7 @@ export const layer = (options = {}) => {
                 writable,
                 defectLog,
                 outbox,
+                recorder: currentRecorder,
             }),
             ...committedReads({
                 registrations,
@@ -683,15 +750,18 @@ export const layer = (options = {}) => {
             deliverProgress: (message) => deliverProgress(message).pipe(Effect.ignoreCause, Effect.provideContext(services)),
             transport,
             holder,
+            live,
             hibernate: (ref) => Effect.flatMap(entityId(ref), (id) => owners.get(ref.actor)?.hibernate(id) ?? Effect.void).pipe(Effect.provideContext(services)),
             execute: (request) => Effect.gen(function* () {
                 const reply = yield* Deferred.make();
                 const pending = new Set();
-                yield* admission
+                const settle = (exit) => Deferred.done(reply, Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) ? shuttingDown : exit);
+                const admitted = yield* admission
                     .admit(dispatch(request, true, pending).pipe(Effect.tap(observe), Effect.onExit((exit) => [...pending].some((fiber) => fiber.pollUnsafe() === undefined)
-                    ? Deferred.done(reply, exit)
+                    ? settle(exit)
                     : Effect.void), Effect.ensuring(Effect.suspend(() => Effect.forEach(pending, Fiber.await, { discard: true })))))
-                    .pipe(Effect.onExit((exit) => Deferred.done(reply, exit)), Effect.ignoreCause, Effect.forkIn(scope));
+                    .pipe(Effect.onExit(settle), Effect.ignoreCause, Effect.forkIn(scope));
+                admitted.addObserver(() => Deferred.doneUnsafe(reply, shuttingDown));
                 return yield* Deferred.await(reply);
             }),
             overloaded: admission.full,
@@ -739,9 +809,10 @@ export const layer = (options = {}) => {
             if (!registration.blobs.some((declared) => declared.name === blob && isContent(declared)))
                 return Option.none();
             const sql = yield* SqlClient.SqlClient;
+            const key = routingKey({ ref, placement: registration.placement });
             const [found] = yield* sql `
             SELECT hash, size::float8 AS size FROM actor_content_refs
-            WHERE routing_key = ${routingKey({ ref, placement: registration.placement })}
+            WHERE routing_key = ${key}
               AND tenant_id = ${ref.tenant} AND actor_type = ${ref.actor} AND actor_id = ${ref.id}
               AND blob = ${blob} AND name = ${name}`;
             const timeoutMs = "policy" in registration ? registration.policy.executionMs : registration.timeoutMs;
@@ -794,7 +865,7 @@ export const layer = (options = {}) => {
         yield* checkRunnerConfiguration(wiring);
         const sharding = (wiring?.sharding ?? Sharding.layer.pipe(Layer.provide(Runners.layerNoop))).pipe(Layer.merge(Layer.effect(RunnerReadiness, wiring?.production === true
             ? acquiredShards
-            : Effect.succeed({ acquired: () => Effect.succeed(true) }))), Layer.provideMerge(directMessages), Layer.provideMerge(mailboxRefusals), Layer.provide([
+            : Effect.succeed({ acquired: () => Effect.succeed(true) })).pipe(Layer.merge(Layer.effect(RunnerPeers, runnerPeers)))), Layer.provideMerge(directMessages), Layer.provideMerge(mailboxRefusals), Layer.provide([
             runnerStorage === "memory"
                 ? Layer.effect(RunnerStorage.RunnerStorage, Effect.map(RunnerStorage.makeMemory, keepAcquiredShards))
                 : Layer.effect(RunnerStorage.RunnerStorage, coordinatedRunnerStorage.pipe(Effect.map(wiring?.storage ?? ((storage) => storage)), Effect.map(keepAcquiredShards))).pipe(Layer.orDie),
@@ -826,51 +897,48 @@ export const layer = (options = {}) => {
         })));
     }));
 };
+/**
+ * A runner holds three primary pools, plus optional replica and coordination
+ * pools. Turns lease sessions from the turn pool, `maxConnections` (default
+ * 50): a command holds one session until its transaction ends, so a pool
+ * smaller than the commands in flight queues callers behind
+ * it. Queries read from the query pool, `queryConnections` (default 10),
+ * except those of types with owned tables or blobs. Command admission,
+ * receipt replays, the relay, migrations, and cluster storage use the
+ * off-turn pool, `offTurnConnections` (default 10), unless `coordination`
+ * moves Cluster storage and deployment locks to an independent unsharded
+ * primary. All runners must designate the same authority. Every pool opens
+ * connections only as load needs them, refuses checkouts past its
+ * connections plus a bounded waiter allowance, and hands its connections out
+ * first come, first served. Keep the sum across runners below the server's
+ * `max_connections`.
+ *
+ * `replica` is this runner's nearest streaming replica of the same primary.
+ * Queries read there once it has replayed the commit version their caller
+ * last saw, and read the primary when it is behind or fails. Its pool
+ * (`maxConnections` default 10) opens connections only as queries need them.
+ * Every pool requests server TCP keepalives at 5 seconds idle, 2 seconds
+ * between probes, and 3 probes. `startupParameters` overrides these defaults
+ * independently on the primary and replica configurations.
+ *
+ * Registers a `regclass` codec because the pinned driver lacks one and the
+ * migrator needs it on restart; remove once Effect #8309 lands.
+ */
+function postgres(options) {
+    const types = PgTypes.makeRegistry();
+    types.register(2205, {
+        encode: (value) => PgTypes.encode(value, PgTypes.OID.oid),
+        decode: (bytes) => bytes.length === 4
+            ? Result.succeed(new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(0))
+            : Result.fail(new PgTypes.CodecError({ message: "Invalid regclass value" })),
+    });
+    const { offTurnConnections, queryConnections, replica, coordination, ...configured } = options;
+    const pool = withKeepalives(configured);
+    return Layer.mergeAll(boundedLayer({ ...pool, maxConnections: offTurnConnections ?? 10, types }), turnGroups.pipe(Layer.provideMerge(turnConnections({ ...pool, maxConnections: pool.maxConnections ?? 50, types }))), queryPoolLayer({ ...pool, maxConnections: queryConnections ?? 10, types }), replicaLayer(replica === undefined ? undefined : { ...withKeepalives(replica), types }), coordinationLayer(coordination === undefined ? undefined : { ...withKeepalives(coordination), types }));
+}
 /** Database layers for `Actors.layer`: `postgres` for real deployments, `pglite` for embedded and test use. */
 export const Database = {
-    /**
-     * A runner holds three primary pools, plus optional replica and coordination
-     * pools. Turns lease sessions from the turn pool, `maxConnections` (default
-     * 50): a command holds one session until its transaction ends, so a pool
-     * smaller than the commands in flight queues callers behind
-     * it. Queries read from the query pool, `queryConnections` (default 10),
-     * except those of types with owned tables or blobs. Command admission,
-     * receipt replays, the relay, migrations, and cluster storage use the
-     * off-turn pool, `offTurnConnections` (default 10), unless `coordination`
-     * moves Cluster storage and deployment locks to an independent unsharded
-     * primary. All runners must designate the same authority. Every pool opens
-     * connections only as load needs them, refuses checkouts past its
-     * connections plus a bounded waiter allowance, and hands its connections out
-     * first come, first served. Keep the sum across runners below the server's
-     * `max_connections`.
-     *
-     * `replica` is this runner's nearest streaming replica of the same primary.
-     * Queries read there once it has replayed the commit version their caller
-     * last saw, and read the primary when it is behind or fails. Its pool
-     * (`maxConnections` default 10) opens connections only as queries need them.
-     * Every pool requests server TCP keepalives at 5 seconds idle, 2 seconds
-     * between probes, and 3 probes. `startupParameters` overrides these defaults
-     * independently on the primary and replica configurations.
-     *
-     * Registers a `regclass` codec because the pinned driver lacks one and the
-     * migrator needs it on restart; remove once Effect #8309 lands.
-     */
-    postgres: (options) => {
-        const types = PgTypes.makeRegistry();
-        types.register(2205, {
-            encode: (value) => PgTypes.encode(value, PgTypes.OID.oid),
-            decode: (bytes) => bytes.length === 4
-                ? Result.succeed(new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(0))
-                : Result.fail(new PgTypes.CodecError({ message: "Invalid regclass value" })),
-        });
-        const { offTurnConnections, queryConnections, replica, neki, coordination, ...configured } = options;
-        const pool = withKeepalives(configured);
-        const database = Layer.mergeAll(boundedLayer({ ...pool, maxConnections: offTurnConnections ?? 10, types }), neki === true
-            ? turnConnections({ ...pool, maxConnections: pool.maxConnections ?? 50, types })
-            : turnGroups.pipe(Layer.provideMerge(turnConnections({ ...pool, maxConnections: pool.maxConnections ?? 50, types }))), queryPoolLayer({ ...pool, maxConnections: queryConnections ?? 10, types }), replicaLayer(replica === undefined ? undefined : { ...withKeepalives(replica), types }), coordinationLayer(coordination === undefined ? undefined : { ...withKeepalives(coordination), types }));
-        return neki === undefined
-            ? database
-            : database.pipe(Layer.provideMerge(Layer.succeed(NekiTurnSessions, neki)));
-    },
+    postgres,
     pglite,
+    schemaChange,
 };

package/dist/runtime/operators/audit.d.ts

@@ -1,6 +1,7 @@
 import { Crypto, Effect, Option, Schema } from "effect";
 import { SqlClient } from "effect/sql";
 import { type Placement } from "../storage/codec.ts";
+import { type BucketRange } from "../database/shards.ts";
 import { Capability, type OperatorAction } from "./grants.ts";
 /** One operator action as the audit log records it. */
 export interface AuditEntry {
@@ -49,6 +50,8 @@ export type AuditRecord = typeof AuditRecord.Type;
 export declare const listAudit: (args_0: {
     readonly tenant: string;
     readonly limit: number;
+    /** The buckets read, when each shard is read on its own. */
+    readonly range?: BucketRange;
 }) => Effect.Effect<readonly {
     readonly auditId: string;
     readonly atMs: number;

package/dist/runtime/operators/audit.js

@@ -1,6 +1,8 @@
 import { Crypto, Effect, Option, Schema } from "effect";
 import { SqlClient } from "effect/sql";
 import { routingKey } from "../storage/codec.js";
+import { withinRange } from "../database/shards.js";
+import { BUCKETS } from "../turn/outbox.js";
 import { databaseTime } from "../turn/admission.js";
 import { Capability } from "./grants.js";
 const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Json));
@@ -45,9 +47,11 @@ export const AuditRecord = Schema.Struct({
     outcome: Schema.String,
 });
 /** A tenant's newest audit rows, or every tenant's for `"*"`, newest first. */
-export const listAudit = Effect.fnUntraced(function* ({ tenant, limit, }) {
+export const listAudit = Effect.fnUntraced(function* ({ tenant, limit, range = BUCKETS, }) {
     const sql = yield* SqlClient.SqlClient;
-    const filter = tenant === "*" ? sql `` : sql `WHERE tenant_id = ${tenant}`;
+    const filter = tenant === "*"
+        ? withinRange({ sql, range, column: "routing_key", keyword: "WHERE" })
+        : sql `WHERE tenant_id = ${tenant} ${withinRange({ sql, range, column: "routing_key" })}`;
     return yield* sql `SELECT audit_id AS "auditId", at_ms::float8 AS "atMs", operator,
       action, tenant_id AS tenant, actor_type AS "actorType", actor_id AS "actorId", target,
       capability, reason, outcome

package/dist/runtime/operators/repair.js

@@ -5,6 +5,7 @@ import { ActorRef, System } from "../../identity/caller.js";
 import * as Queries from "../inspector/queries.js";
 import { TenantScope, withTenant } from "../database/tenancy.js";
 import { routingKey } from "../storage/codec.js";
+import { forEachRange, withinRange } from "../database/shards.js";
 import { recordedPlacement } from "../storage/placements.js";
 import { databaseTime, FrameworkClock } from "../turn/admission.js";
 import { OutboxRuntime, outboxStatements } from "../turn/outbox.js";
@@ -31,6 +32,16 @@ export class JobNotServed extends Schema.TaggedError()("JobNotServed", {
  */
 export class OperatorRuntime extends Context.Service()("@rikalabs/akter/runtime/operators/repair/OperatorRuntime") {
 }
+const encoder = new TextEncoder();
+/** Compares two strings by their UTF-8 bytes, as Postgres orders them under `COLLATE "C"`. */
+const byteOrder = (left, right) => {
+    const a = encoder.encode(left);
+    const b = encoder.encode(right);
+    for (let index = 0; index < Math.min(a.length, b.length); index++)
+        if (a[index] !== b[index])
+            return a[index] - b[index];
+    return a.length - b.length;
+};
 /**
  * Builds the service over the runtime's SQL, crypto, clock, and outbox, and
  * its job registrations, which say whether a retried job is capped.
@@ -98,7 +109,7 @@ export const operatorRuntime = (deps) => {
                 expiresAtMs: found.expires_at_ms,
             }));
         }).pipe(Effect.catchTag("OperatorNotFound", () => Effect.succeedNone), provided),
-        exportSeed: (target) => exportActor(target).pipe(Effect.catchIf(SqlError.isSqlError, Effect.die), provided),
+        exportSeed: (target) => keyOf(target).pipe(Effect.andThen(exportActor(target)), Effect.catchTag("OperatorNotFound", () => Effect.succeedNone), Effect.catchIf(SqlError.isSqlError, Effect.die), provided),
         retry: ({ target, jobId, providerChecked, audit }) => Effect.gen(function* () {
             const ref = ActorRef.make({
                 tenant: target.tenant,
@@ -192,7 +203,7 @@ export const operatorRuntime = (deps) => {
             yield* deps.wake;
             return { through };
         }).pipe(provided),
-        lagging: ({ tenant, minAttempts, limit }) => Queries.readOnly(tenant)(Effect.gen(function* () {
+        lagging: ({ tenant, minAttempts, limit }) => forEachRange((range) => Queries.readOnly(tenant)(Effect.gen(function* () {
             const sql = yield* SqlClient.SqlClient;
             return yield* sql `
             SELECT s.source_type AS "sourceType", s.source_id AS "sourceId",
@@ -206,21 +217,30 @@ export const operatorRuntime = (deps) => {
               AND g.tenant_id = s.tenant_id AND g.actor_type = s.source_type
               AND g.actor_id = s.source_id
             WHERE s.tenant_id = ${tenant} AND s.active AND s.last_error IS NOT NULL
-              AND s.attempts >= ${minAttempts}
+              AND s.attempts >= ${minAttempts} ${withinRange({ sql, range, column: "s.routing_key" })}
             ORDER BY s.attempts DESC, g.event_sequence - s.delivered DESC,
               s.source_type COLLATE "C", s.source_id COLLATE "C",
               s.subscriber_type COLLATE "C", s.subscription COLLATE "C",
               s.subscriber_id COLLATE "C"
             LIMIT ${limit}`;
-        })).pipe(provided, Effect.orDie),
-        audit: (page) => Queries.readOnly(page.tenant)(listAudit(page)).pipe(provided, Effect.orDie),
+        }))).pipe(Effect.map((shards) => shards
+            .flat()
+            .sort((a, b) => b.attempts - a.attempts ||
+            Number(BigInt(b.lag) - BigInt(a.lag)) ||
+            byteOrder(a.sourceType, b.sourceType) ||
+            byteOrder(a.sourceId, b.sourceId) ||
+            byteOrder(a.subscriberType, b.subscriberType) ||
+            byteOrder(a.subscription, b.subscription) ||
+            byteOrder(a.subscriberId, b.subscriberId))
+            .slice(0, limit)), provided, Effect.orDie),
+        audit: (page) => forEachRange((range) => Queries.readOnly(page.tenant)(listAudit({ ...page, range }))).pipe(Effect.map((shards) => shards
+            .flat()
+            .sort((a, b) => b.atMs - a.atMs || byteOrder(a.auditId, b.auditId))
+            .slice(0, page.limit)), provided, Effect.orDie),
         record: (entry, outcome) => Effect.gen(function* () {
             const found = entry.actorType === undefined ? Option.none() : yield* placement(entry.actorType);
-            yield* writeAudit({
-                entry,
-                key: auditRoutingKey({ entry, placement: Option.getOrUndefined(found) }),
-                outcome,
-            });
+            const key = auditRoutingKey({ entry, placement: Option.getOrUndefined(found) });
+            yield* writeAudit({ entry, key, outcome });
         }).pipe(provided, Effect.orDie),
     });
 };

package/dist/runtime/operators/seed.js

@@ -78,7 +78,8 @@ export const seedRuntime = (deps) => Effect.fnUntraced(function* ({ ref, caller,
     if (placement === undefined)
         return yield* Effect.die(new Error(`Actor ${ref.actor} is not registered`));
     const key = routingKey({ ref, placement });
-    yield* withTenant(ref.tenant)(sql.withTransaction(Effect.gen(function* () {
+    yield* sql
+        .withTransaction(Effect.gen(function* () {
         const [inserted] = yield* sql `
             INSERT INTO actor_generations (routing_key, tenant_id, actor_type, actor_id, created)
             VALUES (${key}, ${ref.tenant}, ${ref.actor}, ${ref.id}, ${seed.created})
@@ -118,6 +119,7 @@ export const seedRuntime = (deps) => Effect.fnUntraced(function* ({ ref, caller,
         }, databaseTime);
         for (const statement of staged.statements)
             yield* statement;
-    })));
+    }))
+        .pipe(withTenant(ref.tenant));
     yield* deps.wake;
 }, (effect) => effect.pipe(Effect.catchIf(SqlError.isSqlError, Effect.die), Effect.provideService(FrameworkClock, deps.clock), Effect.provideService(OutboxRuntime, deps.outbox), Effect.provideService(TenantScope, deps.tenantScope), Effect.provideContext(deps.services)));

package/dist/runtime/payloads/versions.d.ts

@@ -17,7 +17,7 @@ export declare const formatPayloadProblem: (problem: PayloadProblem) => string;
  * Compares declared chains with the versions the database may hold. A
  * version recorded above a chain's current version is a rollback past a
  * schema change. A chain that starts above version `v` can't read values of
- * `v`: an event version counts as stored until `durable payloads clear` marks
+ * `v`: an event version counts as stored until `akter payloads clear` marks
  * it cleared. Every stored event has a recorded version, because a runtime
  * records what it writes before it takes a shard and the migration refused
  * databases that held rows before it. Job versions are read from the
@@ -39,9 +39,9 @@ export declare const recordPayloadVersions: (declarations: readonly PayloadDecla
 export declare const refreshWriters: (runtimeId: string, windowMs: number, declarations: readonly PayloadDeclaration[]) => Effect.Effect<void, import("effect/sql/SqlError").SqlError, SqlClient.SqlClient>;
 /** Drops a stopping runtime's writer rows, so a clear need not wait out its window. */
 export declare const dropWriters: (runtimeId: string) => Effect.Effect<void, import("effect/sql/SqlError").SqlError, SqlClient.SqlClient>;
-/** `durable payloads check`: the startup check for `actors`, read-only. */
+/** `akter payloads check`: the startup check for `actors`, read-only. */
 export declare const checkPayloads: (actors: ReadonlyArray<object>) => Effect.Effect<PayloadProblem[], import("effect/sql/SqlError").SqlError, SqlClient.SqlClient>;
-/** What `durable payloads clear` found for one superseded event version. */
+/** What `akter payloads clear` found for one superseded event version. */
 export interface ClearResult {
     readonly actorType: string;
     readonly tag: string;
@@ -54,7 +54,7 @@ export interface ClearResult {
     readonly outcome: "cleared" | "writer" | "stored";
 }
 /**
- * `durable payloads clear`: marks each superseded event version of `actors`
+ * `akter payloads clear`: marks each superseded event version of `actors`
  * cleared once its retention horizon has passed, no runtime can still write
  * it, and no event of it remains. The version row is locked first, the
  * writer rows are read again after the scan, and each statement of the

package/dist/runtime/payloads/versions.js

@@ -1,5 +1,7 @@
 import { Effect } from "effect";
 import { SqlClient } from "effect/sql";
+import { registry } from "../database/coordination.js";
+import { forEachRange, withinRange } from "../database/shards.js";
 import { descriptorOf } from "../../actor/descriptor.js";
 import {} from "../../members/payload.js";
 import { inReadOnlySnapshot } from "../database/snapshot.js";
@@ -14,7 +16,7 @@ const keyOf = (declared) => `${declared.actorType}\u0000${declared.kind}\u0000${
  * Compares declared chains with the versions the database may hold. A
  * version recorded above a chain's current version is a rollback past a
  * schema change. A chain that starts above version `v` can't read values of
- * `v`: an event version counts as stored until `durable payloads clear` marks
+ * `v`: an event version counts as stored until `akter payloads clear` marks
  * it cleared. Every stored event has a recorded version, because a runtime
  * records what it writes before it takes a shard and the migration refused
  * databases that held rows before it. Job versions are read from the
@@ -60,19 +62,24 @@ export const findPayloadProblems = Effect.fnUntraced(function* (declarations, wr
         if (declared.kind === "event") {
             for (const row of versions)
                 if (row.version < first && !row.cleared)
-                    problem(`version ${row.version} may still be stored below this chain's first version ${first}; run durable payloads clear once its events are gone`);
+                    problem(`version ${row.version} may still be stored below this chain's first version ${first}; run akter payloads clear once its events are gone`);
         }
         else {
-            const [stored] = yield* sql `
+            const shards = (yield* forEachRange((range) => sql `
         SELECT min(v)::int AS version, count(*)::int AS rows FROM (
           SELECT payload_version AS v FROM actor_outbox
           WHERE kind = 'job' AND actor_type = ${declared.actorType} AND command = ${declared.tag}
-            AND payload_version < ${first}
+            AND payload_version < ${first} ${withinRange({ sql, range, column: "routing_key" })}
           UNION ALL
           SELECT payload_version FROM actor_dead_letters
           WHERE actor_type = ${declared.actorType} AND job = ${declared.tag}
-            AND payload_version < ${first}) stored`;
-            if (stored !== undefined && stored.version !== null)
+            AND payload_version < ${first} ${withinRange({ sql, range, column: "routing_key" })}) stored`)).flat();
+            const versions = shards.flatMap((row) => (row.version === null ? [] : [row.version]));
+            const stored = {
+                version: versions.length === 0 ? null : Math.min(...versions),
+                rows: shards.reduce((sum, row) => sum + row.rows, 0),
+            };
+            if (stored.version !== null)
                 problem(`version ${stored.version} stored in ${stored.rows} pending job or dead letter row${stored.rows === 1 ? "" : "s"} below this chain's first version ${first}`);
         }
     }
@@ -85,14 +92,15 @@ export const findPayloadProblems = Effect.fnUntraced(function* (declarations, wr
                 .map((row) => row.tag)),
         ];
         for (const tag of removed) {
-            const [pending] = yield* sql `
+            const [pending] = (yield* forEachRange((range) => sql `
         SELECT s.subscriber_type, s.subscription FROM actor_subscriptions s
         WHERE s.source_type = ${writer.actorType} AND ${tag} = ANY(s.events)
           AND EXISTS (SELECT 1 FROM actor_events e
             WHERE e.routing_key = s.routing_key AND e.tenant_id = s.tenant_id
               AND e.actor_type = s.source_type AND e.actor_id = s.source_id
               AND e.event = ${tag} AND e.sequence > s.delivered)
-        LIMIT 1`;
+          ${withinRange({ sql, range, column: "s.routing_key" })}
+        LIMIT 1`)).flat();
             if (pending !== undefined)
                 problems.push({
                     actorType: writer.actorType,
@@ -159,7 +167,7 @@ const definitionsOf = (actors) => actors.flatMap((actor) => {
     const payloads = descriptorOf(actor)?.definitionPayloads;
     return payloads === undefined ? [] : [payloads];
 });
-/** `durable payloads check`: the startup check for `actors`, read-only. */
+/** `akter payloads check`: the startup check for `actors`, read-only. */
 export const checkPayloads = (actors) => Effect.suspend(() => {
     const definitions = definitionsOf(actors);
     return inReadOnlySnapshot(findPayloadProblems(definitions.flatMap((d) => d.declarations), definitions.flatMap((d) => {
@@ -177,7 +185,7 @@ export const checkPayloads = (actors) => Effect.suspend(() => {
     })));
 });
 /**
- * `durable payloads clear`: marks each superseded event version of `actors`
+ * `akter payloads clear`: marks each superseded event version of `actors`
  * cleared once its retention horizon has passed, no runtime can still write
  * it, and no event of it remains. The version row is locked first, the
  * writer rows are read again after the scan, and each statement of the
@@ -186,7 +194,8 @@ export const checkPayloads = (actors) => Effect.suspend(() => {
  * its window and then the longest command timeout have passed.
  */
 export const clearPayloads = (actors) => Effect.gen(function* () {
-    const sql = yield* SqlClient.SqlClient;
+    const data = yield* SqlClient.SqlClient;
+    const sql = yield* registry;
     const definitions = definitionsOf(actors);
     const commandTimeoutMs = Math.max(0, ...definitions.map((d) => d.commandTimeoutMs));
     const results = [];
@@ -219,11 +228,11 @@ export const clearPayloads = (actors) => Effect.gen(function* () {
                         return undefined;
                     if ((yield* writing).length > 0)
                         return "writer";
-                    const stored = yield* sql `SELECT 1 FROM actor_events
+                    const stored = yield* forEachRange((range) => data `SELECT 1 FROM actor_events
                 WHERE actor_type = ${declared.actorType} AND event = ${declared.tag}
-                  AND payload_version = ${version}
-                LIMIT 1`;
-                    if (stored.length > 0)
+                  AND payload_version = ${version} ${withinRange({ sql: data, range, column: "routing_key" })}
+                LIMIT 1`);
+                    if (stored.some((rows) => rows.length > 0))
                         return "stored";
                     if ((yield* writing).length > 0)
                         return "writer";

package/dist/runtime/queries.js

@@ -33,9 +33,10 @@ export const committedReads = ({ registrations, queryRegistrations, services, al
                 reason: ActorUnavailable.make({ cause: new Error("Actor not registered") }),
             });
         const sql = yield* SqlClient.SqlClient;
+        const key = routingKey({ ref, placement: registration.placement });
         const rows = yield* sql `
           SELECT 1 FROM actor_generations
-          WHERE routing_key = ${routingKey({ ref, placement: registration.placement })}
+          WHERE routing_key = ${key}
             AND tenant_id = ${ref.tenant} AND actor_type = ${ref.actor} AND actor_id = ${ref.id}`.pipe(withTenant(ref.tenant));
         return rows.length > 0;
     }, Effect.provideContext(services), Effect.catchIf(SqlError.isSqlError, (cause) => Effect.fail(ActorError.make({ reason: ActorUnavailable.make({ cause }) }))));
@@ -151,9 +152,10 @@ export const committedReads = ({ registrations, queryRegistrations, services, al
                 });
             yield* allow(request);
             const sql = yield* SqlClient.SqlClient;
+            const key = routingKey({ ref: request.ref, placement: registration.placement });
             const [row] = yield* sql `
           SELECT status, result FROM actor_workflow_executions
-          WHERE routing_key = ${routingKey({ ref: request.ref, placement: registration.placement })}
+          WHERE routing_key = ${key}
             AND execution_id = ${request.payload} AND tenant_id = ${request.ref.tenant}
             AND actor_type = ${request.ref.actor} AND actor_id = ${request.ref.id}
             AND workflow = ${request.command}`.pipe(withTenant(request.ref.tenant));

package/dist/runtime/registration.d.ts

@@ -17,6 +17,7 @@ import type { SubscriptionRelay } from "./subscriptions/relay.ts";
 import { DefectLog } from "./telemetry/defects.ts";
 import { FrameworkClock } from "./turn/admission.ts";
 import { OutboxRuntime } from "./turn/outbox.ts";
+import type { LiveRecorder } from "./telemetry/live.ts";
 /**
  * Registers actor, query, and job layers with the runtime. Each
  * registration is refused unless its placement, tables, content, workflows,
@@ -25,13 +26,13 @@ import { OutboxRuntime } from "./turn/outbox.ts";
  * connection holder, and the sweeps, and forgotten when its layer's scope
  * closes.
  */
-export declare const actorRegistration: ({ registrations, queryRegistrations, jobRegistrations, residency, diagnostics, owners, heldTypes, heldType, sweepsWorkflows, checked, services, frameworkClock, content, retryWindowMs, tableRole, adoptionRole, enforcedTypes, writerDeclarations, refreshPayloadWriters, subscriptions, transport, authorize, gate, writable, defectLog, outbox, }: {
+export declare const actorRegistration: ({ registrations, queryRegistrations, jobRegistrations, residency, diagnostics, owners, heldTypes, heldType, sweepsWorkflows, checked, services, frameworkClock, content, retryWindowMs, tableRole, adoptionRole, enforcedTypes, writerDeclarations, refreshPayloadWriters, subscriptions, transport, authorize, gate, writable, defectLog, outbox, recorder, }: {
     readonly registrations: Map<string, Registration>;
     readonly queryRegistrations: Map<string, QueryRegistration>;
     readonly jobRegistrations: Map<string, JobRegistration>;
     readonly residency: Map<string, (entityId: string) => boolean>;
     /** Each actor type's view of its activations, for a delivery that timed out. */
-    readonly diagnostics: Map<string, Pick<Effect.Success<ReturnType<typeof registerActor>>, "diagnose" | "restarting">>;
+    readonly diagnostics: Map<string, Pick<Effect.Success<ReturnType<typeof registerActor>>, "diagnose" | "restarting" | "resident">>;
     readonly owners: Map<string, Owner>;
     readonly heldTypes: Map<string, HeldActorType>;
     readonly heldType: (registration: Registration) => HeldActorType;
@@ -55,4 +56,6 @@ export declare const actorRegistration: ({ registrations, queryRegistrations, jo
     readonly writable: Effect.Effect<void, ActorError>;
     readonly defectLog: DefectLog["Service"];
     readonly outbox: (typeof OutboxRuntime)["Service"];
+    /** The live recorder, once the inspector enabled one. */
+    readonly recorder: () => LiveRecorder | undefined;
 }) => Pick<InternalActors["Service"], "register" | "registerQueries" | "registerJobs">;