@rikalabs/akter 0.1.0-alpha.0

package/dist/runtime/layer.js

@@ -0,0 +1,876 @@
+import { PgClient, PgTypes } from "@effect/sql-pg";
+import { PgliteClient } from "@effect/sql-pglite";
+import { Cause, Clock, Semaphore, Context, Crypto, Duration, Deferred, Effect, Fiber, Layer, Option, Result, Schedule, Schema, Stream, } from "effect";
+import { ClusterError, Entity, EntityId, RunnerHealth, Runners, RunnerStorage, Sharding, ShardingConfig, } from "effect/cluster";
+import { SqlClient, SqlError } from "effect/sql";
+import { ActorError, ActorUnavailable, InvalidInput, Unauthorized, Timeout, MailboxFull, RunnerAtCapacity, } from "../errors/actor.js";
+import { Actors } from "../handles/actors.js";
+import {} from "./members.js";
+import { DeliveryFailed } from "./request.js";
+import { InternalActors } from "./actors.js";
+import { System } from "../identity/caller.js";
+import { deriveMintId } from "../identity/mint.js";
+import { migrate } from "./database/migrations.js";
+import { NekiTurnSessions } from "./database/neki/session.js";
+import { queryPoolLayer, ReadReplica, replicaLayer } from "./database/replica.js";
+import { Coordination, coordinationLayer } from "./database/coordination.js";
+import { withKeepalives } from "./database/keepalive.js";
+import { checkRowLevelSecurity, TenantScope } from "./database/tenancy.js";
+import { retryDelay } from "./retry.js";
+import { withoutDatabase } from "./jobs/isolation.js";
+import { pglite } from "./database/pglite.js";
+import { commandEntity, connectionEntity, encodeEntityId } from "./entity/register.js";
+import { connectionHolder } from "./connections/holder.js";
+import { holderShardGroups, holderTransport } from "./connections/transport.js";
+import { ProgressSink, ProgressTap } from "./jobs/progress.js";
+import { FEED_MEMBER, isWatchMember, watchedQuery } from "./connections/protocol.js";
+import { checkIdentity, databaseTime, FrameworkClock } from "./turn/admission.js";
+import { routingKey } from "./storage/codec.js";
+import { recordedPlacement } from "./storage/placements.js";
+import { TurnHooks } from "./turn/hooks.js";
+import { requestAttributes, SpanNames } from "./telemetry/spans.js";
+import { DefectLog, boundedDefectLog } from "./telemetry/defects.js";
+import { OperatorRuntime, operatorRuntime } from "./operators/repair.js";
+import { seedRuntime } from "./operators/seed.js";
+import { TelemetrySampler } from "./telemetry/sampler.js";
+import { turnGroups } from "./turn/group.js";
+import { TurnConnections, turnConnections } from "./turn/pipeline.js";
+import { outboxRelay } from "./turn/relay.js";
+import { subscriptionRelay, } from "./subscriptions/relay.js";
+import { DEFAULT_WRITER_WINDOW_MS, refreshWriters } from "./payloads/versions.js";
+import { ExecutionTarget, INTERRUPT } from "../handles/workflow.js";
+import { decodeExecutionId } from "../identity/execution.js";
+import { coordinatedRunnerStorage, keepAcquiredShards, ShardLease, tableShardLease, } from "./topology/locks.js";
+import { directMessages } from "./topology/messages.js";
+import { MailboxRefusals, mailboxRefusals } from "./topology/admission.js";
+import { bindBlobs } from "./turn/blobs.js";
+import { ContentStore } from "../handles/content.js";
+import { isContent } from "../members/blob.js";
+import { grantKeys } from "./content/grant.js";
+import { MAX_CONTENT_BYTES, tenantContent } from "./content/store.js";
+import { ContentHooks } from "./turn/hooks.js";
+import { bindTables, rowsDatabase } from "./turn/rows.js";
+import { admissionLimit, isOverloaded, overloaded } from "./admission.js";
+import { boundedLayer, isPoolRefusal } from "./database/bounded.js";
+import { RuntimeControl, runtimeControl, turnGate } from "./drain.js";
+import { eventFeeds } from "./feeds.js";
+import { committedReads } from "./queries.js";
+import { servingReadiness } from "./readiness.js";
+import { acquiredShards, checkRunnerConfiguration, RunnerReadiness, LocalRequestSerialization, RunnerWiring, } from "./runner.js";
+import { actorRegistration } from "./registration.js";
+import { startSweeps } from "./sweeps.js";
+import { checkFleet } from "./fleet/checks.js";
+import { maintain } from "./fleet/maintainer.js";
+import { fleetSubscriptions } from "./fleet/subscribe.js";
+const Count = Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 1_000_000 }));
+const Millis = Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 2_147_483_647 }));
+const millis = (duration) => Millis.make(Math.floor(Duration.toMillis(Duration.fromInputUnsafe(duration))));
+const decodeTarget = Schema.decodeEffect(ExecutionTarget);
+/** How often a fleet subscription is authorized again when its source's actor type is not registered here. */
+const DEFAULT_REAUTHORIZE_MS = 60_000;
+/** Added to the longest turn a claimed intent's receiver may take. */
+const CLAIM_MARGIN_MS = 5000;
+/** The claim lease when no actor type is registered: default policies' 30 s + 2 s + margin. */
+const DEFAULT_CLAIM_LEASE_MS = 37_000;
+/** How long a progress send may take before it is given up as a lost frame. */
+const PROGRESS_SEND_TIMEOUT = "5 seconds";
+/**
+ * An entity type with no messages, registered before any actor. The cluster's
+ * idle sweep takes its interval from the first entity type registered and
+ * sleeps that long before it reads any later one, so an actor type with a
+ * 60-second `hibernateAfter` registering first held every activation for up to
+ * 30 seconds past its own `hibernateAfter`. This type's 5-second idle time, the
+ * sweep's shortest interval, makes every sweep 5 seconds apart from the start.
+ */
+const IdleSweep = Entity.make("akter/IdleSweep", []);
+/**
+ * Builds the runtime: migrates and checks the database, registers every actor,
+ * effect, query, and subscription layer, and starts the relay and background
+ * sweeps. The returned layer provides `RuntimeControl`, and fails to build when
+ * the schema or declared payloads are incompatible.
+ *
+ * Constraints the wiring keeps:
+ * - Turns run through the drain gate, so a drain refuses new turns and
+ *   interrupts the rest. The runtime scope, not the caller, owns an in-flight
+ *   turn: interrupting a waiter never cancels it.
+ * - Commands are direct, so Cluster keeps no messages and the receipt is the
+ *   only admission record; a restarted activation or lost runner drops the
+ *   uncommitted attempt, the handle retries with the same command id, and the
+ *   receipt replays anything that committed. Durable intents use the
+ *   actor-shard outbox. Retries wait from the error's own `retryAfter`, bounded
+ *   in total by the delivery timeout.
+ * - Intents are admitted by their sending turn, so internal delivery skips the
+ *   external access and expiry checks: revocation stops new commands, not
+ *   committed obligations. A draining runner admits no new external work. Only
+ *   the relay presents a mint proof or delivers a subscription envelope.
+ * - A turn runs only in a resident activation. After a capacity rejection with
+ *   none resident, the latest attempt was not admitted, though an earlier one
+ *   may have committed.
+ * - Queries and feeds read committed rows on the caller's node without an
+ *   activation, generation fence, receipt, or command id. State and events are
+ *   read against one event head so they describe one moment; a replica answers
+ *   only once it has replayed the caller's commit version, else the primary
+ *   does. Types with owned tables or blobs read their state on the primary,
+ *   one server per query. No `statement_timeout` bounds query reads;
+ *   interrupting one past `commandTimeout` cancels its statement on the server.
+ *   Access is rechecked after the handler, so a result is released only to a
+ *   caller still allowed. A failed replay read is unavailability, not a defect.
+ * - Effect progress is fire-and-forget to the performing actor's connection
+ *   entity: no retry, no acknowledgment, and a lost message is a lost frame.
+ *   Each effect's last frame is awaited so its close never overtakes it.
+ * - Payload versions this runtime writes are recorded and heartbeat before a
+ *   type takes any shard, refreshed one at a time with the time taken before
+ *   the statement is sent, so the gate's window can only end early.
+ * - Content grants are bound to the database's deployment id. The content
+ *   sweep waits out the longest turn that may attach content across every
+ *   runner, and each tenant is swept at most once an hour.
+ * - Exactly one runner of the deployment runs the telemetry sampler; runtimes
+ *   sharing one Sharding in a process share its one sampler.
+ * - Readiness on PGlite always answers from the layer's own lifetime, since a
+ *   single connection held by a turn would make a probe report unready; on
+ *   Postgres the database answers at most once a second.
+ * - PGlite keeps runner bookkeeping in memory because SqlRunnerStorage would
+ *   reserve its single connection for the layer's lifetime. Postgres uses
+ *   advisory locks, held by a live session, so no second runner can take a
+ *   shard while its holder runs.
+ * - An unreachable database fails a read as `ActorUnavailable`, which callers
+ *   retry like any delivery failure; it is not a defect.
+ */
+export const layer = (options = {}) => {
+    const enforcedTypes = new Set();
+    const retryWindowMs = Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 2_592_000_000 })).make(options.retryWindowMs ?? 86_400_000);
+    const maxResidentActors = Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 2_147_483_647 })).make(options.maxResidentActors ?? 10_000);
+    const admissionSettings = {
+        concurrency: Count.make(options.admission?.concurrency ?? 64),
+        wait: Duration.millis(millis(options.admission?.wait ?? "100 millis")),
+        requests: Count.make(options.admission?.requests ?? 64),
+    };
+    const executorLeaseMs = millis(options.executors?.lease ?? "60 seconds");
+    if (executorLeaseMs < 3000)
+        throw new Error("executors.lease must be at least 3 seconds");
+    const cancelCheckMs = Math.min(options.executors?.cancelCheck === undefined
+        ? executorLeaseMs / 3
+        : millis(options.executors.cancelCheck), executorLeaseMs / 3);
+    if (cancelCheckMs < 1000)
+        throw new Error("executors.cancelCheck must be at least 1 second");
+    const claimLeaseMs = options.relay?.claimLease === undefined ? undefined : millis(options.relay.claimLease);
+    const relaySettings = {
+        pollMs: millis(options.relay?.poll ?? "1 second"),
+        passLimit: Count.make(options.relay?.passLimit ?? 256),
+        deliveryConcurrency: Count.make(options.relay?.deliveryConcurrency ?? 16),
+        maxBackoffMs: millis(options.relay?.maxBackoff ?? "256 seconds"),
+        executorConcurrency: Count.make(options.executors?.concurrency ?? 64),
+        executorLeaseMs,
+        retryWindowMs,
+        cancelCheckMs,
+    };
+    const defectCapacity = Count.make(options.observability?.defects ?? 1000);
+    const sampleEveryMs = millis(options.observability?.sampleEvery ?? "15 seconds");
+    const contentGraceMs = Duration.toMillis(Duration.fromInputUnsafe(options.content?.grace ?? "24 hours"));
+    const contentSkewMs = Duration.toMillis(Duration.fromInputUnsafe(options.content?.skew ?? "60 seconds"));
+    if (!Number.isSafeInteger(contentGraceMs) ||
+        contentGraceMs < 0 ||
+        !Number.isSafeInteger(contentSkewMs) ||
+        contentSkewMs < 0)
+        throw new Error("content.grace and content.skew must be finite, non-negative whole milliseconds");
+    const subscriptionConcurrency = Count.make(options.relay?.subscriptionConcurrency ?? 16);
+    const writerWindowMs = options.payloadWriterWindow === undefined
+        ? DEFAULT_WRITER_WINDOW_MS
+        : millis(options.payloadWriterWindow);
+    if (writerWindowMs < 1000)
+        throw new Error("payloadWriterWindow must be at least 1 second");
+    const subscriptionBatch = Count.make(options.relay?.subscriptionBatch ?? 16);
+    const runtime = Layer.effectContext(Effect.gen(function* () {
+        const crypto = yield* Crypto.Crypto;
+        const scope = yield* Effect.scope;
+        const sharding = yield* Sharding.Sharding;
+        yield* sharding.registerEntity(IdleSweep, Effect.succeed(IdleSweep.of({})), {
+            maxIdleTime: "5 seconds",
+        });
+        const registrations = new Map();
+        const residency = new Map();
+        const refusals = yield* MailboxRefusals;
+        refusals.refuse = (address) => {
+            const registration = registrations.get(address.entityType);
+            if (registration === undefined ||
+                residency.get(address.entityType)?.(address.entityId) !== true)
+                return undefined;
+            return registration.policy.mailboxCapacity === "unbounded"
+                ? overloaded("activation")
+                : ActorError.make({ reason: MailboxFull.make({}) });
+        };
+        const diagnostics = new Map();
+        const owners = new Map();
+        const sweepsWorkflows = new Set();
+        const queryRegistrations = new Map();
+        const jobRegistrations = new Map();
+        const runtimeId = yield* crypto.randomUUIDv4.pipe(Effect.orDie);
+        const frameworkClock = yield* FrameworkClock;
+        const writerDeclarations = [];
+        let refreshedAt;
+        const services = yield* Effect.context();
+        const defectLog = boundedDefectLog(defectCapacity);
+        const database = yield* rowsDatabase;
+        const clockOffset = yield* FrameworkClock;
+        const content = options.content === undefined
+            ? undefined
+            : tenantContent({
+                grants: yield* grantKeys(options.content.keys, (yield* (yield* SqlClient.SqlClient) `
+                  SELECT deployment_id FROM actor_deployment`.pipe(Effect.orDie))[0].deployment_id),
+                graceMs: contentGraceMs,
+                skewMs: contentSkewMs,
+                singleConnection: Option.isSome(yield* Effect.serviceOption(PgliteClient.PgliteClient)),
+                offset: () => clockOffset.offsetMillis(),
+                hooks: yield* ContentHooks,
+            });
+        const contentBinding = content === undefined
+            ? undefined
+            : { store: content, skewMs: contentSkewMs, offset: () => clockOffset.offsetMillis() };
+        const primary = Context.get(services, SqlClient.SqlClient);
+        const replica = yield* ReadReplica;
+        let observed;
+        const observe = (executed) => Effect.sync(() => {
+            const version = executed.version;
+            if (version !== undefined &&
+                (observed === undefined || BigInt(version) > BigInt(observed)))
+                observed = version;
+        });
+        const gate = turnGate();
+        const admission = admissionLimit({
+            limit: admissionSettings.concurrency,
+            wait: admissionSettings.wait,
+        });
+        const requests = admissionLimit({ limit: admissionSettings.requests, wait: Duration.zero });
+        let holder;
+        const transport = yield* holderTransport((message) => Effect.suspend(() => holder.deliver(message)));
+        const connectionCall = (effect) => Effect.suspend(() => effect.pipe(Effect.forkIn(scope))).pipe(Effect.flatMap(Fiber.join), Effect.catchCause((cause) => {
+            const failure = Cause.findErrorOption(cause);
+            if (Option.isSome(failure) && Schema.is(ActorError)(failure.value))
+                return Effect.fail(failure.value);
+            return Effect.fail(ActorError.make({ reason: ActorUnavailable.make({ cause: Cause.squash(cause) }) }));
+        }));
+        const heldTypes = new Map();
+        const shardLockMs = Duration.toMillis(Option.match(yield* Effect.serviceOption(ShardingConfig.ShardingConfig), {
+            onNone: () => ShardingConfig.defaults.shardLockExpiration,
+            onSome: (config) => config.shardLockExpiration,
+        }));
+        const heldType = (registration) => {
+            const entity = connectionEntity(registration.name);
+            const client = (ref) => Effect.gen(function* () {
+                const make = yield* sharding.makeClient(entity);
+                return make(yield* entityId(ref));
+            });
+            return {
+                deliveryMs: registration.policy.deliveryMs,
+                takeoverMs: shardLockMs + registration.policy.deliveryMs,
+                reauthorizeMs: registration.policy.reauthorizeMs,
+                retryWindowMs,
+                placement: registration.placement,
+                routingKey: (ref) => routingKey({ ref, placement: registration.placement }),
+                hasResync: (member) => member === FEED_MEMBER ||
+                    isWatchMember(member) ||
+                    (registration.connections.get(member)?.hasResync ?? false),
+                hasMember: (member) => member === FEED_MEMBER
+                    ? registration.feeds.size > 0
+                    : isWatchMember(member)
+                        ? registration.watches.has(watchedQuery(member))
+                        : registration.connections.has(member),
+                channel: {
+                    open: (request) => connectionCall(Effect.flatMap(client(request.ref), (c) => c.Open(request))),
+                    frame: (request) => connectionCall(Effect.flatMap(client(request.ref), (c) => c.Frame(request))),
+                    close: (request) => connectionCall(Effect.flatMap(client(request.ref), (c) => c.Close(request))),
+                    resync: (request) => connectionCall(Effect.flatMap(client(request.ref), (c) => c.Resync(request))),
+                },
+            };
+        };
+        /**
+         * Whether the request is allowed: the global `authorize` and the
+         * actor's `access` both when both exist, either alone when only one
+         * does, and with neither, a `System` caller. A registration the runner
+         * does not hold has no `access`, so only the global hook or the default
+         * applies.
+         */
+        const permitted = Effect.fnUntraced(function* (request) {
+            const access = (registrations.get(request.ref.actor) ?? queryRegistrations.get(request.ref.actor))?.access;
+            if (options.authorize === undefined && access === undefined)
+                return Schema.is(System)(request.caller);
+            if (options.authorize !== undefined && !(yield* options.authorize(request)))
+                return false;
+            if (access === undefined)
+                return true;
+            const allowed = access(request);
+            return Effect.isEffect(allowed) ? yield* allowed : allowed;
+        });
+        holder = yield* connectionHolder({
+            transport: () => transport,
+            actorType: (name) => heldTypes.get(name),
+            authorize: permitted,
+        });
+        const checked = new Set();
+        const authorizedAs = (request) => request.command !== INTERRUPT
+            ? Effect.succeed(request)
+            : decodeTarget(request.payload).pipe(Effect.flatMap(({ executionId }) => decodeExecutionId(executionId)), Effect.map(({ workflow }) => ({ ...request, command: workflow })), Effect.orElseSucceed(() => request));
+        const allow = Effect.fnUntraced(function* (request, kind = "command") {
+            const { caller, ref, command } = yield* authorizedAs(request);
+            if (!(yield* permitted({ caller, ref, command, kind })))
+                return yield* ActorError.make({ reason: Unauthorized.make({ code: "access_denied" }) });
+        });
+        /**
+         * Rechecks access and expiry before an external caller gets its outcome.
+         * `endedAtMs` is the clock the turn read after its transaction ended;
+         * an outcome without one, a defect, reads the clock again. A reply may
+         * already have committed, so a full clock-read pool must wait, never
+         * report pre-turn refusal.
+         */
+        const authorize = Effect.fnUntraced(function* (request, endedAtMs) {
+            yield* allow(request, "command");
+            yield* checkIdentity(request.commandId, retryWindowMs, endedAtMs === undefined
+                ? yield* databaseTime.pipe(Effect.retry({
+                    while: isPoolRefusal,
+                    schedule: Schedule.spaced("10 millis"),
+                }))
+                : endedAtMs + frameworkClock.offsetMillis());
+        });
+        const refreshing = Semaphore.makeUnsafe(1);
+        const refreshPayloadWriters = Effect.gen(function* () {
+            if (writerDeclarations.length === 0)
+                return;
+            const sentAt = yield* Clock.currentTimeMillis;
+            yield* refreshWriters(runtimeId, writerWindowMs, writerDeclarations);
+            refreshedAt = sentAt;
+        }).pipe(refreshing.withPermits(1), Effect.provideContext(services), Effect.provideService(FrameworkClock, frameworkClock));
+        /**
+         * A runtime that could not refresh its writer rows within the window
+         * may already count as gone to `durable payloads clear`, so it starts
+         * no turn until a refresh succeeds. The check is local.
+         */
+        const writable = Effect.gen(function* () {
+            if (refreshedAt === undefined)
+                return;
+            if ((yield* Clock.currentTimeMillis) - refreshedAt > writerWindowMs)
+                return yield* ActorError.make({
+                    reason: ActorUnavailable.make({
+                        cause: new Error("This runtime's payload writer rows are older than its window; turns wait for a refresh"),
+                    }),
+                });
+        });
+        const entityId = (ref) => encodeEntityId([ref.tenant, ref.id]).pipe(Effect.orDie);
+        const databaseNow = databaseTime.pipe(Effect.provideContext(services), Effect.catchIf(SqlError.isSqlError, (cause) => Effect.fail(ActorError.make({ reason: ActorUnavailable.make({ cause }) }))));
+        const mintCommandId = Effect.gen(function* () {
+            const now = yield* databaseNow;
+            const uuid = yield* crypto.randomUUIDv4.pipe(Effect.orDie);
+            return `v1.${now}.${now + retryWindowMs}.${uuid}`;
+        });
+        const publicActors = Actors.of({ mintCommandId });
+        /**
+         * Logs what this runner knows of a delivery's target when the delivery
+         * waits out `deliveryTimeout`: the target activation's handlers, worker
+         * phase, mailbox and batch, the turn pool's leased and waiting sessions,
+         * and any other activation on the runner being rebuilt or restarted, so
+         * a stall names its cause from one log line.
+         */
+        const deliveryTimedOut = (request, address) => Effect.gen(function* () {
+            const target = diagnostics.get(request.ref.actor);
+            const turns = Option.getOrUndefined(yield* Effect.serviceOption(TurnConnections));
+            yield* Effect.logWarning("Delivery timed out").pipe(Effect.annotateLogs({
+                actor: request.ref.actor,
+                id: request.ref.id,
+                tenant: request.ref.tenant,
+                command: request.command,
+                commandId: request.commandId,
+                activation: target?.diagnose(address),
+                turnSessions: turns?.sessions(),
+                restarting: [...diagnostics].flatMap(([actor, { restarting }]) => restarting()
+                    .filter((entityId) => actor !== request.ref.actor || entityId !== address)
+                    .map((entityId) => `${actor}:${entityId}`)),
+            }));
+        });
+        const makeCommandClient = (actor) => sharding.makeClient(commandEntity(actor));
+        const commandClients = new Map();
+        /** Cluster keeps one client per entity for the runtime's life, so its lookup is done once per actor type. */
+        const commandClient = (actor) => {
+            const cached = commandClients.get(actor);
+            if (cached !== undefined)
+                return Effect.succeed(cached);
+            return Effect.tap(makeCommandClient(actor), (make) => Effect.sync(() => {
+                commandClients.set(actor, make);
+            }));
+        };
+        const dispatch = Effect.fnUntraced(function* (request, external, pending) {
+            const registration = registrations.get(request.ref.actor);
+            if (registration === undefined)
+                return yield* ActorError.make({
+                    reason: ActorUnavailable.make({ cause: new Error("Actor not registered") }),
+                });
+            if (external && !gate.open)
+                return yield* ActorError.make({
+                    reason: ActorUnavailable.make({ cause: new Error("Runner is draining") }),
+                });
+            const address = yield* entityId(request.ref);
+            const isResident = () => residency.get(request.ref.actor)?.(address) === true;
+            let rejectedAtCapacity = false;
+            return yield* Effect.gen(function* () {
+                if (external &&
+                    (request.intent !== undefined ||
+                        request.delivery !== undefined ||
+                        request.redelivered !== undefined ||
+                        request.clockOffset !== undefined ||
+                        (Schema.is(System)(request.caller) &&
+                            (request.caller.mint !== undefined || request.caller.source === "subscription"))))
+                    return yield* ActorError.make({
+                        reason: Unauthorized.make({ code: "access_denied" }),
+                    });
+                if (external) {
+                    yield* allow(request, "command");
+                    yield* checkIdentity(request.commandId, retryWindowMs);
+                }
+                const client = (yield* commandClient(request.ref.actor))(address);
+                yield* (yield* TurnHooks).at("beforeDelivery", request);
+                let admitted = false;
+                const deliver = Clock.currentTimeMillis.pipe(Effect.flatMap((queuedAtMs) => client
+                    .Execute(!external
+                    ? { ...request, queuedAtMs }
+                    : !admitted
+                        ? {
+                            ...request,
+                            external,
+                            clockOffset: frameworkClock.offsetMillis(),
+                            queuedAtMs,
+                        }
+                        : {
+                            ...request,
+                            external,
+                            redelivered: true,
+                            clockOffset: frameworkClock.offsetMillis(),
+                            queuedAtMs,
+                        })
+                    .pipe(Effect.interruptible, Effect.forkIn(scope), Effect.tap((fiber) => Effect.sync(() => {
+                    pending?.add(fiber);
+                })), Effect.uninterruptible)), Effect.flatMap(Fiber.join), Effect.catchCause((cause) => {
+                    const failure = Cause.findErrorOption(cause);
+                    if (Option.isSome(failure) && Schema.is(DeliveryFailed)(failure.value)) {
+                        admitted ||= failure.value.admitted;
+                        return Effect.fail(failure.value.error);
+                    }
+                    if (Option.isSome(failure) && Schema.is(ActorError)(failure.value))
+                        return Effect.fail(failure.value);
+                    if (Option.isSome(failure) && Schema.is(ClusterError.MailboxFull)(failure.value)) {
+                        if (isResident())
+                            return Effect.fail(registration.policy.mailboxCapacity === "unbounded"
+                                ? overloaded("activation")
+                                : ActorError.make({ reason: MailboxFull.make({}) }));
+                        rejectedAtCapacity = true;
+                        return Effect.fail(ActorError.make({ reason: RunnerAtCapacity.make({}) }));
+                    }
+                    return Effect.fail(ActorError.make({
+                        reason: ActorUnavailable.make({ cause: Cause.squash(cause) }),
+                    }));
+                }));
+                const retrying = (attempt) => deliver.pipe(Effect.catchIf((error) => (Schema.is(ActorUnavailable)(error.reason) && !isOverloaded(error)) ||
+                    Schema.is(RunnerAtCapacity)(error.reason), (error) => Effect.sleep(retryDelay(attempt)(error)).pipe(Effect.andThen(Effect.suspend(() => retrying(attempt + 1))))));
+                const executed = yield* retrying(0);
+                if (external)
+                    yield* authorize(request, executed.endedAtMs);
+                return executed;
+            }).pipe(Effect.timeoutOrElse({
+                duration: registration.policy.deliveryMs,
+                orElse: () => rejectedAtCapacity && !isResident()
+                    ? Effect.fail(ActorError.make({ reason: RunnerAtCapacity.make({}) }))
+                    : deliveryTimedOut(request, address).pipe(Effect.andThen(Effect.fail(ActorError.make({
+                        reason: Timeout.make({ commandId: request.commandId }),
+                    })))),
+            }));
+        }, Effect.provideContext(services), Effect.catchIf(SqlError.isSqlError, (cause) => Effect.fail(ActorError.make({ reason: ActorUnavailable.make({ cause }) }))), (effect, ...[request, external]) => effect.pipe(Effect.withSpan(SpanNames.admission, { attributes: { ...requestAttributes(request), "admission.external": external } }, { captureStackTrace: false })));
+        const leaseForTurns = () => {
+            let longest = 0;
+            for (const { policy } of registrations.values())
+                longest = Math.max(longest, policy.executionMs + policy.lockWaitMs);
+            return longest === 0 ? DEFAULT_CLAIM_LEASE_MS : longest + CLAIM_MARGIN_MS;
+        };
+        const progressTap = yield* ProgressTap;
+        const utf8Decoder = new TextDecoder();
+        const ownerOf = (ref) => Effect.gen(function* () {
+            const make = yield* sharding.makeClient(connectionEntity(ref.actor));
+            return make(yield* entityId(ref));
+        });
+        const fireAndForget = (send) => send.pipe(Effect.timeout(PROGRESS_SEND_TIMEOUT), Effect.ignoreCause, Effect.forkIn(scope));
+        const inflight = new Map();
+        const deliverProgress = (message) => Effect.flatMap(ownerOf(message.ref), (client) => client.Progress({ ...message, frame: utf8Decoder.decode(message.frame) }, { discard: true }));
+        const progressSink = ProgressSink.of({
+            wants: (actor, job) => jobRegistrations.get(actor)?.progress.has(job) === true,
+            send: (message) => Effect.flatMap(progressTap.send(message), (deliver) => deliver
+                ? fireAndForget(deliverProgress(message)).pipe(Effect.flatMap((fiber) => Effect.sync(() => {
+                    inflight.set(message.jobId, fiber);
+                    fiber.addObserver(() => {
+                        if (inflight.get(message.jobId) === fiber)
+                            inflight.delete(message.jobId);
+                    });
+                })))
+                : Effect.void),
+            closed: (message) => Effect.flatMap(progressTap.closed(message), (deliver) => deliver
+                ? fireAndForget(Effect.suspend(() => {
+                    const last = inflight.get(message.jobId);
+                    return last === undefined ? Effect.void : Fiber.await(last);
+                }).pipe(Effect.andThen(ownerOf(message.ref)), Effect.flatMap((client) => client.ProgressClosed(message, { discard: true })))).pipe(Effect.asVoid)
+                : Effect.void),
+        });
+        const localSubscriptions = () => [...registrations.values()].flatMap((registration) => registration.subscriptions.map((subscription) => ({
+            subscriberType: registration.name,
+            subscription,
+        })));
+        const placements = new Map();
+        const placementOf = (actorType) => Effect.gen(function* () {
+            const known = registrations.get(actorType)?.placement ??
+                queryRegistrations.get(actorType)?.placement ??
+                placements.get(actorType);
+            if (known !== undefined)
+                return known;
+            const recorded = yield* recordedPlacement(actorType);
+            if (recorded !== undefined)
+                placements.set(actorType, recorded);
+            return recorded;
+        }).pipe(Effect.provideContext(services), Effect.orDie);
+        const relayDeliver = (request) => Effect.map(dispatch(request, false), (executed) => executed.outcome);
+        const subscriptions = yield* subscriptionRelay({
+            deliver: relayDeliver,
+            local: localSubscriptions,
+            placementOf,
+            wake: Effect.suspend(() => relay.wake),
+            settings: {
+                concurrency: subscriptionConcurrency,
+                batch: subscriptionBatch,
+                claimLeaseMs: () => claimLeaseMs ?? leaseForTurns(),
+                maxBackoffMs: relaySettings.maxBackoffMs,
+                retryWindowMs,
+            },
+        });
+        const relay = yield* outboxRelay(relayDeliver, () => [...jobRegistrations.values()].flatMap((registration) => [...registration.jobs].map(([job, registered]) => ({
+            actor: registration.name,
+            job,
+            registered: {
+                ...registered,
+                execute: (payload, version, context) => registered
+                    .execute(payload, version, context)
+                    .pipe(withoutDatabase(registration.services)),
+            },
+        }))), { ...relaySettings, claimLeaseMs: () => claimLeaseMs ?? leaseForTurns() }, {
+            concurrency: subscriptionConcurrency,
+            claim: subscriptions.claim,
+            decode: subscriptions.decode,
+            run: (work, handoff) => subscriptions.run(work, handoff).pipe(Effect.provideContext(services)),
+        }, () => new Map(Array.from(registrations.values(), ({ name, cron, policy }) => [
+            name,
+            { entries: cron, skipMs: policy.cronSkipMs },
+        ]))).pipe(Effect.provideService(ProgressSink, progressSink));
+        yield* relay.run.pipe(Effect.forkIn(scope));
+        let fleetSubscribe;
+        if (options.fleet !== undefined && options.fleet.length > 0) {
+            if (Option.isSome(yield* Effect.serviceOption(PgliteClient.PgliteClient)))
+                return yield* Effect.die(new Error("Fleet.view needs Postgres with wal_level=logical; PGlite has no logical replication"));
+            const views = yield* checkFleet(options.fleet, options.rowLevelSecurity?.role).pipe(Effect.provideContext(services), Effect.orDie);
+            yield* maintain(views).pipe(Effect.provideContext(services), Effect.forkIn(scope));
+            fleetSubscribe = yield* fleetSubscriptions({
+                views,
+                role: options.rowLevelSecurity?.role,
+                permitted,
+                reauthorizeMs: (actorType) => registrations.get(actorType)?.policy.reauthorizeMs ?? DEFAULT_REAUTHORIZE_MS,
+            }).pipe(Effect.provideContext(services));
+        }
+        const { cleanup, sweeping, sample } = yield* startSweeps({
+            registrations,
+            sweepsWorkflows,
+            retryWindowMs,
+            subscriptions,
+            content,
+            services,
+            frameworkClock,
+            refreshPayloadWriters,
+            writerWindowMs,
+            runtimeId,
+            sampleEveryMs,
+            sharding,
+            scope,
+        });
+        const routed = (sourceType) => localSubscriptions().flatMap(({ subscriberType, subscription }) => subscription.routed !== undefined && subscription.sourceType === sourceType
+            ? [{ ...subscription, subscriberType }]
+            : []);
+        const outbox = { retryWindowMs, wake: relay.wake, cancelled: relay.cancelled, routed };
+        const tenantScope = yield* TenantScope;
+        const operators = operatorRuntime({
+            services,
+            clock: frameworkClock,
+            outbox,
+            jobOf: (actorType, job) => jobRegistrations.get(actorType)?.jobs.get(job),
+            wake: relay.wake,
+            tenantScope,
+        });
+        const seeding = seedRuntime({
+            services,
+            clock: frameworkClock,
+            outbox,
+            jobOf: (actorType, job) => jobRegistrations.get(actorType)?.jobs.get(job),
+            createdBy: (actorType) => registrations.get(actorType)?.policy.createdBy !== undefined,
+            wake: relay.wake,
+            tenantScope,
+        });
+        const internalActors = InternalActors.of({
+            fleet: (request) => fleetSubscribe === undefined
+                ? ActorError.make({ reason: InvalidInput.make({ code: "unknown_route" }) })
+                : fleetSubscribe(request),
+            fleetViews: new Set((options.fleet ?? []).map(({ name }) => name)),
+            seed: seeding,
+            mintActorId: crypto.randomUUIDv7.pipe(Effect.orDie),
+            mintChildId: (input) => deriveMintId(input).pipe(Effect.provideService(Crypto.Crypto, crypto)),
+            retryWindowMs,
+            databaseNow,
+            mintCommandId,
+            tables: (scope, write) => bindTables(database, scope, write, checked).pipe(Effect.provideService(SqlClient.SqlClient, primary), Effect.provideService(TenantScope, tenantScope)),
+            blobs: (scope, write) => bindBlobs(scope, write, contentBinding).pipe(Effect.provideService(SqlClient.SqlClient, primary), Effect.provideService(TenantScope, tenantScope)),
+            sweepContent: content === undefined
+                ? Effect.succeed(0)
+                : content.sweep(true).pipe(Effect.provideContext(services), Effect.orDie),
+            registered: (actor) => ({
+                commands: registrations.has(actor),
+                queries: queryRegistrations.has(actor),
+            }),
+            declaredBlobs: (actor) => (registrations.get(actor) ?? queryRegistrations.get(actor))?.blobs.map((blob) => blob.name) ?? [],
+            ...actorRegistration({
+                registrations,
+                queryRegistrations,
+                jobRegistrations,
+                residency,
+                diagnostics,
+                owners,
+                heldTypes,
+                heldType,
+                sweepsWorkflows,
+                checked,
+                services,
+                frameworkClock,
+                content,
+                retryWindowMs,
+                tableRole: options.rowLevelSecurity?.role,
+                adoptionRole: options.adoption?.role,
+                enforcedTypes,
+                writerDeclarations,
+                refreshPayloadWriters,
+                subscriptions,
+                transport,
+                authorize: permitted,
+                gate,
+                writable,
+                defectLog,
+                outbox,
+            }),
+            ...committedReads({
+                registrations,
+                queryRegistrations,
+                services,
+                allow,
+                primary,
+                replica,
+                holder,
+            }),
+            ...eventFeeds({
+                registrations,
+                services,
+                allow,
+                sharding,
+                entityId,
+                frameworkClock,
+                transport,
+            }),
+            deliverProgress: (message) => deliverProgress(message).pipe(Effect.ignoreCause, Effect.provideContext(services)),
+            transport,
+            holder,
+            hibernate: (ref) => Effect.flatMap(entityId(ref), (id) => owners.get(ref.actor)?.hibernate(id) ?? Effect.void).pipe(Effect.provideContext(services)),
+            execute: (request) => Effect.gen(function* () {
+                const reply = yield* Deferred.make();
+                const pending = new Set();
+                yield* admission
+                    .admit(dispatch(request, true, pending).pipe(Effect.tap(observe), Effect.onExit((exit) => [...pending].some((fiber) => fiber.pollUnsafe() === undefined)
+                    ? Deferred.done(reply, exit)
+                    : Effect.void), Effect.ensuring(Effect.suspend(() => Effect.forEach(pending, Fiber.await, { discard: true })))))
+                    .pipe(Effect.onExit((exit) => Deferred.done(reply, exit)), Effect.ignoreCause, Effect.forkIn(scope));
+                return yield* Deferred.await(reply);
+            }),
+            overloaded: admission.full,
+            admitRequest: requests.admit,
+            deliver: (request) => dispatch(request, false).pipe(Effect.tap(observe), Effect.map((executed) => executed.outcome)),
+            observedVersion: () => observed,
+            drainOutbox: relay.drain,
+            cleanup: cleanup.pipe(Effect.orDie),
+            extendOutboxLeases: relay.extendLeases,
+            shardId: (ref) => entityId(ref).pipe(Effect.flatMap((id) => commandEntity(ref.actor).getShardId(EntityId.make(id))), Effect.map(String), Effect.provideService(Sharding.Sharding, sharding)),
+        });
+        const { serving } = yield* servingReadiness({
+            sharding,
+            services,
+            registrations,
+            queryRegistrations,
+            jobRegistrations,
+        });
+        const control = runtimeControl({
+            gate,
+            stopClaims: relay.stop,
+            attemptsIdle: relay.attemptsIdle,
+            interruptAttempts: relay.interruptAttempts,
+            stopBackground: sweeping === undefined ? Effect.void : Fiber.interrupt(sweeping),
+            serving,
+            scope,
+        });
+        const toUnavailable = (cause) => Effect.fail(ActorError.make({ reason: ActorUnavailable.make({ cause }) }));
+        const configured = Effect.suspend(() => content === undefined
+            ? Effect.die(new Error("Content needs the runtime's content.keys"))
+            : Effect.succeed(content));
+        const contentEntry = Effect.fnUntraced(function* (ref, caller, blob, name, operation) {
+            const registration = registrations.get(ref.actor) ?? queryRegistrations.get(ref.actor);
+            if (registration === undefined)
+                return yield* ActorError.make({
+                    reason: ActorUnavailable.make({ cause: new Error("Actor not registered") }),
+                });
+            if (!(yield* permitted({
+                caller,
+                ref,
+                command: `${blob}.${operation}`,
+                kind: "content",
+            })))
+                return yield* ActorError.make({ reason: Unauthorized.make({ code: "access_denied" }) });
+            if (!registration.blobs.some((declared) => declared.name === blob && isContent(declared)))
+                return Option.none();
+            const sql = yield* SqlClient.SqlClient;
+            const [found] = yield* sql `
+            SELECT hash, size::float8 AS size FROM actor_content_refs
+            WHERE routing_key = ${routingKey({ ref, placement: registration.placement })}
+              AND tenant_id = ${ref.tenant} AND actor_type = ${ref.actor} AND actor_id = ${ref.id}
+              AND blob = ${blob} AND name = ${name}`;
+            const timeoutMs = "policy" in registration ? registration.policy.executionMs : registration.timeoutMs;
+            return Option.map(Option.fromUndefinedOr(found), (row) => ({ ...row, timeoutMs }));
+        }, Effect.provideContext(services), Effect.catchTag("SqlError", toUnavailable));
+        const contentStore = ContentStore.of({
+            uploadBytes: (tenant, bytes) => Effect.flatMap(configured, (store) => store.uploadBytes(tenant, bytes)).pipe(Effect.provideContext(services), Effect.catchTag("SqlError", toUnavailable)),
+            upload: (tenant, body, limit) => Effect.flatMap(configured, (store) => store.uploadStream(tenant, body, Math.min(limit, MAX_CONTENT_BYTES))).pipe(Effect.provideContext(services), Effect.catchTag("SqlError", toUnavailable)),
+            grant: (ref, caller, blob, name) => Effect.gen(function* () {
+                const store = yield* configured;
+                const found = yield* contentEntry(ref, caller, blob, name, "grant");
+                if (Option.isNone(found))
+                    return Option.none();
+                return yield* store
+                    .grant(ref.tenant, found.value.hash, found.value.size)
+                    .pipe(Effect.provideContext(services), Effect.catchTag("SqlError", toUnavailable));
+            }),
+            download: (ref, caller, blob, name) => Effect.gen(function* () {
+                const store = yield* configured;
+                const found = yield* contentEntry(ref, caller, blob, name, "get");
+                return Option.map(found, ({ hash, size, timeoutMs }) => ({
+                    size,
+                    bytes: store.stream(ref.tenant, hash, size, timeoutMs).pipe(Stream.provideContext(services), Stream.mapError((cause) => ActorError.make({ reason: ActorUnavailable.make({ cause }) }))),
+                }));
+            }),
+        });
+        return Context.make(Actors, publicActors).pipe(Context.add(ContentStore, contentStore), Context.add(InternalActors, internalActors), Context.add(RuntimeControl, control), Context.add(DefectLog, defectLog), Context.add(TelemetrySampler, TelemetrySampler.of({ sample })), Context.add(OperatorRuntime, operators));
+    }));
+    return Layer.unwrap(Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+        const coordination = (yield* Coordination) ?? sql;
+        const wiring = Option.getOrUndefined(yield* Effect.serviceOption(RunnerWiring));
+        yield* migrate;
+        if (options.rowLevelSecurity !== undefined &&
+            options.adoption !== undefined &&
+            options.rowLevelSecurity.role !== options.adoption.role)
+            return yield* Effect.die(new Error(`rowLevelSecurity.role ${options.rowLevelSecurity.role} and adoption.role ${options.adoption.role} must name the same role: a turn takes one role`));
+        if (options.rowLevelSecurity !== undefined)
+            yield* checkRowLevelSecurity(options.rowLevelSecurity.role);
+        yield* sql `INSERT INTO actor_deployment (protocol, retry_window_ms) VALUES (1, ${retryWindowMs}) ON CONFLICT DO NOTHING`;
+        const rows = yield* sql `SELECT protocol, retry_window_ms::text AS retry_window_ms FROM actor_deployment`;
+        if (rows[0].protocol !== 1 || Number(rows[0].retry_window_ms) !== retryWindowMs) {
+            return yield* Effect.die(new Error("Actor command protocol/retry window differs from the deployment; migrate explicitly"));
+        }
+        const runnerStorage = Option.isSome(yield* Effect.serviceOption(PgliteClient.PgliteClient))
+            ? "memory"
+            : "sql";
+        if (wiring?.production === true && runnerStorage === "memory")
+            return yield* Effect.die(new Error("Runner.socket requires Postgres; PGlite cannot share ownership between processes"));
+        yield* checkRunnerConfiguration(wiring);
+        const sharding = (wiring?.sharding ?? Sharding.layer.pipe(Layer.provide(Runners.layerNoop))).pipe(Layer.merge(Layer.effect(RunnerReadiness, wiring?.production === true
+            ? acquiredShards
+            : Effect.succeed({ acquired: () => Effect.succeed(true) }))), Layer.provideMerge(directMessages), Layer.provideMerge(mailboxRefusals), Layer.provide([
+            runnerStorage === "memory"
+                ? Layer.effect(RunnerStorage.RunnerStorage, Effect.map(RunnerStorage.makeMemory, keepAcquiredShards))
+                : Layer.effect(RunnerStorage.RunnerStorage, coordinatedRunnerStorage.pipe(Effect.map(wiring?.storage ?? ((storage) => storage)), Effect.map(keepAcquiredShards))).pipe(Layer.orDie),
+            RunnerHealth.layerNoop,
+        ]), Layer.provideMerge(ShardingConfig.layer({
+            shardsPerGroup: 1,
+            simulateRemoteSerialization: yield* LocalRequestSerialization,
+            maxResidentEntities: maxResidentActors,
+            ...wiring?.config,
+            ...holderShardGroups(wiring?.config ?? {}),
+        })));
+        const config = wiring?.config;
+        const address = config?.runnerAddress;
+        const lease = runnerStorage === "sql" &&
+            config?.shardLockDisableAdvisory === true &&
+            address !== undefined &&
+            Option.isSome(address)
+            ? Layer.succeed(ShardLease, tableShardLease({
+                sql: coordination,
+                address: address.value,
+                expiration: Duration.fromInputUnsafe(config.shardLockExpiration ?? ShardingConfig.defaults.shardLockExpiration),
+            }))
+            : Layer.empty;
+        return runtime.pipe(Layer.provide(sharding), Layer.provide(lease), Layer.provide(Layer.succeed(TenantScope, {
+            role: options.rowLevelSecurity?.role,
+            adoption: options.adoption === undefined
+                ? undefined
+                : { role: options.adoption.role, enforced: enforcedTypes },
+        })));
+    }));
+};
+/** Database layers for `Actors.layer`: `postgres` for real deployments, `pglite` for embedded and test use. */
+export const Database = {
+    /**
+     * A runner holds three primary pools, plus optional replica and coordination
+     * pools. Turns lease sessions from the turn pool, `maxConnections` (default
+     * 50): a command holds one session until its transaction ends, so a pool
+     * smaller than the commands in flight queues callers behind
+     * it. Queries read from the query pool, `queryConnections` (default 10),
+     * except those of types with owned tables or blobs. Command admission,
+     * receipt replays, the relay, migrations, and cluster storage use the
+     * off-turn pool, `offTurnConnections` (default 10), unless `coordination`
+     * moves Cluster storage and deployment locks to an independent unsharded
+     * primary. All runners must designate the same authority. Every pool opens
+     * connections only as load needs them, refuses checkouts past its
+     * connections plus a bounded waiter allowance, and hands its connections out
+     * first come, first served. Keep the sum across runners below the server's
+     * `max_connections`.
+     *
+     * `replica` is this runner's nearest streaming replica of the same primary.
+     * Queries read there once it has replayed the commit version their caller
+     * last saw, and read the primary when it is behind or fails. Its pool
+     * (`maxConnections` default 10) opens connections only as queries need them.
+     * Every pool requests server TCP keepalives at 5 seconds idle, 2 seconds
+     * between probes, and 3 probes. `startupParameters` overrides these defaults
+     * independently on the primary and replica configurations.
+     *
+     * Registers a `regclass` codec because the pinned driver lacks one and the
+     * migrator needs it on restart; remove once Effect #8309 lands.
+     */
+    postgres: (options) => {
+        const types = PgTypes.makeRegistry();
+        types.register(2205, {
+            encode: (value) => PgTypes.encode(value, PgTypes.OID.oid),
+            decode: (bytes) => bytes.length === 4
+                ? Result.succeed(new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(0))
+                : Result.fail(new PgTypes.CodecError({ message: "Invalid regclass value" })),
+        });
+        const { offTurnConnections, queryConnections, replica, neki, coordination, ...configured } = options;
+        const pool = withKeepalives(configured);
+        const database = Layer.mergeAll(boundedLayer({ ...pool, maxConnections: offTurnConnections ?? 10, types }), neki === true
+            ? turnConnections({ ...pool, maxConnections: pool.maxConnections ?? 50, types })
+            : turnGroups.pipe(Layer.provideMerge(turnConnections({ ...pool, maxConnections: pool.maxConnections ?? 50, types }))), queryPoolLayer({ ...pool, maxConnections: queryConnections ?? 10, types }), replicaLayer(replica === undefined ? undefined : { ...withKeepalives(replica), types }), coordinationLayer(coordination === undefined ? undefined : { ...withKeepalives(coordination), types }));
+        return neki === undefined
+            ? database
+            : database.pipe(Layer.provideMerge(Layer.succeed(NekiTurnSessions, neki)));
+    },
+    pglite,
+};