@rikalabs/akter 0.1.0-alpha.0 → 0.1.0-alpha.3

package/dist/runtime/inspector/layer.js

@@ -1,8 +1,11 @@
-import { Effect, Schema } from "effect";
-import { HttpRouter } from "effect/http";
+import { Clock, DateTime, Effect, Option, Schema, Stream } from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/http";
 import { SqlClient } from "effect/sql";
-import { ActorError, Unauthorized } from "../../errors/actor.js";
+import { ActorError, RunnerAtCapacity, Unauthorized } from "../../errors/actor.js";
 import { Anonymous, User } from "../../identity/caller.js";
+import { InternalActors } from "../actors.js";
+import { StreamMessage } from "../telemetry/live.js";
+import { message } from "../../serve/sessions/sse.js";
 import { CREDENTIAL_BYTES, oversizedCredential, readsCookies, withinLimits, } from "../../serve/auth.js";
 import { invalidInput, undecodable } from "../../serve/wire.js";
 import { foundOrNotFound, operatorResponse, refuseCrossOrigin } from "./http.js";
@@ -61,7 +64,26 @@ const WorkflowsParams = Schema.Struct({
     afterStartedAtMs: Schema.optional(Millis),
     afterExecutionId: Schema.optional(Schema.String),
 }).check(whole("afterStartedAtMs", "afterExecutionId"));
+const Window = Schema.optional(Schema.Literals(["1h", "24h", "7d"]));
+const LiveTypeParams = Schema.Struct({ type: Schema.optional(Schema.String), window: Window });
+/** The most actors one live actors read names. */
+const MAX_LIVE_ACTORS = 100;
+const LiveActorsParams = Schema.Struct({
+    type: Schema.String,
+    ids: Schema.fromJsonString(Schema.Array(Schema.String).check(Schema.isMaxLength(MAX_LIVE_ACTORS))),
+});
+const StreamParams = Schema.Struct({
+    type: Schema.optional(Schema.String),
+    outcome: Schema.optional(Schema.Literals(["Success", "Failure"])),
+    after: Schema.optional(Schema.String),
+});
 const isPrincipal = Schema.is(Schema.Union([User, Anonymous]));
+/**
+ * How often a command stream with no commands sends a comment. A proxy in
+ * front of the inspector, such as the edge or the control plane's client,
+ * closes a connection idle for 10 seconds, so this stays well under it.
+ */
+const STREAM_KEEPALIVE_MS = 4_000;
 /**
  * Serves read-only JSON over the `durable` inspection views as routes on the
  * application's `HttpRouter`. A request from another browser origin is
@@ -87,6 +109,22 @@ const isPrincipal = Schema.is(Schema.Union([User, Anonymous]));
  * - `GET /outbox`, `/jobs`, `/dead-letters?afterDeadAtMs&afterJobId`,
  *   `/workflows?status=open|all|running|suspended|finished|completed|failed&afterStartedAtMs&afterExecutionId`:
  *   tenant-wide lists; the paged ones answer the `next` cursor to pass back.
+ *
+ * Inside a runtime it also serves the schedules and this runner's own memory,
+ * each live answer naming the runner, its region, when it began recording and
+ * how many other runners the cluster lists:
+ *
+ * - `GET /schedules`: each declared cron entry with its pending ticks and last committed tick.
+ * - `GET /live/overview`: recent rates, resident activations and the deepest mailbox, by type.
+ * - `GET /live/activity?type&window`, `/live/latency?type&window`: one type's, or every type's,
+ *   committed turns over `1h`, `24h` (default) or `7d`.
+ * - `GET /live/actors?type&ids`: whether each named actor is resident, its
+ *   mailbox, sockets and feeds; `ids` is a JSON array of at most 100.
+ * - `GET /live/connections`: open sockets and SSE responses, by type.
+ * - `GET /commands/stream?type&outcome&after`: SSE of each command this runner
+ *   commits for the tenant, `id` its position, resuming after `after` while the
+ *   runner still holds it and sending `gap` when it does not, until the
+ *   credential expires.
  */
 const serve = (options) => HttpRouter.use(Effect.fnUntraced(function* (router) {
     const basePath = (options.basePath ?? "/inspector").replace(/\/+$/, "");
@@ -109,11 +147,11 @@ const serve = (options) => HttpRouter.use(Effect.fnUntraced(function* (router) {
             return yield* ActorError.make({
                 reason: Unauthorized.make({ code: "invalid_credentials" }),
             });
-        return authenticated.tenant;
+        return { tenant: authenticated.tenant, expiresAt: authenticated.expiresAt };
     });
     const route = (path, params, read) => router.add("GET", `${basePath}${path}`, (request) => Effect.gen(function* () {
         yield* refuseCrossOrigin(request);
-        const tenant = yield* authenticate(request);
+        const { tenant } = yield* authenticate(request);
         const decoded = yield* HttpRouter.schemaParams(params).pipe(Effect.mapError(undecodable));
         return foundOrNotFound(yield* Queries.readOnly(tenant)(read(tenant, decoded)).pipe(Effect.provideService(SqlClient.SqlClient, sql), Effect.orDie));
     }).pipe(operatorResponse("Inspector request failed")));
@@ -165,6 +203,184 @@ const serve = (options) => HttpRouter.use(Effect.fnUntraced(function* (router) {
                 commandId: afterCommandId,
             },
     }));
+    /**
+     * The live reads, the schedules and the command stream, which need the
+     * runtime itself; mounted only where the inspector runs inside one.
+     */
+    const mountLive = Effect.fnUntraced(function* (actors) {
+        const recorder = yield* actors.live.enable;
+        const scope = Effect.gen(function* () {
+            const peers = yield* actors.live.peers;
+            return {
+                runner: options.runner ?? null,
+                region: options.region ?? null,
+                startedAtMs: recorder.startedAtMs,
+                peers: peers ?? null,
+            };
+        });
+        /** A read of this runner's memory alone, answered as JSON with the scope it covers. */
+        const live = (path, params, read) => router.add("GET", `${basePath}${path}`, (request) => Effect.gen(function* () {
+            yield* refuseCrossOrigin(request);
+            const { tenant } = yield* authenticate(request);
+            const decoded = yield* HttpRouter.schemaParams(params).pipe(Effect.mapError(undecodable));
+            return HttpServerResponse.jsonUnsafe(yield* read(tenant, decoded, yield* Clock.currentTimeMillis, yield* scope));
+        }).pipe(operatorResponse("Inspector request failed")));
+        yield* live("/live/overview", Schema.Struct({}), (tenant, _params, nowMs, covered) => Effect.sync(() => {
+            const resident = actors.live.resident(tenant);
+            const types = new Set([
+                ...recorder.actorTypes(tenant),
+                ...resident.map((r) => r.actorType),
+            ]);
+            const deepest = (activations) => activations.reduce((found, activation) => activation.mailbox > 0 &&
+                (found === undefined || activation.mailbox > found.mailbox)
+                ? activation
+                : found, undefined);
+            const rates = (actorType) => {
+                const found = recorder.rates(tenant, actorType, nowMs);
+                return {
+                    perSecond: found?.perSecond ?? null,
+                    p50Ms: found?.p50Ms ?? null,
+                    p99Ms: found?.p99Ms ?? null,
+                };
+            };
+            const max = deepest(resident);
+            return {
+                scope: covered,
+                total: {
+                    ...rates(undefined),
+                    awake: resident.length,
+                    maxMailbox: max === undefined
+                        ? null
+                        : { depth: max.mailbox, actorType: max.actorType, actorId: max.actorId },
+                },
+                actorTypes: [...types].sort().map((actorType) => {
+                    const ofType = resident.filter((activation) => activation.actorType === actorType);
+                    const typeMax = deepest(ofType);
+                    return {
+                        actorType,
+                        ...rates(actorType),
+                        awake: ofType.length,
+                        maxMailbox: typeMax === undefined
+                            ? null
+                            : { depth: typeMax.mailbox, actorId: typeMax.actorId },
+                    };
+                }),
+            };
+        }));
+        yield* live("/live/activity", LiveTypeParams, (tenant, { type, window }, nowMs, covered) => Effect.sync(() => {
+            const found = recorder.activity(tenant, type, window ?? "24h", nowMs);
+            return {
+                scope: covered,
+                activity: found === undefined
+                    ? null
+                    : { sinceMs: found.since, points: found.points, commands: found.commands },
+            };
+        }));
+        yield* live("/live/latency", LiveTypeParams, (tenant, { type, window }, nowMs, covered) => Effect.sync(() => {
+            const found = recorder.latency(tenant, type, window ?? "24h", nowMs);
+            return {
+                scope: covered,
+                latency: found === undefined
+                    ? null
+                    : {
+                        sinceMs: found.since,
+                        count: found.count,
+                        buckets: found.buckets,
+                        p50Ms: found.p50Ms,
+                        p95Ms: found.p95Ms,
+                        p99Ms: found.p99Ms,
+                    },
+            };
+        }));
+        yield* live("/live/actors", LiveActorsParams, (tenant, { type, ids }, _nowMs, covered) => Effect.sync(() => {
+            const resident = actors.live
+                .resident(tenant)
+                .filter((activation) => activation.actorType === type);
+            const open = actors.live
+                .connections(tenant)
+                .filter((connection) => connection.actorType === type);
+            return {
+                scope: covered,
+                actors: ids.map((actorId) => {
+                    const activation = resident.find((found) => found.actorId === actorId);
+                    const mine = open.filter((connection) => connection.actorId === actorId);
+                    const feeds = new Map();
+                    for (const connection of mine)
+                        if (connection.kind === "feed")
+                            for (const event of connection.events)
+                                feeds.set(event, (feeds.get(event) ?? 0) + 1);
+                    return {
+                        actorId,
+                        awake: activation !== undefined,
+                        mailbox: activation?.mailbox ?? null,
+                        sockets: mine.filter((connection) => connection.kind === "socket").length,
+                        feeds: [...feeds]
+                            .sort(([left], [right]) => (left < right ? -1 : 1))
+                            .map(([event, subscribers]) => ({ event, subscribers })),
+                    };
+                }),
+            };
+        }));
+        yield* live("/live/connections", Schema.Struct({}), (tenant, _params, _nowMs, covered) => Effect.sync(() => {
+            const open = actors.live.connections(tenant);
+            const counts = (connections) => ({
+                sockets: connections.filter((connection) => connection.kind === "socket").length,
+                feeds: connections.filter((connection) => connection.kind === "feed").length,
+                streams: connections.filter((connection) => connection.kind === "stream").length,
+                watches: connections.filter((connection) => connection.kind === "watch").length,
+            });
+            return {
+                scope: covered,
+                ...counts(open),
+                byActorType: [...new Set(open.map((connection) => connection.actorType))]
+                    .sort()
+                    .map((actorType) => ({
+                    actorType,
+                    ...counts(open.filter((connection) => connection.actorType === actorType)),
+                })),
+            };
+        }));
+        yield* route("/schedules", Schema.Struct({}), (tenant) => Effect.asSome(Queries.schedules({ tenant, declared: actors.live.schedules() })));
+        yield* router.add("GET", `${basePath}/commands/stream`, (request) => Effect.gen(function* () {
+            yield* refuseCrossOrigin(request);
+            const { tenant, expiresAt } = yield* authenticate(request);
+            const { type, outcome, after } = yield* HttpRouter.schemaParams(StreamParams).pipe(Effect.mapError(undecodable));
+            const nowMs = yield* Clock.currentTimeMillis;
+            if (recorder.full(tenant))
+                return yield* ActorError.make({ reason: RunnerAtCapacity.make({}) });
+            const opened = recorder.subscribe(tenant, {
+                actorType: type,
+                failed: outcome === undefined ? undefined : outcome === "Failure",
+            }, after);
+            const until = expiresAt === undefined
+                ? Effect.never
+                : Effect.sleep(Math.max(0, DateTime.toEpochMillis(expiresAt) - nowMs));
+            const messages = opened.pipe(Stream.mapEffect(StreamMessage.$match({
+                gap: () => message({ event: "gap", data: null }),
+                command: ({ entry }) => message({
+                    event: "command",
+                    id: entry.id,
+                    data: {
+                        id: entry.id,
+                        commandId: entry.commandId,
+                        atMs: entry.atMs,
+                        durationMs: entry.durationMs,
+                        actorType: entry.actorType,
+                        actorId: entry.actorId,
+                        command: entry.command,
+                        callerKey: Queries.decodeText(entry.callerKey),
+                        outcomeTag: entry.failed ? "Failure" : "Success",
+                        errorTag: entry.errorTag,
+                        payloadPreview: entry.payloadPreview,
+                    },
+                }),
+            })), Stream.interruptWhen(until), Stream.concat(Stream.fromEffect(message({ event: "end", data: null }))));
+            return HttpServerResponse.stream(Stream.concat(Stream.succeed(": open\n\n"), messages).pipe(Stream.merge(Stream.tick(STREAM_KEEPALIVE_MS).pipe(Stream.drop(1), Stream.map(() => ": keepalive\n\n")), { haltStrategy: "left" }), Stream.encodeText), {
+                contentType: "text/event-stream",
+                headers: { "x-accel-buffering": "no" },
+            });
+        }).pipe(operatorResponse("Inspector request failed")));
+    });
     yield* route("/latest-events", LatestEventsParams, (tenant, { limit, type, id, after }) => Queries.latestEvents({ tenant, limit, actorType: type, actorId: id, after }));
     yield* route("/timeline", TimelineParams, (tenant, { limit, type, id, beforeSequence, beforeKind }) => Queries.timeline({
         tenant,
@@ -175,6 +391,9 @@ const serve = (options) => HttpRouter.use(Effect.fnUntraced(function* (router) {
             ? undefined
             : { sequence: beforeSequence, kind: beforeKind },
     }));
+    const runtime = yield* Effect.serviceOption(InternalActors);
+    if (Option.isSome(runtime))
+        yield* mountLive(runtime.value);
 }));
 /** The read-only inspector: `serve` mounts its routes. */
 export const Inspector = { serve };

package/dist/runtime/inspector/queries.d.ts

@@ -52,19 +52,32 @@ interface ActorsPage extends Page {
     readonly after?: ActorIdentity | undefined;
 }
 /**
- * The tenant's actors in `(actor_type, actor_id)` order, one page after `after`. The cursor
+ * The receipts an actor's last command is looked for among: its greatest
+ * command ids, which the receipts' primary key reads in order and which for a
+ * minted id are the newest issued, so the read is bounded however many
+ * receipts the actor retains.
+ */
+export declare const LAST_COMMAND_RECEIPTS = 256;
+/**
+ * The tenant's actors in `(actor_type, actor_id)` order, one page after `after`,
+ * each with its last command: the receipt with the newest recorded commit
+ * time among its `LAST_COMMAND_RECEIPTS` greatest command ids. The cursor
  * comparison uses the same `C` collation as the ORDER BY, or a cursor could skip or repeat actors.
  * A prefix that names a whole type, `type/` and more, matches that type's ids by their own prefix,
  * so it reads one type rather than every actor of the tenant.
  */
 export declare const actors: (page: ActorsPage) => Effect.Effect<{
-    actors: readonly {
-        readonly actorType: string;
-        readonly actorId: string;
-        readonly placement: string | null;
-        readonly generation: number;
-        readonly created: boolean;
-        readonly lastEventSequence: number;
+    actors: {
+        actorType: string;
+        actorId: string;
+        placement: string | null;
+        generation: number;
+        created: boolean;
+        lastEventSequence: number;
+        lastCommand: {
+            command: string;
+            committedAtMs: number;
+        } | null;
     }[];
     next: {
         actorType: string;
@@ -709,3 +722,36 @@ export declare const timeline: (page: TimelinePage) => Effect.Effect<Option.None
         kind: "command" | "event";
     } | null;
 }>, import("effect/sql/SqlError").SqlError, SqlClient.SqlClient>;
+/** The newest receipts of a type a schedule's last run is looked for among. */
+export declare const SCHEDULE_RECEIPTS = 10000;
+/**
+ * The tenant's state of each declared cron entry: its pending ticks and the
+ * soonest one's due time, and its newest committed tick (a receipt of the
+ * entry's command whose caller is the cron source) among the type's newest
+ * `SCHEDULE_RECEIPTS` receipts by expiry, which the expiry index reads in
+ * order, so a type with more receipts than that answers no last run rather
+ * than scan them all.
+ */
+export declare const schedules: ({ tenant, declared, }: {
+    readonly tenant: string;
+    readonly declared: ReadonlyArray<{
+        readonly actorType: string;
+        readonly key: string;
+        readonly command: string;
+    }>;
+}) => Effect.Effect<{
+    schedules: {
+        actorType: string;
+        key: string;
+        expression: string;
+        command: string;
+        pending: number;
+        nextDueAtMs: number | null;
+        lastRun: {
+            commandId: string;
+            committedAtMs: number;
+            durationMs: number | null;
+            outcomeTag: string | null;
+        } | null;
+    }[];
+}, import("effect/sql/SqlError").SqlError, SqlClient.SqlClient>;

package/dist/runtime/inspector/queries.js

@@ -2,6 +2,7 @@ import { Effect, Option, Predicate, Schema } from "effect";
 import { SqlClient } from "effect/sql";
 import { inReadOnlySnapshot } from "../database/snapshot.js";
 import { decompress } from "../storage/codec.js";
+import { CRON_PREFIX } from "../cron/key.js";
 const parseJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Json));
 /** Decodes a JSON `text` column such as a payload, caller, or outcome. */
 export const decodeText = (text) => {
@@ -61,37 +62,62 @@ export const overview = ({ tenant }) => Effect.gen(function* () {
     };
 });
 /**
- * The tenant's actors in `(actor_type, actor_id)` order, one page after `after`. The cursor
+ * The receipts an actor's last command is looked for among: its greatest
+ * command ids, which the receipts' primary key reads in order and which for a
+ * minted id are the newest issued, so the read is bounded however many
+ * receipts the actor retains.
+ */
+export const LAST_COMMAND_RECEIPTS = 256;
+/**
+ * The tenant's actors in `(actor_type, actor_id)` order, one page after `after`,
+ * each with its last command: the receipt with the newest recorded commit
+ * time among its `LAST_COMMAND_RECEIPTS` greatest command ids. The cursor
  * comparison uses the same `C` collation as the ORDER BY, or a cursor could skip or repeat actors.
  * A prefix that names a whole type, `type/` and more, matches that type's ids by their own prefix,
  * so it reads one type rather than every actor of the tenant.
  */
 export const actors = (page) => Effect.gen(function* () {
     const sql = yield* SqlClient.SqlClient;
-    const byType = page.actorType === undefined ? sql `TRUE` : sql `actor_type = ${page.actorType}`;
+    const byType = page.actorType === undefined ? sql `TRUE` : sql `a.actor_type = ${page.actorType}`;
     const slash = page.prefix?.indexOf("/") ?? -1;
     const byPrefix = page.prefix === undefined
         ? sql `TRUE`
         : slash === -1
-            ? sql `starts_with(actor_type, ${page.prefix})`
-            : sql `actor_type = ${page.prefix.slice(0, slash)}
-              AND starts_with(actor_id, ${page.prefix.slice(slash + 1)})`;
+            ? sql `starts_with(a.actor_type, ${page.prefix})`
+            : sql `a.actor_type = ${page.prefix.slice(0, slash)}
+              AND starts_with(a.actor_id, ${page.prefix.slice(slash + 1)})`;
     const after = page.after === undefined
         ? sql `TRUE`
-        : sql `(actor_type COLLATE "C", actor_id COLLATE "C") > (${page.after.actorType}, ${page.after.actorId})`;
+        : sql `(a.actor_type COLLATE "C", a.actor_id COLLATE "C") > (${page.after.actorType}, ${page.after.actorId})`;
     const rows = yield* sql `
-      SELECT actor_type AS "actorType", actor_id AS "actorId", placement,
-        generation::float8 AS generation, created,
-        last_event_sequence::float8 AS "lastEventSequence"
-      FROM durable.actors
-      WHERE tenant_id = ${page.tenant} AND ${byType} AND ${byPrefix} AND ${after}
-      ORDER BY actor_type COLLATE "C", actor_id COLLATE "C"
+      SELECT a.actor_type AS "actorType", a.actor_id AS "actorId",
+        a.generation::float8 AS generation, a.created, a.placement,
+        a.last_event_sequence::float8 AS "lastEventSequence",
+        l.command AS "lastCommand", l.committed_at_ms::float8 AS "lastCommittedAtMs"
+      FROM durable.actors a
+      LEFT JOIN LATERAL (
+        SELECT n.command, n.committed_at_ms FROM (
+          SELECT r.command, r.command_id, r.committed_at_ms FROM durable.receipts r
+          WHERE r.routing_key = a.routing_key AND r.tenant_id = a.tenant_id
+            AND r.actor_type = a.actor_type AND r.actor_id = a.actor_id
+          ORDER BY r.command_id DESC
+          LIMIT ${LAST_COMMAND_RECEIPTS}) n
+        WHERE n.committed_at_ms IS NOT NULL
+        ORDER BY n.committed_at_ms DESC, n.command_id COLLATE "C" DESC
+        LIMIT 1) l ON TRUE
+      WHERE a.tenant_id = ${page.tenant} AND ${byType} AND ${byPrefix} AND ${after}
+      ORDER BY a.actor_type COLLATE "C", a.actor_id COLLATE "C"
       LIMIT ${page.limit + 1}`;
     const more = rows.length > page.limit;
     const items = more ? rows.slice(0, page.limit) : rows;
     const last = items.at(-1);
     return {
-        actors: items,
+        actors: items.map(({ lastCommand, lastCommittedAtMs, ...actor }) => ({
+            ...actor,
+            lastCommand: lastCommand === null || lastCommittedAtMs === null
+                ? null
+                : { command: lastCommand, committedAtMs: lastCommittedAtMs },
+        })),
         next: more && last !== undefined ? { actorType: last.actorType, actorId: last.actorId } : null,
     };
 });
@@ -99,11 +125,11 @@ const findActor = ({ tenant, actorType, actorId }) => Effect.gen(function* () {
     const sql = yield* SqlClient.SqlClient;
     const [row] = yield* sql `
       SELECT actor_type AS "actorType", actor_id AS "actorId", routing_key::text AS "routingKey",
-        placement, generation::float8 AS generation, created,
+        generation::float8 AS generation, created, placement,
         last_event_sequence::float8 AS "lastEventSequence"
       FROM durable.actors
       WHERE tenant_id = ${tenant} AND actor_type = ${actorType} AND actor_id = ${actorId}`;
-    return Option.fromNullishOr(row);
+    return Option.fromUndefinedOr(row);
 });
 const outboxOf = (row) => ({
     ...row,
@@ -179,6 +205,7 @@ export const actor = (page) => Effect.gen(function* () {
     const receipts = yield* sql `
       SELECT r.command_id AS "commandId", r.command, r.caller_key AS "callerKey",
         r.outcome_tag AS "outcomeTag", r.outcome, r.expires_at_ms::float8 AS "expiresAtMs",
+        r.started_at_ms::float8 AS "startedAtMs", r.committed_at_ms::float8 AS "committedAtMs",
         (SELECT string_agg(e.sequence::text, ',' ORDER BY e.sequence) FROM durable.events e
           WHERE e.routing_key = r.routing_key AND e.tenant_id = r.tenant_id
             AND e.actor_type = r.actor_type AND e.actor_id = r.actor_id
@@ -387,6 +414,7 @@ export const receipts = (page) => Effect.gen(function* () {
       SELECT r.actor_type AS "actorType", r.actor_id AS "actorId",
         r.command_id AS "commandId", r.command, r.caller_key AS "callerKey",
         r.outcome_tag AS "outcomeTag", r.outcome, r.expires_at_ms::float8 AS "expiresAtMs",
+        r.started_at_ms::float8 AS "startedAtMs", r.committed_at_ms::float8 AS "committedAtMs",
         (SELECT string_agg(e.sequence::text, ',' ORDER BY e.sequence) FROM durable.events e
           WHERE e.routing_key = r.routing_key AND e.tenant_id = r.tenant_id
             AND e.actor_type = r.actor_type AND e.actor_id = r.actor_id
@@ -492,3 +520,44 @@ export const timeline = (page) => Effect.gen(function* () {
             : null,
     });
 });
+/** The newest receipts of a type a schedule's last run is looked for among. */
+export const SCHEDULE_RECEIPTS = 10_000;
+/**
+ * The tenant's state of each declared cron entry: its pending ticks and the
+ * soonest one's due time, and its newest committed tick (a receipt of the
+ * entry's command whose caller is the cron source) among the type's newest
+ * `SCHEDULE_RECEIPTS` receipts by expiry, which the expiry index reads in
+ * order, so a type with more receipts than that answers no last run rather
+ * than scan them all.
+ */
+export const schedules = ({ tenant, declared, }) => Effect.forEach(declared, (entry) => Effect.gen(function* () {
+    const sql = yield* SqlClient.SqlClient;
+    const [timers] = yield* sql `
+        SELECT count(*)::int AS pending, min(due_at_ms)::float8 AS "dueAtMs" FROM durable.timers
+        WHERE tenant_id = ${tenant} AND actor_type = ${entry.actorType}
+          AND timer_key = ${entry.key}`;
+    const [last] = yield* sql `
+        SELECT command_id AS "commandId", committed_at_ms::float8 AS "committedAtMs",
+          duration_ms::float8 AS "durationMs", outcome_tag AS "outcomeTag"
+        FROM (
+          SELECT command_id, command, caller_key, committed_at_ms, duration_ms, outcome_tag
+          FROM durable.receipts
+          WHERE tenant_id = ${tenant} AND actor_type = ${entry.actorType}
+          ORDER BY expires_at_ms DESC
+          LIMIT ${SCHEDULE_RECEIPTS}) r
+        WHERE command = ${entry.command} AND starts_with(caller_key, '["System","cron",')
+          AND committed_at_ms IS NOT NULL
+        ORDER BY committed_at_ms DESC, command_id COLLATE "C" DESC
+        LIMIT 1`;
+    return {
+        actorType: entry.actorType,
+        key: entry.key,
+        expression: entry.key.startsWith(CRON_PREFIX)
+            ? entry.key.slice(CRON_PREFIX.length)
+            : entry.key,
+        command: entry.command,
+        pending: timers?.pending ?? 0,
+        nextDueAtMs: timers?.dueAtMs ?? null,
+        lastRun: last ?? null,
+    };
+})).pipe(Effect.map((found) => ({ schedules: found })));

package/dist/runtime/layer.d.ts

@@ -81,7 +81,7 @@ export interface Options {
     };
     /** Telemetry this runner keeps beside the spans and metrics it reports. */
     readonly observability?: {
-        /** Defect spans the runner keeps for `durable defects list`, newest last. Default 1,000. */
+        /** Defect spans the runner keeps for `akter defects list`, newest last. Default 1,000. */
         readonly defects?: number;
         /**
          * How often one runner of the deployment samples the database gauges
@@ -94,7 +94,7 @@ export interface Options {
      * How long this runtime keeps writing event and effect payload versions
      * without refreshing its writer rows; it refreshes every half window and
      * refuses new turns once a window passes without a refresh, so
-     * `durable payloads clear` can tell when no turn still writes an old
+     * `akter payloads clear` can tell when no turn still writes an old
      * version. Default 2 minutes, at least 1 second.
      */
     readonly payloadWriterWindow?: Duration.Input;
@@ -159,7 +159,7 @@ export interface Options {
     };
     /**
      * The fleet views this runtime maintains and serves. They need Postgres with
-     * `wal_level=logical`, a login with `REPLICATION`, and `durable fleet setup`;
+     * `wal_level=logical`, a login with `REPLICATION`, and `akter fleet setup`;
      * startup refuses otherwise, and PGlite refuses any view. One runner at a
      * time maintains them, off the turn path.
      */
@@ -174,7 +174,10 @@ export interface Options {
  * Constraints the wiring keeps:
  * - Turns run through the drain gate, so a drain refuses new turns and
  *   interrupts the rest. The runtime scope, not the caller, owns an in-flight
- *   turn: interrupting a waiter never cancels it.
+ *   turn: interrupting a waiter never cancels it. Closing that scope does, and
+ *   a command it cuts off, or one sent after it starts closing, fails with a
+ *   retryable `ActorUnavailable` instead of an interrupt or a wait that never
+ *   ends; delivery stops retrying once the scope is closed.
  * - Commands are direct, so Cluster keeps no messages and the receipt is the
  *   only admission record; a restarted activation or lost runner drops the
  *   uncommitted attempt, the handle retries with the same command id, and the
@@ -219,42 +222,48 @@ export interface Options {
  *   retry like any delivery failure; it is not a defect.
  */
 export declare const layer: (options?: Options) => Layer.Layer<Actors | ContentStore | DefectLog | InternalActors | OperatorRuntime | RuntimeControl | TelemetrySampler, import("effect/sql/Migrator").MigrationError | SqlError.SqlError, Crypto.Crypto | SqlClient.SqlClient>;
+type PostgresOptions = Omit<PgClient.PgPoolConfig, "types"> & {
+    readonly offTurnConnections?: number;
+    readonly queryConnections?: number;
+    readonly replica?: Omit<PgClient.PgPoolConfig, "types"> | undefined;
+    /** An unsharded primary shared by every runner; owns coordination rows and Cluster and fleet locks. */
+    readonly coordination?: Omit<PgClient.PgPoolConfig, "types"> | undefined;
+};
+/**
+ * A runner holds three primary pools, plus optional replica and coordination
+ * pools. Turns lease sessions from the turn pool, `maxConnections` (default
+ * 50): a command holds one session until its transaction ends, so a pool
+ * smaller than the commands in flight queues callers behind
+ * it. Queries read from the query pool, `queryConnections` (default 10),
+ * except those of types with owned tables or blobs. Command admission,
+ * receipt replays, the relay, migrations, and cluster storage use the
+ * off-turn pool, `offTurnConnections` (default 10), unless `coordination`
+ * moves Cluster storage and deployment locks to an independent unsharded
+ * primary. All runners must designate the same authority. Every pool opens
+ * connections only as load needs them, refuses checkouts past its
+ * connections plus a bounded waiter allowance, and hands its connections out
+ * first come, first served. Keep the sum across runners below the server's
+ * `max_connections`.
+ *
+ * `replica` is this runner's nearest streaming replica of the same primary.
+ * Queries read there once it has replayed the commit version their caller
+ * last saw, and read the primary when it is behind or fails. Its pool
+ * (`maxConnections` default 10) opens connections only as queries need them.
+ * Every pool requests server TCP keepalives at 5 seconds idle, 2 seconds
+ * between probes, and 3 probes. `startupParameters` overrides these defaults
+ * independently on the primary and replica configurations.
+ *
+ * Registers a `regclass` codec because the pinned driver lacks one and the
+ * migrator needs it on restart; remove once Effect #8309 lands.
+ */
+declare function postgres(options: PostgresOptions): Layer.Layer<PgClient.PgClient | SqlClient.SqlClient | TurnConnections, SqlError.SqlError>;
 /** Database layers for `Actors.layer`: `postgres` for real deployments, `pglite` for embedded and test use. */
 export declare const Database: {
-    /**
-     * A runner holds three primary pools, plus optional replica and coordination
-     * pools. Turns lease sessions from the turn pool, `maxConnections` (default
-     * 50): a command holds one session until its transaction ends, so a pool
-     * smaller than the commands in flight queues callers behind
-     * it. Queries read from the query pool, `queryConnections` (default 10),
-     * except those of types with owned tables or blobs. Command admission,
-     * receipt replays, the relay, migrations, and cluster storage use the
-     * off-turn pool, `offTurnConnections` (default 10), unless `coordination`
-     * moves Cluster storage and deployment locks to an independent unsharded
-     * primary. All runners must designate the same authority. Every pool opens
-     * connections only as load needs them, refuses checkouts past its
-     * connections plus a bounded waiter allowance, and hands its connections out
-     * first come, first served. Keep the sum across runners below the server's
-     * `max_connections`.
-     *
-     * `replica` is this runner's nearest streaming replica of the same primary.
-     * Queries read there once it has replayed the commit version their caller
-     * last saw, and read the primary when it is behind or fails. Its pool
-     * (`maxConnections` default 10) opens connections only as queries need them.
-     * Every pool requests server TCP keepalives at 5 seconds idle, 2 seconds
-     * between probes, and 3 probes. `startupParameters` overrides these defaults
-     * independently on the primary and replica configurations.
-     *
-     * Registers a `regclass` codec because the pinned driver lacks one and the
-     * migrator needs it on restart; remove once Effect #8309 lands.
-     */
-    postgres: (options: Omit<PgClient.PgPoolConfig, "types"> & {
-        readonly offTurnConnections?: number;
-        readonly queryConnections?: number;
-        readonly neki?: boolean;
-        readonly replica?: Omit<PgClient.PgPoolConfig, "types"> | undefined;
-        /** An unsharded primary shared by every runner; owns coordination rows and Cluster and fleet locks. */
-        readonly coordination?: Omit<PgClient.PgPoolConfig, "types"> | undefined;
-    }) => Layer.Layer<PgClient.PgClient | SqlClient.SqlClient | TurnConnections, SqlError.SqlError, never>;
+    postgres: typeof postgres;
     pglite: typeof pglite;
+    schemaChange: {
+        (lock: number): <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E | SqlError.SqlError, R | SqlClient.SqlClient>;
+        <A, E, R>(effect: Effect.Effect<A, E, R>, lock: number): Effect.Effect<A, E | SqlError.SqlError, R | SqlClient.SqlClient>;
+    };
 };
+export {};