@nylorun/runtime 0.21.1-beta → 0.23.0-beta

package/dist/store/postgres/store.js

@@ -15,7 +15,7 @@
  *
  * Every `tx` is one READ COMMITTED transaction on a pooled connection (Drizzle's
  * `transaction` on postgres.js `begin`). Tables are addressed by schema-qualified names,
- * never through `search_path`. Commit listeners and `afterCommit` callbacks run after `COMMIT`
+ * never through `search_path`. Commit listeners and outbox deliveries run after `COMMIT`
  * returns and never reject the committed `tx`. Nested `tx` calls are detected with
  * `AsyncLocalStorage` and rejected, and a `Tx` rejects every call once its callback settles.
  * A failed statement rejects `tx` with the driver's error (`driverError`), not Drizzle's
@@ -50,7 +50,7 @@
  */
 import { AsyncLocalStorage } from "node:async_hooks";
 import { randomUUID } from "node:crypto";
-import { and, count, eq, getTableColumns, gte, inArray, isNotNull, isNull, lt, lte, ne, or, sql, } from "drizzle-orm";
+import { and, count, eq, getTableColumns, gte, gt, inArray, isNotNull, isNull, lt, lte, ne, or, sql, } from "drizzle-orm";
 import { appendEvent, appendSandboxEvent } from "../../record/index.js";
 import { OwnershipLostError, PrincipalRoleConflict } from "../ownership.js";
 import { followControlSignals, pruneSignals, writeSignal } from "./control.js";
@@ -58,7 +58,7 @@ import { database, driverError } from "./db.js";
 import { expectedSchemaVersion, readSchemaVersion } from "./migrate.js";
 import { createPostgresRecordReader } from "./record.js";
 import { postgresRecordWriter, postgresSandboxRecordWriter } from "./record-writer.js";
-import { artifacts, artifactVersions, artifactContent, definitionFiles, definitionFileUses, commands, definitions, effects, links, modelBudgets, modelUsage, principals, sandboxes, sandboxEvents, sandboxResources, sessionEvents, sessionLogHeads, sessions, toJson, toolCrossings, signingKeys, tenant, tenantSettings, vaultAudit, vaultCredentials, vaultIdempotency, vaults, wakes, } from "./schema.js";
+import { artifacts, artifactVersions, artifactContent, definitionFiles, definitionFileUses, commands, definitions, effects, links, modelBudgets, modelUsage, principals, sandboxes, sandboxEvents, sandboxResources, sessionEvents, sessionLogHeads, sessions, toJson, toolCrossings, signingKeys, tenant, tenantSettings, vaultAudit, vaultCredentials, vaultIdempotency, vaults, sandboxSignals, wakes, } from "./schema.js";
 export function createPostgresSessionStore(options) {
     return new PostgresSessionStore(options);
 }
@@ -88,6 +88,9 @@ class PostgresSessionStore {
     active = new AsyncLocalStorage();
     listeners = new Set();
     inflight = new Set();
+    /** Deliveries of committed outbox requests still running (`deliverTo`). */
+    deliveries = new Set();
+    deliver;
     now;
     onError;
     closed = false;
@@ -111,7 +114,7 @@ class PostgresSessionStore {
         let t;
         // Wrapped so `begin` does not treat an array result as queries to await.
         const run = this.db.transaction(async (db) => {
-            t = new PostgresTx(db, this.tenantId, this.now, this.deleteWake);
+            t = new PostgresTx(db, this.tenantId, this.now);
             try {
                 return { value: await this.active.run(this, () => fn(t)) };
             }
@@ -144,20 +147,51 @@ class PostgresSessionStore {
                 }
             }
         }
-        for (const callback of t.callbacks) {
-            try {
-                await callback();
-            }
-            catch (error) {
-                this.onError(error);
-            }
-        }
+        if (t.outbox.length > 0)
+            this.send(t.outbox);
         return result.value;
     }
-    /** Deletes a delivered outbox request, outside any transaction (`Tx.wake`). */
-    deleteWake = async (id) => {
-        await this.db.delete(wakes).where(eq(wakes.id, id));
-    };
+    deliverTo(deliver) {
+        this.deliver = deliver;
+    }
+    async delivered() {
+        while (this.deliveries.size > 0)
+            await Promise.allSettled([...this.deliveries]);
+    }
+    /**
+     * Hands a commit's outbox requests to the delivery, one after the other, without the caller
+     * waiting: a delivered one's row is deleted, and one that was not is due at the sweep's next
+     * pass.
+     */
+    send(committed) {
+        const deliver = this.deliver;
+        if (!deliver)
+            return;
+        const run = (async () => {
+            for (const { kind, id, request } of committed) {
+                let sent = false;
+                try {
+                    sent = await deliver(request);
+                }
+                catch (error) {
+                    this.onError(error);
+                }
+                const table = OUTBOX[kind];
+                try {
+                    if (sent === false)
+                        await this.db.update(table).set({ retryAt: sql `clock_timestamp()` }).where(eq(table.id, id));
+                    else
+                        await this.db.delete(table).where(eq(table.id, id));
+                }
+                catch (error) {
+                    // The row stays as it was: the sweep sends it again, under the same key.
+                    this.onError(error);
+                }
+            }
+        })();
+        this.deliveries.add(run);
+        void run.finally(() => this.deliveries.delete(run));
+    }
     onCommit(listener) {
         this.listeners.add(listener);
         return () => this.listeners.delete(listener);
@@ -186,10 +220,14 @@ class PostgresSessionStore {
             throw new Error("SessionStore is closed");
         return followControlSignals(this.sql, this.db, onSignal, options);
     }
-    /** Rejects new transactions and waits for running ones. Does not end the pool. */
+    /**
+     * Rejects new transactions and waits for running ones, and for the deliveries of committed
+     * outbox requests. Does not end the pool.
+     */
     async close() {
         this.closed = true;
         await Promise.allSettled([...this.inflight]);
+        await this.delivered();
     }
 }
 function storedSession(row) {
@@ -262,26 +300,49 @@ const SIGNING_KEY_STAMP = {
 /** The audit columns a row has (`ord` only orders them). */
 const { ord: _ord, ...AUDIT } = getTableColumns(vaultAudit);
 const SESSION_TABLES = [sessions, commands, effects, links, wakes];
-/** An outbox row as it is delivered: the wake's own dedupe key, or one made from the row id. */
+/**
+ * The outbox's tables, by the kind of request each holds. Their delivery columns (`id`,
+ * `created_at`, `attempts`, `retry_at`, `parked_at`) are the same.
+ */
+const OUTBOX = {
+    wake: wakes,
+    sandbox: sandboxSignals,
+};
+/** A wake outbox row as it is delivered: its own dedupe key, or one made from the row id. */
 function outboxWake(id, reason, dedupeKey) {
     return { reason, dedupeKey: dedupeKey ?? `wake:${id}` };
 }
+/** A sandbox signal outbox row as it is delivered: a reconcile is keyed by the row. */
+function outboxSignal(id, row) {
+    return row.kind === "arm"
+        ? { kind: "arm", timer: row.timer, at: row.at }
+        : { kind: "reconcile", key: `signal:${id}` };
+}
+/** When an outbox row of `table` is due, after a grace of `graceMs` (the database's clock). */
+function dueAt(table, graceMs) {
+    const grace = sql `make_interval(secs => ${Math.max(0, graceMs) / 1000})`;
+    return sql `coalesce(${table.retryAt}, ${table.createdAt} + ${grace})`;
+}
+/** Due rows of `table`, soonest first: their epoch seconds, to merge two tables' rows. */
+const DUE = (table, graceMs) => ({
+    due: dueAt(table, graceMs),
+    dueSeconds: sql `extract(epoch from ${dueAt(table, graceMs)})::float8`,
+});
 /** The inserted row's `column` (`ON CONFLICT … DO UPDATE`). */
 const excluded = (column) => sql.raw(`excluded.${column}`);
 class PostgresTx {
     db;
     tenantId;
     now;
-    deleteWake;
     closed = false;
     events = [];
     generations = [];
-    callbacks = [];
-    constructor(db, tenantId, now, deleteWake) {
+    /** The outbox requests this transaction wrote, delivered after it commits. */
+    outbox = [];
+    constructor(db, tenantId, now) {
         this.db = db;
         this.tenantId = tenantId;
         this.now = now;
-        this.deleteWake = deleteWake;
     }
     check() {
         if (this.closed)
@@ -352,12 +413,8 @@ class PostgresTx {
         this.generations.push(generation);
         return structuredClone(event);
     }
-    afterCommit(fn) {
-        this.check();
-        this.callbacks.push(fn);
-    }
     // --- wake outbox ---------------------------------------------------------
-    async wake(sessionId, wake, deliver) {
+    async wake(sessionId, wake) {
         this.check();
         const id = randomUUID();
         await this.db.insert(wakes).values({
@@ -366,34 +423,90 @@ class PostgresTx {
             reason: wake.reason,
             dedupeKey: wake.dedupeKey ?? null,
         });
-        const delivered = outboxWake(id, wake.reason, wake.dedupeKey ?? null);
-        // A rejection goes to the store's error hook (as every callback's), and the row stays.
-        this.callbacks.push(async () => {
-            if ((await deliver(sessionId, delivered)) === false)
-                return;
-            await this.deleteWake(id);
+        this.outbox.push({
+            kind: "wake",
+            id,
+            request: { kind: "wake", sessionId, wake: outboxWake(id, wake.reason, wake.dedupeKey ?? null) },
         });
     }
-    async pendingWakes(before, limit) {
+    async signalSandbox(sandboxId, signal) {
         this.check();
-        const rows = await this.db
-            .select()
+        const id = randomUUID();
+        const row = {
+            kind: signal.kind,
+            timer: signal.kind === "arm" ? signal.timer : null,
+            at: signal.kind === "arm" ? signal.at : null,
+        };
+        await this.db.insert(sandboxSignals).values({ id, sandboxId, ...row });
+        this.outbox.push({
+            kind: "sandbox",
+            id,
+            request: { kind: "sandbox", sandboxId, signal: outboxSignal(id, row) },
+        });
+    }
+    async pendingOutbox(graceMs, limit) {
+        this.check();
+        // Compared on the database's clock, which wrote `created_at` and `retry_at`.
+        const now = sql `clock_timestamp()`;
+        const wake = DUE(wakes, graceMs);
+        const signal = DUE(OUTBOX.sandbox, graceMs);
+        const wakeRows = await this.db
+            .select({ ...getTableColumns(wakes), dueSeconds: wake.dueSeconds })
             .from(wakes)
-            .where(lt(wakes.createdAt, before))
-            .orderBy(wakes.createdAt, wakes.id)
+            .where(and(isNull(wakes.parkedAt), lte(wake.due, now)))
+            .orderBy(wake.due, wakes.id)
             .limit(limit);
-        return rows.map((row) => ({
-            id: row.id,
-            sessionId: row.sessionId,
-            wake: outboxWake(row.id, row.reason, row.dedupeKey),
-            createdAt: row.createdAt.toISOString(),
-        }));
+        const signalRows = await this.db
+            .select({ ...getTableColumns(sandboxSignals), dueSeconds: signal.dueSeconds })
+            .from(sandboxSignals)
+            .where(and(isNull(sandboxSignals.parkedAt), lte(signal.due, now)))
+            .orderBy(signal.due, sandboxSignals.id)
+            .limit(limit);
+        const rows = [
+            ...wakeRows.map((row) => ({
+                kind: "wake",
+                id: row.id,
+                request: {
+                    kind: "wake",
+                    sessionId: row.sessionId,
+                    wake: outboxWake(row.id, row.reason, row.dedupeKey),
+                },
+                createdAt: row.createdAt.toISOString(),
+                attempts: row.attempts,
+                due: row.dueSeconds,
+            })),
+            ...signalRows.map((row) => ({
+                kind: "sandbox",
+                id: row.id,
+                request: { kind: "sandbox", sandboxId: row.sandboxId, signal: outboxSignal(row.id, row) },
+                createdAt: row.createdAt.toISOString(),
+                attempts: row.attempts,
+                due: row.dueSeconds,
+            })),
+        ];
+        rows.sort((a, b) => a.due - b.due || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+        return rows.slice(0, limit).map(({ due: _due, ...row }) => row);
+    }
+    async deleteOutbox(rows) {
+        this.check();
+        for (const kind of ["wake", "sandbox"]) {
+            const ids = rows.filter((row) => row.kind === kind).map((row) => row.id);
+            if (ids.length > 0)
+                await this.db.delete(OUTBOX[kind]).where(inArray(OUTBOX[kind].id, ids));
+        }
     }
-    async deleteWakes(ids) {
+    async failOutbox(row, retry) {
         this.check();
-        if (ids.length === 0)
-            return;
-        await this.db.delete(wakes).where(inArray(wakes.id, [...ids]));
+        const table = OUTBOX[row.kind];
+        const secs = Math.max(0, retry.retryInMs) / 1000;
+        await this.db
+            .update(table)
+            .set({
+            attempts: sql `${table.attempts} + 1`,
+            retryAt: sql `clock_timestamp() + make_interval(secs => ${secs})`,
+            ...(retry.park ? { parkedAt: sql `clock_timestamp()` } : {}),
+        })
+            .where(eq(table.id, row.id));
     }
     // --- ownership -----------------------------------------------------------
     async takeOwnership(sessionId, claim) {
@@ -470,12 +583,11 @@ class PostgresTx {
             .orderBy(sessions.id);
         return rows.map((row) => storedSession(row));
     }
-    async orphanedSessions(now, limit) {
+    async orphanedSessions(now, limit, after) {
         this.check();
         const rows = await this.sessions()
-            .where(and(inArray(sessions.status, OPEN_SESSION), or(isNull(sessions.owner), isNull(sessions.ownerExpiresAt), lte(sessions.ownerExpiresAt, now))))
-            // Never-owned sessions first, then the longest expired.
-            .orderBy(sql `CASE WHEN ${sessions.owner} IS NULL THEN NULL ELSE ${sessions.ownerExpiresAt} END ASC NULLS FIRST`, sessions.id)
+            .where(and(inArray(sessions.status, OPEN_SESSION), or(isNull(sessions.owner), isNull(sessions.ownerExpiresAt), lte(sessions.ownerExpiresAt, now)), after === undefined ? undefined : gt(sessions.id, after)))
+            .orderBy(sessions.id)
             .limit(limit);
         return rows.map((row) => storedSession(row));
     }
@@ -1120,6 +1232,7 @@ class PostgresTx {
             await db.delete(sandboxes);
             await db.delete(sandboxResources);
             await db.delete(sandboxEvents);
+            await db.delete(sandboxSignals);
         }
         if (scope === "all") {
             await db.delete(definitions);

package/dist/store/types.d.ts

@@ -10,8 +10,8 @@
  * 1. **One transaction per `tx` call, READ COMMITTED or stronger.** Nothing a
  *    transaction wrote is visible to others before it commits, and nothing is
  *    kept when `fn` throws: document writes, event sequences, record rows,
- *    and `afterCommit` callbacks are all discarded, and `tx` rejects with the
- *    error `fn` threw.
+ *    and outbox requests are all discarded, and `tx` rejects with the error `fn`
+ *    threw.
  * 2. **Session-scoped writes lock the session row first.** `lockSession` takes
  *    a row lock (`SELECT … FOR UPDATE`) held until the transaction ends. Effect
  *    intent and outcome, checkpoint settlement and event
@@ -25,18 +25,22 @@
  *    generation, `streams/basin.ts`), and the cursor is
  *    `base64url("<sessionId>:<seq>")` (see `record/cursor.ts`).
  * 4. **No external I/O inside `fn`.** No model, tool, MCP, sandbox, Restate or
- *    S2 call, and no `fetch`, runs inside a transaction. A wake is written in the
- *    transaction (`Tx.wake`, the wake outbox) and delivered after commit, and events
- *    are delivered to commit listeners after commit (seam rule 1 and 2).
+ *    S2 call, and no `fetch`, runs inside a transaction. What a transaction asks of
+ *    Durable Session Execution (a wake, a pod sandbox's reconcile or timer) is written
+ *    to the outbox in the transaction (`Tx.wake`, `Tx.signalSandbox`) and delivered
+ *    after commit; what it tells the other processes goes on the control bus
+ *    (`Tx.signal`); and events are delivered to commit listeners after commit (seam
+ *    rule 1 and 2).
  * 5. **No nested transactions.** Calling `store.tx` from inside `fn` rejects.
  *    A `Tx` must not be used after its `tx` call settles.
  * 6. **Post-commit order.** After a commit, the store first calls every commit
- *    listener once with the transaction's events (in allocation order), then runs the `afterCommit` callbacks in registration
- *    order and awaits them. Listener and callback failures are reported to the
- *    store's error hook; they never reject `tx`, because the commit stands.
- *    Recovery from a lost post-commit step is the Tenant sweep's job: a wake's
- *    delivery is a callback too, and its outbox row is what the sweep delivers
- *    again (`Tx.wake`).
+ *    listener once with the transaction's events (in allocation order), then hands
+ *    the transaction's outbox requests to the delivery (`deliverTo`), in the order
+ *    they were written, without the caller waiting. Listener and delivery failures
+ *    are reported to the store's error hook; they never reject `tx`, because the
+ *    commit stands. A request whose delivery fails, or that a crash cuts off, stays
+ *    in the outbox, which the Tenant sweep delivers again. Nothing else runs after
+ *    commit: a step that is not written down is lost to a crash.
  * 7. **Documents are values.** `get` and queries return fresh copies; mutating
  *    them changes nothing until `put`.
  * 8. **Ownership columns are store-managed.** `owner`, `epoch` and
@@ -58,7 +62,7 @@
  */
 import type { KeyRole } from "@nylorun/core/compatibility";
 import type { RecordReader } from "../streams/relay/types.js";
-import type { Wake } from "../execution/types.js";
+import type { SandboxSignal, Wake } from "../execution/types.js";
 import type { EventPayload, EventType, LiveEvent, SandboxEvent, SandboxEventPayload, SandboxEventType, SandboxKind, SandboxPodDesired, SandboxPodObserved, SessionEventOf } from "@nylorun/core/contracts";
 import type { SandboxManifest } from "@nylorun/core/define";
 import type { HostEffect } from "@nylorun/harness/run";
@@ -272,6 +276,14 @@ export interface SessionStore {
      * throw; what it throws goes to `onError`, as do listener failures.
      */
     followSignals(onSignal: (signal: ControlSignal) => void, options?: FollowSignalsOptions): Promise<SignalFollower>;
+    /**
+     * Sets where committed outbox requests go (`Tx.wake`, `Tx.signalSandbox`): after each
+     * commit the store hands them to `deliver` without waiting for it, so a slow execution
+     * never holds up the caller. Without one, they wait in the outbox for the Tenant sweep.
+     */
+    deliverTo(deliver: OutboxDelivery | undefined): void;
+    /** Resolves once every delivery started so far has settled (`close` waits for them too). */
+    delivered(): Promise<void>;
     close(): Promise<void>;
 }
 /** A wake as the outbox delivers it: its `dedupeKey` is the request's idempotency key. */
@@ -279,18 +291,38 @@ export type OutboxWake = Wake & {
     dedupeKey: string;
 };
 /**
- * Hands an outbox request to Durable Session Execution (`Tx.wake`). Resolving means the
- * execution accepted it; `false` means it was not delivered and stays in the outbox, as a
- * rejection does, without an error.
+ * What a committed transaction asked of Durable Session Execution, as the outbox delivers it:
+ * a wake (`Tx.wake`), or a pod sandbox's reconcile or timer (`Tx.signalSandbox`; a reconcile
+ * carries its idempotency key, `signal:<row id>`).
  */
-export type WakeDelivery = (sessionId: string, wake: OutboxWake) => boolean | void | Promise<boolean | void>;
-/** A wake outbox request not yet delivered (`Tx.pendingWakes`). */
-export interface PendingWake {
-    id: string;
+export type OutboxRequest = {
+    kind: "wake";
     sessionId: string;
     wake: OutboxWake;
+} | {
+    kind: "sandbox";
+    sandboxId: string;
+    signal: SandboxSignal;
+};
+/**
+ * Hands an outbox request to Durable Session Execution (`SessionStore.deliverTo`). Resolving
+ * means the execution accepted it; `false` means it was not delivered and stays in the
+ * outbox, due at the sweep's next pass, as after a rejection, which also goes to the store's
+ * error hook.
+ */
+export type OutboxDelivery = (request: OutboxRequest) => boolean | void | Promise<boolean | void>;
+/** An outbox row: which table (`kind`) and its id. */
+export type OutboxRef = {
+    kind: OutboxRequest["kind"];
+    id: string;
+};
+/** An outbox request not yet delivered (`Tx.pendingOutbox`). */
+export interface PendingRequest extends OutboxRef {
+    request: OutboxRequest;
     /** ISO time it was written. */
     createdAt: string;
+    /** The sweep's failed deliveries of it so far (`failOutbox`). */
+    attempts: number;
 }
 /**
  * A signal on the control bus (D21, D48): one process tells the others with the Tenant open.
@@ -307,6 +339,15 @@ export type ControlSignal = {
     /** The Tenant's sessions were reset and it moved to basin generation `generation`. */
     type: "sessions.reset";
     generation: number;
+} | {
+    /**
+     * Pod sandbox `sandboxId`'s host epoch moved to `epoch` (a join, a stop, a reset, a
+     * deletion, a loss): every process closes its connections hosting it at an older epoch
+     * (`HarnessApiServer.revokeHost`).
+     */
+    type: "host.revoked";
+    sandboxId: string;
+    epoch: number;
 };
 export interface FollowSignalsOptions {
     /** How often recent signals are read back. Default 5 s. */
@@ -327,7 +368,7 @@ export interface SessionStoreOptions {
     tenantId: string;
     /** Clock for event `time`. Defaults to `() => new Date()`. */
     now?: () => Date;
-    /** Receives listener and `afterCommit` failures, which never reject `tx`. */
+    /** Receives listener and outbox delivery failures, which never reject `tx`. */
     onError?: (error: unknown) => void;
 }
 /** The transaction handle. Valid only inside its `tx` callback. */
@@ -366,28 +407,43 @@ export interface Tx {
      * cursor is final.
      */
     event<T extends EventType>(sessionId: string, turnId: string | null, type: T, payload: EventPayload<T>): Promise<SessionEventOf<T>>;
-    /**
-     * Runs `fn` after a successful commit (never on rollback): a sandbox signal or a host
-     * revocation. Nothing writes it down, so a crash after commit loses it. A wake goes
-     * through `wake`, which does.
-     */
-    afterCommit(fn: () => void | Promise<void>): void;
     /**
      * Asks for an advance of `sessionId` once this transaction commits. The request is a row
      * of the wake outbox written in this transaction, so it commits, or rolls back, with what
-     * caused it: a commit never loses its wake. After commit, in `afterCommit` order, the
-     * store calls `deliver` (`DurableExecution.wake`, through the Tenant) and deletes the row
-     * once it resolves to anything but `false`. A request `deliver` declines (`false`) or
-     * rejects, or one a crash cut off, stays in the outbox until the Tenant sweep delivers it
-     * (`pendingWakes`, `deleteWakes`). `deliver` gets the request's idempotency key as
-     * `dedupeKey`: the wake's own, or `wake:<row id>` without one, so a request delivered
-     * twice still causes one advance.
+     * caused it: a commit never loses its wake. After commit the store hands it to the
+     * delivery (`SessionStore.deliverTo`, `DurableExecution.wake` through the Tenant) without
+     * waiting, and deletes the row once that resolves to anything but `false`. A request the
+     * delivery declines or rejects is due at the sweep's next pass; one a crash cut off, or
+     * still being sent, is due once it is older than the sweep's grace (`pendingOutbox`). The
+     * delivery gets the request's idempotency key as `dedupeKey`: the wake's own, or
+     * `wake:<row id>` without one, so a request delivered twice still causes one advance.
+     */
+    wake(sessionId: string, wake: Wake): Promise<void>;
+    /**
+     * Asks for a pod sandbox's reconcile, or sets one of its timers, once this transaction
+     * commits (`DurableExecution.sandbox`): a row of the sandbox signal outbox, delivered as a
+     * wake's is. A reconcile is delivered with the key `signal:<row id>`, so one delivered
+     * twice runs once.
+     */
+    signalSandbox(sandboxId: string, signal: SandboxSignal): Promise<void>;
+    /**
+     * Outbox requests that are due, by the database's clock: written more than `graceMs` ago,
+     * or whose delivery failed and whose retry time has come (`failOutbox`), and not parked.
+     * Oldest first, a retried one by its retry time, so a request that keeps failing never
+     * holds the head of the queue.
      */
-    wake(sessionId: string, wake: Wake, deliver: WakeDelivery): Promise<void>;
-    /** Outbox requests written before `before`, oldest first. */
-    pendingWakes(before: Date, limit: number): Promise<PendingWake[]>;
+    pendingOutbox(graceMs: number, limit: number): Promise<PendingRequest[]>;
     /** Deletes delivered outbox requests. */
-    deleteWakes(ids: readonly string[]): Promise<void>;
+    deleteOutbox(rows: readonly OutboxRef[]): Promise<void>;
+    /**
+     * Records a failed delivery of outbox request `row`: one more attempt, due again `retryInMs`
+     * from now (the database's clock). With `park` it is never due again: it stays in the
+     * outbox, with when it was parked, for an operator to send again or delete.
+     */
+    failOutbox(row: OutboxRef, retry: {
+        retryInMs: number;
+        park: boolean;
+    }): Promise<void>;
     /**
      * Locks the session and takes ownership when it has no owner or the owner's
      * lease ended at or before `now`; then increments `epoch` and sets the lease
@@ -416,9 +472,10 @@ export interface Tx {
     sessionsWithStatus<T extends SessionDoc = SessionDoc>(statuses: readonly (SessionStatus | (string & {}))[]): Promise<StoredSession<T>[]>;
     /**
      * Sessions `running` or `runnable` with no owner or an owner whose lease
-     * ended at or before `now`, oldest lease first. The sweep re-wakes these.
+     * ended at or before `now`, by id, the first `limit` after `after` (a page:
+     * pass the last id of the previous one). The sweep re-wakes these.
      */
-    orphanedSessions<T extends SessionDoc = SessionDoc>(now: Date, limit: number): Promise<StoredSession<T>[]>;
+    orphanedSessions<T extends SessionDoc = SessionDoc>(now: Date, limit: number, after?: string): Promise<StoredSession<T>[]>;
     /** Sessions, optionally of one agent and one owner (`ownerUserId`), by id. */
     listSessions<T extends SessionDoc = SessionDoc>(filter?: {
         agentId?: string;

package/dist/streams/relay/core.js

@@ -22,6 +22,7 @@
  *
  * The relay never writes the record and never deletes from S2.
  */
+import { sleep } from "../../retry.js";
 import { basinOf } from "../basin.js";
 import { sessionStream } from "../types.js";
 const RETRY_MIN_MS = 50;
@@ -351,16 +352,3 @@ export function createStreamRelay(options) {
     };
 }
 const messageOf = (error) => (error instanceof Error ? error.message : String(error));
-function sleep(ms, signal) {
-    return new Promise((resolve) => {
-        if (signal.aborted)
-            return resolve();
-        const done = () => {
-            clearTimeout(timer);
-            signal.removeEventListener("abort", done);
-            resolve();
-        };
-        const timer = setTimeout(done, ms);
-        signal.addEventListener("abort", done, { once: true });
-    });
-}

package/dist/tenant/advance.js

@@ -42,6 +42,7 @@
 import { runDurable, runFlowDurable, } from "@nylorun/harness/run";
 import { aggregateWaits, cancelSiblingWork, isWorkflowManifest, wakeLinkedWorkflow, } from "../core/flow-host.js";
 import { isOwnershipLost, ownedTx } from "../store/ownership.js";
+import { endAdvance } from "./scheduler.js";
 import { isGateToolEffect, recoversModelCalls, recoversToolCalls } from "./effects.js";
 import { slimModelEffects } from "./slim.js";
 import { applyUpdates, leanState, transcriptShadow, withTranscript, } from "./history.js";
@@ -169,10 +170,7 @@ export async function advance(ctx, id, signal) {
         heartbeat.stop();
         dropRunGrant(ctx, lease);
         signal.removeEventListener("abort", forward);
-        if (ctx.work.running.get(id) === controller) {
-            ctx.work.running.delete(id);
-            ctx.work.runningTurns.delete(id);
-        }
+        endAdvance(ctx.work, id, controller);
         // Best effort: a release that fails leaves a lease that simply expires.
         if (release)
             await ctx.store
@@ -395,7 +393,6 @@ async function wakeWorkflowOf(ctx, t, agent, turnId, type, payload) {
             error: type === "turn.failed"
                 ? String(payload.message ?? "")
                 : undefined,
-            schedule: ctx.wake,
         });
     else if (type === "turn.cancelled")
         await wakeLinkedWorkflow({
@@ -404,7 +401,6 @@ async function wakeWorkflowOf(ctx, t, agent, turnId, type, payload) {
             turnId,
             cancelled: true,
             error: "Agent turn was cancelled",
-            schedule: ctx.wake,
         });
 }
 /**
@@ -473,7 +469,10 @@ async function settle(ctx, lease, s, result, updates = [], cursor) {
             await slimModelEffects(t, id, s.activeTurnId);
             await t.put("sessions", id, current);
             const segment = finished.segment + 1;
-            await t.wake(id, { reason: "rollover", dedupeKey: `rollover:${s.activeTurnId}:${segment}` }, ctx.wake);
+            await t.wake(id, {
+                reason: "rollover",
+                dedupeKey: `rollover:${s.activeTurnId}:${segment}`,
+            });
             return siblings;
         }
         const flow = isWorkflowManifest(current.manifest);

package/dist/tenant/commands.js

@@ -231,7 +231,7 @@ export async function command(ctx, id, input, scope) {
                 reason: command.type,
                 dedupeKey: `${command.type}:${s.activeTurnId}:${cause}`,
             };
-            await t.wake(id, commandWake, ctx.wake);
+            await t.wake(id, commandWake);
             event = await t.event(id, s.activeTurnId, `command.${command.type}`, command);
         }
         await t.put("sessions", id, s);
@@ -244,7 +244,6 @@ export async function command(ctx, id, input, scope) {
                 turnId: cancelledTurnId,
                 cancelled: true,
                 error: "Agent turn was cancelled",
-                schedule: ctx.wake,
             });
         const response = {
             status: "accepted",