@nylorun/runtime 0.21.1-beta → 0.23.0-beta

package/dist/tenant/context.d.ts

@@ -5,8 +5,9 @@
  *
  * Business code changes state only inside `ctx.store.tx(async (t) => …)` and follows the
  * seam rules: events through `t.event(...)` (the relay appends them to Durable Streams after
- * commit), and advances through `await t.wake(id, { reason, dedupeKey }, ctx.wake)`, which
- * writes the wake to the outbox in the transaction and delivers it after commit. It never
+ * commit), and advances through `await t.wake(id, { reason, dedupeKey })`, which writes the
+ * wake to the outbox in the transaction; the store delivers it after commit (`ctx.wake`,
+ * wired with `SessionStore.deliverTo`). It never
  * publishes or notifies itself; `runtime.ts` wires the streams with `wireStreams()`. No
  * external I/O runs inside a tx.
  *
@@ -18,7 +19,7 @@ import type { IssuerScope, SubjectScope, TenantEnvelope } from "@nylorun/core/co
 import type { DurableCheckpoint, FlowCheckpoint } from "@nylorun/harness/run";
 import type { AgentManifest, JsonValue, SandboxManifest } from "@nylorun/core/define";
 import type { CredentialSelection } from "@nylorun/core/contracts";
-import type { SessionStore, Tx } from "../store/types.js";
+import type { OutboxRequest, SessionStore, Tx } from "../store/types.js";
 import type { FlowLimits } from "../core/limits.js";
 import type { ModelProvider } from "../core/provider.js";
 import type { ModelGate } from "../gates/model-gate.js";
@@ -212,13 +213,18 @@ export interface TenantContext {
     /** How long an advance's ownership lease lasts; the heartbeat renews it. */
     readonly ownerLeaseMs: number;
     /**
-     * Seam: hand a wake to `DurableExecution.wake`, outside any tx. Business code passes it to
-     * `t.wake` (the outbox), which calls it after commit; the sweep calls it for outbox rows
-     * left undelivered and for orphaned sessions. Resolves `true` once the execution accepted
-     * the wake, `false` while the Tenant is closing: the wake is not sent, and its outbox row
-     * stays for the next sweep.
+     * Seam: hand a wake to `DurableExecution.wake`, outside any tx: the sweep's orphan scan,
+     * and `send`. Resolves `true` once the execution accepted the wake, `false` while the
+     * Tenant is closing: the wake is not sent.
      */
     wake(sessionId: string, wake: Wake): Promise<boolean>;
+    /**
+     * Seam: hand an outbox request to Durable Session Execution (`wake` or `sandboxSignal`),
+     * outside any tx. The store calls it after commit for what business code wrote with
+     * `t.wake` and `t.signalSandbox` (`SessionStore.deliverTo`); the sweep calls it for rows
+     * left undelivered. `false` (the Tenant is closing) leaves the row for the next sweep.
+     */
+    send(request: OutboxRequest): Promise<boolean>;
     /**
      * Seam: abort the advance of a session running in this process, if any. With `turnId`, only
      * an advance of that turn (a cancel signal names the turn it cancelled).
@@ -230,10 +236,12 @@ export interface TenantContext {
      */
     readonly pods?: TenantPods;
     /**
-     * Seam: reconcile a pod sandbox, or arm one of its timers (`DurableExecution.sandbox`). Call
-     * it from `t.afterCommit`. Dropped while the Tenant is closing; the sweep re-sends reconciles.
+     * Seam: reconcile a pod sandbox, or arm one of its timers (`DurableExecution.sandbox`),
+     * outside any tx. What a transaction causes goes through `t.signalSandbox` instead, which
+     * writes it down; a direct call is best effort. `false` while the Tenant is closing (not
+     * sent); `true` without sandbox pods, where there is nothing to send.
      */
-    sandboxSignal(sandboxId: string, signal: SandboxSignal): Promise<void>;
+    sandboxSignal(sandboxId: string, signal: SandboxSignal): Promise<boolean>;
     /** This Tenant's execution invocations that need an operator, for Tenant status. */
     readonly stuckInvocations?: () => Promise<StuckInvocation[]>;
     /** Callbacks the Tenant sweep runs after its own steps (`sweep.ts`). */

package/dist/tenant/control.d.ts

@@ -8,9 +8,16 @@
  * - `sessions.reset` is written by the reset's transaction (`reset.ts`): every process moves
  *   its session streams to the new basin generation and ends those of deleted sessions
  *   (`checkSessionStreams`).
+ * - `host.revoked` is written by the transaction that moves a pod sandbox's host epoch
+ *   (`sandboxes.ts`, `sandbox/join.ts`, `sandbox/pods/reconcile.ts`): every process closes its
+ *   connections hosting that sandbox at an older epoch, wherever the host connected
+ *   (`HarnessApiServer.revokeHost`).
  *
  * A lost signal costs latency, never correctness: the advance checks the Session Store before
- * every effect, and the session streams are checked periodically (`streams.ts`).
+ * every effect, the session streams are checked periodically (`streams.ts`), and a revoked
+ * host's writes are fenced by its runs' lease epochs and its token by the sandbox's row.
+ * Signals are read back from their rows for two minutes, so a process whose listener was
+ * down catches up.
  */
 import type { TenantContext } from "./context.js";
 export interface WireControlOptions {

package/dist/tenant/control.js

@@ -16,6 +16,8 @@ export async function wireControl(ctx, options = {}) {
     const follower = await ctx.store.followSignals((signal) => {
         if (signal.type === "session.cancel")
             ctx.abortLocal(signal.sessionId, signal.turnId);
+        else if (signal.type === "host.revoked")
+            ctx.harness.revokeHost(signal.sandboxId, signal.epoch);
         else
             void checkSessionStreams(ctx).catch(report("session stream check failed"));
     }, {

package/dist/tenant/runtime.js

@@ -268,10 +268,14 @@ export class TenantRuntime {
                 abortLocal: (sessionId, turnId) => abortLocal(ctx, sessionId, "cancel", turnId),
                 ...(hooks.pods ? { pods: hooks.pods } : {}),
                 sandboxSignal: async (sandboxId, signal) => {
-                    if (ctx.closing || ctx.closed || !execution.sandbox)
-                        return;
-                    await execution.sandbox(config.tenantId, sandboxId, signal);
+                    if (ctx.closing || ctx.closed)
+                        return false;
+                    await execution.sandbox?.(config.tenantId, sandboxId, signal);
+                    return true;
                 },
+                send: (request) => request.kind === "wake"
+                    ? ctx.wake(request.sessionId, request.wake)
+                    : ctx.sandboxSignal(request.sandboxId, request.signal),
                 ...(hooks.execution?.stuckInvocations
                     ? {
                         stuckInvocations: () => hooks.execution.stuckInvocations(config.tenantId),
@@ -283,6 +287,22 @@ export class TenantRuntime {
                     return () => sweepHooks.delete(hook);
                 },
             };
+            // What a commit asks of the execution goes without the request waiting; a send that
+            // fails is the sweep's to send again, so it is only a warning here.
+            opened.deliverTo(async (request) => {
+                try {
+                    return await ctx.send(request);
+                }
+                catch (error) {
+                    config.logger.warn("outbox delivery after commit failed; the sweep sends it again", {
+                        ...(request.kind === "wake"
+                            ? { sessionId: request.sessionId, reason: request.wake.reason }
+                            : { sandboxId: request.sandboxId, signal: request.signal.kind }),
+                        message: error instanceof Error ? error.message : String(error),
+                    });
+                    return false;
+                }
+            });
             harnessServer = createHarnessApiServer(ctx, { sandboxPreference: preference });
             if (mcp && manager)
                 harness = await startInProcessHarness(ctx, { mcp, sandbox: manager }, {

package/dist/tenant/sandboxes.d.ts

@@ -19,7 +19,8 @@
  *   `sandbox-pods`): an agent-sandbox pod on the Tenant's cluster, created at once, whose engine
  *   runs the turns of the sessions attached to it. Its lifecycle (`sandbox/pods/lifecycle.ts`)
  *   runs in the `Sandbox` object: this module records what is wanted (`desired`, `rev`) and asks
- *   for a reconcile after the commit (`ctx.sandboxSignal`). Without sandbox pods kind `pod` is
+ *   for a reconcile in the same transaction (`Tx.signalSandbox`, delivered after the commit),
+ *   and a host epoch it moves goes on the control bus (`host.revoked`). Without sandbox pods kind `pod` is
  *   `409 sandbox_unavailable`. `POST .../stop` suspends a pod (the volume is kept; the next turn
  *   starts it), `POST .../reset` gives it a new pod on a new volume. An expired pod refuses
  *   turns (`sandbox_expired`) until a `PUT` sets a longer `lifecycle.ttl`; a lost one

package/dist/tenant/sandboxes.js

@@ -19,7 +19,8 @@
  *   `sandbox-pods`): an agent-sandbox pod on the Tenant's cluster, created at once, whose engine
  *   runs the turns of the sessions attached to it. Its lifecycle (`sandbox/pods/lifecycle.ts`)
  *   runs in the `Sandbox` object: this module records what is wanted (`desired`, `rev`) and asks
- *   for a reconcile after the commit (`ctx.sandboxSignal`). Without sandbox pods kind `pod` is
+ *   for a reconcile in the same transaction (`Tx.signalSandbox`, delivered after the commit),
+ *   and a host epoch it moves goes on the control bus (`host.revoked`). Without sandbox pods kind `pod` is
  *   `409 sandbox_unavailable`. `POST .../stop` suspends a pod (the volume is kept; the next turn
  *   starts it), `POST .../reset` gives it a new pod on a new volume. An expired pod refuses
  *   turns (`sandbox_expired`) until a `PUT` sets a longer `lifecycle.ttl`; a lost one
@@ -161,9 +162,13 @@ function specMismatch(existing, body) {
         return "storage";
     return undefined;
 }
-/** Asks for a pod's reconcile once the transaction commits. */
-function reconcileAfter(ctx, t, id) {
-    t.afterCommit(() => ctx.sandboxSignal(id, { kind: "reconcile" }));
+/** Asks for a pod's reconcile once the transaction commits (the sandbox signal outbox). */
+function reconcileAfter(t, id) {
+    return t.signalSandbox(id, { kind: "reconcile" });
+}
+/** Tells every process that pod sandbox `id`'s host epoch moved to `epoch`, at commit. */
+function revokeHost(t, id, epoch) {
+    return t.signal({ type: "host.revoked", sandboxId: id, epoch });
 }
 /** Asks the pod to run (again): a new `rev`, so the service applies it. */
 async function wantRunning(t, sandbox, now) {
@@ -227,7 +232,7 @@ export async function putSandbox(ctx, id, body, scope) {
                     });
                 if (revive || pod.desired !== "running" || pod.observed === "failed") {
                     await wantRunning(t, current, now);
-                    reconcileAfter(ctx, t, id);
+                    await reconcileAfter(t, id);
                 }
                 return viewOf(ctx, t, (await t.sandboxResource(id)) ?? current, ownerOf(scope));
             }
@@ -278,9 +283,9 @@ export async function putSandbox(ctx, id, body, scope) {
         await t.sandboxEvent(id, "sandbox.created", { kind, labels: row.labels });
         if (row.pod) {
             // Created at once; idle from now if no turn comes.
-            reconcileAfter(ctx, t, id);
+            await reconcileAfter(t, id);
             const idleAt = Date.parse(now) + podLifecycleConfig(config).idleMs;
-            t.afterCommit(() => ctx.sandboxSignal(id, { kind: "arm", timer: "idle", at: idleAt }));
+            await t.signalSandbox(id, { kind: "arm", timer: "idle", at: idleAt });
         }
         return viewOf(ctx, t, row, ownerOf(scope));
     });
@@ -330,8 +335,8 @@ export async function deleteSandbox(ctx, id, scope) {
         if (sandbox.pod) {
             const epoch = sandbox.pod.hostEpoch + 1;
             await t.updateSandboxPod(id, { desired: "deleted", observed: "deleting", rev: sandbox.pod.rev + 1, hostEpoch: epoch }, new Date().toISOString());
-            reconcileAfter(ctx, t, id);
-            t.afterCommit(() => ctx.harness.revokeHost(id, epoch));
+            await reconcileAfter(t, id);
+            await revokeHost(t, id, epoch);
         }
         else
             await t.deleteSandboxResource(id);
@@ -364,8 +369,8 @@ export async function stopSandbox(ctx, id, scope) {
             const epoch = pod.hostEpoch + 1;
             await t.updateSandboxPod(id, { desired: "suspended", rev: pod.rev + 1, hostEpoch: epoch }, now);
             await t.sandboxEvent(id, "sandbox.suspended", { reason: "stop" });
-            reconcileAfter(ctx, t, id);
-            t.afterCommit(() => ctx.harness.revokeHost(id, epoch));
+            await reconcileAfter(t, id);
+            await revokeHost(t, id, epoch);
         }
         return viewOf(ctx, t, (await t.sandboxResource(id)) ?? sandbox, ownerOf(scope));
     });
@@ -408,8 +413,8 @@ export async function resetSandbox(ctx, id, scope) {
             reason: null,
         }, now);
         await t.sandboxEvent(id, "sandbox.reset", { volumeGeneration: volumeGen });
-        reconcileAfter(ctx, t, id);
-        t.afterCommit(() => ctx.harness.revokeHost(id, epoch));
+        await reconcileAfter(t, id);
+        await revokeHost(t, id, epoch);
         return viewOf(ctx, t, (await t.sandboxResource(id)) ?? sandbox, ownerOf(scope));
     });
 }
@@ -480,7 +485,7 @@ export async function checkSandboxTurn(t, session, scope, ctx) {
     const now = new Date().toISOString();
     if (pod.desired !== "running" || pod.observed === "failed") {
         await wantRunning(t, sandbox, now);
-        reconcileAfter(ctx, t, id);
+        await reconcileAfter(t, id);
     }
     else
         await t.updateSandboxPod(id, { lastActiveAt: now }, now);
@@ -505,16 +510,15 @@ export async function detachAllSessions(t) {
  */
 export async function touchPodSandbox(ctx, id) {
     try {
-        const at = await ctx.store.tx(async (t) => {
+        await ctx.store.tx(async (t) => {
             const sandbox = await t.sandboxResource(id, { lock: true });
             if (!sandbox?.pod || sandbox.pod.desired !== "running")
-                return undefined;
+                return;
             const now = new Date();
             await t.updateSandboxPod(id, { lastActiveAt: now.toISOString() }, now.toISOString());
-            return now.getTime() + podLifecycleConfig(await readSandboxConfig(t)).idleMs;
+            const at = now.getTime() + podLifecycleConfig(await readSandboxConfig(t)).idleMs;
+            await t.signalSandbox(id, { kind: "arm", timer: "idle", at });
         });
-        if (at !== undefined)
-            await ctx.sandboxSignal(id, { kind: "arm", timer: "idle", at });
     }
     catch (error) {
         ctx.config.logger.warn("sandbox idle timer not armed", {

package/dist/tenant/scheduler.d.ts

@@ -5,8 +5,15 @@ export interface WorkState {
     readonly running: Map<string, AbortController>;
     /** The turn each running advance belongs to, by session id (`null`: none). */
     readonly runningTurns: Map<string, string | null>;
+    /** Called once when the last running advance ends (`endAdvance`), then forgotten. */
+    readonly idleWaiters: Set<() => void>;
 }
 export declare function createWorkState(): WorkState;
+/**
+ * Advance `id`'s end on this process: forgets its controller (unless a later advance of the
+ * session replaced it) and, when it was the last, settles every `idle` wait.
+ */
+export declare function endAdvance(work: WorkState, id: string, controller: AbortController): void;
 /**
  * Abort the advance of `id` if it runs on this process.
  * Cancel calls it after committing `cancelled`, and so does the control bus for cancels made

package/dist/tenant/scheduler.js

@@ -10,10 +10,45 @@
  * advance instead of settling it.
  */
 import { randomUUID } from "node:crypto";
+import { sleep } from "../retry.js";
 import { command } from "./commands.js";
 import { AdvanceAbort } from "./worker.js";
 export function createWorkState() {
-    return { running: new Map(), runningTurns: new Map() };
+    return { running: new Map(), runningTurns: new Map(), idleWaiters: new Set() };
+}
+/**
+ * Advance `id`'s end on this process: forgets its controller (unless a later advance of the
+ * session replaced it) and, when it was the last, settles every `idle` wait.
+ */
+export function endAdvance(work, id, controller) {
+    if (work.running.get(id) !== controller)
+        return;
+    work.running.delete(id);
+    work.runningTurns.delete(id);
+    if (work.running.size > 0)
+        return;
+    for (const waiter of work.idleWaiters)
+        waiter();
+    work.idleWaiters.clear();
+}
+/** Resolves once no advance runs on this process, or after `timeoutMs`, whichever is first. */
+async function idle(work, timeoutMs) {
+    if (work.running.size === 0)
+        return;
+    const timer = new AbortController();
+    let settle;
+    const ended = new Promise((resolve) => (settle = resolve));
+    work.idleWaiters.add(settle);
+    try {
+        await Promise.race([
+            ended,
+            sleep(Math.max(0, timeoutMs), timer.signal),
+        ]);
+    }
+    finally {
+        timer.abort();
+        work.idleWaiters.delete(settle);
+    }
 }
 const MESSAGES = {
     cancel: "Turn cancelled",
@@ -56,9 +91,7 @@ export async function drain(ctx, activeWork, timeoutMs) {
                     principalId: "drain",
                 });
     }
-    const deadline = Date.now() + timeoutMs;
-    while (ctx.work.running.size > 0 && Date.now() < deadline)
-        await new Promise((resolve) => setTimeout(resolve, 10));
+    await idle(ctx.work, timeoutMs);
 }
 /** Abort every advance running on this process; closing the Tenant is a `shutdown`. */
 export function abortAll(ctx, kind = "shutdown") {
@@ -72,9 +105,7 @@ export function abortAll(ctx, kind = "shutdown") {
  * (§11.4); the epoch fences anything they still try to write. Returns their session ids.
  */
 export async function waitForIdle(ctx, timeoutMs) {
-    const deadline = Date.now() + timeoutMs;
-    while (ctx.work.running.size > 0 && Date.now() < deadline)
-        await new Promise((resolve) => setTimeout(resolve, 10));
+    await idle(ctx.work, timeoutMs);
     const abandoned = [...ctx.work.running.keys()];
     if (abandoned.length > 0)
         ctx.config.logger.warn("tenant closed with advances still running", {

package/dist/tenant/session-streams.d.ts

@@ -120,5 +120,4 @@ export declare function checkSessionStreams(ctx: TenantContext): Promise<void>;
 export declare function clearObservers(hub: SessionStreams): void;
 /** Close: end every session observer. */
 export declare function endAllStreams(hub: SessionStreams): void;
-export declare function sleep(ms: number, signal: AbortSignal): Promise<void>;
 export {};

package/dist/tenant/session-streams.js

@@ -20,6 +20,7 @@
  */
 import { historyCursor, historyResume } from "../reads/cursor.js";
 import { decodeCursor, encodeCursor } from "../record/index.js";
+import { sleep } from "../retry.js";
 import { basinOf } from "../streams/basin.js";
 import { sessionStream, } from "../streams/types.js";
 import { fail } from "./http.js";
@@ -405,16 +406,3 @@ export function clearObservers(hub) {
 export function endAllStreams(hub) {
     clearObservers(hub);
 }
-export function sleep(ms, signal) {
-    return new Promise((resolve) => {
-        if (signal.aborted)
-            return resolve();
-        const done = () => {
-            clearTimeout(timer);
-            signal.removeEventListener("abort", done);
-            resolve();
-        };
-        const timer = setTimeout(done, ms);
-        signal.addEventListener("abort", done, { once: true });
-    });
-}

package/dist/tenant/sessions.js

@@ -136,7 +136,12 @@ export async function putSession(ctx, id, body, access, options = {}) {
             return prior;
         }
         await ctx.vault.assertAttachment(t, body.ownerUserId, vaultIds, credentialSelections, { opaque });
+        const environment = await ctx.vault.sessionEnvironment(t, vaultIds);
+        if (environment.conflict)
+            fail(409, environment.conflict, { code: "credential_conflict" });
         if (prior) {
+            if (prior.sandboxId !== undefined)
+                await assertSandboxSecrets(ctx, t, prior.sandboxId, id, body.ownerUserId, environment.secretHosts);
             if (sessionIdentity(prior.creation) !== sessionIdentity(body))
                 fail(409, "Session already exists with different creation parameters");
             prior.vaultIds = vaultIds;
@@ -156,6 +161,8 @@ export async function putSession(ctx, id, body, access, options = {}) {
         checkPlacement(await readSandboxConfig(t), sandbox.kind);
         if (sandbox.kind === "pod")
             await requirePods(ctx.pods, `Sandbox ${sandbox.sandboxId}`);
+        if (sandbox.sandboxId !== undefined)
+            await assertSandboxSecrets(ctx, t, sandbox.sandboxId, id, body.ownerUserId, environment.secretHosts);
         const pinned = sandbox.manifest === undefined ? platformTools(definition) : sandbox;
         const created = {
             id,
@@ -186,6 +193,25 @@ export async function putSession(ctx, id, body, access, options = {}) {
         return created;
     });
 }
+/**
+ * A sandbox's shell secrets must be unambiguous (R2c, D50): egress-gate sets the header from the
+ * one `environment_secret` bound to a host among the sessions using the sandbox. So while any of
+ * them has secrets, they all have one owner, and no host is bound to two credentials.
+ */
+async function assertSandboxSecrets(ctx, t, sandboxId, sessionId, ownerUserId, secretHosts) {
+    const conflict = (message) => fail(409, message, { code: "credential_conflict" });
+    const mine = Object.keys(secretHosts).length > 0;
+    for (const other of await t.sessionsOnSandbox(sandboxId)) {
+        if (other.id === sessionId)
+            continue;
+        const theirs = (await ctx.vault.sessionEnvironment(t, other.vaultIds ?? [])).secretHosts;
+        if (other.ownerUserId !== ownerUserId && (mine || Object.keys(theirs).length > 0))
+            conflict(`Sandbox ${sandboxId} serves environment_secret credentials of one owner at a time`);
+        for (const [host, credentialId] of Object.entries(secretHosts))
+            if (theirs[host] !== undefined && theirs[host] !== credentialId)
+                conflict(`Two environment_secret credentials would be bound to ${host} in sandbox ${sandboxId}`);
+    }
+}
 function isShare(value) {
     return typeof value === "object" && value !== null && "session" in value;
 }

package/dist/tenant/streams.js

@@ -1,6 +1,7 @@
 import { basinOf } from "../streams/basin.js";
 import { createStreamRelay, } from "../streams/relay/core.js";
-import { checkSessionStreams, currentBasin, sleep, } from "./session-streams.js";
+import { retry } from "../retry.js";
+import { checkSessionStreams, currentBasin } from "./session-streams.js";
 const RETRY_MIN_MS = 100;
 const RETRY_MAX_MS = 2000;
 /** The longest wait between two `ensureTenant` attempts while the basin is being repaired. */
@@ -92,7 +93,11 @@ export async function wireStreams(ctx, options) {
     /** Deletes a retired generation's basin, then forgets it. A failure retries on next open. */
     async function deleteRetired(generation) {
         try {
-            await withRetries(() => streams.deleteTenant(basinOf(tenantId, generation)), 5);
+            await retry(() => streams.deleteTenant(basinOf(tenantId, generation)), {
+                minMs: RETRY_MIN_MS,
+                maxMs: RETRY_MAX_MS,
+                attempts: 5,
+            });
             await store.tx((t) => t.forgetRetiredGeneration(generation));
             logger.info("retired stream basin deleted", { generation });
         }
@@ -176,21 +181,6 @@ async function probe(streams, timeoutMs) {
 export async function closeStreams(ctx) {
     await ctx.sessionStreams.wiring?.close();
 }
-// ---------------------------------------------------------------------------
-async function withRetries(task, attempts) {
-    let delay = RETRY_MIN_MS;
-    for (let attempt = 1;; attempt += 1) {
-        try {
-            return await task();
-        }
-        catch (error) {
-            if (attempt >= attempts)
-                throw error;
-            await new Promise((resolve) => setTimeout(resolve, delay));
-            delay = Math.min(delay * 2, RETRY_MAX_MS);
-        }
-    }
-}
 function basinKeeper(streams, basinKey, signal, onError) {
     let ready = false;
     let readyFor;
@@ -230,20 +220,15 @@ function basinKeeper(streams, basinKey, signal, onError) {
             repairing = true;
             ready = false;
             void (async () => {
-                let delay = RETRY_MIN_MS;
-                while (!signal.aborted) {
-                    try {
-                        await ensure();
-                        repairedAt = Date.now();
-                        break;
-                    }
-                    catch (error) {
+                await retry(ensure, {
+                    minMs: RETRY_MIN_MS,
+                    maxMs: BASIN_RETRY_MAX_MS,
+                    signal,
+                    onError: (error) => {
                         if (failures === 1 || failures % 10 === 0)
                             onError(error);
-                    }
-                    await sleep(delay, signal);
-                    delay = Math.min(delay * 2, BASIN_RETRY_MAX_MS);
-                }
+                    },
+                }).then(() => (repairedAt = Date.now()), () => undefined);
                 repairing = false;
             })();
         },

package/dist/tenant/sweep.d.ts

@@ -1,20 +1,47 @@
+import type { PendingRequest } from "../store/types.js";
 import type { TenantContext } from "./context.js";
 /**
- * How old a wake outbox row is before the sweep delivers it: longer than a delivery after
- * commit takes, so the sweep seldom sends a wake its commit is still sending. Sending one
- * twice is harmless either way (the idempotency key).
+ * How old an outbox row is before the sweep delivers it, unless its delivery after commit
+ * already failed (then it is due at once): longer than that delivery can take with its
+ * retries, so the sweep does not send again, and double the load on, an execution that is
+ * only slow. Sending one twice is harmless either way (the idempotency key).
+ */
+export declare const OUTBOX_GRACE_MS = 30000;
+/**
+ * How often the orphan scan, and the stale pod sandbox check, run after the first pass of a
+ * Tenant opened on this process.
  */
-export declare const WAKE_GRACE_MS = 2000;
-/** How often the orphan scan runs after the first pass of a Tenant opened on this process. */
 export declare const ORPHAN_SCAN_MS = 60000;
+/**
+ * How long one pass may spend draining the outbox, and paging through orphaned sessions.
+ * A backlog is worked off over several passes rather than holding one (and with it the
+ * Tenant's sweep object in the execution) for long.
+ */
+export declare const DRAIN_BUDGET_MS = 2000;
 export declare function sweep(ctx: TenantContext): Promise<void>;
+/** Failed deliveries after which the sweep parks an outbox row (`Tx.failOutbox`). */
+export declare const MAX_OUTBOX_ATTEMPTS = 10;
 /**
- * Deliver the wake outbox rows written before `before` (oldest first, one batch) and delete
- * the delivered ones. A row the Tenant declines (it is closing) stays. Returns the ids of the
- * rows delivered.
+ * Deliver the outbox rows that are due (`Tx.pendingOutbox`: older than `graceMs`, or whose
+ * delivery failed), oldest first, batch after batch until none is left or `budgetMs` is
+ * spent, and delete the delivered ones. Each row is delivered on its own: one that fails
+ * (Restate refuses it, or a Runtime that does not know a wake's reason) is tried again later,
+ * with backoff, so it never holds back the rows behind it, and after `MAX_OUTBOX_ATTEMPTS`
+ * failures it is parked: logged once, at error, and kept in the outbox for an operator. A
+ * parked wake's session is still re-woken by the orphan scan while it is runnable, and a
+ * parked sandbox signal's pod by the stale pod check. A row the Tenant declines (it is
+ * closing) stays as it is, and ends the drain. Never throws for a delivery; returns the rows
+ * delivered.
  */
-export declare function deliverPendingWakes(ctx: Pick<TenantContext, "store" | "wake">, before?: Date): Promise<string[]>;
+export declare function deliverOutbox(ctx: Pick<TenantContext, "store" | "send" | "config">, graceMs?: number, budgetMs?: number): Promise<PendingRequest[]>;
 /** Settle pending workflow `agent` effects whose linked turn already finished. */
-export declare function reconcileLinkedAgents(ctx: Pick<TenantContext, "store" | "wake">): Promise<void>;
-/** Wake `running`/`runnable` sessions without a live owner. Returns the ids woken. */
-export declare function wakeOrphanedSessions(ctx: Pick<TenantContext, "store" | "wake">, now?: Date): Promise<string[]>;
+export declare function reconcileLinkedAgents(ctx: Pick<TenantContext, "store">): Promise<void>;
+/**
+ * Wake `running`/`runnable` sessions without a live owner, page after page from the first id
+ * after `after`, until none is left or `budgetMs` is spent. Returns the ids woken, and the last
+ * id when the budget ran out first (`next`, where the next scan goes on).
+ */
+export declare function wakeOrphanedSessions(ctx: Pick<TenantContext, "store" | "wake">, now?: Date, after?: string, budgetMs?: number): Promise<{
+    woken: string[];
+    next?: string;
+}>;