@nylorun/runtime 0.13.0-beta → 0.14.0-beta

package/dist/tenant/module.d.ts

@@ -1,3 +1,4 @@
+import type { StreamRelayStatus } from "@nylorun/core/contracts";
 import { isQuarantineError, quarantine, TenantBusyError, TenantConflictError } from "./quarantine.js";
 import { TenantNotFoundError, TenantUnavailableError, type Logger, type OpenTenantRuntime, type TenantConfig, type TenantModule, type TenantStore } from "./types.js";
 export interface CreateTenantModuleOptions {
@@ -21,6 +22,8 @@ export interface CreateTenantModuleOptions {
      * Durable Streams basin, its armed sweep). A failure is logged; the Tenant stays deleted.
      */
     onDeleted?: (tenantId: string) => Promise<void>;
+    /** This process's stream relay, for the Host aggregate (a Host with S2 runs one). */
+    relayStatus?: () => Promise<StreamRelayStatus>;
 }
 /**
  * Deep Tenant module (§8): discovery, create, resolve, list, delete and summarize over an

package/dist/tenant/module.js

@@ -7,7 +7,7 @@ import { TenantNotFoundError, TenantUnavailableError, } from "./types.js";
 const OPEN_TIMEOUT_MS = 30_000;
 function hasLiveWork(summary) {
     return (summary.runningSessions > 0 ||
-        summary.connectedExecutors > 0 ||
+        summary.inFlightDeliveries > 0 ||
         summary.pendingActions > 0);
 }
 function toAdmin(id, entry, envelope) {
@@ -284,28 +284,22 @@ export function createTenantModule(options) {
         },
         async summarize() {
             let runningSessions = 0;
-            let connectedExecutors = 0;
+            let inFlightDeliveries = 0;
             let pendingActions = 0;
             let uncertainEffects = 0;
-            let outboxDepth;
-            let relayLagMs;
             for (const s of await Promise.all(openEntries().map((e) => e.handle.summary()))) {
                 runningSessions += s.runningSessions;
-                connectedExecutors += s.connectedExecutors;
+                inFlightDeliveries += s.inFlightDeliveries;
                 pendingActions += s.pendingActions;
                 uncertainEffects += s.uncertainEffects;
-                if (s.outboxDepth !== undefined)
-                    outboxDepth = (outboxDepth ?? 0) + s.outboxDepth;
-                if (s.relayLagMs !== undefined)
-                    relayLagMs = Math.max(relayLagMs ?? 0, s.relayLagMs);
             }
+            const relay = await options.relayStatus?.().catch(() => undefined);
             return {
                 runningSessions,
-                connectedExecutors,
+                inFlightDeliveries,
                 pendingActions,
                 uncertainEffects,
-                ...(outboxDepth !== undefined ? { outboxDepth } : {}),
-                ...(relayLagMs !== undefined ? { relayLagMs } : {}),
+                ...(relay ? { relay } : {}),
             };
         },
         async close() {

package/dist/tenant/outbound.d.ts

@@ -0,0 +1,44 @@
+import { type IncomingHttpHeaders } from "node:http";
+/** How the Host lets the Runtime call developer URLs (Host settings, `TenantConfig.delivery`). */
+export interface OutboundPolicy {
+    /** `http` URLs as well as `https`. Default true (OSS). */
+    allowHttp?: boolean;
+    /** Private, loopback and link-local addresses. Default `allow` (OSS); Cloud refuses. */
+    privateAddresses?: "allow" | "refuse";
+    /**
+     * `docker-host`: the Runtime runs in the local stack's container, so `localhost`,
+     * `127.0.0.1` and `[::1]` mean the machine that runs Docker (`host.docker.internal`).
+     */
+    loopback?: "docker-host";
+}
+export type OutboundResult = {
+    kind: "response";
+    status: number;
+    headers: IncomingHttpHeaders;
+    body: Buffer;
+}
+/** The answer was larger than `maxResponseBytes`. */
+ | {
+    kind: "too_large";
+    status: number;
+} | {
+    kind: "not_sent";
+    code: string;
+    message: string;
+} | {
+    kind: "lost";
+    code: string;
+    message: string;
+};
+/** The largest answer read from an endpoint. */
+export declare const MAX_RESPONSE_BYTES: number;
+/** True when `address` is private, loopback, link-local or unspecified (IPv4-mapped included). */
+export declare function isPrivateAddress(address: string): boolean;
+/** Why the Host refuses `url`, or `undefined` when it may be called. */
+export declare function refusal(url: URL, policy: OutboundPolicy): string | undefined;
+/** POSTs `body` to `url` under `policy`. Never throws; `signal` aborts the request. */
+export declare function post(target: string, body: string, headers: Record<string, string>, options: {
+    policy: OutboundPolicy;
+    signal: AbortSignal;
+    maxResponseBytes?: number;
+}): Promise<OutboundResult>;

package/dist/tenant/outbound.js

@@ -0,0 +1,127 @@
+/**
+ * The Runtime's requests to Action endpoints (design: Action endpoints §8.3): one POST, no
+ * redirects, a bounded answer, and the Host's address policy checked on the address actually
+ * connected to, so a DNS answer cannot steer a delivery somewhere the Host forbids.
+ *
+ * The result says whether anything reached the endpoint. A request that failed before its body
+ * was flushed (refused connection, unknown host, TLS failure, a refused address) was `not_sent`
+ * and is safe to send again, even for a tool; one that failed after (reset, timeout) is `lost`.
+ */
+import { lookup as dnsLookup } from "node:dns";
+import { request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
+import { BlockList, isIP } from "node:net";
+/** The largest answer read from an endpoint. */
+export const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
+const PRIVATE = new BlockList();
+for (const [network, prefix] of [
+    ["0.0.0.0", 8],
+    ["10.0.0.0", 8],
+    ["100.64.0.0", 10],
+    ["127.0.0.0", 8],
+    ["169.254.0.0", 16],
+    ["172.16.0.0", 12],
+    ["192.168.0.0", 16],
+])
+    PRIVATE.addSubnet(network, prefix, "ipv4");
+for (const [network, prefix] of [
+    ["::", 128],
+    ["::1", 128],
+    ["fc00::", 7],
+    ["fe80::", 10],
+])
+    PRIVATE.addSubnet(network, prefix, "ipv6");
+/** True when `address` is private, loopback, link-local or unspecified (IPv4-mapped included). */
+export function isPrivateAddress(address) {
+    const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(address);
+    if (mapped)
+        return PRIVATE.check(mapped[1], "ipv4");
+    const family = isIP(address);
+    if (family === 0)
+        return false;
+    return PRIVATE.check(address, family === 4 ? "ipv4" : "ipv6");
+}
+class RefusedAddress extends Error {
+    code = "ENDPOINT_ADDRESS_REFUSED";
+}
+/** Why the Host refuses `url`, or `undefined` when it may be called. */
+export function refusal(url, policy) {
+    if (url.protocol !== "https:" && !(url.protocol === "http:" && policy.allowHttp !== false))
+        return `This Runtime calls only ${policy.allowHttp === false ? "https" : "http and https"} endpoints`;
+    const host = url.hostname.replace(/^\[|\]$/g, "");
+    if (policy.privateAddresses === "refuse" && isIP(host) && isPrivateAddress(host))
+        return "This Runtime does not call private addresses; expose the endpoint through a public URL (a tunnel, for a local app)";
+    return undefined;
+}
+/** POSTs `body` to `url` under `policy`. Never throws; `signal` aborts the request. */
+export async function post(target, body, headers, options) {
+    const { policy, signal } = options;
+    const limit = options.maxResponseBytes ?? MAX_RESPONSE_BYTES;
+    const url = new URL(target);
+    const refused = refusal(url, policy);
+    if (refused)
+        return { kind: "not_sent", code: "ENDPOINT_ADDRESS_REFUSED", message: refused };
+    const originalHost = url.host;
+    if (policy.loopback === "docker-host" && ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname))
+        url.hostname = "host.docker.internal";
+    const lookup = (hostname, lookupOptions, callback) => dnsLookup(hostname, { ...lookupOptions, all: true }, (error, addresses) => {
+        if (error)
+            return callback(error, []);
+        const allowed = policy.privateAddresses === "refuse"
+            ? addresses.filter((a) => !isPrivateAddress(a.address))
+            : addresses;
+        if (allowed.length === 0)
+            return callback(new RefusedAddress(`${hostname} resolves only to private addresses, which this Runtime does not call`), []);
+        if (lookupOptions.all)
+            return callback(null, allowed);
+        return callback(null, allowed[0].address, allowed[0].family);
+    });
+    return new Promise((resolve) => {
+        let sent = false;
+        let settled = false;
+        const finish = (result) => {
+            if (settled)
+                return;
+            settled = true;
+            resolve(result);
+        };
+        const failed = (error) => finish({
+            kind: sent ? "lost" : "not_sent",
+            code: signal.aborted ? "ABORTED" : (error.code ?? error.name),
+            message: signal.aborted ? "The delivery was aborted" : error.message,
+        });
+        const request = (url.protocol === "https:" ? httpsRequest : httpRequest)(url, {
+            method: "POST",
+            headers: {
+                ...headers,
+                host: originalHost,
+                "content-type": "application/json",
+                "content-length": String(Buffer.byteLength(body)),
+            },
+            lookup: lookup,
+            signal,
+        }, (response) => {
+            const status = response.statusCode ?? 0;
+            const chunks = [];
+            let size = 0;
+            response.on("data", (chunk) => {
+                size += chunk.length;
+                if (size > limit) {
+                    // Settle first: destroying the answer reports an error of its own.
+                    finish({ kind: "too_large", status });
+                    response.destroy();
+                    return;
+                }
+                chunks.push(chunk);
+            });
+            response.on("end", () => finish({ kind: "response", status, headers: response.headers, body: Buffer.concat(chunks) }));
+            response.on("error", failed);
+            response.on("aborted", () => failed(Object.assign(new Error("The answer was cut off"), { code: "ECONNRESET" })));
+        });
+        request.on("finish", () => {
+            sent = true;
+        });
+        request.on("error", failed);
+        request.end(body);
+    });
+}

package/dist/tenant/pool.d.ts

@@ -1,5 +1,3 @@
-/** Run async work over items with a fixed concurrency limit. */
-export declare function mapPool<T, R>(items: readonly T[], concurrency: number, worker: (item: T, index: number) => Promise<R>): Promise<R[]>;
 export declare class TimeoutError extends Error {
     constructor(message?: string);
 }

package/dist/tenant/pool.js

@@ -1,21 +1,3 @@
-/** Run async work over items with a fixed concurrency limit. */
-export async function mapPool(items, concurrency, worker) {
-    if (items.length === 0)
-        return [];
-    const limit = Math.max(1, Math.min(concurrency, items.length));
-    const results = new Array(items.length);
-    let next = 0;
-    async function run() {
-        for (;;) {
-            const index = next++;
-            if (index >= items.length)
-                return;
-            results[index] = await worker(items[index], index);
-        }
-    }
-    await Promise.all(Array.from({ length: limit }, () => run()));
-    return results;
-}
 export class TimeoutError extends Error {
     constructor(message = "operation timed out") {
         super(message);

package/dist/tenant/reset.d.ts

@@ -1,17 +1,13 @@
 import type { ResetScope, SessionStore } from "../store/types.js";
-import type { ExecutorRegistry } from "../core/executors.js";
 import type { SandboxManager } from "../sandbox/manager.js";
 import type { TenantPaths } from "./types.js";
 export type { ResetScope } from "../store/types.js";
 export interface ResetTenantContext {
     store: SessionStore;
-    registry: ExecutorRegistry;
     sandbox: SandboxManager;
     paths: TenantPaths;
     /** Clear in-memory session observers after the store wipe. */
     clearSessionState: () => void;
-    /** Clear executor SSE streams when registrations are removed. */
-    clearExecutorStreams: () => void;
 }
 /**
  * Reset Tenant durable state for `scope` (A17).

package/dist/tenant/reset.js

@@ -42,13 +42,8 @@ export async function resetTenant(ctx, scope) {
         await ctx.sandbox.reconcile(() => false);
     }
     await ctx.store.tx((t) => t.reset(scope));
-    if (clearAll)
-        for (const row of ctx.registry.list())
-            ctx.registry.remove(row.agentId);
     if (clearSessions)
         ctx.clearSessionState();
-    if (clearAll)
-        ctx.clearExecutorStreams();
     if (clearSandboxes)
         replaceDirectory(ctx.paths.sandboxes);
     if (clearAll) {

package/dist/tenant/runtime.d.ts

@@ -1,25 +1,8 @@
-/**
- * Composition root of an open Tenant: validates the config, opens the Session Store and the
- * vault key, builds the services and the `TenantContext`, wires the seams, registers the
- * Tenant's Worker handlers and arms its sweep, and implements `TenantHandle` (handle,
- * summary, drain, close) by delegating to the Tenant modules.
- *
- * Seams wired here: `wireStreams()` connects the store's commits to Durable Streams (the
- * relay, and the history, SSE, work and control readers); `wake` goes to the
- * `DurableExecution`, whose handlers (`worker.ts`) call `advance` and `sweep`; `abortLocal`
- * aborts an advance running on this process. The sweep also drains the outbox.
- *
- * Execution: the Host passes one `TenantExecution` for every Tenant it opens
- * (`TenantOpenHooks.execution`); without one, the Tenant runs its own in-process
- * `MemoryExecution`. No lock file: several processes may open a Tenant, and ownership of each
- * session (§10.6) keeps its advances apart.
- */
-import type { ServerResponse, IncomingMessage } from "node:http";
 import type { TenantEnvelope } from "@nylorun/core/contracts";
 import { type ModelProvider } from "../core/provider.js";
 import { type AuthorizeResult } from "../vault/service.js";
 import type { DurableStreams } from "../streams/types.js";
-import type { OpenedTenant, TenantConfig, TenantHandle, TenantSummary } from "./types.js";
+import type { NodeBindings, OpenedTenant, TenantConfig, TenantHandle, TenantSummary } from "./types.js";
 import { type StreamsStatus } from "./streams.js";
 import { type TenantExecution, type TenantWorker } from "./worker.js";
 /** TENANTS-CCR: test/injection hooks until TenantConfig gains them. */
@@ -42,6 +25,13 @@ export type TenantOpenHooks = {
      * survive a restart (tests, and a Host without S2).
      */
     streams?: DurableStreams;
+    /**
+     * The Host's stream relay feeds `streams` from the record for every Tenant. Without it the
+     * Tenant relays its own commits.
+     */
+    hostRelay?: boolean;
+    /** How long a retired stream basin is kept after a reset (tests). Default 60 s. */
+    retireGraceMs?: number;
 } & OpenedTenant;
 export declare class TenantRuntime implements TenantHandle {
     private readonly ctx;
@@ -53,7 +43,7 @@ export declare class TenantRuntime implements TenantHandle {
     private readonly closeGraceMs;
     private constructor();
     static open(config: TenantConfig, hooks: TenantOpenHooks): Promise<TenantRuntime>;
-    handle(request: IncomingMessage, response: ServerResponse, url?: URL): Promise<void>;
+    fetch(request: Request, node: NodeBindings): Promise<Response>;
     authorize(sessionId: string, request: {
         url: string;
         serverName?: string;
@@ -67,7 +57,7 @@ export declare class TenantRuntime implements TenantHandle {
     abortLocal(sessionId: string): void;
     /**
      * The Durable Streams seam's status for Tenant status and readiness: S2 reachability, the
-     * basin, outbox depth and relay lag (`streamsStatus` in `tenant/streams.ts`).
+     * basin, its generation and the Tenant's relay (`streamsStatus` in `tenant/streams.ts`).
      */
     streamsStatus(): Promise<StreamsStatus>;
     /** Adds a callback to the Tenant sweep. Returns a function that removes it. */

package/dist/tenant/runtime.js

@@ -1,8 +1,8 @@
 import { mkdirSync } from "node:fs";
 import { resolveFlowLimits } from "../core/limits.js";
-import { loadExecutorRegistry } from "../core/executors.js";
 import { scriptedModel, gatewayModel, toolFixtureModel, } from "../core/provider.js";
 import { createKekFile, readVaultKek } from "../vault/kek.js";
+import { SigningKeys } from "./signing-keys.js";
 import { VaultService } from "../vault/service.js";
 import { McpPool } from "../mcp/pool.js";
 import { SandboxManager } from "../sandbox/manager.js";
@@ -10,14 +10,15 @@ import { defaultSandboxBackends } from "../sandbox/select.js";
 import { MemoryExecution } from "../execution/memory.js";
 import { QuarantineError } from "./quarantine-error.js";
 import { MemoryStreams } from "../streams/memory.js";
-import { connectedExecutorCount, createLiveHub, endAllStreams, } from "./live.js";
-import { closeStreams, drainOutbox, streamsStatus, wireStreams, } from "./streams.js";
+import { createSessionStreams, endAllStreams, } from "./session-streams.js";
+import { closeStreams, streamsStatus, wireStreams, } from "./streams.js";
 import { abortAll, abortLocal, createWorkState, drain, waitForIdle, } from "./scheduler.js";
 import { advance } from "./advance.js";
 import { sweep } from "./sweep.js";
+import { deliverAction } from "./delivery.js";
 import { TenantWorkers, WORKER_ID, } from "./worker.js";
 import { authorize } from "./effects.js";
-import { handle } from "./routes.js";
+import { tenantApi } from "../api/http/app.js";
 /** Default ownership lease of an advance; the heartbeat renews it every third. */
 const DEFAULT_OWNER_LEASE_MS = 30_000;
 function validateConfig(config) {
@@ -70,8 +71,9 @@ export class TenantRuntime {
                 vaultKek: hooks.vaultKek,
                 vaultKekPath: paths.kek,
             });
-            if ((await store.tx((t) => t.countCredentials())) > 0 && !kek) {
-                throw new QuarantineError("kek-missing", "Vault key-encryption key is missing for ciphertext in this Tenant", "restore the vault-kek file in the Tenant directory");
+            const sealed = await store.tx(async (t) => (await t.countCredentials()) + (await t.countSigningKeys()));
+            if (sealed > 0 && !kek) {
+                throw new QuarantineError("kek-missing", "Vault key-encryption key is missing for ciphertext (vault credentials or signing keys) in this Tenant", "restore the vault-kek file in the Tenant directory");
             }
             const createKekIfMissing = hooks.createKekIfMissing !== false;
             const ensureKek = () => {
@@ -83,7 +85,6 @@ export class TenantRuntime {
                 return kek;
             };
             const opened = store;
-            const registry = await loadExecutorRegistry(opened);
             // A seeded Tenant setting wins over the Host default (A18).
             const seededBackend = await opened.tx((t) => t.getSetting("sandbox.backend"));
             const vault = new VaultService({
@@ -141,7 +142,7 @@ export class TenantRuntime {
             }
             else
                 modelProvider = scriptedModel();
-            const live = createLiveHub();
+            const sessionStreams = createSessionStreams();
             const local = hooks.execution
                 ? undefined
                 : new MemoryExecution({
@@ -160,7 +161,6 @@ export class TenantRuntime {
                 envelope,
                 store: opened,
                 vault,
-                registry,
                 mcp,
                 sandbox,
                 flowLimits,
@@ -169,7 +169,8 @@ export class TenantRuntime {
                 closing: false,
                 closed: false,
                 work: createWorkState(),
-                live,
+                sessionStreams,
+                signingKeys: new SigningKeys({ tenantId: config.tenantId, kek: ensureKek }),
                 workerId: hooks.workerId ?? WORKER_ID,
                 ownerLeaseMs: config.ownerLeaseMs ?? DEFAULT_OWNER_LEASE_MS,
                 wake: async (sessionId, wake) => {
@@ -178,6 +179,11 @@ export class TenantRuntime {
                     await execution.wake(config.tenantId, sessionId, wake);
                 },
                 abortLocal: (sessionId) => abortLocal(ctx, sessionId),
+                deliver: async (actionId) => {
+                    if (ctx.closing || ctx.closed)
+                        return;
+                    await execution.deliver(config.tenantId, actionId);
+                },
                 ...(hooks.execution?.stuckInvocations
                     ? {
                         stuckInvocations: () => hooks.execution.stuckInvocations(config.tenantId),
@@ -194,21 +200,15 @@ export class TenantRuntime {
                 streams: hooks.streams ?? new MemoryStreams(),
                 ownsStreams: !hooks.streams,
                 tenantId: config.tenantId,
-            });
-            // Outbox rows a lost relay step left behind are appended by the sweep.
-            sweepHooks.add(async () => {
-                await drainOutbox(ctx);
+                ...(hooks.hostRelay ? { hostRelay: true } : {}),
+                ...(hooks.retireGraceMs !== undefined ? { retireGraceMs: hooks.retireGraceMs } : {}),
             });
             // Register the handlers, then arm the sweep: its first pass runs at once and re-wakes
             // sessions a previous process left runnable or running (takeover handles the rest).
-            let afterOpen = true;
             const worker = {
                 advance: (sessionId, signal) => advance(ctx, sessionId, signal),
-                sweep: async () => {
-                    const first = afterOpen;
-                    afterOpen = false;
-                    await sweep(ctx, { afterOpen: first });
-                },
+                deliver: (actionId, signal) => deliverAction(ctx, actionId, signal),
+                sweep: () => sweep(ctx),
             };
             const unregister = workers.register(config.tenantId, worker);
             detach = async () => {
@@ -227,28 +227,21 @@ export class TenantRuntime {
             throw error;
         }
     }
-    handle(request, response, url) {
-        return handle(this.ctx, request, response, url);
+    async fetch(request, node) {
+        return await tenantApi().fetch(request, { ...node, tenant: this.ctx });
     }
     authorize(sessionId, request) {
         return authorize(this.ctx, sessionId, request);
     }
     async summary() {
-        const { store, live } = this.ctx;
-        const { counts, outbox } = await store.tx(async (t) => ({
-            counts: await t.counts(),
-            outbox: await t.outboxStats(),
-        }));
+        const { store } = this.ctx;
+        const counts = await store.tx((t) => t.counts());
         return {
             ready: !this.ctx.closing && !this.ctx.closed,
             runningSessions: counts.runningSessions,
-            connectedExecutors: connectedExecutorCount(live),
+            inFlightDeliveries: [...this.ctx.work.deliveries.values()].reduce((n, set) => n + set.size, 0),
             pendingActions: counts.pendingActions,
             uncertainEffects: counts.uncertainEffects,
-            outboxDepth: outbox.depth,
-            relayLagMs: outbox.oldestCreatedAt === null
-                ? 0
-                : Math.max(0, Date.now() - Date.parse(outbox.oldestCreatedAt)),
         };
     }
     drain(activeWork, timeoutMs = 30_000) {
@@ -263,7 +256,7 @@ export class TenantRuntime {
     }
     /**
      * The Durable Streams seam's status for Tenant status and readiness: S2 reachability, the
-     * basin, outbox depth and relay lag (`streamsStatus` in `tenant/streams.ts`).
+     * basin, its generation and the Tenant's relay (`streamsStatus` in `tenant/streams.ts`).
      */
     streamsStatus() {
         return streamsStatus(this.ctx);
@@ -278,15 +271,39 @@ export class TenantRuntime {
         // A shutdown abort: the advances leave their sessions to the next advance, unsettled.
         abortAll(ctx, "shutdown");
         const idleBy = Date.now() + this.closeGraceMs;
-        await this.detach();
-        await ctx.mcp.close();
-        endAllStreams(ctx.live);
-        // Bounded: an advance that ignores its abort is abandoned; its lease lapses (§11.4).
-        await waitForIdle(ctx, Math.max(0, idleBy - Date.now()));
-        await ctx.sandbox.close();
-        ctx.closed = true;
-        await closeStreams(ctx);
-        await ctx.store.close();
+        // Every step runs even when one before it fails, so nothing is left running; the first
+        // error is rethrown at the end.
+        const steps = [
+            ["detach", () => this.detach()],
+            ["mcp", () => ctx.mcp.close()],
+            ["session streams", () => endAllStreams(ctx.sessionStreams)],
+            // Bounded: an advance that ignores its abort is abandoned; its lease lapses (§11.4).
+            ["idle", () => waitForIdle(ctx, Math.max(0, idleBy - Date.now()))],
+            ["sandbox", () => ctx.sandbox.close()],
+            [
+                "streams",
+                () => {
+                    ctx.closed = true;
+                    return closeStreams(ctx);
+                },
+            ],
+            ["store", () => ctx.store.close()],
+        ];
+        let failure;
+        for (const [step, run] of steps) {
+            try {
+                await run();
+            }
+            catch (error) {
+                ctx.config.logger.warn("tenant close step failed", {
+                    step,
+                    message: error instanceof Error ? error.message : String(error),
+                });
+                failure ??= { error };
+            }
+        }
+        if (failure)
+            throw failure.error;
     }
 }
 /**

package/dist/tenant/scheduler.d.ts

@@ -3,11 +3,16 @@ import { type AdvanceAbortKind } from "./worker.js";
 export interface WorkState {
     /** Advances running on this process, by session id. */
     readonly running: Map<string, AbortController>;
+    /** Deliveries to Action endpoints in flight on this process, by their Action's session id. */
+    readonly deliveries: Map<string, Set<AbortController>>;
+    /** When each Action last reported a failed delivery (`action.delivery_failed` throttling). */
+    readonly deliveryNotices: Map<string, number>;
 }
 export declare function createWorkState(): WorkState;
 /**
- * Abort the advance of `id` if it runs on this process. Cancel calls it after committing
- * `cancelled`, and so does the control stream for cancels made elsewhere.
+ * Abort the advance of `id` if it runs on this process, and its deliveries to Action endpoints.
+ * Cancel calls it after committing `cancelled`, and so does the control stream for cancels made
+ * elsewhere.
  */
 export declare function abortLocal(ctx: TenantContext, id: string, kind?: AdvanceAbortKind): void;
 /** Reset: abort every advance running on this process; their sessions are being cleared. */

package/dist/tenant/scheduler.js

@@ -13,7 +13,7 @@ import { randomUUID } from "node:crypto";
 import { command } from "./commands.js";
 import { AdvanceAbort } from "./worker.js";
 export function createWorkState() {
-    return { running: new Map() };
+    return { running: new Map(), deliveries: new Map(), deliveryNotices: new Map() };
 }
 const MESSAGES = {
     cancel: "Turn cancelled",
@@ -26,13 +26,17 @@ function abortWith(controller, kind) {
         controller.abort(new AdvanceAbort(kind, MESSAGES[kind]));
 }
 /**
- * Abort the advance of `id` if it runs on this process. Cancel calls it after committing
- * `cancelled`, and so does the control stream for cancels made elsewhere.
+ * Abort the advance of `id` if it runs on this process, and its deliveries to Action endpoints.
+ * Cancel calls it after committing `cancelled`, and so does the control stream for cancels made
+ * elsewhere.
  */
 export function abortLocal(ctx, id, kind = "cancel") {
     const controller = ctx.work.running.get(id);
     if (controller)
         abortWith(controller, kind);
+    // A cancel also closes the session's deliveries, which aborts the tools' `ctx.signal`.
+    for (const delivery of ctx.work.deliveries.get(id) ?? [])
+        abortWith(delivery, kind);
 }
 /** Reset: abort every advance running on this process; their sessions are being cleared. */
 export function clearWork(ctx) {