@nylorun/runtime 0.13.0-beta → 0.15.0-beta

package/dist/tenant/runtime.js

@@ -1,23 +1,27 @@
 import { mkdirSync } from "node:fs";
 import { resolveFlowLimits } from "../core/limits.js";
-import { loadExecutorRegistry } from "../core/executors.js";
-import { scriptedModel, gatewayModel, toolFixtureModel, } from "../core/provider.js";
+import { scriptedModel, toolFixtureModel, } from "../core/provider.js";
+import { tenantModelGate } from "../gates/in-process.js";
 import { createKekFile, readVaultKek } from "../vault/kek.js";
+import { SigningKeys } from "./signing-keys.js";
 import { VaultService } from "../vault/service.js";
 import { McpPool } from "../mcp/pool.js";
 import { SandboxManager } from "../sandbox/manager.js";
 import { defaultSandboxBackends } from "../sandbox/select.js";
 import { MemoryExecution } from "../execution/memory.js";
-import { QuarantineError } from "./quarantine-error.js";
+import { openError } from "./cause.js";
 import { MemoryStreams } from "../streams/memory.js";
-import { connectedExecutorCount, createLiveHub, endAllStreams, } from "./live.js";
-import { closeStreams, drainOutbox, streamsStatus, wireStreams, } from "./streams.js";
+import { createSessionStreams, endAllStreams, } from "./session-streams.js";
+import { closeStreams, streamsStatus, wireStreams, } from "./streams.js";
 import { abortAll, abortLocal, createWorkState, drain, waitForIdle, } from "./scheduler.js";
 import { advance } from "./advance.js";
 import { sweep } from "./sweep.js";
+import { deliverAction } from "./delivery.js";
 import { TenantWorkers, WORKER_ID, } from "./worker.js";
 import { authorize } from "./effects.js";
-import { handle } from "./routes.js";
+import { inProcessToolGate } from "../gates/tool-gate.js";
+import { inProcessKeys } from "../keys/keys.js";
+import { tenantApi } from "../api/http/app.js";
 /** Default ownership lease of an advance; the heartbeat renews it every third. */
 const DEFAULT_OWNER_LEASE_MS = 30_000;
 function validateConfig(config) {
@@ -66,24 +70,32 @@ export class TenantRuntime {
         let wired;
         let detach;
         try {
-            let kek = readVaultKek({
-                vaultKek: hooks.vaultKek,
-                vaultKekPath: paths.kek,
-            });
-            if ((await store.tx((t) => t.countCredentials())) > 0 && !kek) {
-                throw new QuarantineError("kek-missing", "Vault key-encryption key is missing for ciphertext in this Tenant", "restore the vault-kek file in the Tenant directory");
+            // With the keys service (F4.2) the key lives in the gateway: this process never reads,
+            // creates or holds it, and the gateway reports a missing key on its readiness.
+            let kek = hooks.keys
+                ? undefined
+                : readVaultKek({
+                    vaultKek: hooks.vaultKek,
+                    vaultKekPath: paths.kek,
+                });
+            if (!hooks.keys) {
+                const sealed = await store.tx(async (t) => (await t.countCredentials()) + (await t.countSigningKeys()));
+                if (sealed > 0 && !kek) {
+                    throw openError("kek-missing", "Vault key-encryption key is missing for ciphertext (vault credentials or signing keys) in this Tenant");
+                }
             }
             const createKekIfMissing = hooks.createKekIfMissing !== false;
             const ensureKek = () => {
+                if (hooks.keys)
+                    throw new Error("The vault key lives in the gateway's keys service: this process never reads it");
                 if (kek)
                     return kek;
                 if (!createKekIfMissing)
-                    throw new QuarantineError("kek-missing", "Vault key-encryption key is required", "restore the vault-kek file in the Tenant directory");
+                    throw openError("kek-missing", "Vault key-encryption key is required");
                 kek = createKekFile(paths.kek);
                 return kek;
             };
             const opened = store;
-            const registry = await loadExecutorRegistry(opened);
             // A seeded Tenant setting wins over the Host default (A18).
             const seededBackend = await opened.tx((t) => t.getSetting("sandbox.backend"));
             const vault = new VaultService({
@@ -93,10 +105,14 @@ export class TenantRuntime {
             });
             // `ctx` is assigned below; these callbacks only run once the Tenant is open.
             let ctx;
+            const toolGate = hooks.toolGate ?? inProcessToolGate(config.delivery ?? {});
+            const signingKeys = new SigningKeys({ tenantId: config.tenantId, kek: ensureKek });
+            const keys = hooks.keys ?? inProcessKeys({ store: opened, vault, signingKeys, kek: ensureKek });
             const mcp = new McpPool({
                 pluginData: paths.pluginData,
                 childEnv: config.childEnv,
                 authorize: (sessionId, request) => authorize(ctx, sessionId, request),
+                ...(toolGate.openMcp ? { openRemote: (server) => toolGate.openMcp(server) } : {}),
             });
             const ephemeral = config.mode === "ephemeral";
             const sandbox = new SandboxManager({
@@ -125,23 +141,19 @@ export class TenantRuntime {
                 modelProvider = scriptedModel(config.model.output);
             else if (config.model.kind === "fixture")
                 modelProvider = toolFixtureModel();
-            else if (config.model.kind === "gateway") {
-                const gateway = config.model;
-                modelProvider = async (effect, signal) => {
-                    const secret = await vault.readHostModel();
-                    const token = typeof secret?.credential.key === "string"
-                        ? secret.credential.key
-                        : "";
-                    return gatewayModel({
-                        url: gateway.url,
-                        model: gateway.model,
-                        token,
-                    })(effect, signal);
-                };
-            }
             else
                 modelProvider = scriptedModel();
-            const live = createLiveHub();
+            // With the gates service the loop never reads a model credential: only a Runtime
+            // without one (embedding, the ephemeral Runtime, tests) reads it here.
+            const modelGate = hooks.modelGate ??
+                tenantModelGate({
+                    store: opened,
+                    kek: ensureKek,
+                    root: paths.home,
+                    logger: config.logger,
+                    ...(config.modelCall ? { settings: config.modelCall } : {}),
+                });
+            const sessionStreams = createSessionStreams();
             const local = hooks.execution
                 ? undefined
                 : new MemoryExecution({
@@ -160,16 +172,19 @@ export class TenantRuntime {
                 envelope,
                 store: opened,
                 vault,
-                registry,
                 mcp,
                 sandbox,
                 flowLimits,
                 modelProvider,
                 useVaultModel,
+                modelGate,
+                toolGate,
                 closing: false,
                 closed: false,
                 work: createWorkState(),
-                live,
+                sessionStreams,
+                signingKeys,
+                keys,
                 workerId: hooks.workerId ?? WORKER_ID,
                 ownerLeaseMs: config.ownerLeaseMs ?? DEFAULT_OWNER_LEASE_MS,
                 wake: async (sessionId, wake) => {
@@ -177,7 +192,12 @@ export class TenantRuntime {
                         return;
                     await execution.wake(config.tenantId, sessionId, wake);
                 },
-                abortLocal: (sessionId) => abortLocal(ctx, sessionId),
+                abortLocal: (sessionId, turnId) => abortLocal(ctx, sessionId, "cancel", turnId),
+                deliver: async (actionId) => {
+                    if (ctx.closing || ctx.closed)
+                        return;
+                    await execution.deliver(config.tenantId, actionId);
+                },
                 ...(hooks.execution?.stuckInvocations
                     ? {
                         stuckInvocations: () => hooks.execution.stuckInvocations(config.tenantId),
@@ -194,21 +214,15 @@ export class TenantRuntime {
                 streams: hooks.streams ?? new MemoryStreams(),
                 ownsStreams: !hooks.streams,
                 tenantId: config.tenantId,
-            });
-            // Outbox rows a lost relay step left behind are appended by the sweep.
-            sweepHooks.add(async () => {
-                await drainOutbox(ctx);
+                ...(hooks.hostRelay ? { hostRelay: true } : {}),
+                ...(hooks.retireGraceMs !== undefined ? { retireGraceMs: hooks.retireGraceMs } : {}),
             });
             // Register the handlers, then arm the sweep: its first pass runs at once and re-wakes
             // sessions a previous process left runnable or running (takeover handles the rest).
-            let afterOpen = true;
             const worker = {
                 advance: (sessionId, signal) => advance(ctx, sessionId, signal),
-                sweep: async () => {
-                    const first = afterOpen;
-                    afterOpen = false;
-                    await sweep(ctx, { afterOpen: first });
-                },
+                deliver: (actionId, signal) => deliverAction(ctx, actionId, signal),
+                sweep: () => sweep(ctx),
             };
             const unregister = workers.register(config.tenantId, worker);
             detach = async () => {
@@ -227,28 +241,21 @@ export class TenantRuntime {
             throw error;
         }
     }
-    handle(request, response, url) {
-        return handle(this.ctx, request, response, url);
+    async fetch(request, node) {
+        return await tenantApi().fetch(request, { ...node, tenant: this.ctx });
     }
     authorize(sessionId, request) {
         return authorize(this.ctx, sessionId, request);
     }
     async summary() {
-        const { store, live } = this.ctx;
-        const { counts, outbox } = await store.tx(async (t) => ({
-            counts: await t.counts(),
-            outbox: await t.outboxStats(),
-        }));
+        const { store } = this.ctx;
+        const counts = await store.tx((t) => t.counts());
         return {
             ready: !this.ctx.closing && !this.ctx.closed,
             runningSessions: counts.runningSessions,
-            connectedExecutors: connectedExecutorCount(live),
+            inFlightDeliveries: [...this.ctx.work.deliveries.values()].reduce((n, set) => n + set.size, 0),
             pendingActions: counts.pendingActions,
             uncertainEffects: counts.uncertainEffects,
-            outboxDepth: outbox.depth,
-            relayLagMs: outbox.oldestCreatedAt === null
-                ? 0
-                : Math.max(0, Date.now() - Date.parse(outbox.oldestCreatedAt)),
         };
     }
     drain(activeWork, timeoutMs = 30_000) {
@@ -263,7 +270,7 @@ export class TenantRuntime {
     }
     /**
      * The Durable Streams seam's status for Tenant status and readiness: S2 reachability, the
-     * basin, outbox depth and relay lag (`streamsStatus` in `tenant/streams.ts`).
+     * basin, its generation and the Tenant's relay (`streamsStatus` in `tenant/streams.ts`).
      */
     streamsStatus() {
         return streamsStatus(this.ctx);
@@ -278,15 +285,39 @@ export class TenantRuntime {
         // A shutdown abort: the advances leave their sessions to the next advance, unsettled.
         abortAll(ctx, "shutdown");
         const idleBy = Date.now() + this.closeGraceMs;
-        await this.detach();
-        await ctx.mcp.close();
-        endAllStreams(ctx.live);
-        // Bounded: an advance that ignores its abort is abandoned; its lease lapses (§11.4).
-        await waitForIdle(ctx, Math.max(0, idleBy - Date.now()));
-        await ctx.sandbox.close();
-        ctx.closed = true;
-        await closeStreams(ctx);
-        await ctx.store.close();
+        // Every step runs even when one before it fails, so nothing is left running; the first
+        // error is rethrown at the end.
+        const steps = [
+            ["detach", () => this.detach()],
+            ["mcp", () => ctx.mcp.close()],
+            ["session streams", () => endAllStreams(ctx.sessionStreams)],
+            // Bounded: an advance that ignores its abort is abandoned; its lease lapses (§11.4).
+            ["idle", () => waitForIdle(ctx, Math.max(0, idleBy - Date.now()))],
+            ["sandbox", () => ctx.sandbox.close()],
+            [
+                "streams",
+                () => {
+                    ctx.closed = true;
+                    return closeStreams(ctx);
+                },
+            ],
+            ["store", () => ctx.store.close()],
+        ];
+        let failure;
+        for (const [step, run] of steps) {
+            try {
+                await run();
+            }
+            catch (error) {
+                ctx.config.logger.warn("tenant close step failed", {
+                    step,
+                    message: error instanceof Error ? error.message : String(error),
+                });
+                failure ??= { error };
+            }
+        }
+        if (failure)
+            throw failure.error;
     }
 }
 /**

package/dist/tenant/scheduler.d.ts

@@ -3,13 +3,20 @@ import { type AdvanceAbortKind } from "./worker.js";
 export interface WorkState {
     /** Advances running on this process, by session id. */
     readonly running: Map<string, AbortController>;
+    /** The turn each running advance belongs to, by session id (`null`: none). */
+    readonly runningTurns: Map<string, string | null>;
+    /** Deliveries to Action endpoints in flight on this process, by their Action's session id. */
+    readonly deliveries: Map<string, Set<AbortController>>;
+    /** When each Action last reported a failed delivery (`action.delivery_failed` throttling). */
+    readonly deliveryNotices: Map<string, number>;
 }
 export declare function createWorkState(): WorkState;
 /**
- * Abort the advance of `id` if it runs on this process. Cancel calls it after committing
- * `cancelled`, and so does the control stream for cancels made elsewhere.
+ * Abort the advance of `id` if it runs on this process, and its deliveries to Action endpoints.
+ * Cancel calls it after committing `cancelled`, and so does the control stream for cancels made
+ * elsewhere. With `turnId`, an advance of another turn keeps running: the signal came late.
  */
-export declare function abortLocal(ctx: TenantContext, id: string, kind?: AdvanceAbortKind): void;
+export declare function abortLocal(ctx: TenantContext, id: string, kind?: AdvanceAbortKind, turnId?: string): void;
 /** Reset: abort every advance running on this process; their sessions are being cleared. */
 export declare function clearWork(ctx: TenantContext): void;
 /** Stop new advances; cancel active turns if asked; wait for running advances until the timeout. */

package/dist/tenant/scheduler.js

@@ -13,7 +13,7 @@ import { randomUUID } from "node:crypto";
 import { command } from "./commands.js";
 import { AdvanceAbort } from "./worker.js";
 export function createWorkState() {
-    return { running: new Map() };
+    return { running: new Map(), runningTurns: new Map(), deliveries: new Map(), deliveryNotices: new Map() };
 }
 const MESSAGES = {
     cancel: "Turn cancelled",
@@ -26,13 +26,17 @@ function abortWith(controller, kind) {
         controller.abort(new AdvanceAbort(kind, MESSAGES[kind]));
 }
 /**
- * Abort the advance of `id` if it runs on this process. Cancel calls it after committing
- * `cancelled`, and so does the control stream for cancels made elsewhere.
+ * Abort the advance of `id` if it runs on this process, and its deliveries to Action endpoints.
+ * Cancel calls it after committing `cancelled`, and so does the control stream for cancels made
+ * elsewhere. With `turnId`, an advance of another turn keeps running: the signal came late.
  */
-export function abortLocal(ctx, id, kind = "cancel") {
+export function abortLocal(ctx, id, kind = "cancel", turnId) {
     const controller = ctx.work.running.get(id);
-    if (controller)
+    if (controller && (turnId === undefined || ctx.work.runningTurns.get(id) === turnId))
         abortWith(controller, kind);
+    // A cancel also closes the session's deliveries, which aborts the tools' `ctx.signal`.
+    for (const delivery of ctx.work.deliveries.get(id) ?? [])
+        abortWith(delivery, kind);
 }
 /** Reset: abort every advance running on this process; their sessions are being cleared. */
 export function clearWork(ctx) {

package/dist/tenant/session-streams.d.ts

@@ -0,0 +1,127 @@
+/**
+ * Live delivery over Durable Streams (architecture §12.4, "Reading"): session history
+ * (`GET /v1/sessions/:id/items`) and session SSE (`GET /v1/sessions/:id/events`).
+ *
+ * - **Streams.** A session's events are in `sessions/<id>` in the Tenant's current basin
+ *   generation (`currentBasin`); cursors carry only the session id and sequence.
+ * - **History** reads the session's stream from the cursor up to the tail seen when the read
+ *   starts. The `agent` filter runs here, and the response cursor is the last record read. If
+ *   the streams fail, history answers `503`.
+ * - **SSE** shares one stream read per observed session in this process (a `SessionStream`)
+ *   among that session's observers. Each observer keeps the next sequence it needs and skips
+ *   what it already has, so a client resuming from `Last-Event-ID` sees no gap and no
+ *   duplicate. An observer behind the feed restarts the shared read from its own position.
+ *   A feed follows one basin generation: it ends once its session is gone or the Tenant moved
+ *   to a new generation (a reset, possibly on another node), checked when a client joins, when
+ *   its read fails, on a `sessions.reset` signal, and periodically (`checkSessionStreams`).
+ *
+ * Business code never writes here: events reach streams only through the stream relay, from
+ * the record, after commit.
+ */
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { LiveEvent } from "@nylorun/core/contracts";
+import type { TenantContext } from "./context.js";
+import type { StreamsWiring } from "./streams.js";
+/** Why the Runtime ended a stream a subject token opened. */
+export type StreamEndReason = "token_expired" | "revoked";
+/** Where an observer's events go: an SSE response, or an in-process reader. */
+export interface ObserverSink {
+    write(event: LiveEvent): void;
+    /** Ends the stream; with a reason, the client is told why before it ends. */
+    end(reason?: StreamEndReason): void;
+}
+/** The subject token a stream was opened with: it ends at expiry or revocation. */
+export interface StreamHolder {
+    readonly subject: string;
+    readonly epoch: number;
+    /** Epoch ms when the token expires. */
+    readonly expiresAt: number;
+}
+/** One client of a session: the next sequence it needs. */
+interface Observer {
+    readonly sink: ObserverSink;
+    next: number;
+    readonly holder?: StreamHolder;
+    /** Ends the observer when its token expires. */
+    deadline?: NodeJS.Timeout;
+}
+/** The shared stream read of one observed session. */
+interface SessionStream {
+    readonly sessionId: string;
+    /** The basin the feed reads (the Tenant's generation when it started). */
+    readonly basin: string;
+    readonly stream: string;
+    /** The sequence the shared read yields next. Every observer's `next` is at or past it. */
+    next: number;
+    readonly observers: Set<Observer>;
+    /** Aborts the current read; replaced when the read restarts from an earlier sequence. */
+    read: AbortController;
+}
+export interface SessionStreams {
+    /** Set once by `wireStreams`. */
+    wiring: StreamsWiring | undefined;
+    /** The Tenant's basin generation as this process last saw it (`wireStreams` keeps it). */
+    generation: number;
+    /** Session id → its shared read and observers. */
+    readonly sessions: Map<string, SessionStream>;
+}
+export declare function createSessionStreams(): SessionStreams;
+export declare function send(response: ServerResponse, data: string): void;
+/** Start an SSE response with keepalives; `onClose` runs once the client goes away. */
+export declare function openSse(request: IncomingMessage, response: ServerResponse, onClose: () => void): void;
+/** The resume cursor: `?cursor=` wins over `Last-Event-ID`. */
+export declare function requestCursor(request: IncomingMessage, url: URL): string | undefined;
+/** The basin the Tenant's session streams are in now. */
+export declare function currentBasin(ctx: TenantContext): string;
+/**
+ * `GET /v1/sessions/:id/items`: the session stream after the cursor, up to its tail when the
+ * read starts. The cursor returned is the last record read (the request's cursor when none
+ * was), whatever the `agent` filter kept.
+ */
+export declare function readHistory(ctx: TenantContext, sessionId: string, cursor: string | undefined, agent: string | undefined): Promise<{
+    items: LiveEvent[];
+    cursor: string | null;
+}>;
+/**
+ * `GET /v1/sessions/:id/events`: joins the session's shared read at the client's cursor and
+ * follows it until the client leaves, the session is reset or deleted, or the Tenant closes.
+ */
+export declare function streamSessionEvents(ctx: TenantContext, request: IncomingMessage, response: ServerResponse, sessionId: string, cursor: string | undefined, holder?: StreamHolder): Promise<void>;
+/** Thrown by `observeSession` when the Runtime ended the stream for its token. */
+export declare class StreamClosed extends Error {
+    readonly reason: StreamEndReason;
+    /** The cursor of the last event delivered, if any. */
+    readonly cursor: string | undefined;
+    constructor(reason: StreamEndReason, 
+    /** The cursor of the last event delivered, if any. */
+    cursor: string | undefined);
+}
+/**
+ * A session's events from `cursor`, in process: the same shared read and deadlines as an SSE
+ * client (`streamSessionEvents`). Ends when `signal` aborts or the feed ends; throws
+ * `StreamClosed` when the Runtime ends it for its token (expiry or revocation).
+ */
+export declare function observeSession(ctx: TenantContext, sessionId: string, cursor: string | undefined, options: {
+    holder?: StreamHolder;
+    signal: AbortSignal;
+}): AsyncGenerator<LiveEvent>;
+/** Internal events (`transcript.updated`) are recorded and streamed but never served. */
+export declare function served(event: LiveEvent): boolean;
+/**
+ * Ends every feed in this process whose session is gone or whose Tenant moved to a new basin
+ * generation, and moves this process to the Tenant's current generation. Run on a
+ * `sessions.reset` signal and periodically, so feeds on nodes other than the one that reset
+ * the Tenant end too.
+ */
+export declare function checkSessionStreams(ctx: TenantContext): Promise<void>;
+/**
+ * Ends the streams of `subject` opened with a token older than `epoch` (a revocation), on
+ * this process. Other processes do the same on the `subject.revoked` signal.
+ */
+export declare function endSubjectStreams(hub: SessionStreams, subject: string, epoch: number): void;
+/** Reset or close: end every session observer and stop every shared read. */
+export declare function clearObservers(hub: SessionStreams): void;
+/** Close: end every session observer. */
+export declare function endAllStreams(hub: SessionStreams): void;
+export declare function sleep(ms: number, signal: AbortSignal): Promise<void>;
+export {};