@nylorun/runtime 0.46.0-beta → 0.46.2-beta

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.46.2-beta
+
+### Patch Changes
+
+- 4b91451: Pods on Agent Substrate (preview):
+
+  - **A crashed actor recovers.** A wake that finds the actor crashed, or that crashes it, reverts the actor to its last snapshot once and resumes it, as Substrate requires. Before, the Agent stayed failed until it was reset.
+  - **A wake during an idle suspend is no longer lost.** The resume waits for the suspend to finish instead of dropping the wake.
+  - **One resume call per wake.** It is no longer cancelled after 2 minutes, which also cancelled the restore in progress. It is tried again only while Substrate is busy, for at most 5 minutes (the Runtime's own ready deadline). It stops as soon as a later request replaces it, and holds no lock during the call.
+  - **Each resume is logged** as `substrate resumed` with its time in `ms`.
+  - **Removed:**
+    - `RES_OPTIONS`, which is no longer set in the actor's environment. The edge's `/etc/hosts` pin already spares the engine any DNS lookup, and the Agent's own resolver is its own.
+    - The shutdown-time sweep, since the Runtime never sends a shutdown time.
+
+## 0.46.1-beta
+
+### Patch Changes
+
+- 3a9933c: A pod's engine rejoins at once after an Agent Substrate restore. When the process resumes from a memory snapshot, the engine notices the jump in the wall clock (a 100 ms timer finding more than 1.5 s gone), drops the connection attempt the freeze caught in flight, its host-token renew or join included, and dials again without a backoff pause. Before, it waited for TCP to resend the packet lost with the old network: 1 s, or 3 s on a second loss, which the Phase 1 exit test measured in 4 of 10 wakes. Only pod engines watch for this (`harness_clock_jumped` in their log); an open connection is never dropped by it.
+
 ## 0.46.0-beta
 
 ### Minor Changes

package/dist/harness/clock-jump.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Notices that the process was frozen and resumed, as an Agent Substrate actor is by a memory
+ * snapshot's restore: a timer that should fire every `intervalMs` finds the wall clock moved on
+ * by more than `jumpMs` beyond that. A pod's engine uses it to drop a connection attempt the
+ * freeze caught in flight (its packets went with the old network, so it would wait for TCP to
+ * send them again, 1 s and then 3 s; P1.8's third run) and to dial again at once.
+ *
+ * An event loop held up as long (a long GC pause) reads as a jump too; the caller only redials,
+ * so that costs one connection attempt, never a working connection.
+ */
+export interface ClockJumpWatch {
+    stop(): void;
+}
+export declare function watchClockJumps(onJump: (jumpedMs: number) => void, options?: {
+    intervalMs?: number;
+    jumpMs?: number;
+    now?: () => number;
+}): ClockJumpWatch;

package/dist/harness/clock-jump.js

@@ -0,0 +1,15 @@
+export function watchClockJumps(onJump, options = {}) {
+    const intervalMs = options.intervalMs ?? 100;
+    const jumpMs = options.jumpMs ?? 1_500;
+    const now = options.now ?? Date.now;
+    let last = now();
+    const timer = setInterval(() => {
+        const at = now();
+        const late = at - last - intervalMs;
+        last = at;
+        if (late > jumpMs)
+            onJump(late);
+    }, intervalMs);
+    timer.unref();
+    return { stop: () => clearInterval(timer) };
+}

package/dist/harness/main.js

@@ -70,13 +70,15 @@ export async function runHarness(stack, baseline) {
         if (!pod.afterSeed)
             await prepare();
     }
-    const token = pod ? () => pod.token() : config.token;
+    const token = pod ? (signal) => pod.token(signal) : config.token;
     if (token === undefined)
         throw new Error("--service harness needs NYLORUN_HARNESS_TOKEN");
     const runTokens = harnessRunTokens();
     const service = startHarnessService({
         url: config.url,
         token,
+        // A pod's engine may be frozen and resumed (an Agent Substrate restore): redial at once.
+        ...(pod ? { redialOnClockJump: true } : {}),
         ...(pod
             ? { sandboxBackends: [localBackend({ env: baseline, proxyEnv: () => pod.proxyEnv(), harnessRoot: config.root })] }
             : {}),

package/dist/harness/pod.d.ts

@@ -23,8 +23,11 @@ export interface DiskSetupReport {
     readonly output?: string;
 }
 export interface PodHost {
-    /** A current host token, for the next connection. Throws when neither renewal nor join works. */
-    token(): Promise<string>;
+    /**
+     * A current host token, for the next connection. Throws when neither renewal nor join works,
+     * or when `signal` drops the attempt.
+     */
+    token(signal?: AbortSignal): Promise<string>;
     /** `HTTPS_PROXY` and friends with the current egress token; none before the first join. */
     proxyEnv(): Record<string, string>;
     /** The host epoch of the tokens held. */

package/dist/harness/pod.js

@@ -38,7 +38,8 @@ export function podHost(config, logger, options = {}) {
     let afterSeed = false;
     /** The CA in the bundle on disk, once one is written. */
     let bundled;
-    const post = async (path, init) => {
+    const post = async (path, init, signal) => {
+        const timeout = AbortSignal.timeout(hostTuning().harness.podJoinTimeoutMs);
         const response = await call(`${config.httpUrl}/nylorun/harness/v1/host/${path}`, {
             method: "POST",
             headers: {
@@ -48,7 +49,7 @@ export function podHost(config, logger, options = {}) {
                 ...(path === "join" && imageEngine ? { [IMAGE_ENGINE_HEADER]: RUNTIME_VERSION } : {}),
             },
             ...(init.body === undefined ? {} : { body: JSON.stringify(init.body) }),
-            signal: AbortSignal.timeout(hostTuning().harness.podJoinTimeoutMs),
+            signal: signal ? AbortSignal.any([timeout, signal]) : timeout,
         });
         const text = await response.text();
         if (response.status === 401 || response.status === 400)
@@ -97,7 +98,7 @@ export function podHost(config, logger, options = {}) {
             return id;
         }
     };
-    const join = async () => {
+    const join = async (signal) => {
         const joinToken = (await readFile(config.joinFile, "utf8")).trim();
         const volume = await volumeId();
         held = await post("join", {
@@ -107,7 +108,7 @@ export function podHost(config, logger, options = {}) {
                 joinToken,
                 ...(volume === undefined ? {} : { volumeId: volume }),
             },
-        });
+        }, signal);
         logger.info("sandbox_host_joined", { sandboxId: config.sandboxId, epoch: held.epoch });
         return held;
     };
@@ -122,11 +123,11 @@ export function podHost(config, logger, options = {}) {
             signal: AbortSignal.timeout(hostTuning().harness.podRequestTimeoutMs),
         });
     };
-    const renew = async () => {
+    const renew = async (signal) => {
         if (!held)
             return undefined;
         try {
-            held = await post("renew", { bearer: held.hostToken });
+            held = await post("renew", { bearer: held.hostToken }, signal);
             return held;
         }
         catch (error) {
@@ -143,8 +144,8 @@ export function podHost(config, logger, options = {}) {
     }, options.renewMs ?? hostTuning().harness.podRenewMs);
     timer.unref();
     return {
-        async token() {
-            const current = (await renew()) ?? (await join());
+        async token(signal) {
+            const current = (await renew(signal)) ?? (await join(signal));
             return current.hostToken;
         },
         proxyEnv() {

package/dist/harness/service.d.ts

@@ -7,6 +7,7 @@ import { type EngineTuning } from "../tuning.js";
 export interface HarnessServiceOptions {
     readonly url: string;
     readonly token: HarnessClientOptions["token"];
+    readonly redialOnClockJump?: boolean;
     /** Where workspaces (and their records) live. */
     readonly paths: {
         readonly sandboxes: string;

package/dist/harness/service.js

@@ -124,6 +124,7 @@ export function startHarnessService(options) {
     client = connectHarness({
         url: options.url,
         token: options.token,
+        ...(options.redialOnClockJump ? { redialOnClockJump: true } : {}),
         logger,
         executors,
         serve,

package/dist/harness/ws-client.d.ts

@@ -5,18 +5,24 @@
  * core's `workspace.*` requests (`serve`). When the connection is lost its runs stop (core gives
  * their segments to the next advance, which resumes them by replay) and the client connects
  * again after a jittered backoff (`harness.reconnect` in the Host tuning, 0.5 to 10 s).
+ *
+ * A pod's engine (`redialOnClockJump`) also watches for a freeze and resume (an Agent Substrate
+ * restore): it drops the connection attempt in flight, token request included, and dials again
+ * at once, instead of waiting for TCP to resend what the freeze lost.
  */
 import { type HarnessChannel, type RequestHandler } from "@nylorun/core/harness-api";
 import { type HarnessOptions } from "@nylorun/harness/api";
 import { type WsPortOptions } from "./ws-port.js";
+import { watchClockJumps } from "./clock-jump.js";
 export interface HarnessClientOptions extends Omit<HarnessOptions, "channel" | "logger"> {
     /** The Harness API, e.g. `ws://runtime:4200/nylorun/harness/v1` (`NYLORUN_HARNESS_URL`). */
     readonly url: string;
     /**
      * `NYLORUN_HARNESS_TOKEN`, or, for a pod sandbox's engine (F7.2), a function that answers
-     * a current host token before each connection (it renews, or joins again).
+     * a current host token before each connection (it renews, or joins again). The signal aborts
+     * it when the attempt is dropped.
      */
-    readonly token: string | (() => Promise<string>);
+    readonly token: string | ((signal?: AbortSignal) => Promise<string>);
     readonly logger: {
         info(message: string, fields?: Record<string, unknown>): void;
         warn(message: string, fields?: Record<string, unknown>): void;
@@ -29,6 +35,13 @@ export interface HarnessClientOptions extends Omit<HarnessOptions, "channel" | "
         readonly maxMs: number;
     };
     readonly liveness?: Omit<WsPortOptions, "ping">;
+    /**
+     * Drop an attempt in flight and dial again at once when the process was frozen and resumed
+     * (`clock-jump.ts`): a pod's engine, which Agent Substrate suspends to a memory snapshot.
+     */
+    readonly redialOnClockJump?: boolean;
+    /** The clock and timings `redialOnClockJump` watches with (tests set them). */
+    readonly clockJumps?: Parameters<typeof watchClockJumps>[1];
 }
 export interface HarnessClient {
     /** Resolves once the first connection's `hello` is answered. */

package/dist/harness/ws-client.js

@@ -5,6 +5,10 @@
  * core's `workspace.*` requests (`serve`). When the connection is lost its runs stop (core gives
  * their segments to the next advance, which resumes them by replay) and the client connects
  * again after a jittered backoff (`harness.reconnect` in the Host tuning, 0.5 to 10 s).
+ *
+ * A pod's engine (`redialOnClockJump`) also watches for a freeze and resume (an Agent Substrate
+ * restore): it drops the connection attempt in flight, token request included, and dials again
+ * at once, instead of waiting for TCP to resend what the freeze lost.
  */
 import { createChannel } from "@nylorun/core/harness-api";
 import { createHarness } from "@nylorun/harness/api";
@@ -12,6 +16,7 @@ import { WebSocket } from "ws";
 import { HARNESS_API_HEADER, HARNESS_API_TRANSPORT, HARNESS_MAX_PAYLOAD } from "@nylorun/core/harness-api";
 import { wsPort } from "./ws-port.js";
 import { hostTuning } from "../tuning.js";
+import { watchClockJumps } from "./clock-jump.js";
 export function connectHarness(options) {
     const { logger } = options;
     const tuning = hostTuning().harness;
@@ -20,30 +25,49 @@ export function connectHarness(options) {
     let connected = false;
     let current;
     let wake;
+    /** The connection attempt in flight (token, then handshake), until it opens or fails. */
+    let attempting;
+    let attempt = 0;
     let markReady;
     const ready = new Promise((resolve) => (markReady = resolve));
     const open = async () => {
-        const token = typeof options.token === "string" ? options.token : await options.token();
-        return new Promise((resolve, reject) => {
-            const socket = new WebSocket(options.url, {
-                headers: { authorization: `Bearer ${token}`, [HARNESS_API_HEADER]: HARNESS_API_TRANSPORT },
-                maxPayload: HARNESS_MAX_PAYLOAD,
-                handshakeTimeout: tuning.handshakeTimeoutMs,
-            });
-            socket.once("open", () => {
-                socket.removeAllListeners("error");
-                resolve(socket);
-            });
-            socket.once("unexpected-response", (_request, response) => {
-                socket.removeAllListeners("error");
-                socket.on("error", () => undefined);
-                reject(new Error(`The Harness API refused the connection: ${response.statusCode ?? 0}`));
-                response.resume();
-                socket.terminate();
-            });
-            socket.once("error", reject);
-        });
+        const controller = new AbortController();
+        attempting = controller;
+        try {
+            const token = typeof options.token === "string" ? options.token : await options.token(controller.signal);
+            controller.signal.throwIfAborted();
+            return await dial(token, controller.signal);
+        }
+        finally {
+            if (attempting === controller)
+                attempting = undefined;
+        }
     };
+    const dial = (token, signal) => new Promise((resolve, reject) => {
+        const socket = new WebSocket(options.url, {
+            headers: { authorization: `Bearer ${token}`, [HARNESS_API_HEADER]: HARNESS_API_TRANSPORT },
+            maxPayload: HARNESS_MAX_PAYLOAD,
+            handshakeTimeout: tuning.handshakeTimeoutMs,
+        });
+        socket.once("open", () => {
+            socket.removeAllListeners("error");
+            resolve(socket);
+        });
+        socket.once("unexpected-response", (_request, response) => {
+            socket.removeAllListeners("error");
+            socket.on("error", () => undefined);
+            reject(new Error(`The Harness API refused the connection: ${response.statusCode ?? 0}`));
+            response.resume();
+            socket.terminate();
+        });
+        socket.once("error", reject);
+        signal.addEventListener("abort", () => {
+            socket.removeAllListeners("error");
+            socket.on("error", () => undefined);
+            socket.terminate();
+            reject(signal.reason instanceof Error ? signal.reason : new Error("connection attempt dropped"));
+        }, { once: true });
+    });
     const pause = (ms) => new Promise((resolve) => {
         const timer = setTimeout(done, ms);
         timer.unref();
@@ -81,8 +105,16 @@ export function connectHarness(options) {
             current = undefined;
         }
     };
+    const watch = options.redialOnClockJump
+        ? watchClockJumps((jumpedMs) => {
+            const inFlight = attempting !== undefined;
+            logger.info("harness_clock_jumped", { jumpedMs, redial: inFlight || wake !== undefined });
+            attempt = 0;
+            attempting?.abort(new Error(`the process resumed after ${jumpedMs} ms`));
+            wake?.();
+        }, options.clockJumps)
+        : undefined;
     const loop = (async () => {
-        let attempt = 0;
         while (!stopped) {
             const started = Date.now();
             try {
@@ -113,6 +145,8 @@ export function connectHarness(options) {
         channel: () => current?.channel,
         async stop(waitMs = tuning.stopMs) {
             stopped = true;
+            watch?.stop();
+            attempting?.abort(new Error("harness stopped"));
             wake?.();
             const live = current;
             if (live) {

package/dist/management-openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.2.0",
   "info": {
     "title": "Nylorun Management API",
-    "version": "0.46.0-beta",
+    "version": "0.46.2-beta",
     "x-nylorun-protocol": {
       "min": 16,
       "max": 18,

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.2.0",
   "info": {
     "title": "Nylorun Runtime API",
-    "version": "0.46.0-beta",
+    "version": "0.46.2-beta",
     "x-nylorun-protocol": {
       "min": 16,
       "max": 18,

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const RUNTIME_VERSION = "0.46.0-beta";
+export declare const RUNTIME_VERSION = "0.46.2-beta";

package/dist/version.js

@@ -1,3 +1,3 @@
 // Kept in step with package.json by runtime/server/scripts/check-package.mjs, which runs in npm run check.
 // release:prepare also rewrites this when it bumps @nylorun/runtime.
-export const RUNTIME_VERSION = "0.46.0-beta";
+export const RUNTIME_VERSION = "0.46.2-beta";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nylorun/runtime",
-  "version": "0.46.0-beta",
+  "version": "0.46.2-beta",
   "description": "Nylorun Runtime: the agent execution Host on Postgres, Restate and S2, and model providers.",
   "type": "module",
   "license": "Apache-2.0",
@@ -59,7 +59,7 @@
     "db:studio": "drizzle-kit studio"
   },
   "dependencies": {
-    "@ag-ui/core": "~1.0.3",
+    "@ag-ui/core": "~1.0.4",
     "@asteasolutions/zod-to-openapi": "^9.1.0",
     "@earendil-works/pi-ai": "1.3.0",
     "@hono/node-server": "^2.1.5",
@@ -85,7 +85,7 @@
   },
   "devDependencies": {
     "@a2a-js/sdk": "~1.3.0",
-    "@ag-ui/client": "1.0.3",
+    "@ag-ui/client": "1.0.4",
     "@scalar/openapi-parser": "^0.29.13",
     "@types/node": "^26.6.5",
     "@types/pg": "^8.23.1",