@nylorun/runtime 0.46.0-beta → 0.46.1-beta

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.46.1-beta
+
+### Patch Changes
+
+- 3a9933c: A pod's engine rejoins at once after an Agent Substrate restore. When the process resumes from a memory snapshot, the engine notices the jump in the wall clock (a 100 ms timer finding more than 1.5 s gone), drops the connection attempt the freeze caught in flight, its host-token renew or join included, and dials again without a backoff pause. Before, it waited for TCP to resend the packet lost with the old network: 1 s, or 3 s on a second loss, which the Phase 1 exit test measured in 4 of 10 wakes. Only pod engines watch for this (`harness_clock_jumped` in their log); an open connection is never dropped by it.
+
 ## 0.46.0-beta
 
 ### Minor Changes

package/dist/harness/clock-jump.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Notices that the process was frozen and resumed, as an Agent Substrate actor is by a memory
+ * snapshot's restore: a timer that should fire every `intervalMs` finds the wall clock moved on
+ * by more than `jumpMs` beyond that. A pod's engine uses it to drop a connection attempt the
+ * freeze caught in flight (its packets went with the old network, so it would wait for TCP to
+ * send them again, 1 s and then 3 s; P1.8's third run) and to dial again at once.
+ *
+ * An event loop held up as long (a long GC pause) reads as a jump too; the caller only redials,
+ * so that costs one connection attempt, never a working connection.
+ */
+export interface ClockJumpWatch {
+    stop(): void;
+}
+export declare function watchClockJumps(onJump: (jumpedMs: number) => void, options?: {
+    intervalMs?: number;
+    jumpMs?: number;
+    now?: () => number;
+}): ClockJumpWatch;

package/dist/harness/clock-jump.js

@@ -0,0 +1,15 @@
+export function watchClockJumps(onJump, options = {}) {
+    const intervalMs = options.intervalMs ?? 100;
+    const jumpMs = options.jumpMs ?? 1_500;
+    const now = options.now ?? Date.now;
+    let last = now();
+    const timer = setInterval(() => {
+        const at = now();
+        const late = at - last - intervalMs;
+        last = at;
+        if (late > jumpMs)
+            onJump(late);
+    }, intervalMs);
+    timer.unref();
+    return { stop: () => clearInterval(timer) };
+}

package/dist/harness/main.js

@@ -70,13 +70,15 @@ export async function runHarness(stack, baseline) {
         if (!pod.afterSeed)
             await prepare();
     }
-    const token = pod ? () => pod.token() : config.token;
+    const token = pod ? (signal) => pod.token(signal) : config.token;
     if (token === undefined)
         throw new Error("--service harness needs NYLORUN_HARNESS_TOKEN");
     const runTokens = harnessRunTokens();
     const service = startHarnessService({
         url: config.url,
         token,
+        // A pod's engine may be frozen and resumed (an Agent Substrate restore): redial at once.
+        ...(pod ? { redialOnClockJump: true } : {}),
         ...(pod
             ? { sandboxBackends: [localBackend({ env: baseline, proxyEnv: () => pod.proxyEnv(), harnessRoot: config.root })] }
             : {}),

package/dist/harness/pod.d.ts

@@ -23,8 +23,11 @@ export interface DiskSetupReport {
     readonly output?: string;
 }
 export interface PodHost {
-    /** A current host token, for the next connection. Throws when neither renewal nor join works. */
-    token(): Promise<string>;
+    /**
+     * A current host token, for the next connection. Throws when neither renewal nor join works,
+     * or when `signal` drops the attempt.
+     */
+    token(signal?: AbortSignal): Promise<string>;
     /** `HTTPS_PROXY` and friends with the current egress token; none before the first join. */
     proxyEnv(): Record<string, string>;
     /** The host epoch of the tokens held. */

package/dist/harness/pod.js

@@ -38,7 +38,8 @@ export function podHost(config, logger, options = {}) {
     let afterSeed = false;
     /** The CA in the bundle on disk, once one is written. */
     let bundled;
-    const post = async (path, init) => {
+    const post = async (path, init, signal) => {
+        const timeout = AbortSignal.timeout(hostTuning().harness.podJoinTimeoutMs);
         const response = await call(`${config.httpUrl}/nylorun/harness/v1/host/${path}`, {
             method: "POST",
             headers: {
@@ -48,7 +49,7 @@ export function podHost(config, logger, options = {}) {
                 ...(path === "join" && imageEngine ? { [IMAGE_ENGINE_HEADER]: RUNTIME_VERSION } : {}),
             },
             ...(init.body === undefined ? {} : { body: JSON.stringify(init.body) }),
-            signal: AbortSignal.timeout(hostTuning().harness.podJoinTimeoutMs),
+            signal: signal ? AbortSignal.any([timeout, signal]) : timeout,
         });
         const text = await response.text();
         if (response.status === 401 || response.status === 400)
@@ -97,7 +98,7 @@ export function podHost(config, logger, options = {}) {
             return id;
         }
     };
-    const join = async () => {
+    const join = async (signal) => {
         const joinToken = (await readFile(config.joinFile, "utf8")).trim();
         const volume = await volumeId();
         held = await post("join", {
@@ -107,7 +108,7 @@ export function podHost(config, logger, options = {}) {
                 joinToken,
                 ...(volume === undefined ? {} : { volumeId: volume }),
             },
-        });
+        }, signal);
         logger.info("sandbox_host_joined", { sandboxId: config.sandboxId, epoch: held.epoch });
         return held;
     };
@@ -122,11 +123,11 @@ export function podHost(config, logger, options = {}) {
             signal: AbortSignal.timeout(hostTuning().harness.podRequestTimeoutMs),
         });
     };
-    const renew = async () => {
+    const renew = async (signal) => {
         if (!held)
             return undefined;
         try {
-            held = await post("renew", { bearer: held.hostToken });
+            held = await post("renew", { bearer: held.hostToken }, signal);
             return held;
         }
         catch (error) {
@@ -143,8 +144,8 @@ export function podHost(config, logger, options = {}) {
     }, options.renewMs ?? hostTuning().harness.podRenewMs);
     timer.unref();
     return {
-        async token() {
-            const current = (await renew()) ?? (await join());
+        async token(signal) {
+            const current = (await renew(signal)) ?? (await join(signal));
             return current.hostToken;
         },
         proxyEnv() {

package/dist/harness/service.d.ts

@@ -7,6 +7,7 @@ import { type EngineTuning } from "../tuning.js";
 export interface HarnessServiceOptions {
     readonly url: string;
     readonly token: HarnessClientOptions["token"];
+    readonly redialOnClockJump?: boolean;
     /** Where workspaces (and their records) live. */
     readonly paths: {
         readonly sandboxes: string;

package/dist/harness/service.js

@@ -124,6 +124,7 @@ export function startHarnessService(options) {
     client = connectHarness({
         url: options.url,
         token: options.token,
+        ...(options.redialOnClockJump ? { redialOnClockJump: true } : {}),
         logger,
         executors,
         serve,

package/dist/harness/ws-client.d.ts

@@ -5,18 +5,24 @@
  * core's `workspace.*` requests (`serve`). When the connection is lost its runs stop (core gives
  * their segments to the next advance, which resumes them by replay) and the client connects
  * again after a jittered backoff (`harness.reconnect` in the Host tuning, 0.5 to 10 s).
+ *
+ * A pod's engine (`redialOnClockJump`) also watches for a freeze and resume (an Agent Substrate
+ * restore): it drops the connection attempt in flight, token request included, and dials again
+ * at once, instead of waiting for TCP to resend what the freeze lost.
  */
 import { type HarnessChannel, type RequestHandler } from "@nylorun/core/harness-api";
 import { type HarnessOptions } from "@nylorun/harness/api";
 import { type WsPortOptions } from "./ws-port.js";
+import { watchClockJumps } from "./clock-jump.js";
 export interface HarnessClientOptions extends Omit<HarnessOptions, "channel" | "logger"> {
     /** The Harness API, e.g. `ws://runtime:4200/nylorun/harness/v1` (`NYLORUN_HARNESS_URL`). */
     readonly url: string;
     /**
      * `NYLORUN_HARNESS_TOKEN`, or, for a pod sandbox's engine (F7.2), a function that answers
-     * a current host token before each connection (it renews, or joins again).
+     * a current host token before each connection (it renews, or joins again). The signal aborts
+     * it when the attempt is dropped.
      */
-    readonly token: string | (() => Promise<string>);
+    readonly token: string | ((signal?: AbortSignal) => Promise<string>);
     readonly logger: {
         info(message: string, fields?: Record<string, unknown>): void;
         warn(message: string, fields?: Record<string, unknown>): void;
@@ -29,6 +35,13 @@ export interface HarnessClientOptions extends Omit<HarnessOptions, "channel" | "
         readonly maxMs: number;
     };
     readonly liveness?: Omit<WsPortOptions, "ping">;
+    /**
+     * Drop an attempt in flight and dial again at once when the process was frozen and resumed
+     * (`clock-jump.ts`): a pod's engine, which Agent Substrate suspends to a memory snapshot.
+     */
+    readonly redialOnClockJump?: boolean;
+    /** The clock and timings `redialOnClockJump` watches with (tests set them). */
+    readonly clockJumps?: Parameters<typeof watchClockJumps>[1];
 }
 export interface HarnessClient {
     /** Resolves once the first connection's `hello` is answered. */

package/dist/harness/ws-client.js

@@ -5,6 +5,10 @@
  * core's `workspace.*` requests (`serve`). When the connection is lost its runs stop (core gives
  * their segments to the next advance, which resumes them by replay) and the client connects
  * again after a jittered backoff (`harness.reconnect` in the Host tuning, 0.5 to 10 s).
+ *
+ * A pod's engine (`redialOnClockJump`) also watches for a freeze and resume (an Agent Substrate
+ * restore): it drops the connection attempt in flight, token request included, and dials again
+ * at once, instead of waiting for TCP to resend what the freeze lost.
  */
 import { createChannel } from "@nylorun/core/harness-api";
 import { createHarness } from "@nylorun/harness/api";
@@ -12,6 +16,7 @@ import { WebSocket } from "ws";
 import { HARNESS_API_HEADER, HARNESS_API_TRANSPORT, HARNESS_MAX_PAYLOAD } from "@nylorun/core/harness-api";
 import { wsPort } from "./ws-port.js";
 import { hostTuning } from "../tuning.js";
+import { watchClockJumps } from "./clock-jump.js";
 export function connectHarness(options) {
     const { logger } = options;
     const tuning = hostTuning().harness;
@@ -20,30 +25,49 @@ export function connectHarness(options) {
     let connected = false;
     let current;
     let wake;
+    /** The connection attempt in flight (token, then handshake), until it opens or fails. */
+    let attempting;
+    let attempt = 0;
     let markReady;
     const ready = new Promise((resolve) => (markReady = resolve));
     const open = async () => {
-        const token = typeof options.token === "string" ? options.token : await options.token();
-        return new Promise((resolve, reject) => {
-            const socket = new WebSocket(options.url, {
-                headers: { authorization: `Bearer ${token}`, [HARNESS_API_HEADER]: HARNESS_API_TRANSPORT },
-                maxPayload: HARNESS_MAX_PAYLOAD,
-                handshakeTimeout: tuning.handshakeTimeoutMs,
-            });
-            socket.once("open", () => {
-                socket.removeAllListeners("error");
-                resolve(socket);
-            });
-            socket.once("unexpected-response", (_request, response) => {
-                socket.removeAllListeners("error");
-                socket.on("error", () => undefined);
-                reject(new Error(`The Harness API refused the connection: ${response.statusCode ?? 0}`));
-                response.resume();
-                socket.terminate();
-            });
-            socket.once("error", reject);
-        });
+        const controller = new AbortController();
+        attempting = controller;
+        try {
+            const token = typeof options.token === "string" ? options.token : await options.token(controller.signal);
+            controller.signal.throwIfAborted();
+            return await dial(token, controller.signal);
+        }
+        finally {
+            if (attempting === controller)
+                attempting = undefined;
+        }
     };
+    const dial = (token, signal) => new Promise((resolve, reject) => {
+        const socket = new WebSocket(options.url, {
+            headers: { authorization: `Bearer ${token}`, [HARNESS_API_HEADER]: HARNESS_API_TRANSPORT },
+            maxPayload: HARNESS_MAX_PAYLOAD,
+            handshakeTimeout: tuning.handshakeTimeoutMs,
+        });
+        socket.once("open", () => {
+            socket.removeAllListeners("error");
+            resolve(socket);
+        });
+        socket.once("unexpected-response", (_request, response) => {
+            socket.removeAllListeners("error");
+            socket.on("error", () => undefined);
+            reject(new Error(`The Harness API refused the connection: ${response.statusCode ?? 0}`));
+            response.resume();
+            socket.terminate();
+        });
+        socket.once("error", reject);
+        signal.addEventListener("abort", () => {
+            socket.removeAllListeners("error");
+            socket.on("error", () => undefined);
+            socket.terminate();
+            reject(signal.reason instanceof Error ? signal.reason : new Error("connection attempt dropped"));
+        }, { once: true });
+    });
     const pause = (ms) => new Promise((resolve) => {
         const timer = setTimeout(done, ms);
         timer.unref();
@@ -81,8 +105,16 @@ export function connectHarness(options) {
             current = undefined;
         }
     };
+    const watch = options.redialOnClockJump
+        ? watchClockJumps((jumpedMs) => {
+            const inFlight = attempting !== undefined;
+            logger.info("harness_clock_jumped", { jumpedMs, redial: inFlight || wake !== undefined });
+            attempt = 0;
+            attempting?.abort(new Error(`the process resumed after ${jumpedMs} ms`));
+            wake?.();
+        }, options.clockJumps)
+        : undefined;
     const loop = (async () => {
-        let attempt = 0;
         while (!stopped) {
             const started = Date.now();
             try {
@@ -113,6 +145,8 @@ export function connectHarness(options) {
         channel: () => current?.channel,
         async stop(waitMs = tuning.stopMs) {
             stopped = true;
+            watch?.stop();
+            attempting?.abort(new Error("harness stopped"));
             wake?.();
             const live = current;
             if (live) {

package/dist/management-openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.2.0",
   "info": {
     "title": "Nylorun Management API",
-    "version": "0.46.0-beta",
+    "version": "0.46.1-beta",
     "x-nylorun-protocol": {
       "min": 16,
       "max": 18,

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.2.0",
   "info": {
     "title": "Nylorun Runtime API",
-    "version": "0.46.0-beta",
+    "version": "0.46.1-beta",
     "x-nylorun-protocol": {
       "min": 16,
       "max": 18,

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const RUNTIME_VERSION = "0.46.0-beta";
+export declare const RUNTIME_VERSION = "0.46.1-beta";

package/dist/version.js

@@ -1,3 +1,3 @@
 // Kept in step with package.json by runtime/server/scripts/check-package.mjs, which runs in npm run check.
 // release:prepare also rewrites this when it bumps @nylorun/runtime.
-export const RUNTIME_VERSION = "0.46.0-beta";
+export const RUNTIME_VERSION = "0.46.1-beta";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nylorun/runtime",
-  "version": "0.46.0-beta",
+  "version": "0.46.1-beta",
   "description": "Nylorun Runtime: the agent execution Host on Postgres, Restate and S2, and model providers.",
   "type": "module",
   "license": "Apache-2.0",