@nylorun/runtime 0.50.0-beta → 0.51.1-beta

package/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## 0.51.1-beta
+
+### Patch Changes
+
+- 8451138: Pods on Agent Substrate (preview): a drain is recorded as soon as the moved pod's engine connects again. After suspend before drain, the restored engine redials with its host token (no join), and nothing asked the Runtime to reconcile the pod, so `agent.suspended` with reason `drain` waited for the next turn or idle timer (E13 on kind: not within two minutes). A pod engine's connection now asks for its sandbox's reconcile.
+
+## 0.51.0-beta
+
+### Minor Changes
+
+- 874cc83: Pods on Agent Substrate (preview): suspend before drain, following Substrate's eviction contract. A worker pod that stops (a drain, a GKE node upgrade or scale-down, a WorkerPool rollout, a deleted pod) sends each of its actors SIGTERM and gives them 30 minutes to be suspended; until now the engine exited on that signal, so the actor crashed and went back to its last suspend, losing everything since.
+
+  - `@nylorun/runtime`: on Substrate (`NYLORUN_SANDBOX_SUBSTRATE=1`, which the sandboxes service's template sets), the engine logs `harness_worker_stopping` on SIGTERM and keeps running, with its runs. The sandboxes service watches worker pods (label `ate.dev/worker-pool`) and, when one turns Terminating, suspends each of the Tenant's actors on it synchronously (eight at a time, trying again for up to 29 minutes), records `drainedAt`, and resumes a sandbox whose request is `Running` on another worker. While it moves, its status reason is `Moving`, and the pod lifecycle waits instead of failing it at the ready deadline. Each drain is recorded once, as `agent.suspended` with reason `drain` (the pod row's `drained_at`, store migration 0048, nullable, run at start). Not covered: a spot reclaim or a node that dies, which still crash and revert. An actor created by an earlier release keeps its engine (#643) until it is reset.
+  - `@nylorun/core`: `agent.suspended`'s `reason` may be `drain`.
+  - `nylorun`: `local sandbox enable --backend substrate` lets the sandboxes service read pods cluster-wide (`get`, `list`, `watch`), for the worker pods it watches.
+
+### Patch Changes
+
+- Pin core to the tested release.
+- Pin harness to the tested release.
+- Updated dependencies [874cc83]
+- Updated dependencies
+  - @nylorun/core@0.44.0-beta
+  - @nylorun/harness@0.36.4-beta
+
 ## 0.50.0-beta
 
 ### Minor Changes

package/dist/harness/main.js

@@ -14,7 +14,9 @@
  * pod's join token instead of a harness token (`pod.ts`), runs its disk's setup on a new volume
  * (`setup.ts`), and runs its sandbox's tools in the pod itself (the `local` backend), with egress
  * through egress-gate. SIGTERM (tini forwards
- * it; the pod's grace is 10 s by default) gives its runs back and exits.
+ * it; the pod's grace is 10 s by default) gives its runs back and exits. On Agent Substrate
+ * (`NYLORUN_SANDBOX_SUBSTRATE=1`), SIGTERM means the worker pod is stopping: the engine logs it and
+ * keeps running, so the sandboxes service can suspend the actor within Substrate's 30 minutes.
  */
 import { existsSync, mkdirSync } from "node:fs";
 import { createServer } from "node:http";
@@ -106,8 +108,18 @@ export async function runHarness(stack, baseline) {
     });
     logger.info("harness_ready", { health: `http://127.0.0.1:${config.healthPort}/health` });
     let stopping = false;
+    let workerStopping = false;
     for (const signal of ["SIGINT", "SIGTERM"])
         process.on(signal, () => {
+            // On Agent Substrate, SIGTERM means the worker pod is stopping: the actor has 30 minutes to
+            // be suspended, and the sandboxes service suspends it. A process that exits can't be, so the
+            // engine keeps running, and keeps its runs (Substrate's eviction contract).
+            if (signal === "SIGTERM" && config.pod?.substrate) {
+                if (!workerStopping)
+                    logger.info("harness_worker_stopping", { sandboxId: config.pod.sandboxId });
+                workerStopping = true;
+                return;
+            }
             if (stopping)
                 return;
             stopping = true;

package/dist/host/stack-config.d.ts

@@ -169,6 +169,12 @@ export interface PodHostConfig {
      * for the hosts whose keys Substrate's egress adds.
      */
     egressCaFile?: string;
+    /**
+     * `NYLORUN_SANDBOX_SUBSTRATE=1`: the pod is an Agent Substrate actor. A stopping worker sends
+     * it SIGTERM and gives it 30 minutes to be suspended, which the sandboxes service does; the
+     * engine stays up instead of exiting (suspend before drain).
+     */
+    substrate?: boolean;
     /** Addresses the pod's NetworkPolicy must block, probed before the engine starts. */
     blocked: BlockedAddress[];
 }

package/dist/host/stack-config.js

@@ -647,6 +647,7 @@ function parsePodHost(env, url) {
             httpUrl: origin,
             ...(egress ? { egressProxy: egress.replace(/\/+$/, "") } : {}),
             ...(egressCa ? { egressCaFile: egressCa } : {}),
+            ...(read(env, "NYLORUN_SANDBOX_SUBSTRATE") === "1" ? { substrate: true } : {}),
             blocked: blockedAddresses(env),
         },
     };

package/dist/management-openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.2.0",
   "info": {
     "title": "Nylorun Management API",
-    "version": "0.50.0-beta",
+    "version": "0.51.1-beta",
     "x-nylorun-protocol": {
       "min": 16,
       "max": 18,

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.2.0",
   "info": {
     "title": "Nylorun Runtime API",
-    "version": "0.50.0-beta",
+    "version": "0.51.1-beta",
     "x-nylorun-protocol": {
       "min": 16,
       "max": 18,

package/dist/sandbox/pods/client.d.ts

@@ -63,6 +63,11 @@ export interface PodStatus {
     readonly shutdownTime?: string;
     readonly reason?: string;
     readonly message?: string;
+    /**
+     * Agent Substrate: when the sandbox was last suspended because its worker stopped. While it
+     * moves to another worker, `reason` is `Moving`.
+     */
+    readonly drainedAt?: string;
     /** With `?wait=`: whether the state was reached before the timeout. */
     readonly met?: boolean;
 }

package/dist/sandbox/pods/lifecycle.d.ts

@@ -25,6 +25,10 @@
  *   (`agent.failed`, saying what the Agent lost); a crash an idle suspend finds is recorded the
  *   same way. The next turn start asks for `Running`, and the sandboxes service reverts the
  *   sandbox to its last suspend before resuming it.
+ * - **Drained** (Agent Substrate): a worker that stops gives its actors 30 minutes to be
+ *   suspended. The sandboxes service suspends each, records `drainedAt` and resumes it on
+ *   another worker; a new `drainedAt` is `agent.suspended` (reason `drain`), once, and a pod
+ *   that is `Moving` waits rather than fail at the ready deadline.
  * - **Lost**: an incarnation that has run (a pod joined) whose Sandbox is gone or whose volume
  *   is missing is `lost` (`agent.lost`): its host epoch moves, its Sandbox is deleted, and
  *   only a reset brings the Environment back. agent-sandbox recreates a deleted claim empty, so
@@ -60,6 +64,11 @@ export declare function repositoryOf(reference: string): string;
  * the node made up for an image loaded onto it, which no registry serves.
  */
 export declare function imageDigestOf(imageID: string | undefined, requested: string): string | undefined;
+/**
+ * The sandboxes service's reason for a sandbox suspended because its Agent Substrate worker
+ * stopped, until it runs on another one.
+ */
+export declare const MOVING = "Moving";
 /** A setup failure's `reason` starts with this (`sandbox/setup.ts`); it is sticky. */
 export declare const SETUP_FAILED = "setup:";
 /** A crash's `reason` (see `crashReasonOf`) starts with this; it stays until a turn starts the pod. */

package/dist/sandbox/pods/lifecycle.js

@@ -34,6 +34,11 @@ export function imageDigestOf(imageID, requested) {
         return undefined;
     return repositoryOf(id) === repositoryOf(requested) ? id : undefined;
 }
+/**
+ * The sandboxes service's reason for a sandbox suspended because its Agent Substrate worker
+ * stopped, until it runs on another one.
+ */
+export const MOVING = "Moving";
 /** A setup failure's `reason` starts with this (`sandbox/setup.ts`); it is sticky. */
 export const SETUP_FAILED = "setup:";
 /** A crash's `reason` (see `crashReasonOf`) starts with this; it stays until a turn starts the pod. */
@@ -158,6 +163,12 @@ export function decide(facts) {
         again(pollMs);
         return done();
     }
+    // Agent Substrate: its worker stopped, and the sandboxes service suspended it with everything it
+    // held and resumes it on another worker (suspend before drain). Recorded once per drain.
+    if (status.drainedAt !== undefined && status.drainedAt !== pod.drainedAt) {
+        patch.drainedAt = status.drainedAt;
+        events.push(event("agent.suspended", { reason: "drain" }));
+    }
     const opId = opIdOf({ k8sName: pod.k8sName, rev });
     if (desired === "suspended" && !status.exists) {
         if (pod.observed !== "suspended")
@@ -217,6 +228,11 @@ export function decide(facts) {
             events.push(event("agent.failed", { reason: crash }));
             return done();
         }
+        // On its way to another worker after a drain: neither failed nor starting.
+        if (status.reason === MOVING) {
+            again(pollMs);
+            return done();
+        }
         if (now - startedAt >= readyDeadlineMs) {
             const reason = crash ??
                 ([status.reason, status.message].filter(Boolean).join(": ") ||

package/dist/store/postgres/drizzle/0048_sandbox_pod_drained_at.sql

@@ -0,0 +1 @@
+ALTER TABLE "nylorun"."agents" ADD COLUMN "drained_at" text;

package/dist/store/postgres/drizzle/meta/_journal.json

@@ -337,6 +337,13 @@
       "when": 1794630124577,
       "tag": "0047_model_usage_attempts",
       "breakpoints": true
+    },
+    {
+      "idx": 48,
+      "version": "7",
+      "when": 1794635246579,
+      "tag": "0048_sandbox_pod_drained_at",
+      "breakpoints": true
     }
   ]
 }

package/dist/store/postgres/schema.d.ts

@@ -1321,6 +1321,23 @@ export declare const agents: import("drizzle-orm/pg-core").PgTableWithColumns<{
         }, {}, {
             pgColumnBuilderBrand: 'PgCustomColumnBuilderBrand';
         }>;
+        drainedAt: import("drizzle-orm/pg-core").PgColumn<{
+            name: "drainedAt";
+            tableName: "agents";
+            dataType: "string";
+            columnType: "PgText";
+            data: string;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
         diskOp: import("drizzle-orm/pg-core").PgColumn<{
             name: "diskOp";
             tableName: "agents";

package/dist/store/postgres/schema.js

@@ -173,6 +173,8 @@ export const agents = nylorun.table("agents", {
     imageDigest: textC(),
     /** The engine the image carries (ADR-99): `{ version, digest? }`, from a join of its engine. */
     engineImage: jsonText(),
+    /** When its Agent Substrate worker last stopped under it (the sandboxes service's `drainedAt`). */
+    drainedAt: text(),
     /**
      * The disk lease (R2d step 3): `{ kind, version?, startedAt, until }` while a snapshot, a
      * restore or a seed copies the disk; turns are refused until it ends or `until` passes.

package/dist/store/postgres/store.js

@@ -284,6 +284,7 @@ function environmentOf(row) {
                     ...(row.volumeId !== null ? { volumeId: row.volumeId } : {}),
                     ...(row.imageDigest !== null ? { imageDigest: row.imageDigest } : {}),
                     ...(row.engineImage ? { engineImage: row.engineImage } : {}),
+                    ...(row.drainedAt !== null ? { drainedAt: row.drainedAt } : {}),
                 },
             }
             : {}),
@@ -319,6 +320,7 @@ function podColumns(patch) {
         volumeId: patch.volumeId,
         imageDigest: patch.imageDigest,
         engineImage: patch.engineImage,
+        drainedAt: patch.drainedAt,
     };
     return Object.fromEntries(Object.entries(columns).filter(([, value]) => value !== undefined));
 }

package/dist/store/types.d.ts

@@ -269,6 +269,11 @@ export interface SandboxPodState {
         version: string;
         digest?: string;
     };
+    /**
+     * When the sandbox was last suspended because its Agent Substrate worker stopped (the sandboxes
+     * service's `drainedAt`), so each drain is recorded once (`agent.suspended`, reason `drain`).
+     */
+    drainedAt?: string;
 }
 /** A change to a pod's columns: `null` clears an optional one. */
 export type SandboxPodPatch = {

package/dist/tenant/runtime.js

@@ -392,7 +392,20 @@ export class TenantRuntime {
         return this.ctx.onSweep(hook);
     }
     attachHarness(channel, peer) {
-        return this.ctx.harness.attach(channel, peer);
+        const detach = this.ctx.harness.attach(channel, peer);
+        // A pod's engine that (re)connects asks for its sandbox's reconcile: after a drain (Agent
+        // Substrate's eviction contract) the restored engine redials with its host token and no join,
+        // and this is what lets the Runtime see the move (`drainedAt`, `agent.suspended` reason
+        // `drain`) at once rather than at the next turn or idle timer.
+        const sandboxId = peer.host?.sandboxId;
+        if (sandboxId !== undefined)
+            void this.ctx.store
+                .tx((t) => t.signalSandbox(sandboxId, { kind: "reconcile" }))
+                .catch((error) => this.ctx.config.logger.warn("sandbox reconcile after connect failed", {
+                sandboxId,
+                error: error instanceof Error ? error.message : String(error),
+            }));
+        return detach;
     }
     harnessStatus() {
         return harnessStatusOf(this.ctx);

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const RUNTIME_VERSION = "0.50.0-beta";
+export declare const RUNTIME_VERSION = "0.51.1-beta";

package/dist/version.js

@@ -1,3 +1,3 @@
 // Kept in step with package.json by runtime/server/scripts/check-package.mjs, which runs in npm run check.
 // release:prepare also rewrites this when it bumps @nylorun/runtime.
-export const RUNTIME_VERSION = "0.50.0-beta";
+export const RUNTIME_VERSION = "0.51.1-beta";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nylorun/runtime",
-  "version": "0.50.0-beta",
+  "version": "0.51.1-beta",
   "description": "Nylorun Runtime: the agent execution Host on Postgres, Restate and S2, and model providers.",
   "type": "module",
   "license": "Apache-2.0",
@@ -61,39 +61,39 @@
   "dependencies": {
     "@ag-ui/core": "~1.0.4",
     "@asteasolutions/zod-to-openapi": "^9.1.0",
-    "@earendil-works/pi-ai": "1.3.0",
-    "@hono/node-server": "^2.1.5",
+    "@earendil-works/pi-ai": "1.5.2",
+    "@hono/node-server": "^2.1.8",
     "@hono/zod-openapi": "^1.6.3",
-    "@modelcontextprotocol/sdk": "^1.32.1",
-    "@nylorun/core": "0.43.0-beta",
-    "@nylorun/harness": "0.36.3-beta",
-    "@restatedev/restate-sdk": "1.17.3",
-    "@restatedev/restate-sdk-clients": "1.17.3",
-    "@s2-dev/streamstore": "0.28.0",
+    "@modelcontextprotocol/sdk": "^1.32.2",
+    "@nylorun/core": "0.44.0-beta",
+    "@nylorun/harness": "0.36.4-beta",
+    "@restatedev/restate-sdk": "1.18.3",
+    "@restatedev/restate-sdk-clients": "1.18.3",
+    "@s2-dev/streamstore": "0.30.0",
     "aws4fetch": "1.0.20",
     "croner": "^9.1.0",
     "drizzle-orm": "^0.45.4",
-    "hono": "^4.13.13",
-    "jose": "^6.2.12",
+    "hono": "^4.13.18",
+    "jose": "^6.2.17",
     "just-bash": "3.6.0",
     "pg": "8.23.1",
-    "pg-logical-replication": "2.5.0",
+    "pg-logical-replication": "2.6.0",
     "postgres": "^3.4.9",
-    "ws": "^8.22.0",
+    "ws": "^8.22.2",
     "yaml": "^2.9.1",
     "zod": "^4.6.7"
   },
   "devDependencies": {
     "@a2a-js/sdk": "~1.3.0",
     "@ag-ui/client": "1.0.4",
-    "@scalar/openapi-parser": "^0.29.13",
-    "@types/node": "^26.6.5",
+    "@scalar/openapi-parser": "^0.29.21",
+    "@types/node": "^26.6.6",
     "@types/pg": "^8.23.1",
     "@types/ws": "^8.18.2",
     "@typescript/native": "npm:typescript@^7.0.2",
     "drizzle-kit": "^0.31.11",
     "esbuild": "^0.28.2",
     "typescript": "npm:@typescript/typescript6@^6.0.2",
-    "vitest": "^5.0.4"
+    "vitest": "^5.1.0"
   }
 }