@nylorun/runtime 0.42.0-beta → 0.44.0-beta

package/dist/tenant/sweep.js

@@ -43,7 +43,11 @@
  *    the delivery after commit, sends), and configs of tasks long finished are deleted. A push
  *    row that fails is retried with the Host tuning's `a2a.pushRetry` and dropped once it is
  *    `a2a.pushMaxAgeMs` old, never parked: the change feed covers what it said.
- * 7. **Hooks.** Callbacks registered with `ctx.onSweep`.
+ * 7. **Queued messages** (Host feature `queue-recovery`): at most every `sweep.queueRetryMs`, the
+ *    turn start of every session queue an Agent's refusal left waiting is tried again
+ *    (`retryQueuedStarts`); a message past the Runtime limit `queuedStartMs` fails
+ *    (`queue.failed`). The end of a disk copy tries its Agent's sessions at once.
+ * 8. **Hooks.** Callbacks registered with `ctx.onSweep`.
  *
  * Every step runs one transaction per session it changes (a linked agent and its workflow
  * share one, child first), so the sweep follows the lock order in `store/types.ts`. A failing
@@ -51,12 +55,14 @@
  * reports it.
  */
 import { sweepUnusedAgents } from "./agents.js";
+import { retryQueuedStarts } from "./commands.js";
 import { reconcileTriggers } from "./triggers.js";
 import { streamMissedRuns } from "./runs-stream.js";
 import { sweepPendingSeeds } from "../artifacts/disk.js";
 import { pendingAgentEffects, reconcilePendingAgentEffect, } from "../core/flow-host.js";
 import { hostTuning } from "../tuning.js";
-import { sweepStartChecks } from "./agent-versions.js";
+import { idleMsOf, sweepStartChecks } from "./agent-versions.js";
+import { readSandboxConfig } from "../sandbox/tenant-config.js";
 import { pumpPushes, pushRetry } from "../api/a2a/push.js";
 /*
  * The sweep's timings are the Host tuning's `sweep` group (`tuning.ts`):
@@ -69,6 +75,7 @@ import { pumpPushes, pushRetry } from "../api/a2a/push.js";
  * - `drainBudgetMs`: how long one pass may spend draining the outbox, and paging through
  *   orphaned sessions. A backlog is worked off over several passes rather than holding one (and
  *   with it the Tenant's sweep object in the execution) for long.
+ * - `queueRetryMs`: how often the queues an Agent's refusal left waiting are tried again.
  * - `batchSize`, `signalRetentionMs`, `outboxAttempts`, `outboxRetry`, `stalePodMs`.
  */
 /**
@@ -88,6 +95,7 @@ export async function sweep(ctx) {
         pass.orphansAfter !== undefined ||
         now.getTime() - pass.orphansAt >= hostTuning().sweep.orphanScanMs;
     const checkPods = pass.podsAt === undefined || now.getTime() - pass.podsAt >= hostTuning().sweep.orphanScanMs;
+    const retryQueues = pass.queuesAt === undefined || now.getTime() - pass.queuesAt >= hostTuning().sweep.queueRetryMs;
     const steps = [
         ["outbox", () => deliverOutbox(ctx, first ? 0 : hostTuning().sweep.outboxGraceMs)],
         ["linked", () => reconcileLinkedAgents(ctx)],
@@ -142,6 +150,17 @@ export async function sweep(ctx) {
             "signals",
             () => ctx.store.tx((t) => t.pruneSignals(new Date(now.getTime() - hostTuning().sweep.signalRetentionMs))),
         ],
+        ...(retryQueues
+            ? [
+                [
+                    "queues",
+                    async () => {
+                        pass.queuesAt = now.getTime();
+                        await retryQueuedStarts(ctx, { now, batchSize: hostTuning().sweep.batchSize });
+                    },
+                ],
+            ]
+            : []),
         ...[...ctx.sweepHooks].map((hook) => ["hook", hook]),
     ];
     let failure;
@@ -287,7 +306,8 @@ export async function wakeOrphanedSessions(ctx, now = new Date(), after, budgetM
 }
 /**
  * Pod sandboxes (F7.2) whose change has not finished (creating, deleting, an old volume still
- * to delete, an expiry not yet recorded), or that should run but whose engine is not connected,
+ * to delete, an expiry not yet recorded), that should run but whose engine is not connected, or
+ * that run past their idle limit (a wake that armed no idle timer: the reconcile arms it),
  * and whose row has not moved for `sweep.stalePodMs` (a minute), get a reconcile: a pod or volume deleted outside
  * the Runtime is found, and a change goes on whose reconcile the execution itself lost (its
  * retry timer, an invocation an operator killed, Restate's state wiped) or whose outbox row
@@ -296,18 +316,27 @@ export async function wakeOrphanedSessions(ctx, now = new Date(), after, budgetM
 async function reconcileStalePods(ctx, now) {
     if (!ctx.pods)
         return;
-    const stale = await ctx.store.tx(async (t) => (await t.listEnvironments()).filter((row) => {
-        const pod = row.pod;
-        if (!pod || now.getTime() - Date.parse(row.updatedAt) < hostTuning().sweep.stalePodMs)
-            return false;
-        return (pod.desired === "deleted" ||
-            pod.retiring !== undefined ||
-            pod.observed === "creating" ||
-            pod.observed === "deleting" ||
-            (pod.desired === "suspended" && pod.observed !== "suspended") ||
-            // Running, but its engine is not connected: the pod may be gone, or its volume.
-            (pod.desired === "running" && pod.observed === "running" && !ctx.harness.hosting(row.id)));
-    }));
+    const stale = await ctx.store.tx(async (t) => {
+        const config = await readSandboxConfig(t);
+        return (await t.listEnvironments()).filter((row) => {
+            const pod = row.pod;
+            if (!pod || now.getTime() - Date.parse(row.updatedAt) < hostTuning().sweep.stalePodMs)
+                return false;
+            return (pod.desired === "deleted" ||
+                pod.retiring !== undefined ||
+                pod.observed === "creating" ||
+                pod.observed === "deleting" ||
+                (pod.desired === "suspended" && pod.observed !== "suspended") ||
+                // Running, but its engine is not connected: the pod may be gone, or its volume.
+                (pod.desired === "running" && pod.observed === "running" && !ctx.harness.hosting(row.id)) ||
+                // Running past its idle limit: its idle timer is missing (or a turn holds it, which the
+                // reconcile leaves alone).
+                (pod.desired === "running" &&
+                    pod.observed === "running" &&
+                    now.getTime() - Date.parse(pod.lastActiveAt ?? row.createdAt) >
+                        idleMsOf(config, row) + hostTuning().sweep.stalePodMs));
+        });
+    });
     for (const row of stale)
         await ctx.sandboxSignal(row.id, { kind: "reconcile" });
 }

package/dist/tuning.d.ts

@@ -121,6 +121,7 @@ export declare const HostTuningSchema: z.ZodObject<{
             maxMs: z.ZodDefault<z.ZodNumber>;
         }, z.core.$strict>>;
         stalePodMs: z.ZodDefault<z.ZodNumber>;
+        queueRetryMs: z.ZodDefault<z.ZodNumber>;
     }, z.core.$strict>>;
     a2a: z.ZodPrefault<z.ZodObject<{
         pushRetry: z.ZodPrefault<z.ZodObject<{
@@ -297,6 +298,8 @@ export declare const HostTuningSchema: z.ZodObject<{
         networkProbeMs: z.ZodDefault<z.ZodNumber>;
         networkPauseMs: z.ZodDefault<z.ZodNumber>;
         networkConfirmations: z.ZodDefault<z.ZodNumber>;
+        wakeNetworkProbeMs: z.ZodDefault<z.ZodNumber>;
+        wakeNetworkConfirmations: z.ZodDefault<z.ZodNumber>;
         setupRetryPollMs: z.ZodDefault<z.ZodNumber>;
         setupAfterSeedPollMs: z.ZodDefault<z.ZodNumber>;
         setupReportTailChars: z.ZodDefault<z.ZodNumber>;

package/dist/tuning.js

@@ -207,6 +207,11 @@ export const HostTuningSchema = z
         outboxRetry: backoff(5_000, 10 * MINUTE),
         /** How long a pod row may go unchanged before the sweep checks it. */
         stalePodMs: ms(MINUTE),
+        /**
+         * How often the sweep tries again the turn start of queued messages their Agent refused
+         * (Host feature `queue-recovery`); they fail after the Runtime limit `queuedStartMs`.
+         */
+        queueRetryMs: ms(30_000),
     }),
     /** The A2A server's push notifications (Host feature `a2a-profile-v2`). */
     a2a: group({
@@ -495,6 +500,13 @@ export const HostTuningSchema = z
         networkProbeMs: ms(1_000),
         networkPauseMs: ms(250, 0),
         networkConfirmations: count(3),
+        /**
+         * The same check on a wake (the disk already holds the engine's volume id): each probe, and
+         * the failed probes in a row that prove an address blocked. A probe that connects turns the
+         * wake's check into the full one.
+         */
+        wakeNetworkProbeMs: ms(150),
+        wakeNetworkConfirmations: count(1),
         /** Environment setup: how often a failed setup is checked for a retry, and after a seed. */
         setupRetryPollMs: ms(5_000),
         setupAfterSeedPollMs: ms(1_000),

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const RUNTIME_VERSION = "0.42.0-beta";
+export declare const RUNTIME_VERSION = "0.44.0-beta";

package/dist/version.js

@@ -1,3 +1,3 @@
 // Kept in step with package.json by runtime/server/scripts/check-package.mjs, which runs in npm run check.
 // release:prepare also rewrites this when it bumps @nylorun/runtime.
-export const RUNTIME_VERSION = "0.42.0-beta";
+export const RUNTIME_VERSION = "0.44.0-beta";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nylorun/runtime",
-  "version": "0.42.0-beta",
+  "version": "0.44.0-beta",
   "description": "Nylorun Runtime: the agent execution Host on Postgres, Restate and S2, and model providers.",
   "type": "module",
   "license": "Apache-2.0",
@@ -65,8 +65,8 @@
     "@hono/node-server": "^2.1.5",
     "@hono/zod-openapi": "^1.6.3",
     "@modelcontextprotocol/sdk": "^1.32.1",
-    "@nylorun/core": "0.36.0-beta",
-    "@nylorun/harness": "0.33.0-beta",
+    "@nylorun/core": "0.38.0-beta",
+    "@nylorun/harness": "0.34.1-beta",
     "@restatedev/restate-sdk": "1.17.3",
     "@restatedev/restate-sdk-clients": "1.17.3",
     "@s2-dev/streamstore": "0.28.0",