@nylorun/runtime 0.43.0-beta → 0.44.1-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -55,8 +55,8 @@ export type MessageAnswer = {
55
55
  /**
56
56
  * Takes a resolved message on session `s` (locked) in transaction `t`: it starts a turn when the
57
57
  * session takes messages and none wait, else it waits in the queue behind them (protocol 17), and
58
- * an idle session whose queue was left waiting (its Agent was unavailable when the last turn
59
- * ended) starts a turn with all of them, this one last. Past `sessionQueuedMessages` it is
58
+ * an idle session whose queue was left waiting (its Agent refused the start when the last turn
59
+ * ended, `startQueuedTurn`) tries the start again with all of them, this one last. Past `sessionQueuedMessages` it is
60
60
  * `429 queue_full`. `waiting` is the queue's length, read under the session's lock. `asked`
61
61
  * `steer` (D85) marks a waiting message so the running turn may take it before a step; a
62
62
  * workflow session can't, so it waits as `queue`.
@@ -76,16 +76,45 @@ export declare function takeSteeredMessages(t: Tx, s: Session, id: string, step:
76
76
  export declare const QUEUE_SCOPE: AuthScope;
77
77
  /**
78
78
  * Starts the next turn of session `s` (locked) from its queue, when it takes messages and messages
79
- * wait (protocol 17, D76): in the transaction that ended the last turn, or a cancel's. Our engine
80
- * takes every queued message that carries the same turn manifest as the first, a workflow one.
81
- * Writes `queue.taken`. Returns nothing when nothing started: an empty queue, or an Agent that
82
- * cannot take a turn now, whose messages wait for the next message or turn end.
79
+ * wait (protocol 17, D76): in the transaction that ended the last turn, a cancel's, a message's
80
+ * that joined a queue left waiting, or the Tenant sweep's (`retryQueuedStarts`). Our engine takes
81
+ * every queued message that carries the same turn manifest as the first, a workflow one. Writes
82
+ * `queue.taken`. Returns nothing when nothing started: an empty queue, or an Agent that refused
83
+ * the start.
84
+ *
85
+ * A refusal a later try may get past (`RETRIED_REFUSALS`) leaves the messages waiting, with
86
+ * `s.queueStalled` saying since when and why (written here); the next message, the sweep and the
87
+ * end of a disk copy try again. A message that has waited longer than the Runtime limit
88
+ * `queuedStartMs` since the stall began (or since it was queued, when later) fails with
89
+ * `queue.failed` (`queue.start_expired`); a refusal no retry changes fails the messages it refused
90
+ * at once (`queue.start_refused`). Either way the messages behind them are tried next (Host
91
+ * feature `queue-recovery`).
83
92
  */
84
- export declare function startQueuedTurn(ctx: TenantContext, t: Tx, s: Session, id: string): Promise<{
93
+ export declare function startQueuedTurn(ctx: TenantContext, t: Tx, s: Session, id: string, now?: Date): Promise<{
85
94
  turnId: string;
86
95
  taken: string[];
87
96
  cursor: string;
88
97
  } | undefined>;
98
+ /**
99
+ * Removes queued message `queuedMessageId` from session `id`'s queue, as `scope` (Host feature
100
+ * `queue-recovery`): `queue.cancelled`, written by the caller. A turn running or not, the message
101
+ * never starts one. Returns the event's cursor, or undefined when the message no longer waits (a
102
+ * turn took it, or it failed or was cancelled already).
103
+ */
104
+ export declare function cancelQueuedMessage(ctx: TenantContext, id: string, queuedMessageId: string, scope: AuthScope, reason?: string): Promise<{
105
+ cursor: string;
106
+ } | undefined>;
107
+ /**
108
+ * Tries again the turn start of every queue a refused start left waiting, one transaction per
109
+ * session (Host feature `queue-recovery`): the Tenant sweep's step, and what the end of an Agent's
110
+ * disk copy runs for that Agent's sessions (`environmentId`). Each start fails its messages past
111
+ * `queuedStartMs` (`startQueuedTurn`). Returns the sessions whose queue started a turn.
112
+ */
113
+ export declare function retryQueuedStarts(ctx: TenantContext, options?: {
114
+ environmentId?: string;
115
+ now?: Date;
116
+ batchSize?: number;
117
+ }): Promise<string[]>;
89
118
  /**
90
119
  * Who sent a command (R4.9), from its credential and never its body: a client over the Runtime
91
120
  * API (an application, or a person through it or a token), or core itself.
@@ -19,7 +19,9 @@
19
19
  * (`session_queue`, `message.queued`) unless it asks `delivery: "reject"`, which is
20
20
  * `409 session_busy`. A turn that ends takes every queued message in the same transaction
21
21
  * (`startQueuedTurn`): client messages and queued ones start a turn through one door,
22
- * `beginMessageTurn`, which writes each one's `command.message`. A message that asks
22
+ * `beginMessageTurn`, which writes each one's `command.message`. A start the Agent refuses leaves
23
+ * the queue waiting, tried again by the Tenant sweep until the Runtime limit `queuedStartMs`, then
24
+ * failed (`queue.failed`, Host feature `queue-recovery`). A message that asks
23
25
  * `delivery: "steer"` (D85) waits marked `steer`, and the running turn takes it before a step
24
26
  * (`takeSteeredMessages`, the `inputs` effect); one it doesn't take starts the next turn like
25
27
  * any other.
@@ -300,8 +302,8 @@ export const takesMessages = (s) => TAKES_MESSAGES.includes(s.status);
300
302
  /**
301
303
  * Takes a resolved message on session `s` (locked) in transaction `t`: it starts a turn when the
302
304
  * session takes messages and none wait, else it waits in the queue behind them (protocol 17), and
303
- * an idle session whose queue was left waiting (its Agent was unavailable when the last turn
304
- * ended) starts a turn with all of them, this one last. Past `sessionQueuedMessages` it is
305
+ * an idle session whose queue was left waiting (its Agent refused the start when the last turn
306
+ * ended, `startQueuedTurn`) tries the start again with all of them, this one last. Past `sessionQueuedMessages` it is
305
307
  * `429 queue_full`. `waiting` is the queue's length, read under the session's lock. `asked`
306
308
  * `steer` (D85) marks a waiting message so the running turn may take it before a step; a
307
309
  * workflow session can't, so it waits as `queue`.
@@ -464,6 +466,8 @@ async function beginMessageTurn(t, s, id, messages) {
464
466
  });
465
467
  }
466
468
  s.turnStartState = s.state;
469
+ // A turn holds the session now: its queue, if any, waits for the turn, not for the Agent.
470
+ delete s.queueStalled;
467
471
  // A new turn folds from the latest snapshot: a cancel or failure only reverts this turn.
468
472
  if (s.history?.snapshot !== undefined)
469
473
  s.history = { ...s.history, from: s.history.snapshot };
@@ -499,52 +503,166 @@ async function beginMessageTurn(t, s, id, messages) {
499
503
  }
500
504
  /** Who checks a queued message's turn start: core, since the sender's grants were checked when it was queued. */
501
505
  export const QUEUE_SCOPE = { kind: "application", principalId: "queue" };
506
+ /**
507
+ * The refusals of a queued turn's start that a later try may get past (Host feature
508
+ * `queue-recovery`): the Agent's disk is being copied or it is being deleted (`agent_busy`), it
509
+ * can't run now or was deleted and may be made again (`agent_unavailable`), its pod was lost and
510
+ * may be reset (`agent_lost`). Any other refusal (a turn request the version no longer resolves,
511
+ * another harness, another owner's Agent) fails the messages at once.
512
+ */
513
+ const RETRIED_REFUSALS = new Set(["agent_busy", "agent_unavailable", "agent_lost"]);
514
+ const retried = (error) => error.status === 429 ||
515
+ error.status === 503 ||
516
+ (error.status === 409 && RETRIED_REFUSALS.has(error.rejection.code ?? ""));
502
517
  /**
503
518
  * Starts the next turn of session `s` (locked) from its queue, when it takes messages and messages
504
- * wait (protocol 17, D76): in the transaction that ended the last turn, or a cancel's. Our engine
505
- * takes every queued message that carries the same turn manifest as the first, a workflow one.
506
- * Writes `queue.taken`. Returns nothing when nothing started: an empty queue, or an Agent that
507
- * cannot take a turn now, whose messages wait for the next message or turn end.
519
+ * wait (protocol 17, D76): in the transaction that ended the last turn, a cancel's, a message's
520
+ * that joined a queue left waiting, or the Tenant sweep's (`retryQueuedStarts`). Our engine takes
521
+ * every queued message that carries the same turn manifest as the first, a workflow one. Writes
522
+ * `queue.taken`. Returns nothing when nothing started: an empty queue, or an Agent that refused
523
+ * the start.
524
+ *
525
+ * A refusal a later try may get past (`RETRIED_REFUSALS`) leaves the messages waiting, with
526
+ * `s.queueStalled` saying since when and why (written here); the next message, the sweep and the
527
+ * end of a disk copy try again. A message that has waited longer than the Runtime limit
528
+ * `queuedStartMs` since the stall began (or since it was queued, when later) fails with
529
+ * `queue.failed` (`queue.start_expired`); a refusal no retry changes fails the messages it refused
530
+ * at once (`queue.start_refused`). Either way the messages behind them are tried next (Host
531
+ * feature `queue-recovery`).
508
532
  */
509
- export async function startQueuedTurn(ctx, t, s, id) {
533
+ export async function startQueuedTurn(ctx, t, s, id, now = new Date()) {
510
534
  if (s.closedReason !== undefined || !takesMessages(s) || s.activeTurnId !== null)
511
535
  return undefined;
512
- const queued = await t.queuedMessages(id);
513
- if (queued.length === 0)
514
- return undefined;
515
- // Messages that ask the same of their turn's manifest (a variant, `removeTools`) share a turn.
516
- const manifestOf = (row) => canonical(turnManifestRequestOf(row.command) ?? null);
517
- const batch = isWorkflowManifest(s.manifest) ? [queued[0]] : [];
518
- if (batch.length === 0)
519
- for (const row of queued) {
520
- if (manifestOf(row) !== manifestOf(queued[0]))
521
- break;
522
- batch.push(row);
536
+ for (;;) {
537
+ const queued = await t.queuedMessages(id);
538
+ if (queued.length === 0) {
539
+ if (s.queueStalled !== undefined) {
540
+ delete s.queueStalled;
541
+ await t.put("sessions", id, s);
542
+ }
543
+ return undefined;
523
544
  }
524
- try {
525
- await checkAgentTurn(t, s, QUEUE_SCOPE, ctx);
526
- await beginMessageTurn(t, s, id, batch.map((row) => ({
527
- command: row.command,
528
- input: row.input,
529
- fileArtifacts: row.fileArtifacts,
530
- writer: row.writer,
531
- queuedMessageId: row.id,
532
- ...(row.trace ? { trace: row.trace } : {}),
533
- ...(row.cause ? { cause: row.cause } : {}),
534
- })));
535
- }
536
- catch (error) {
537
- if (error instanceof HttpError)
545
+ // Messages that ask the same of their turn's manifest (a variant, `removeTools`) share a turn.
546
+ const manifestOf = (row) => canonical(turnManifestRequestOf(row.command) ?? null);
547
+ const batch = isWorkflowManifest(s.manifest) ? [queued[0]] : [];
548
+ if (batch.length === 0)
549
+ for (const row of queued) {
550
+ if (manifestOf(row) !== manifestOf(queued[0]))
551
+ break;
552
+ batch.push(row);
553
+ }
554
+ try {
555
+ await checkAgentTurn(t, s, QUEUE_SCOPE, ctx);
556
+ await beginMessageTurn(t, s, id, batch.map((row) => ({
557
+ command: row.command,
558
+ input: row.input,
559
+ fileArtifacts: row.fileArtifacts,
560
+ writer: row.writer,
561
+ queuedMessageId: row.id,
562
+ ...(row.trace ? { trace: row.trace } : {}),
563
+ ...(row.cause ? { cause: row.cause } : {}),
564
+ })));
565
+ }
566
+ catch (error) {
567
+ if (!(error instanceof HttpError))
568
+ throw error;
569
+ const reason = error.rejection.code ?? "request_rejected";
570
+ if (!retried(error)) {
571
+ await failQueued(t, id, batch, {
572
+ code: "queue.start_refused",
573
+ message: `The Agent refused these messages' turn: ${error.message}`,
574
+ reason,
575
+ });
576
+ continue;
577
+ }
578
+ const since = s.queueStalled?.since ?? now.toISOString();
579
+ const { queuedStartMs } = await readRuntimeLimits(t);
580
+ const waited = (row) => now.getTime() - Math.max(Date.parse(since), Date.parse(row.createdAt));
581
+ const expired = batch.filter((row) => waited(row) >= queuedStartMs);
582
+ if (expired[0]?.id === batch[0].id) {
583
+ await failQueued(t, id, expired, {
584
+ code: "queue.start_expired",
585
+ message: `The Agent refused these messages' turn for longer than the Runtime limit queuedStartMs (${queuedStartMs} ms): ${error.message}`,
586
+ reason,
587
+ });
588
+ continue;
589
+ }
590
+ s.queueStalled = { since, code: reason, message: error.message };
591
+ await t.put("sessions", id, s);
538
592
  return undefined;
539
- throw error;
593
+ }
594
+ const taken = batch.map((row) => row.id);
595
+ await t.deleteQueuedMessages(taken);
596
+ const event = await t.event(id, s.activeTurnId, "queue.taken", {
597
+ turnId: s.activeTurnId,
598
+ queuedMessageIds: taken,
599
+ });
600
+ return { turnId: s.activeTurnId, taken, cursor: event.cursor };
540
601
  }
541
- const taken = batch.map((row) => row.id);
542
- await t.deleteQueuedMessages(taken);
543
- const event = await t.event(id, s.activeTurnId, "queue.taken", {
544
- turnId: s.activeTurnId,
545
- queuedMessageIds: taken,
602
+ }
603
+ /** Removes queued messages no turn will take, with `queue.failed` (Host feature `queue-recovery`). */
604
+ async function failQueued(t, id, rows, error) {
605
+ const ids = rows.map((row) => row.id);
606
+ await t.deleteQueuedMessages(ids);
607
+ await t.event(id, null, "queue.failed", { queuedMessageIds: ids, error });
608
+ }
609
+ /**
610
+ * Removes queued message `queuedMessageId` from session `id`'s queue, as `scope` (Host feature
611
+ * `queue-recovery`): `queue.cancelled`, written by the caller. A turn running or not, the message
612
+ * never starts one. Returns the event's cursor, or undefined when the message no longer waits (a
613
+ * turn took it, or it failed or was cancelled already).
614
+ */
615
+ export async function cancelQueuedMessage(ctx, id, queuedMessageId, scope, reason) {
616
+ return ctx.store.tx(async (t) => {
617
+ const s = await lockedSession(t, id, accessOf(scope));
618
+ const queued = await t.queuedMessages(id);
619
+ if (!queued.some((row) => row.id === queuedMessageId))
620
+ return undefined;
621
+ await t.deleteQueuedMessages([queuedMessageId]);
622
+ const event = await t.event(id, null, "queue.cancelled", { queuedMessageIds: [queuedMessageId], ...(reason === undefined ? {} : { reason }) }, commandWriter(scope));
623
+ if (queued.length === 1 && s.queueStalled !== undefined) {
624
+ delete s.queueStalled;
625
+ await t.put("sessions", id, s);
626
+ }
627
+ return { cursor: event.cursor };
546
628
  });
547
- return { turnId: s.activeTurnId, taken, cursor: event.cursor };
629
+ }
630
+ /**
631
+ * Tries again the turn start of every queue a refused start left waiting, one transaction per
632
+ * session (Host feature `queue-recovery`): the Tenant sweep's step, and what the end of an Agent's
633
+ * disk copy runs for that Agent's sessions (`environmentId`). Each start fails its messages past
634
+ * `queuedStartMs` (`startQueuedTurn`). Returns the sessions whose queue started a turn.
635
+ */
636
+ export async function retryQueuedStarts(ctx, options = {}) {
637
+ const started = [];
638
+ const limit = options.batchSize ?? 100;
639
+ let after;
640
+ for (;;) {
641
+ const ids = await ctx.store.tx((t) => t.queuedSessions({
642
+ statuses: TAKES_MESSAGES,
643
+ limit,
644
+ ...(after === undefined ? {} : { after }),
645
+ ...(options.environmentId === undefined ? {} : { environmentId: options.environmentId }),
646
+ }));
647
+ for (const id of ids) {
648
+ if (ctx.closing || ctx.closed)
649
+ return started;
650
+ const turn = await ctx.store.tx(async (t) => {
651
+ const s = await t.lockSession(id);
652
+ if (!s)
653
+ return undefined;
654
+ const begun = await startQueuedTurn(ctx, t, s, id, options.now ?? new Date());
655
+ if (begun)
656
+ await t.put("sessions", id, s);
657
+ return begun;
658
+ });
659
+ if (turn)
660
+ started.push(id);
661
+ }
662
+ if (ids.length < limit)
663
+ return started;
664
+ after = ids.at(-1);
665
+ }
548
666
  }
549
667
  /** Core's own commands (R4.9): the flow host's, and a drain's cancels. */
550
668
  const CORE_PRINCIPALS = { "flow-host": "flow-host", drain: "core" };
@@ -113,6 +113,17 @@ export interface Session {
113
113
  triggerId: string;
114
114
  runId: string;
115
115
  };
116
+ /**
117
+ * Set while the session's queued messages wait for an Agent that refused their turn's start
118
+ * (Host feature `queue-recovery`, `startQueuedTurn`): since when, and the last refusal. The
119
+ * Tenant sweep tries the start again, and fails them once `queuedStartMs` has passed since.
120
+ * Cleared when a turn starts, or when the queue empties.
121
+ */
122
+ queueStalled?: {
123
+ since: string;
124
+ code: string;
125
+ message: string;
126
+ };
116
127
  creation: unknown;
117
128
  mcpSnapshot?: McpSnapshot;
118
129
  mcpDiagnostics?: readonly McpDiagnostic[];
@@ -43,7 +43,11 @@
43
43
  * the delivery after commit, sends), and configs of tasks long finished are deleted. A push
44
44
  * row that fails is retried with the Host tuning's `a2a.pushRetry` and dropped once it is
45
45
  * `a2a.pushMaxAgeMs` old, never parked: the change feed covers what it said.
46
- * 7. **Hooks.** Callbacks registered with `ctx.onSweep`.
46
+ * 7. **Queued messages** (Host feature `queue-recovery`): at most every `sweep.queueRetryMs`, the
47
+ * turn start of every session queue an Agent's refusal left waiting is tried again
48
+ * (`retryQueuedStarts`); a message past the Runtime limit `queuedStartMs` fails
49
+ * (`queue.failed`). The end of a disk copy tries its Agent's sessions at once.
50
+ * 8. **Hooks.** Callbacks registered with `ctx.onSweep`.
47
51
  *
48
52
  * Every step runs one transaction per session it changes (a linked agent and its workflow
49
53
  * share one, child first), so the sweep follows the lock order in `store/types.ts`. A failing
@@ -51,12 +55,14 @@
51
55
  * reports it.
52
56
  */
53
57
  import { sweepUnusedAgents } from "./agents.js";
58
+ import { retryQueuedStarts } from "./commands.js";
54
59
  import { reconcileTriggers } from "./triggers.js";
55
60
  import { streamMissedRuns } from "./runs-stream.js";
56
61
  import { sweepPendingSeeds } from "../artifacts/disk.js";
57
62
  import { pendingAgentEffects, reconcilePendingAgentEffect, } from "../core/flow-host.js";
58
63
  import { hostTuning } from "../tuning.js";
59
- import { sweepStartChecks } from "./agent-versions.js";
64
+ import { idleMsOf, sweepStartChecks } from "./agent-versions.js";
65
+ import { readSandboxConfig } from "../sandbox/tenant-config.js";
60
66
  import { pumpPushes, pushRetry } from "../api/a2a/push.js";
61
67
  /*
62
68
  * The sweep's timings are the Host tuning's `sweep` group (`tuning.ts`):
@@ -69,6 +75,7 @@ import { pumpPushes, pushRetry } from "../api/a2a/push.js";
69
75
  * - `drainBudgetMs`: how long one pass may spend draining the outbox, and paging through
70
76
  * orphaned sessions. A backlog is worked off over several passes rather than holding one (and
71
77
  * with it the Tenant's sweep object in the execution) for long.
78
+ * - `queueRetryMs`: how often the queues an Agent's refusal left waiting are tried again.
72
79
  * - `batchSize`, `signalRetentionMs`, `outboxAttempts`, `outboxRetry`, `stalePodMs`.
73
80
  */
74
81
  /**
@@ -88,6 +95,7 @@ export async function sweep(ctx) {
88
95
  pass.orphansAfter !== undefined ||
89
96
  now.getTime() - pass.orphansAt >= hostTuning().sweep.orphanScanMs;
90
97
  const checkPods = pass.podsAt === undefined || now.getTime() - pass.podsAt >= hostTuning().sweep.orphanScanMs;
98
+ const retryQueues = pass.queuesAt === undefined || now.getTime() - pass.queuesAt >= hostTuning().sweep.queueRetryMs;
91
99
  const steps = [
92
100
  ["outbox", () => deliverOutbox(ctx, first ? 0 : hostTuning().sweep.outboxGraceMs)],
93
101
  ["linked", () => reconcileLinkedAgents(ctx)],
@@ -142,6 +150,17 @@ export async function sweep(ctx) {
142
150
  "signals",
143
151
  () => ctx.store.tx((t) => t.pruneSignals(new Date(now.getTime() - hostTuning().sweep.signalRetentionMs))),
144
152
  ],
153
+ ...(retryQueues
154
+ ? [
155
+ [
156
+ "queues",
157
+ async () => {
158
+ pass.queuesAt = now.getTime();
159
+ await retryQueuedStarts(ctx, { now, batchSize: hostTuning().sweep.batchSize });
160
+ },
161
+ ],
162
+ ]
163
+ : []),
145
164
  ...[...ctx.sweepHooks].map((hook) => ["hook", hook]),
146
165
  ];
147
166
  let failure;
@@ -287,7 +306,8 @@ export async function wakeOrphanedSessions(ctx, now = new Date(), after, budgetM
287
306
  }
288
307
  /**
289
308
  * Pod sandboxes (F7.2) whose change has not finished (creating, deleting, an old volume still
290
- * to delete, an expiry not yet recorded), or that should run but whose engine is not connected,
309
+ * to delete, an expiry not yet recorded), that should run but whose engine is not connected, or
310
+ * that run past their idle limit (a wake that armed no idle timer: the reconcile arms it),
291
311
  * and whose row has not moved for `sweep.stalePodMs` (a minute), get a reconcile: a pod or volume deleted outside
292
312
  * the Runtime is found, and a change goes on whose reconcile the execution itself lost (its
293
313
  * retry timer, an invocation an operator killed, Restate's state wiped) or whose outbox row
@@ -296,18 +316,27 @@ export async function wakeOrphanedSessions(ctx, now = new Date(), after, budgetM
296
316
  async function reconcileStalePods(ctx, now) {
297
317
  if (!ctx.pods)
298
318
  return;
299
- const stale = await ctx.store.tx(async (t) => (await t.listEnvironments()).filter((row) => {
300
- const pod = row.pod;
301
- if (!pod || now.getTime() - Date.parse(row.updatedAt) < hostTuning().sweep.stalePodMs)
302
- return false;
303
- return (pod.desired === "deleted" ||
304
- pod.retiring !== undefined ||
305
- pod.observed === "creating" ||
306
- pod.observed === "deleting" ||
307
- (pod.desired === "suspended" && pod.observed !== "suspended") ||
308
- // Running, but its engine is not connected: the pod may be gone, or its volume.
309
- (pod.desired === "running" && pod.observed === "running" && !ctx.harness.hosting(row.id)));
310
- }));
319
+ const stale = await ctx.store.tx(async (t) => {
320
+ const config = await readSandboxConfig(t);
321
+ return (await t.listEnvironments()).filter((row) => {
322
+ const pod = row.pod;
323
+ if (!pod || now.getTime() - Date.parse(row.updatedAt) < hostTuning().sweep.stalePodMs)
324
+ return false;
325
+ return (pod.desired === "deleted" ||
326
+ pod.retiring !== undefined ||
327
+ pod.observed === "creating" ||
328
+ pod.observed === "deleting" ||
329
+ (pod.desired === "suspended" && pod.observed !== "suspended") ||
330
+ // Running, but its engine is not connected: the pod may be gone, or its volume.
331
+ (pod.desired === "running" && pod.observed === "running" && !ctx.harness.hosting(row.id)) ||
332
+ // Running past its idle limit: its idle timer is missing (or a turn holds it, which the
333
+ // reconcile leaves alone).
334
+ (pod.desired === "running" &&
335
+ pod.observed === "running" &&
336
+ now.getTime() - Date.parse(pod.lastActiveAt ?? row.createdAt) >
337
+ idleMsOf(config, row) + hostTuning().sweep.stalePodMs));
338
+ });
339
+ });
311
340
  for (const row of stale)
312
341
  await ctx.sandboxSignal(row.id, { kind: "reconcile" });
313
342
  }
package/dist/tuning.d.ts CHANGED
@@ -121,6 +121,7 @@ export declare const HostTuningSchema: z.ZodObject<{
121
121
  maxMs: z.ZodDefault<z.ZodNumber>;
122
122
  }, z.core.$strict>>;
123
123
  stalePodMs: z.ZodDefault<z.ZodNumber>;
124
+ queueRetryMs: z.ZodDefault<z.ZodNumber>;
124
125
  }, z.core.$strict>>;
125
126
  a2a: z.ZodPrefault<z.ZodObject<{
126
127
  pushRetry: z.ZodPrefault<z.ZodObject<{
@@ -297,6 +298,8 @@ export declare const HostTuningSchema: z.ZodObject<{
297
298
  networkProbeMs: z.ZodDefault<z.ZodNumber>;
298
299
  networkPauseMs: z.ZodDefault<z.ZodNumber>;
299
300
  networkConfirmations: z.ZodDefault<z.ZodNumber>;
301
+ wakeNetworkProbeMs: z.ZodDefault<z.ZodNumber>;
302
+ wakeNetworkConfirmations: z.ZodDefault<z.ZodNumber>;
300
303
  setupRetryPollMs: z.ZodDefault<z.ZodNumber>;
301
304
  setupAfterSeedPollMs: z.ZodDefault<z.ZodNumber>;
302
305
  setupReportTailChars: z.ZodDefault<z.ZodNumber>;
package/dist/tuning.js CHANGED
@@ -207,6 +207,11 @@ export const HostTuningSchema = z
207
207
  outboxRetry: backoff(5_000, 10 * MINUTE),
208
208
  /** How long a pod row may go unchanged before the sweep checks it. */
209
209
  stalePodMs: ms(MINUTE),
210
+ /**
211
+ * How often the sweep tries again the turn start of queued messages their Agent refused
212
+ * (Host feature `queue-recovery`); they fail after the Runtime limit `queuedStartMs`.
213
+ */
214
+ queueRetryMs: ms(30_000),
210
215
  }),
211
216
  /** The A2A server's push notifications (Host feature `a2a-profile-v2`). */
212
217
  a2a: group({
@@ -495,6 +500,13 @@ export const HostTuningSchema = z
495
500
  networkProbeMs: ms(1_000),
496
501
  networkPauseMs: ms(250, 0),
497
502
  networkConfirmations: count(3),
503
+ /**
504
+ * The same check on a wake (the disk already holds the engine's volume id): each probe, and
505
+ * the failed probes in a row that prove an address blocked. A probe that connects turns the
506
+ * wake's check into the full one.
507
+ */
508
+ wakeNetworkProbeMs: ms(150),
509
+ wakeNetworkConfirmations: count(1),
498
510
  /** Environment setup: how often a failed setup is checked for a retry, and after a seed. */
499
511
  setupRetryPollMs: ms(5_000),
500
512
  setupAfterSeedPollMs: ms(1_000),
package/dist/version.d.ts CHANGED
@@ -1 +1 @@
1
- export declare const RUNTIME_VERSION = "0.43.0-beta";
1
+ export declare const RUNTIME_VERSION = "0.44.1-beta";
package/dist/version.js CHANGED
@@ -1,3 +1,3 @@
1
1
  // Kept in step with package.json by runtime/server/scripts/check-package.mjs, which runs in npm run check.
2
2
  // release:prepare also rewrites this when it bumps @nylorun/runtime.
3
- export const RUNTIME_VERSION = "0.43.0-beta";
3
+ export const RUNTIME_VERSION = "0.44.1-beta";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@nylorun/runtime",
3
- "version": "0.43.0-beta",
3
+ "version": "0.44.1-beta",
4
4
  "description": "Nylorun Runtime: the agent execution Host on Postgres, Restate and S2, and model providers.",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -65,8 +65,8 @@
65
65
  "@hono/node-server": "^2.1.5",
66
66
  "@hono/zod-openapi": "^1.6.3",
67
67
  "@modelcontextprotocol/sdk": "^1.32.1",
68
- "@nylorun/core": "0.37.0-beta",
69
- "@nylorun/harness": "0.34.0-beta",
68
+ "@nylorun/core": "0.38.0-beta",
69
+ "@nylorun/harness": "0.34.1-beta",
70
70
  "@restatedev/restate-sdk": "1.17.3",
71
71
  "@restatedev/restate-sdk-clients": "1.17.3",
72
72
  "@s2-dev/streamstore": "0.28.0",