@nylorun/runtime 0.43.0-beta → 0.44.1-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/dist/api/a2a/endpoint.js +1 -1
- package/dist/api/a2a/push.js +14 -1
- package/dist/api/a2a/routes.js +40 -26
- package/dist/api/a2a/tasks.d.ts +14 -0
- package/dist/api/a2a/tasks.js +42 -0
- package/dist/api/components.d.ts +1158 -280
- package/dist/artifacts/disk.js +7 -0
- package/dist/harness/main.js +8 -4
- package/dist/management-openapi.json +14 -4
- package/dist/openapi.json +433 -3
- package/dist/sandbox/pods/lifecycle.d.ts +5 -1
- package/dist/sandbox/pods/lifecycle.js +8 -0
- package/dist/sandbox/pods/network-gate.d.ts +6 -0
- package/dist/sandbox/pods/network-gate.js +20 -4
- package/dist/store/postgres/store.js +11 -0
- package/dist/store/types.d.ts +11 -1
- package/dist/tenant/agent-versions.d.ts +5 -1
- package/dist/tenant/agent-versions.js +8 -1
- package/dist/tenant/commands.d.ts +36 -7
- package/dist/tenant/commands.js +158 -40
- package/dist/tenant/context.d.ts +11 -0
- package/dist/tenant/sweep.js +44 -15
- package/dist/tuning.d.ts +3 -0
- package/dist/tuning.js +12 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +3 -3
|
@@ -55,8 +55,8 @@ export type MessageAnswer = {
|
|
|
55
55
|
/**
|
|
56
56
|
* Takes a resolved message on session `s` (locked) in transaction `t`: it starts a turn when the
|
|
57
57
|
* session takes messages and none wait, else it waits in the queue behind them (protocol 17), and
|
|
58
|
-
* an idle session whose queue was left waiting (its Agent
|
|
59
|
-
* ended)
|
|
58
|
+
* an idle session whose queue was left waiting (its Agent refused the start when the last turn
|
|
59
|
+
* ended, `startQueuedTurn`) tries the start again with all of them, this one last. Past `sessionQueuedMessages` it is
|
|
60
60
|
* `429 queue_full`. `waiting` is the queue's length, read under the session's lock. `asked`
|
|
61
61
|
* `steer` (D85) marks a waiting message so the running turn may take it before a step; a
|
|
62
62
|
* workflow session can't, so it waits as `queue`.
|
|
@@ -76,16 +76,45 @@ export declare function takeSteeredMessages(t: Tx, s: Session, id: string, step:
|
|
|
76
76
|
export declare const QUEUE_SCOPE: AuthScope;
|
|
77
77
|
/**
|
|
78
78
|
* Starts the next turn of session `s` (locked) from its queue, when it takes messages and messages
|
|
79
|
-
* wait (protocol 17, D76): in the transaction that ended the last turn,
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
79
|
+
* wait (protocol 17, D76): in the transaction that ended the last turn, a cancel's, a message's
|
|
80
|
+
* that joined a queue left waiting, or the Tenant sweep's (`retryQueuedStarts`). Our engine takes
|
|
81
|
+
* every queued message that carries the same turn manifest as the first, a workflow one. Writes
|
|
82
|
+
* `queue.taken`. Returns nothing when nothing started: an empty queue, or an Agent that refused
|
|
83
|
+
* the start.
|
|
84
|
+
*
|
|
85
|
+
* A refusal a later try may get past (`RETRIED_REFUSALS`) leaves the messages waiting, with
|
|
86
|
+
* `s.queueStalled` saying since when and why (written here); the next message, the sweep and the
|
|
87
|
+
* end of a disk copy try again. A message that has waited longer than the Runtime limit
|
|
88
|
+
* `queuedStartMs` since the stall began (or since it was queued, when later) fails with
|
|
89
|
+
* `queue.failed` (`queue.start_expired`); a refusal no retry changes fails the messages it refused
|
|
90
|
+
* at once (`queue.start_refused`). Either way the messages behind them are tried next (Host
|
|
91
|
+
* feature `queue-recovery`).
|
|
83
92
|
*/
|
|
84
|
-
export declare function startQueuedTurn(ctx: TenantContext, t: Tx, s: Session, id: string): Promise<{
|
|
93
|
+
export declare function startQueuedTurn(ctx: TenantContext, t: Tx, s: Session, id: string, now?: Date): Promise<{
|
|
85
94
|
turnId: string;
|
|
86
95
|
taken: string[];
|
|
87
96
|
cursor: string;
|
|
88
97
|
} | undefined>;
|
|
98
|
+
/**
|
|
99
|
+
* Removes queued message `queuedMessageId` from session `id`'s queue, as `scope` (Host feature
|
|
100
|
+
* `queue-recovery`): `queue.cancelled`, written by the caller. A turn running or not, the message
|
|
101
|
+
* never starts one. Returns the event's cursor, or undefined when the message no longer waits (a
|
|
102
|
+
* turn took it, or it failed or was cancelled already).
|
|
103
|
+
*/
|
|
104
|
+
export declare function cancelQueuedMessage(ctx: TenantContext, id: string, queuedMessageId: string, scope: AuthScope, reason?: string): Promise<{
|
|
105
|
+
cursor: string;
|
|
106
|
+
} | undefined>;
|
|
107
|
+
/**
|
|
108
|
+
* Tries again the turn start of every queue a refused start left waiting, one transaction per
|
|
109
|
+
* session (Host feature `queue-recovery`): the Tenant sweep's step, and what the end of an Agent's
|
|
110
|
+
* disk copy runs for that Agent's sessions (`environmentId`). Each start fails its messages past
|
|
111
|
+
* `queuedStartMs` (`startQueuedTurn`). Returns the sessions whose queue started a turn.
|
|
112
|
+
*/
|
|
113
|
+
export declare function retryQueuedStarts(ctx: TenantContext, options?: {
|
|
114
|
+
environmentId?: string;
|
|
115
|
+
now?: Date;
|
|
116
|
+
batchSize?: number;
|
|
117
|
+
}): Promise<string[]>;
|
|
89
118
|
/**
|
|
90
119
|
* Who sent a command (R4.9), from its credential and never its body: a client over the Runtime
|
|
91
120
|
* API (an application, or a person through it or a token), or core itself.
|
package/dist/tenant/commands.js
CHANGED
|
@@ -19,7 +19,9 @@
|
|
|
19
19
|
* (`session_queue`, `message.queued`) unless it asks `delivery: "reject"`, which is
|
|
20
20
|
* `409 session_busy`. A turn that ends takes every queued message in the same transaction
|
|
21
21
|
* (`startQueuedTurn`): client messages and queued ones start a turn through one door,
|
|
22
|
-
* `beginMessageTurn`, which writes each one's `command.message`. A
|
|
22
|
+
* `beginMessageTurn`, which writes each one's `command.message`. A start the Agent refuses leaves
|
|
23
|
+
* the queue waiting, tried again by the Tenant sweep until the Runtime limit `queuedStartMs`, then
|
|
24
|
+
* failed (`queue.failed`, Host feature `queue-recovery`). A message that asks
|
|
23
25
|
* `delivery: "steer"` (D85) waits marked `steer`, and the running turn takes it before a step
|
|
24
26
|
* (`takeSteeredMessages`, the `inputs` effect); one it doesn't take starts the next turn like
|
|
25
27
|
* any other.
|
|
@@ -300,8 +302,8 @@ export const takesMessages = (s) => TAKES_MESSAGES.includes(s.status);
|
|
|
300
302
|
/**
|
|
301
303
|
* Takes a resolved message on session `s` (locked) in transaction `t`: it starts a turn when the
|
|
302
304
|
* session takes messages and none wait, else it waits in the queue behind them (protocol 17), and
|
|
303
|
-
* an idle session whose queue was left waiting (its Agent
|
|
304
|
-
* ended)
|
|
305
|
+
* an idle session whose queue was left waiting (its Agent refused the start when the last turn
|
|
306
|
+
* ended, `startQueuedTurn`) tries the start again with all of them, this one last. Past `sessionQueuedMessages` it is
|
|
305
307
|
* `429 queue_full`. `waiting` is the queue's length, read under the session's lock. `asked`
|
|
306
308
|
* `steer` (D85) marks a waiting message so the running turn may take it before a step; a
|
|
307
309
|
* workflow session can't, so it waits as `queue`.
|
|
@@ -464,6 +466,8 @@ async function beginMessageTurn(t, s, id, messages) {
|
|
|
464
466
|
});
|
|
465
467
|
}
|
|
466
468
|
s.turnStartState = s.state;
|
|
469
|
+
// A turn holds the session now: its queue, if any, waits for the turn, not for the Agent.
|
|
470
|
+
delete s.queueStalled;
|
|
467
471
|
// A new turn folds from the latest snapshot: a cancel or failure only reverts this turn.
|
|
468
472
|
if (s.history?.snapshot !== undefined)
|
|
469
473
|
s.history = { ...s.history, from: s.history.snapshot };
|
|
@@ -499,52 +503,166 @@ async function beginMessageTurn(t, s, id, messages) {
|
|
|
499
503
|
}
|
|
500
504
|
/** Who checks a queued message's turn start: core, since the sender's grants were checked when it was queued. */
|
|
501
505
|
export const QUEUE_SCOPE = { kind: "application", principalId: "queue" };
|
|
506
|
+
/**
|
|
507
|
+
* The refusals of a queued turn's start that a later try may get past (Host feature
|
|
508
|
+
* `queue-recovery`): the Agent's disk is being copied or it is being deleted (`agent_busy`), it
|
|
509
|
+
* can't run now or was deleted and may be made again (`agent_unavailable`), its pod was lost and
|
|
510
|
+
* may be reset (`agent_lost`). Any other refusal (a turn request the version no longer resolves,
|
|
511
|
+
* another harness, another owner's Agent) fails the messages at once.
|
|
512
|
+
*/
|
|
513
|
+
const RETRIED_REFUSALS = new Set(["agent_busy", "agent_unavailable", "agent_lost"]);
|
|
514
|
+
const retried = (error) => error.status === 429 ||
|
|
515
|
+
error.status === 503 ||
|
|
516
|
+
(error.status === 409 && RETRIED_REFUSALS.has(error.rejection.code ?? ""));
|
|
502
517
|
/**
|
|
503
518
|
* Starts the next turn of session `s` (locked) from its queue, when it takes messages and messages
|
|
504
|
-
* wait (protocol 17, D76): in the transaction that ended the last turn,
|
|
505
|
-
*
|
|
506
|
-
*
|
|
507
|
-
*
|
|
519
|
+
* wait (protocol 17, D76): in the transaction that ended the last turn, a cancel's, a message's
|
|
520
|
+
* that joined a queue left waiting, or the Tenant sweep's (`retryQueuedStarts`). Our engine takes
|
|
521
|
+
* every queued message that carries the same turn manifest as the first, a workflow one. Writes
|
|
522
|
+
* `queue.taken`. Returns nothing when nothing started: an empty queue, or an Agent that refused
|
|
523
|
+
* the start.
|
|
524
|
+
*
|
|
525
|
+
* A refusal a later try may get past (`RETRIED_REFUSALS`) leaves the messages waiting, with
|
|
526
|
+
* `s.queueStalled` saying since when and why (written here); the next message, the sweep and the
|
|
527
|
+
* end of a disk copy try again. A message that has waited longer than the Runtime limit
|
|
528
|
+
* `queuedStartMs` since the stall began (or since it was queued, when later) fails with
|
|
529
|
+
* `queue.failed` (`queue.start_expired`); a refusal no retry changes fails the messages it refused
|
|
530
|
+
* at once (`queue.start_refused`). Either way the messages behind them are tried next (Host
|
|
531
|
+
* feature `queue-recovery`).
|
|
508
532
|
*/
|
|
509
|
-
export async function startQueuedTurn(ctx, t, s, id) {
|
|
533
|
+
export async function startQueuedTurn(ctx, t, s, id, now = new Date()) {
|
|
510
534
|
if (s.closedReason !== undefined || !takesMessages(s) || s.activeTurnId !== null)
|
|
511
535
|
return undefined;
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
if (manifestOf(row) !== manifestOf(queued[0]))
|
|
521
|
-
break;
|
|
522
|
-
batch.push(row);
|
|
536
|
+
for (;;) {
|
|
537
|
+
const queued = await t.queuedMessages(id);
|
|
538
|
+
if (queued.length === 0) {
|
|
539
|
+
if (s.queueStalled !== undefined) {
|
|
540
|
+
delete s.queueStalled;
|
|
541
|
+
await t.put("sessions", id, s);
|
|
542
|
+
}
|
|
543
|
+
return undefined;
|
|
523
544
|
}
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
545
|
+
// Messages that ask the same of their turn's manifest (a variant, `removeTools`) share a turn.
|
|
546
|
+
const manifestOf = (row) => canonical(turnManifestRequestOf(row.command) ?? null);
|
|
547
|
+
const batch = isWorkflowManifest(s.manifest) ? [queued[0]] : [];
|
|
548
|
+
if (batch.length === 0)
|
|
549
|
+
for (const row of queued) {
|
|
550
|
+
if (manifestOf(row) !== manifestOf(queued[0]))
|
|
551
|
+
break;
|
|
552
|
+
batch.push(row);
|
|
553
|
+
}
|
|
554
|
+
try {
|
|
555
|
+
await checkAgentTurn(t, s, QUEUE_SCOPE, ctx);
|
|
556
|
+
await beginMessageTurn(t, s, id, batch.map((row) => ({
|
|
557
|
+
command: row.command,
|
|
558
|
+
input: row.input,
|
|
559
|
+
fileArtifacts: row.fileArtifacts,
|
|
560
|
+
writer: row.writer,
|
|
561
|
+
queuedMessageId: row.id,
|
|
562
|
+
...(row.trace ? { trace: row.trace } : {}),
|
|
563
|
+
...(row.cause ? { cause: row.cause } : {}),
|
|
564
|
+
})));
|
|
565
|
+
}
|
|
566
|
+
catch (error) {
|
|
567
|
+
if (!(error instanceof HttpError))
|
|
568
|
+
throw error;
|
|
569
|
+
const reason = error.rejection.code ?? "request_rejected";
|
|
570
|
+
if (!retried(error)) {
|
|
571
|
+
await failQueued(t, id, batch, {
|
|
572
|
+
code: "queue.start_refused",
|
|
573
|
+
message: `The Agent refused these messages' turn: ${error.message}`,
|
|
574
|
+
reason,
|
|
575
|
+
});
|
|
576
|
+
continue;
|
|
577
|
+
}
|
|
578
|
+
const since = s.queueStalled?.since ?? now.toISOString();
|
|
579
|
+
const { queuedStartMs } = await readRuntimeLimits(t);
|
|
580
|
+
const waited = (row) => now.getTime() - Math.max(Date.parse(since), Date.parse(row.createdAt));
|
|
581
|
+
const expired = batch.filter((row) => waited(row) >= queuedStartMs);
|
|
582
|
+
if (expired[0]?.id === batch[0].id) {
|
|
583
|
+
await failQueued(t, id, expired, {
|
|
584
|
+
code: "queue.start_expired",
|
|
585
|
+
message: `The Agent refused these messages' turn for longer than the Runtime limit queuedStartMs (${queuedStartMs} ms): ${error.message}`,
|
|
586
|
+
reason,
|
|
587
|
+
});
|
|
588
|
+
continue;
|
|
589
|
+
}
|
|
590
|
+
s.queueStalled = { since, code: reason, message: error.message };
|
|
591
|
+
await t.put("sessions", id, s);
|
|
538
592
|
return undefined;
|
|
539
|
-
|
|
593
|
+
}
|
|
594
|
+
const taken = batch.map((row) => row.id);
|
|
595
|
+
await t.deleteQueuedMessages(taken);
|
|
596
|
+
const event = await t.event(id, s.activeTurnId, "queue.taken", {
|
|
597
|
+
turnId: s.activeTurnId,
|
|
598
|
+
queuedMessageIds: taken,
|
|
599
|
+
});
|
|
600
|
+
return { turnId: s.activeTurnId, taken, cursor: event.cursor };
|
|
540
601
|
}
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
602
|
+
}
|
|
603
|
+
/** Removes queued messages no turn will take, with `queue.failed` (Host feature `queue-recovery`). */
|
|
604
|
+
async function failQueued(t, id, rows, error) {
|
|
605
|
+
const ids = rows.map((row) => row.id);
|
|
606
|
+
await t.deleteQueuedMessages(ids);
|
|
607
|
+
await t.event(id, null, "queue.failed", { queuedMessageIds: ids, error });
|
|
608
|
+
}
|
|
609
|
+
/**
|
|
610
|
+
* Removes queued message `queuedMessageId` from session `id`'s queue, as `scope` (Host feature
|
|
611
|
+
* `queue-recovery`): `queue.cancelled`, written by the caller. A turn running or not, the message
|
|
612
|
+
* never starts one. Returns the event's cursor, or undefined when the message no longer waits (a
|
|
613
|
+
* turn took it, or it failed or was cancelled already).
|
|
614
|
+
*/
|
|
615
|
+
export async function cancelQueuedMessage(ctx, id, queuedMessageId, scope, reason) {
|
|
616
|
+
return ctx.store.tx(async (t) => {
|
|
617
|
+
const s = await lockedSession(t, id, accessOf(scope));
|
|
618
|
+
const queued = await t.queuedMessages(id);
|
|
619
|
+
if (!queued.some((row) => row.id === queuedMessageId))
|
|
620
|
+
return undefined;
|
|
621
|
+
await t.deleteQueuedMessages([queuedMessageId]);
|
|
622
|
+
const event = await t.event(id, null, "queue.cancelled", { queuedMessageIds: [queuedMessageId], ...(reason === undefined ? {} : { reason }) }, commandWriter(scope));
|
|
623
|
+
if (queued.length === 1 && s.queueStalled !== undefined) {
|
|
624
|
+
delete s.queueStalled;
|
|
625
|
+
await t.put("sessions", id, s);
|
|
626
|
+
}
|
|
627
|
+
return { cursor: event.cursor };
|
|
546
628
|
});
|
|
547
|
-
|
|
629
|
+
}
|
|
630
|
+
/**
|
|
631
|
+
* Tries again the turn start of every queue a refused start left waiting, one transaction per
|
|
632
|
+
* session (Host feature `queue-recovery`): the Tenant sweep's step, and what the end of an Agent's
|
|
633
|
+
* disk copy runs for that Agent's sessions (`environmentId`). Each start fails its messages past
|
|
634
|
+
* `queuedStartMs` (`startQueuedTurn`). Returns the sessions whose queue started a turn.
|
|
635
|
+
*/
|
|
636
|
+
export async function retryQueuedStarts(ctx, options = {}) {
|
|
637
|
+
const started = [];
|
|
638
|
+
const limit = options.batchSize ?? 100;
|
|
639
|
+
let after;
|
|
640
|
+
for (;;) {
|
|
641
|
+
const ids = await ctx.store.tx((t) => t.queuedSessions({
|
|
642
|
+
statuses: TAKES_MESSAGES,
|
|
643
|
+
limit,
|
|
644
|
+
...(after === undefined ? {} : { after }),
|
|
645
|
+
...(options.environmentId === undefined ? {} : { environmentId: options.environmentId }),
|
|
646
|
+
}));
|
|
647
|
+
for (const id of ids) {
|
|
648
|
+
if (ctx.closing || ctx.closed)
|
|
649
|
+
return started;
|
|
650
|
+
const turn = await ctx.store.tx(async (t) => {
|
|
651
|
+
const s = await t.lockSession(id);
|
|
652
|
+
if (!s)
|
|
653
|
+
return undefined;
|
|
654
|
+
const begun = await startQueuedTurn(ctx, t, s, id, options.now ?? new Date());
|
|
655
|
+
if (begun)
|
|
656
|
+
await t.put("sessions", id, s);
|
|
657
|
+
return begun;
|
|
658
|
+
});
|
|
659
|
+
if (turn)
|
|
660
|
+
started.push(id);
|
|
661
|
+
}
|
|
662
|
+
if (ids.length < limit)
|
|
663
|
+
return started;
|
|
664
|
+
after = ids.at(-1);
|
|
665
|
+
}
|
|
548
666
|
}
|
|
549
667
|
/** Core's own commands (R4.9): the flow host's, and a drain's cancels. */
|
|
550
668
|
const CORE_PRINCIPALS = { "flow-host": "flow-host", drain: "core" };
|
package/dist/tenant/context.d.ts
CHANGED
|
@@ -113,6 +113,17 @@ export interface Session {
|
|
|
113
113
|
triggerId: string;
|
|
114
114
|
runId: string;
|
|
115
115
|
};
|
|
116
|
+
/**
|
|
117
|
+
* Set while the session's queued messages wait for an Agent that refused their turn's start
|
|
118
|
+
* (Host feature `queue-recovery`, `startQueuedTurn`): since when, and the last refusal. The
|
|
119
|
+
* Tenant sweep tries the start again, and fails them once `queuedStartMs` has passed since.
|
|
120
|
+
* Cleared when a turn starts, or when the queue empties.
|
|
121
|
+
*/
|
|
122
|
+
queueStalled?: {
|
|
123
|
+
since: string;
|
|
124
|
+
code: string;
|
|
125
|
+
message: string;
|
|
126
|
+
};
|
|
116
127
|
creation: unknown;
|
|
117
128
|
mcpSnapshot?: McpSnapshot;
|
|
118
129
|
mcpDiagnostics?: readonly McpDiagnostic[];
|
package/dist/tenant/sweep.js
CHANGED
|
@@ -43,7 +43,11 @@
|
|
|
43
43
|
* the delivery after commit, sends), and configs of tasks long finished are deleted. A push
|
|
44
44
|
* row that fails is retried with the Host tuning's `a2a.pushRetry` and dropped once it is
|
|
45
45
|
* `a2a.pushMaxAgeMs` old, never parked: the change feed covers what it said.
|
|
46
|
-
* 7. **
|
|
46
|
+
* 7. **Queued messages** (Host feature `queue-recovery`): at most every `sweep.queueRetryMs`, the
|
|
47
|
+
* turn start of every session queue an Agent's refusal left waiting is tried again
|
|
48
|
+
* (`retryQueuedStarts`); a message past the Runtime limit `queuedStartMs` fails
|
|
49
|
+
* (`queue.failed`). The end of a disk copy tries its Agent's sessions at once.
|
|
50
|
+
* 8. **Hooks.** Callbacks registered with `ctx.onSweep`.
|
|
47
51
|
*
|
|
48
52
|
* Every step runs one transaction per session it changes (a linked agent and its workflow
|
|
49
53
|
* share one, child first), so the sweep follows the lock order in `store/types.ts`. A failing
|
|
@@ -51,12 +55,14 @@
|
|
|
51
55
|
* reports it.
|
|
52
56
|
*/
|
|
53
57
|
import { sweepUnusedAgents } from "./agents.js";
|
|
58
|
+
import { retryQueuedStarts } from "./commands.js";
|
|
54
59
|
import { reconcileTriggers } from "./triggers.js";
|
|
55
60
|
import { streamMissedRuns } from "./runs-stream.js";
|
|
56
61
|
import { sweepPendingSeeds } from "../artifacts/disk.js";
|
|
57
62
|
import { pendingAgentEffects, reconcilePendingAgentEffect, } from "../core/flow-host.js";
|
|
58
63
|
import { hostTuning } from "../tuning.js";
|
|
59
|
-
import { sweepStartChecks } from "./agent-versions.js";
|
|
64
|
+
import { idleMsOf, sweepStartChecks } from "./agent-versions.js";
|
|
65
|
+
import { readSandboxConfig } from "../sandbox/tenant-config.js";
|
|
60
66
|
import { pumpPushes, pushRetry } from "../api/a2a/push.js";
|
|
61
67
|
/*
|
|
62
68
|
* The sweep's timings are the Host tuning's `sweep` group (`tuning.ts`):
|
|
@@ -69,6 +75,7 @@ import { pumpPushes, pushRetry } from "../api/a2a/push.js";
|
|
|
69
75
|
* - `drainBudgetMs`: how long one pass may spend draining the outbox, and paging through
|
|
70
76
|
* orphaned sessions. A backlog is worked off over several passes rather than holding one (and
|
|
71
77
|
* with it the Tenant's sweep object in the execution) for long.
|
|
78
|
+
* - `queueRetryMs`: how often the queues an Agent's refusal left waiting are tried again.
|
|
72
79
|
* - `batchSize`, `signalRetentionMs`, `outboxAttempts`, `outboxRetry`, `stalePodMs`.
|
|
73
80
|
*/
|
|
74
81
|
/**
|
|
@@ -88,6 +95,7 @@ export async function sweep(ctx) {
|
|
|
88
95
|
pass.orphansAfter !== undefined ||
|
|
89
96
|
now.getTime() - pass.orphansAt >= hostTuning().sweep.orphanScanMs;
|
|
90
97
|
const checkPods = pass.podsAt === undefined || now.getTime() - pass.podsAt >= hostTuning().sweep.orphanScanMs;
|
|
98
|
+
const retryQueues = pass.queuesAt === undefined || now.getTime() - pass.queuesAt >= hostTuning().sweep.queueRetryMs;
|
|
91
99
|
const steps = [
|
|
92
100
|
["outbox", () => deliverOutbox(ctx, first ? 0 : hostTuning().sweep.outboxGraceMs)],
|
|
93
101
|
["linked", () => reconcileLinkedAgents(ctx)],
|
|
@@ -142,6 +150,17 @@ export async function sweep(ctx) {
|
|
|
142
150
|
"signals",
|
|
143
151
|
() => ctx.store.tx((t) => t.pruneSignals(new Date(now.getTime() - hostTuning().sweep.signalRetentionMs))),
|
|
144
152
|
],
|
|
153
|
+
...(retryQueues
|
|
154
|
+
? [
|
|
155
|
+
[
|
|
156
|
+
"queues",
|
|
157
|
+
async () => {
|
|
158
|
+
pass.queuesAt = now.getTime();
|
|
159
|
+
await retryQueuedStarts(ctx, { now, batchSize: hostTuning().sweep.batchSize });
|
|
160
|
+
},
|
|
161
|
+
],
|
|
162
|
+
]
|
|
163
|
+
: []),
|
|
145
164
|
...[...ctx.sweepHooks].map((hook) => ["hook", hook]),
|
|
146
165
|
];
|
|
147
166
|
let failure;
|
|
@@ -287,7 +306,8 @@ export async function wakeOrphanedSessions(ctx, now = new Date(), after, budgetM
|
|
|
287
306
|
}
|
|
288
307
|
/**
|
|
289
308
|
* Pod sandboxes (F7.2) whose change has not finished (creating, deleting, an old volume still
|
|
290
|
-
* to delete, an expiry not yet recorded),
|
|
309
|
+
* to delete, an expiry not yet recorded), that should run but whose engine is not connected, or
|
|
310
|
+
* that run past their idle limit (a wake that armed no idle timer: the reconcile arms it),
|
|
291
311
|
* and whose row has not moved for `sweep.stalePodMs` (a minute), get a reconcile: a pod or volume deleted outside
|
|
292
312
|
* the Runtime is found, and a change goes on whose reconcile the execution itself lost (its
|
|
293
313
|
* retry timer, an invocation an operator killed, Restate's state wiped) or whose outbox row
|
|
@@ -296,18 +316,27 @@ export async function wakeOrphanedSessions(ctx, now = new Date(), after, budgetM
|
|
|
296
316
|
async function reconcileStalePods(ctx, now) {
|
|
297
317
|
if (!ctx.pods)
|
|
298
318
|
return;
|
|
299
|
-
const stale = await ctx.store.tx(async (t) =>
|
|
300
|
-
const
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
pod.
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
319
|
+
const stale = await ctx.store.tx(async (t) => {
|
|
320
|
+
const config = await readSandboxConfig(t);
|
|
321
|
+
return (await t.listEnvironments()).filter((row) => {
|
|
322
|
+
const pod = row.pod;
|
|
323
|
+
if (!pod || now.getTime() - Date.parse(row.updatedAt) < hostTuning().sweep.stalePodMs)
|
|
324
|
+
return false;
|
|
325
|
+
return (pod.desired === "deleted" ||
|
|
326
|
+
pod.retiring !== undefined ||
|
|
327
|
+
pod.observed === "creating" ||
|
|
328
|
+
pod.observed === "deleting" ||
|
|
329
|
+
(pod.desired === "suspended" && pod.observed !== "suspended") ||
|
|
330
|
+
// Running, but its engine is not connected: the pod may be gone, or its volume.
|
|
331
|
+
(pod.desired === "running" && pod.observed === "running" && !ctx.harness.hosting(row.id)) ||
|
|
332
|
+
// Running past its idle limit: its idle timer is missing (or a turn holds it, which the
|
|
333
|
+
// reconcile leaves alone).
|
|
334
|
+
(pod.desired === "running" &&
|
|
335
|
+
pod.observed === "running" &&
|
|
336
|
+
now.getTime() - Date.parse(pod.lastActiveAt ?? row.createdAt) >
|
|
337
|
+
idleMsOf(config, row) + hostTuning().sweep.stalePodMs));
|
|
338
|
+
});
|
|
339
|
+
});
|
|
311
340
|
for (const row of stale)
|
|
312
341
|
await ctx.sandboxSignal(row.id, { kind: "reconcile" });
|
|
313
342
|
}
|
package/dist/tuning.d.ts
CHANGED
|
@@ -121,6 +121,7 @@ export declare const HostTuningSchema: z.ZodObject<{
|
|
|
121
121
|
maxMs: z.ZodDefault<z.ZodNumber>;
|
|
122
122
|
}, z.core.$strict>>;
|
|
123
123
|
stalePodMs: z.ZodDefault<z.ZodNumber>;
|
|
124
|
+
queueRetryMs: z.ZodDefault<z.ZodNumber>;
|
|
124
125
|
}, z.core.$strict>>;
|
|
125
126
|
a2a: z.ZodPrefault<z.ZodObject<{
|
|
126
127
|
pushRetry: z.ZodPrefault<z.ZodObject<{
|
|
@@ -297,6 +298,8 @@ export declare const HostTuningSchema: z.ZodObject<{
|
|
|
297
298
|
networkProbeMs: z.ZodDefault<z.ZodNumber>;
|
|
298
299
|
networkPauseMs: z.ZodDefault<z.ZodNumber>;
|
|
299
300
|
networkConfirmations: z.ZodDefault<z.ZodNumber>;
|
|
301
|
+
wakeNetworkProbeMs: z.ZodDefault<z.ZodNumber>;
|
|
302
|
+
wakeNetworkConfirmations: z.ZodDefault<z.ZodNumber>;
|
|
300
303
|
setupRetryPollMs: z.ZodDefault<z.ZodNumber>;
|
|
301
304
|
setupAfterSeedPollMs: z.ZodDefault<z.ZodNumber>;
|
|
302
305
|
setupReportTailChars: z.ZodDefault<z.ZodNumber>;
|
package/dist/tuning.js
CHANGED
|
@@ -207,6 +207,11 @@ export const HostTuningSchema = z
|
|
|
207
207
|
outboxRetry: backoff(5_000, 10 * MINUTE),
|
|
208
208
|
/** How long a pod row may go unchanged before the sweep checks it. */
|
|
209
209
|
stalePodMs: ms(MINUTE),
|
|
210
|
+
/**
|
|
211
|
+
* How often the sweep tries again the turn start of queued messages their Agent refused
|
|
212
|
+
* (Host feature `queue-recovery`); they fail after the Runtime limit `queuedStartMs`.
|
|
213
|
+
*/
|
|
214
|
+
queueRetryMs: ms(30_000),
|
|
210
215
|
}),
|
|
211
216
|
/** The A2A server's push notifications (Host feature `a2a-profile-v2`). */
|
|
212
217
|
a2a: group({
|
|
@@ -495,6 +500,13 @@ export const HostTuningSchema = z
|
|
|
495
500
|
networkProbeMs: ms(1_000),
|
|
496
501
|
networkPauseMs: ms(250, 0),
|
|
497
502
|
networkConfirmations: count(3),
|
|
503
|
+
/**
|
|
504
|
+
* The same check on a wake (the disk already holds the engine's volume id): each probe, and
|
|
505
|
+
* the failed probes in a row that prove an address blocked. A probe that connects turns the
|
|
506
|
+
* wake's check into the full one.
|
|
507
|
+
*/
|
|
508
|
+
wakeNetworkProbeMs: ms(150),
|
|
509
|
+
wakeNetworkConfirmations: count(1),
|
|
498
510
|
/** Environment setup: how often a failed setup is checked for a retry, and after a seed. */
|
|
499
511
|
setupRetryPollMs: ms(5_000),
|
|
500
512
|
setupAfterSeedPollMs: ms(1_000),
|
package/dist/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const RUNTIME_VERSION = "0.
|
|
1
|
+
export declare const RUNTIME_VERSION = "0.44.1-beta";
|
package/dist/version.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
// Kept in step with package.json by runtime/server/scripts/check-package.mjs, which runs in npm run check.
|
|
2
2
|
// release:prepare also rewrites this when it bumps @nylorun/runtime.
|
|
3
|
-
export const RUNTIME_VERSION = "0.
|
|
3
|
+
export const RUNTIME_VERSION = "0.44.1-beta";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nylorun/runtime",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.44.1-beta",
|
|
4
4
|
"description": "Nylorun Runtime: the agent execution Host on Postgres, Restate and S2, and model providers.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -65,8 +65,8 @@
|
|
|
65
65
|
"@hono/node-server": "^2.1.5",
|
|
66
66
|
"@hono/zod-openapi": "^1.6.3",
|
|
67
67
|
"@modelcontextprotocol/sdk": "^1.32.1",
|
|
68
|
-
"@nylorun/core": "0.
|
|
69
|
-
"@nylorun/harness": "0.34.
|
|
68
|
+
"@nylorun/core": "0.38.0-beta",
|
|
69
|
+
"@nylorun/harness": "0.34.1-beta",
|
|
70
70
|
"@restatedev/restate-sdk": "1.17.3",
|
|
71
71
|
"@restatedev/restate-sdk-clients": "1.17.3",
|
|
72
72
|
"@s2-dev/streamstore": "0.28.0",
|