@nylorun/runtime 0.49.2-beta → 0.51.0-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,42 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.51.0-beta
4
+
5
+ ### Minor Changes
6
+
7
+ - 874cc83: Pods on Agent Substrate (preview): suspend before drain, following Substrate's eviction contract. A worker pod that stops (a drain, a GKE node upgrade or scale-down, a WorkerPool rollout, a deleted pod) sends each of its actors SIGTERM and gives them 30 minutes to be suspended; until now the engine exited on that signal, so the actor crashed and went back to its last suspend, losing everything since.
8
+
9
+ - `@nylorun/runtime`: on Substrate (`NYLORUN_SANDBOX_SUBSTRATE=1`, which the sandboxes service's template sets), the engine logs `harness_worker_stopping` on SIGTERM and keeps running, with its runs. The sandboxes service watches worker pods (label `ate.dev/worker-pool`) and, when one turns Terminating, suspends each of the Tenant's actors on it synchronously (eight at a time, trying again for up to 29 minutes), records `drainedAt`, and resumes a sandbox whose request is `Running` on another worker. While it moves, its status reason is `Moving`, and the pod lifecycle waits instead of failing it at the ready deadline. Each drain is recorded once, as `agent.suspended` with reason `drain` (the pod row's `drained_at`, store migration 0048, nullable, run at start). Not covered: a spot reclaim or a node that dies, which still crash and revert. An actor created by an earlier release keeps its engine (#643) until it is reset.
10
+ - `@nylorun/core`: `agent.suspended`'s `reason` may be `drain`.
11
+ - `nylorun`: `local sandbox enable --backend substrate` lets the sandboxes service read pods cluster-wide (`get`, `list`, `watch`), for the worker pods it watches.
12
+
13
+ ### Patch Changes
14
+
15
+ - Pin core to the tested release.
16
+ - Pin harness to the tested release.
17
+ - Updated dependencies [874cc83]
18
+ - Updated dependencies
19
+ - @nylorun/core@0.44.0-beta
20
+ - @nylorun/harness@0.36.4-beta
21
+
22
+ ## 0.50.0-beta
23
+
24
+ ### Minor Changes
25
+
26
+ - 745ef22: A model call's retries are in its event. `message.assistant` and `model.failed` carry `attempts`, the provider calls the gate made for the call (the first included), and `retries`, one `{ attempt, delayMs, error }` for each retry (the error as the provider worded it, cut to 300 characters, with the provider's credential scrubbed). Before, a call that answered after a retry was one `message.assistant` that showed no sign of it, and a call that failed after its attempts showed only its last error; the retries were only in the gates service's `model_retry` log lines.
27
+
28
+ - `@nylorun/core`: `ModelRetrySchema`, and the optional `attempts` and `retries` on the facts `message.assistant`, `model.failed` and `context.compacted` share. `ModelFailureOutcome` takes optional `attempts` and `retries`.
29
+ - `@nylorun/runtime`: the gate's answer reports them in `evidence.extras`, the usage ledger keeps them (`model_usage.attempts` and `retries`, store migration 0047, nullable, run at start), and the call's event takes them from its row; a failed call's from its failure outcome. A trace's chat span has `nylorun.model.attempts`. A retried call is still metered once, as one call. `/calls/model` is unchanged.
30
+
31
+ ### Patch Changes
32
+
33
+ - Pin core to the tested release.
34
+ - Pin harness to the tested release.
35
+ - Updated dependencies [745ef22]
36
+ - Updated dependencies
37
+ - @nylorun/core@0.43.0-beta
38
+ - @nylorun/harness@0.36.3-beta
39
+
3
40
  ## 0.49.2-beta
4
41
 
5
42
  ### Patch Changes
@@ -3502,6 +3502,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
3502
3502
  cachedTokens: z.ZodOptional<z.ZodNumber>;
3503
3503
  costUsd: z.ZodOptional<z.ZodNumber>;
3504
3504
  }, z.core.$loose>>;
3505
+ attempts: z.ZodOptional<z.ZodNumber>;
3506
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
3507
+ attempt: z.ZodNumber;
3508
+ delayMs: z.ZodNumber;
3509
+ error: z.ZodString;
3510
+ }, z.core.$loose>>>;
3505
3511
  invocationId: z.ZodOptional<z.ZodString>;
3506
3512
  trigger: z.ZodEnum<{
3507
3513
  overflow: "overflow";
@@ -4363,6 +4369,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
4363
4369
  cachedTokens: z.ZodOptional<z.ZodNumber>;
4364
4370
  costUsd: z.ZodOptional<z.ZodNumber>;
4365
4371
  }, z.core.$loose>>;
4372
+ attempts: z.ZodOptional<z.ZodNumber>;
4373
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
4374
+ attempt: z.ZodNumber;
4375
+ delayMs: z.ZodNumber;
4376
+ error: z.ZodString;
4377
+ }, z.core.$loose>>>;
4366
4378
  invocationId: z.ZodString;
4367
4379
  text: z.ZodString;
4368
4380
  model: z.ZodOptional<z.ZodObject<{
@@ -4540,6 +4552,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
4540
4552
  cachedTokens: z.ZodOptional<z.ZodNumber>;
4541
4553
  costUsd: z.ZodOptional<z.ZodNumber>;
4542
4554
  }, z.core.$loose>>;
4555
+ attempts: z.ZodOptional<z.ZodNumber>;
4556
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
4557
+ attempt: z.ZodNumber;
4558
+ delayMs: z.ZodNumber;
4559
+ error: z.ZodString;
4560
+ }, z.core.$loose>>>;
4543
4561
  invocationId: z.ZodOptional<z.ZodString>;
4544
4562
  code: z.ZodString;
4545
4563
  message: z.ZodString;
@@ -6735,6 +6753,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
6735
6753
  cachedTokens: z.ZodOptional<z.ZodNumber>;
6736
6754
  costUsd: z.ZodOptional<z.ZodNumber>;
6737
6755
  }, z.core.$loose>>;
6756
+ attempts: z.ZodOptional<z.ZodNumber>;
6757
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
6758
+ attempt: z.ZodNumber;
6759
+ delayMs: z.ZodNumber;
6760
+ error: z.ZodString;
6761
+ }, z.core.$loose>>>;
6738
6762
  invocationId: z.ZodOptional<z.ZodString>;
6739
6763
  trigger: z.ZodEnum<{
6740
6764
  overflow: "overflow";
@@ -7596,6 +7620,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
7596
7620
  cachedTokens: z.ZodOptional<z.ZodNumber>;
7597
7621
  costUsd: z.ZodOptional<z.ZodNumber>;
7598
7622
  }, z.core.$loose>>;
7623
+ attempts: z.ZodOptional<z.ZodNumber>;
7624
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
7625
+ attempt: z.ZodNumber;
7626
+ delayMs: z.ZodNumber;
7627
+ error: z.ZodString;
7628
+ }, z.core.$loose>>>;
7599
7629
  invocationId: z.ZodString;
7600
7630
  text: z.ZodString;
7601
7631
  model: z.ZodOptional<z.ZodObject<{
@@ -7773,6 +7803,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
7773
7803
  cachedTokens: z.ZodOptional<z.ZodNumber>;
7774
7804
  costUsd: z.ZodOptional<z.ZodNumber>;
7775
7805
  }, z.core.$loose>>;
7806
+ attempts: z.ZodOptional<z.ZodNumber>;
7807
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
7808
+ attempt: z.ZodNumber;
7809
+ delayMs: z.ZodNumber;
7810
+ error: z.ZodString;
7811
+ }, z.core.$loose>>>;
7776
7812
  invocationId: z.ZodOptional<z.ZodString>;
7777
7813
  code: z.ZodString;
7778
7814
  message: z.ZodString;
@@ -9971,6 +10007,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
9971
10007
  cachedTokens: z.ZodOptional<z.ZodNumber>;
9972
10008
  costUsd: z.ZodOptional<z.ZodNumber>;
9973
10009
  }, z.core.$loose>>;
10010
+ attempts: z.ZodOptional<z.ZodNumber>;
10011
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
10012
+ attempt: z.ZodNumber;
10013
+ delayMs: z.ZodNumber;
10014
+ error: z.ZodString;
10015
+ }, z.core.$loose>>>;
9974
10016
  invocationId: z.ZodOptional<z.ZodString>;
9975
10017
  trigger: z.ZodEnum<{
9976
10018
  overflow: "overflow";
@@ -10832,6 +10874,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
10832
10874
  cachedTokens: z.ZodOptional<z.ZodNumber>;
10833
10875
  costUsd: z.ZodOptional<z.ZodNumber>;
10834
10876
  }, z.core.$loose>>;
10877
+ attempts: z.ZodOptional<z.ZodNumber>;
10878
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
10879
+ attempt: z.ZodNumber;
10880
+ delayMs: z.ZodNumber;
10881
+ error: z.ZodString;
10882
+ }, z.core.$loose>>>;
10835
10883
  invocationId: z.ZodString;
10836
10884
  text: z.ZodString;
10837
10885
  model: z.ZodOptional<z.ZodObject<{
@@ -11009,6 +11057,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
11009
11057
  cachedTokens: z.ZodOptional<z.ZodNumber>;
11010
11058
  costUsd: z.ZodOptional<z.ZodNumber>;
11011
11059
  }, z.core.$loose>>;
11060
+ attempts: z.ZodOptional<z.ZodNumber>;
11061
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
11062
+ attempt: z.ZodNumber;
11063
+ delayMs: z.ZodNumber;
11064
+ error: z.ZodString;
11065
+ }, z.core.$loose>>>;
11012
11066
  invocationId: z.ZodOptional<z.ZodString>;
11013
11067
  code: z.ZodString;
11014
11068
  message: z.ZodString;
@@ -13204,6 +13258,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
13204
13258
  cachedTokens: z.ZodOptional<z.ZodNumber>;
13205
13259
  costUsd: z.ZodOptional<z.ZodNumber>;
13206
13260
  }, z.core.$loose>>;
13261
+ attempts: z.ZodOptional<z.ZodNumber>;
13262
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
13263
+ attempt: z.ZodNumber;
13264
+ delayMs: z.ZodNumber;
13265
+ error: z.ZodString;
13266
+ }, z.core.$loose>>>;
13207
13267
  invocationId: z.ZodOptional<z.ZodString>;
13208
13268
  trigger: z.ZodEnum<{
13209
13269
  overflow: "overflow";
@@ -14065,6 +14125,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
14065
14125
  cachedTokens: z.ZodOptional<z.ZodNumber>;
14066
14126
  costUsd: z.ZodOptional<z.ZodNumber>;
14067
14127
  }, z.core.$loose>>;
14128
+ attempts: z.ZodOptional<z.ZodNumber>;
14129
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
14130
+ attempt: z.ZodNumber;
14131
+ delayMs: z.ZodNumber;
14132
+ error: z.ZodString;
14133
+ }, z.core.$loose>>>;
14068
14134
  invocationId: z.ZodString;
14069
14135
  text: z.ZodString;
14070
14136
  model: z.ZodOptional<z.ZodObject<{
@@ -14242,6 +14308,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
14242
14308
  cachedTokens: z.ZodOptional<z.ZodNumber>;
14243
14309
  costUsd: z.ZodOptional<z.ZodNumber>;
14244
14310
  }, z.core.$loose>>;
14311
+ attempts: z.ZodOptional<z.ZodNumber>;
14312
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
14313
+ attempt: z.ZodNumber;
14314
+ delayMs: z.ZodNumber;
14315
+ error: z.ZodString;
14316
+ }, z.core.$loose>>>;
14245
14317
  invocationId: z.ZodOptional<z.ZodString>;
14246
14318
  code: z.ZodString;
14247
14319
  message: z.ZodString;
@@ -17390,6 +17462,12 @@ export declare const HistoryPage: z.ZodObject<{
17390
17462
  cachedTokens: z.ZodOptional<z.ZodNumber>;
17391
17463
  costUsd: z.ZodOptional<z.ZodNumber>;
17392
17464
  }, z.core.$loose>>;
17465
+ attempts: z.ZodOptional<z.ZodNumber>;
17466
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
17467
+ attempt: z.ZodNumber;
17468
+ delayMs: z.ZodNumber;
17469
+ error: z.ZodString;
17470
+ }, z.core.$loose>>>;
17393
17471
  invocationId: z.ZodOptional<z.ZodString>;
17394
17472
  trigger: z.ZodEnum<{
17395
17473
  overflow: "overflow";
@@ -18251,6 +18329,12 @@ export declare const HistoryPage: z.ZodObject<{
18251
18329
  cachedTokens: z.ZodOptional<z.ZodNumber>;
18252
18330
  costUsd: z.ZodOptional<z.ZodNumber>;
18253
18331
  }, z.core.$loose>>;
18332
+ attempts: z.ZodOptional<z.ZodNumber>;
18333
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
18334
+ attempt: z.ZodNumber;
18335
+ delayMs: z.ZodNumber;
18336
+ error: z.ZodString;
18337
+ }, z.core.$loose>>>;
18254
18338
  invocationId: z.ZodString;
18255
18339
  text: z.ZodString;
18256
18340
  model: z.ZodOptional<z.ZodObject<{
@@ -18428,6 +18512,12 @@ export declare const HistoryPage: z.ZodObject<{
18428
18512
  cachedTokens: z.ZodOptional<z.ZodNumber>;
18429
18513
  costUsd: z.ZodOptional<z.ZodNumber>;
18430
18514
  }, z.core.$loose>>;
18515
+ attempts: z.ZodOptional<z.ZodNumber>;
18516
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
18517
+ attempt: z.ZodNumber;
18518
+ delayMs: z.ZodNumber;
18519
+ error: z.ZodString;
18520
+ }, z.core.$loose>>>;
18431
18521
  invocationId: z.ZodOptional<z.ZodString>;
18432
18522
  code: z.ZodString;
18433
18523
  message: z.ZodString;
@@ -20623,6 +20713,12 @@ export declare const HistoryPage: z.ZodObject<{
20623
20713
  cachedTokens: z.ZodOptional<z.ZodNumber>;
20624
20714
  costUsd: z.ZodOptional<z.ZodNumber>;
20625
20715
  }, z.core.$loose>>;
20716
+ attempts: z.ZodOptional<z.ZodNumber>;
20717
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
20718
+ attempt: z.ZodNumber;
20719
+ delayMs: z.ZodNumber;
20720
+ error: z.ZodString;
20721
+ }, z.core.$loose>>>;
20626
20722
  invocationId: z.ZodOptional<z.ZodString>;
20627
20723
  trigger: z.ZodEnum<{
20628
20724
  overflow: "overflow";
@@ -21484,6 +21580,12 @@ export declare const HistoryPage: z.ZodObject<{
21484
21580
  cachedTokens: z.ZodOptional<z.ZodNumber>;
21485
21581
  costUsd: z.ZodOptional<z.ZodNumber>;
21486
21582
  }, z.core.$loose>>;
21583
+ attempts: z.ZodOptional<z.ZodNumber>;
21584
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
21585
+ attempt: z.ZodNumber;
21586
+ delayMs: z.ZodNumber;
21587
+ error: z.ZodString;
21588
+ }, z.core.$loose>>>;
21487
21589
  invocationId: z.ZodString;
21488
21590
  text: z.ZodString;
21489
21591
  model: z.ZodOptional<z.ZodObject<{
@@ -21661,6 +21763,12 @@ export declare const HistoryPage: z.ZodObject<{
21661
21763
  cachedTokens: z.ZodOptional<z.ZodNumber>;
21662
21764
  costUsd: z.ZodOptional<z.ZodNumber>;
21663
21765
  }, z.core.$loose>>;
21766
+ attempts: z.ZodOptional<z.ZodNumber>;
21767
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
21768
+ attempt: z.ZodNumber;
21769
+ delayMs: z.ZodNumber;
21770
+ error: z.ZodString;
21771
+ }, z.core.$loose>>>;
21664
21772
  invocationId: z.ZodOptional<z.ZodString>;
21665
21773
  code: z.ZodString;
21666
21774
  message: z.ZodString;
@@ -11,10 +11,12 @@
11
11
  *
12
12
  * The row also keeps the gate's own timing of the call (R7.1): `now()` before the call and
13
13
  * after it, and the adapter's `firstTokenMs` (`evidence.extras`), so a model call's span is
14
- * what the gate measured.
14
+ * what the gate measured; and the adapter's `attempts` and `retries`, so the call's event says
15
+ * how many times the provider was called for it, and why again.
15
16
  */
16
17
  import { randomUUID } from "node:crypto";
17
18
  import { failure } from "../model/classify.js";
19
+ import { attemptFacts } from "../model/retry.js";
18
20
  export function createMeter(options) {
19
21
  const now = options.now ?? (() => new Date());
20
22
  /** Calls running through this gate, per Tenant and budget scope. */
@@ -143,6 +145,8 @@ export function usageRow(request, outcome, at, timing) {
143
145
  ? { firstTokenMs: Math.round(firstTokenMs) }
144
146
  : {}),
145
147
  ...costSplitColumns(outcome.evidence?.extras?.costSplit),
148
+ // How many times the gate called the provider for this answer, and why again.
149
+ ...attemptFacts(outcome.evidence?.extras?.attempts, outcome.evidence?.extras?.retries),
146
150
  };
147
151
  }
148
152
  /** The ledger's cost-by-kind columns from the adapter's `costSplit`; none unless all four are costs. */
@@ -14,7 +14,9 @@
14
14
  * pod's join token instead of a harness token (`pod.ts`), runs its disk's setup on a new volume
15
15
  * (`setup.ts`), and runs its sandbox's tools in the pod itself (the `local` backend), with egress
16
16
  * through egress-gate. SIGTERM (tini forwards
17
- * it; the pod's grace is 10 s by default) gives its runs back and exits.
17
+ * it; the pod's grace is 10 s by default) gives its runs back and exits. On Agent Substrate
18
+ * (`NYLORUN_SANDBOX_SUBSTRATE=1`), SIGTERM means the worker pod is stopping: the engine logs it and
19
+ * keeps running, so the sandboxes service can suspend the actor within Substrate's 30 minutes.
18
20
  */
19
21
  import { existsSync, mkdirSync } from "node:fs";
20
22
  import { createServer } from "node:http";
@@ -106,8 +108,18 @@ export async function runHarness(stack, baseline) {
106
108
  });
107
109
  logger.info("harness_ready", { health: `http://127.0.0.1:${config.healthPort}/health` });
108
110
  let stopping = false;
111
+ let workerStopping = false;
109
112
  for (const signal of ["SIGINT", "SIGTERM"])
110
113
  process.on(signal, () => {
114
+ // On Agent Substrate, SIGTERM means the worker pod is stopping: the actor has 30 minutes to
115
+ // be suspended, and the sandboxes service suspends it. A process that exits can't be, so the
116
+ // engine keeps running, and keeps its runs (Substrate's eviction contract).
117
+ if (signal === "SIGTERM" && config.pod?.substrate) {
118
+ if (!workerStopping)
119
+ logger.info("harness_worker_stopping", { sandboxId: config.pod.sandboxId });
120
+ workerStopping = true;
121
+ return;
122
+ }
111
123
  if (stopping)
112
124
  return;
113
125
  stopping = true;
@@ -169,6 +169,12 @@ export interface PodHostConfig {
169
169
  * for the hosts whose keys Substrate's egress adds.
170
170
  */
171
171
  egressCaFile?: string;
172
+ /**
173
+ * `NYLORUN_SANDBOX_SUBSTRATE=1`: the pod is an Agent Substrate actor. A stopping worker sends
174
+ * it SIGTERM and gives it 30 minutes to be suspended, which the sandboxes service does; the
175
+ * engine stays up instead of exiting (suspend before drain).
176
+ */
177
+ substrate?: boolean;
172
178
  /** Addresses the pod's NetworkPolicy must block, probed before the engine starts. */
173
179
  blocked: BlockedAddress[];
174
180
  }
@@ -647,6 +647,7 @@ function parsePodHost(env, url) {
647
647
  httpUrl: origin,
648
648
  ...(egress ? { egressProxy: egress.replace(/\/+$/, "") } : {}),
649
649
  ...(egressCa ? { egressCaFile: egressCa } : {}),
650
+ ...(read(env, "NYLORUN_SANDBOX_SUBSTRATE") === "1" ? { substrate: true } : {}),
650
651
  blocked: blockedAddresses(env),
651
652
  },
652
653
  };
@@ -2,7 +2,7 @@
2
2
  "openapi": "3.2.0",
3
3
  "info": {
4
4
  "title": "Nylorun Management API",
5
- "version": "0.49.2-beta",
5
+ "version": "0.51.0-beta",
6
6
  "x-nylorun-protocol": {
7
7
  "min": 16,
8
8
  "max": 18,
@@ -25,6 +25,8 @@ export function modelCallSettingsOf(limits) {
25
25
  };
26
26
  }
27
27
  const OPENAI_COMPATIBLE_REASONING = /^reasoning(_content|_text)?$/;
28
+ /** How much of a retried attempt's error the outcome keeps. */
29
+ const RETRY_ERROR_CHARS = 300;
28
30
  const emptyUsage = () => ({
29
31
  input: 0,
30
32
  output: 0,
@@ -200,21 +202,35 @@ export function piModel(options = {}) {
200
202
  clearTimeout(timer);
201
203
  }
202
204
  };
203
- const response = await retryModelCall(attempt, { attempts: settings.attempts, baseDelayMs: settings.retryBaseDelayMs }, context.signal, options.onRetry);
205
+ // Each retry, for the outcome (Model Calls §6): the gate's ledger row and the call's event
206
+ // carry how many times the provider was called, and why again.
207
+ const retries = [];
208
+ const response = await retryModelCall(attempt, { attempts: settings.attempts, baseDelayMs: settings.retryBaseDelayMs }, context.signal, (retry) => {
209
+ retries.push({
210
+ attempt: retry.attempt,
211
+ delayMs: retry.delayMs,
212
+ error: String(scrub(retry.errorMessage, secrets)).slice(0, RETRY_ERROR_CHARS),
213
+ });
214
+ options.onRetry?.(retry);
215
+ });
216
+ const tried = { attempts: retries.length + 1, ...(retries.length ? { retries } : {}) };
204
217
  // An abort is never a failure outcome: the host decides what it means.
205
218
  context.signal.throwIfAborted();
206
219
  if (natives.error)
207
220
  return redacted(failure("invalid_request", natives.error, false));
208
221
  if (response.stopReason === "aborted")
209
- return redacted(failure("transient", "The provider stream was aborted.", true));
222
+ return { ...redacted(failure("transient", "The provider stream was aborted.", true)), ...tried };
210
223
  if (response.stopReason === "error") {
211
224
  const outcome = classifyAssistantError(response, selected.contextWindow);
212
- return redacted(outcome.code === "auth"
213
- ? {
214
- ...outcome,
215
- message: `${outcome.message} Update the provider credential in Studio's Model Settings, or run nylorun tenant models set.`,
216
- }
217
- : outcome);
225
+ return {
226
+ ...redacted(outcome.code === "auth"
227
+ ? {
228
+ ...outcome,
229
+ message: `${outcome.message} Update the provider credential in Studio's Model Settings, or run nylorun tenant models set.`,
230
+ }
231
+ : outcome),
232
+ ...tried,
233
+ };
218
234
  }
219
235
  if (response.stopReason === "deferred")
220
236
  return failure("invalid_request", "The provider deferred the response, which the Runtime does not request.", false);
@@ -272,6 +288,9 @@ export function piModel(options = {}) {
272
288
  maxOutputTokens: selected.maxTokens,
273
289
  // From the call to the provider's first stream event (R7.1); absent when none came.
274
290
  ...(firstEventAt !== undefined ? { firstTokenMs: firstEventAt - calledAt } : {}),
291
+ // The provider calls this answer took, and each retry before it: the ledger keeps them.
292
+ attempts: tried.attempts,
293
+ ...(retries.length ? { retries: retries } : {}),
275
294
  },
276
295
  },
277
296
  };
@@ -34,3 +34,17 @@ export interface ModelRetryPolicy {
34
34
  * delay comes back as the last failure marked `aborted`, as pi-ai's `retryAssistantCall` does.
35
35
  */
36
36
  export declare function retryModelCall(produce: () => Promise<AssistantMessage>, policy: ModelRetryPolicy, signal: AbortSignal, onRetry?: (retry: ModelRetry) => void): Promise<AssistantMessage>;
37
+ /** A retry as the ledger and the call's event keep it (Model Calls §6). */
38
+ export interface RecordedRetry {
39
+ readonly attempt: number;
40
+ readonly delayMs: number;
41
+ readonly error: string;
42
+ }
43
+ /** A retry list as the ledger and the events keep it: well-formed entries only, at most 10. */
44
+ export declare function modelRetries(value: unknown): RecordedRetry[];
45
+ /**
46
+ * `attempts` and `retries` for a model call's event (Model Calls §6): how many times the gate
47
+ * called the provider, and each retry: the ledger's columns and the event's fields alike. None
48
+ * unless `attempts` is a positive whole number (a call a Runtime before 0.50 served).
49
+ */
50
+ export declare function attemptFacts(attempts: unknown, retries: unknown): {};
@@ -61,3 +61,37 @@ function wait(ms, signal) {
61
61
  signal.addEventListener("abort", onAbort, { once: true });
62
62
  });
63
63
  }
64
+ /** The most retries a row keeps; `modelAttempts` is 3 unless a Tenant raised it. */
65
+ const MAX_RETRIES = 10;
66
+ /** A retry list as the ledger and the events keep it: well-formed entries only, at most 10. */
67
+ export function modelRetries(value) {
68
+ if (!Array.isArray(value))
69
+ return [];
70
+ const out = [];
71
+ for (const entry of value) {
72
+ if (!entry || typeof entry !== "object")
73
+ continue;
74
+ const { attempt, delayMs, error } = entry;
75
+ if (typeof attempt !== "number" || !Number.isInteger(attempt) || attempt < 2)
76
+ continue;
77
+ out.push({
78
+ attempt,
79
+ delayMs: typeof delayMs === "number" && Number.isFinite(delayMs) && delayMs >= 0 ? Math.round(delayMs) : 0,
80
+ error: typeof error === "string" ? error.slice(0, 300) : "",
81
+ });
82
+ if (out.length === MAX_RETRIES)
83
+ break;
84
+ }
85
+ return out;
86
+ }
87
+ /**
88
+ * `attempts` and `retries` for a model call's event (Model Calls §6): how many times the gate
89
+ * called the provider, and each retry: the ledger's columns and the event's fields alike. None
90
+ * unless `attempts` is a positive whole number (a call a Runtime before 0.50 served).
91
+ */
92
+ export function attemptFacts(attempts, retries) {
93
+ if (typeof attempts !== "number" || !Number.isInteger(attempts) || attempts < 1)
94
+ return {};
95
+ const kept = modelRetries(retries);
96
+ return { attempts, ...(kept.length ? { retries: kept } : {}) };
97
+ }
package/dist/openapi.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "openapi": "3.2.0",
3
3
  "info": {
4
4
  "title": "Nylorun Runtime API",
5
- "version": "0.49.2-beta",
5
+ "version": "0.51.0-beta",
6
6
  "x-nylorun-protocol": {
7
7
  "min": 16,
8
8
  "max": 18,
@@ -13430,6 +13430,35 @@
13430
13430
  ],
13431
13431
  "additionalProperties": {}
13432
13432
  },
13433
+ "attempts": {
13434
+ "type": "integer",
13435
+ "exclusiveMinimum": 0
13436
+ },
13437
+ "retries": {
13438
+ "type": "array",
13439
+ "items": {
13440
+ "type": "object",
13441
+ "properties": {
13442
+ "attempt": {
13443
+ "type": "integer",
13444
+ "minimum": 2
13445
+ },
13446
+ "delayMs": {
13447
+ "type": "integer",
13448
+ "minimum": 0
13449
+ },
13450
+ "error": {
13451
+ "type": "string"
13452
+ }
13453
+ },
13454
+ "required": [
13455
+ "attempt",
13456
+ "delayMs",
13457
+ "error"
13458
+ ],
13459
+ "additionalProperties": {}
13460
+ }
13461
+ },
13433
13462
  "agent": {
13434
13463
  "type": "object",
13435
13464
  "properties": {
@@ -13716,6 +13745,35 @@
13716
13745
  ],
13717
13746
  "additionalProperties": {}
13718
13747
  },
13748
+ "attempts": {
13749
+ "type": "integer",
13750
+ "exclusiveMinimum": 0
13751
+ },
13752
+ "retries": {
13753
+ "type": "array",
13754
+ "items": {
13755
+ "type": "object",
13756
+ "properties": {
13757
+ "attempt": {
13758
+ "type": "integer",
13759
+ "minimum": 2
13760
+ },
13761
+ "delayMs": {
13762
+ "type": "integer",
13763
+ "minimum": 0
13764
+ },
13765
+ "error": {
13766
+ "type": "string"
13767
+ }
13768
+ },
13769
+ "required": [
13770
+ "attempt",
13771
+ "delayMs",
13772
+ "error"
13773
+ ],
13774
+ "additionalProperties": {}
13775
+ }
13776
+ },
13719
13777
  "agent": {
13720
13778
  "type": "object",
13721
13779
  "properties": {
@@ -14007,6 +14065,35 @@
14007
14065
  ],
14008
14066
  "additionalProperties": {}
14009
14067
  },
14068
+ "attempts": {
14069
+ "type": "integer",
14070
+ "exclusiveMinimum": 0
14071
+ },
14072
+ "retries": {
14073
+ "type": "array",
14074
+ "items": {
14075
+ "type": "object",
14076
+ "properties": {
14077
+ "attempt": {
14078
+ "type": "integer",
14079
+ "minimum": 2
14080
+ },
14081
+ "delayMs": {
14082
+ "type": "integer",
14083
+ "minimum": 0
14084
+ },
14085
+ "error": {
14086
+ "type": "string"
14087
+ }
14088
+ },
14089
+ "required": [
14090
+ "attempt",
14091
+ "delayMs",
14092
+ "error"
14093
+ ],
14094
+ "additionalProperties": {}
14095
+ }
14096
+ },
14010
14097
  "agent": {
14011
14098
  "type": "object",
14012
14099
  "properties": {
@@ -63,6 +63,11 @@ export interface PodStatus {
63
63
  readonly shutdownTime?: string;
64
64
  readonly reason?: string;
65
65
  readonly message?: string;
66
+ /**
67
+ * Agent Substrate: when the sandbox was last suspended because its worker stopped. While it
68
+ * moves to another worker, `reason` is `Moving`.
69
+ */
70
+ readonly drainedAt?: string;
66
71
  /** With `?wait=`: whether the state was reached before the timeout. */
67
72
  readonly met?: boolean;
68
73
  }
@@ -25,6 +25,10 @@
25
25
  * (`agent.failed`, saying what the Agent lost); a crash an idle suspend finds is recorded the
26
26
  * same way. The next turn start asks for `Running`, and the sandboxes service reverts the
27
27
  * sandbox to its last suspend before resuming it.
28
+ * - **Drained** (Agent Substrate): a worker that stops gives its actors 30 minutes to be
29
+ * suspended. The sandboxes service suspends each, records `drainedAt` and resumes it on
30
+ * another worker; a new `drainedAt` is `agent.suspended` (reason `drain`), once, and a pod
31
+ * that is `Moving` waits rather than fail at the ready deadline.
28
32
  * - **Lost**: an incarnation that has run (a pod joined) whose Sandbox is gone or whose volume
29
33
  * is missing is `lost` (`agent.lost`): its host epoch moves, its Sandbox is deleted, and
30
34
  * only a reset brings the Environment back. agent-sandbox recreates a deleted claim empty, so
@@ -60,6 +64,11 @@ export declare function repositoryOf(reference: string): string;
60
64
  * the node made up for an image loaded onto it, which no registry serves.
61
65
  */
62
66
  export declare function imageDigestOf(imageID: string | undefined, requested: string): string | undefined;
67
+ /**
68
+ * The sandboxes service's reason for a sandbox suspended because its Agent Substrate worker
69
+ * stopped, until it runs on another one.
70
+ */
71
+ export declare const MOVING = "Moving";
63
72
  /** A setup failure's `reason` starts with this (`sandbox/setup.ts`); it is sticky. */
64
73
  export declare const SETUP_FAILED = "setup:";
65
74
  /** A crash's `reason` (see `crashReasonOf`) starts with this; it stays until a turn starts the pod. */
@@ -34,6 +34,11 @@ export function imageDigestOf(imageID, requested) {
34
34
  return undefined;
35
35
  return repositoryOf(id) === repositoryOf(requested) ? id : undefined;
36
36
  }
37
+ /**
38
+ * The sandboxes service's reason for a sandbox suspended because its Agent Substrate worker
39
+ * stopped, until it runs on another one.
40
+ */
41
+ export const MOVING = "Moving";
37
42
  /** A setup failure's `reason` starts with this (`sandbox/setup.ts`); it is sticky. */
38
43
  export const SETUP_FAILED = "setup:";
39
44
  /** A crash's `reason` (see `crashReasonOf`) starts with this; it stays until a turn starts the pod. */
@@ -158,6 +163,12 @@ export function decide(facts) {
158
163
  again(pollMs);
159
164
  return done();
160
165
  }
166
+ // Agent Substrate: its worker stopped, and the sandboxes service suspended it with everything it
167
+ // held and resumes it on another worker (suspend before drain). Recorded once per drain.
168
+ if (status.drainedAt !== undefined && status.drainedAt !== pod.drainedAt) {
169
+ patch.drainedAt = status.drainedAt;
170
+ events.push(event("agent.suspended", { reason: "drain" }));
171
+ }
161
172
  const opId = opIdOf({ k8sName: pod.k8sName, rev });
162
173
  if (desired === "suspended" && !status.exists) {
163
174
  if (pod.observed !== "suspended")
@@ -217,6 +228,11 @@ export function decide(facts) {
217
228
  events.push(event("agent.failed", { reason: crash }));
218
229
  return done();
219
230
  }
231
+ // On its way to another worker after a drain: neither failed nor starting.
232
+ if (status.reason === MOVING) {
233
+ again(pollMs);
234
+ return done();
235
+ }
220
236
  if (now - startedAt >= readyDeadlineMs) {
221
237
  const reason = crash ??
222
238
  ([status.reason, status.message].filter(Boolean).join(": ") ||
@@ -0,0 +1,2 @@
1
+ ALTER TABLE "nylorun"."model_usage" ADD COLUMN "attempts" integer;--> statement-breakpoint
2
+ ALTER TABLE "nylorun"."model_usage" ADD COLUMN "retries" json;
@@ -0,0 +1 @@
1
+ ALTER TABLE "nylorun"."agents" ADD COLUMN "drained_at" text;
@@ -330,6 +330,20 @@
330
330
  "when": 1794240246972,
331
331
  "tag": "0046_artifacts_audience",
332
332
  "breakpoints": true
333
+ },
334
+ {
335
+ "idx": 47,
336
+ "version": "7",
337
+ "when": 1794630124577,
338
+ "tag": "0047_model_usage_attempts",
339
+ "breakpoints": true
340
+ },
341
+ {
342
+ "idx": 48,
343
+ "version": "7",
344
+ "when": 1794635246579,
345
+ "tag": "0048_sandbox_pod_drained_at",
346
+ "breakpoints": true
333
347
  }
334
348
  ]
335
349
  }
@@ -1321,6 +1321,23 @@ export declare const agents: import("drizzle-orm/pg-core").PgTableWithColumns<{
1321
1321
  }, {}, {
1322
1322
  pgColumnBuilderBrand: 'PgCustomColumnBuilderBrand';
1323
1323
  }>;
1324
+ drainedAt: import("drizzle-orm/pg-core").PgColumn<{
1325
+ name: "drainedAt";
1326
+ tableName: "agents";
1327
+ dataType: "string";
1328
+ columnType: "PgText";
1329
+ data: string;
1330
+ driverParam: string;
1331
+ notNull: false;
1332
+ hasDefault: false;
1333
+ isPrimaryKey: false;
1334
+ isAutoincrement: false;
1335
+ hasRuntimeDefault: false;
1336
+ enumValues: [string, ...string[]];
1337
+ baseColumn: never;
1338
+ identity: undefined;
1339
+ generated: undefined;
1340
+ }, {}, {}>;
1324
1341
  diskOp: import("drizzle-orm/pg-core").PgColumn<{
1325
1342
  name: "diskOp";
1326
1343
  tableName: "agents";
@@ -3587,7 +3604,9 @@ export declare const signingKeys: import("drizzle-orm/pg-core").PgTableWithColum
3587
3604
  * when the provider sent none. `startedAt`, `endedAt` and `firstTokenMs` are the gate's own
3588
3605
  * timing of the call (R7.1, migration 0022); null on rows written before, and `firstTokenMs`
3589
3606
  * when the provider streamed nothing. The `cost*Usd` columns split `costUsd` by token kind, at the
3590
- * prices the adapter applied (migration 0037); null on rows written before.
3607
+ * prices the adapter applied (migration 0037); null on rows written before. `attempts` and
3608
+ * `retries` say how many times the gate called the provider for the answer, and why again
3609
+ * (migration 0047).
3591
3610
  */
3592
3611
  export declare const modelUsage: import("drizzle-orm/pg-core").PgTableWithColumns<{
3593
3612
  name: "model_usage";
@@ -4047,6 +4066,42 @@ export declare const modelUsage: import("drizzle-orm/pg-core").PgTableWithColumn
4047
4066
  identity: undefined;
4048
4067
  generated: undefined;
4049
4068
  }, {}, {}>;
4069
+ attempts: import("drizzle-orm/pg-core").PgColumn<{
4070
+ name: "attempts";
4071
+ tableName: "model_usage";
4072
+ dataType: "number";
4073
+ columnType: "PgInteger";
4074
+ data: number;
4075
+ driverParam: string | number;
4076
+ notNull: false;
4077
+ hasDefault: false;
4078
+ isPrimaryKey: false;
4079
+ isAutoincrement: false;
4080
+ hasRuntimeDefault: false;
4081
+ enumValues: undefined;
4082
+ baseColumn: never;
4083
+ identity: undefined;
4084
+ generated: undefined;
4085
+ }, {}, {}>;
4086
+ retries: import("drizzle-orm/pg-core").PgColumn<{
4087
+ name: "retries";
4088
+ tableName: "model_usage";
4089
+ dataType: "custom";
4090
+ columnType: "PgCustomColumn";
4091
+ data: unknown;
4092
+ driverParam: unknown;
4093
+ notNull: false;
4094
+ hasDefault: false;
4095
+ isPrimaryKey: false;
4096
+ isAutoincrement: false;
4097
+ hasRuntimeDefault: false;
4098
+ enumValues: undefined;
4099
+ baseColumn: never;
4100
+ identity: undefined;
4101
+ generated: undefined;
4102
+ }, {}, {
4103
+ pgColumnBuilderBrand: 'PgCustomColumnBuilderBrand';
4104
+ }>;
4050
4105
  };
4051
4106
  dialect: 'pg';
4052
4107
  }>;
@@ -6707,7 +6762,7 @@ export type EnvironmentRowColumns = typeof agents.$inferSelect;
6707
6762
  export type ArtifactRow = typeof artifacts.$inferSelect;
6708
6763
  export type ArtifactVersionRow = typeof artifactVersions.$inferSelect;
6709
6764
  export type DefinitionFileRow = typeof definitionFiles.$inferSelect;
6710
- export type ModelUsageWrite = Omit<ModelUsageRow, "duplicate" | "txid" | "tokensReported" | "costKnown" | "startedAt" | "endedAt" | "firstTokenMs" | "costInputUsd" | "costCacheReadUsd" | "costCacheWriteUsd" | "costOutputUsd"> & {
6765
+ export type ModelUsageWrite = Omit<ModelUsageRow, "duplicate" | "txid" | "tokensReported" | "costKnown" | "startedAt" | "endedAt" | "firstTokenMs" | "costInputUsd" | "costCacheReadUsd" | "costCacheWriteUsd" | "costOutputUsd" | "attempts" | "retries"> & {
6711
6766
  tokensReported?: boolean | null;
6712
6767
  costKnown?: boolean | null;
6713
6768
  startedAt?: string | null;
@@ -6717,4 +6772,6 @@ export type ModelUsageWrite = Omit<ModelUsageRow, "duplicate" | "txid" | "tokens
6717
6772
  costCacheReadUsd?: number | null;
6718
6773
  costCacheWriteUsd?: number | null;
6719
6774
  costOutputUsd?: number | null;
6775
+ attempts?: number | null;
6776
+ retries?: unknown;
6720
6777
  };
@@ -173,6 +173,8 @@ export const agents = nylorun.table("agents", {
173
173
  imageDigest: textC(),
174
174
  /** The engine the image carries (ADR-99): `{ version, digest? }`, from a join of its engine. */
175
175
  engineImage: jsonText(),
176
+ /** When its Agent Substrate worker last stopped under it (the sandboxes service's `drainedAt`). */
177
+ drainedAt: text(),
176
178
  /**
177
179
  * The disk lease (R2d step 3): `{ kind, version?, startedAt, until }` while a snapshot, a
178
180
  * restore or a seed copies the disk; turns are refused until it ends or `until` passes.
@@ -455,7 +457,9 @@ export const signingKeys = nylorun.table("signing_keys", {
455
457
  * when the provider sent none. `startedAt`, `endedAt` and `firstTokenMs` are the gate's own
456
458
  * timing of the call (R7.1, migration 0022); null on rows written before, and `firstTokenMs`
457
459
  * when the provider streamed nothing. The `cost*Usd` columns split `costUsd` by token kind, at the
458
- * prices the adapter applied (migration 0037); null on rows written before.
460
+ * prices the adapter applied (migration 0037); null on rows written before. `attempts` and
461
+ * `retries` say how many times the gate called the provider for the answer, and why again
462
+ * (migration 0047).
459
463
  */
460
464
  export const modelUsage = nylorun.table("model_usage", {
461
465
  id: textC().primaryKey(),
@@ -497,6 +501,13 @@ export const modelUsage = nylorun.table("model_usage", {
497
501
  costCacheWriteUsd: doublePrecision(),
498
502
  /** What the output cost. */
499
503
  costOutputUsd: doublePrecision(),
504
+ /**
505
+ * The provider calls the gate made for this answer, the first included (migration 0047);
506
+ * null on rows written before.
507
+ */
508
+ attempts: integer(),
509
+ /** Each retry before the answer: `{ attempt, delayMs, error }` (migration 0047); null when none. */
510
+ retries: jsonText(),
500
511
  }, (t) => [
501
512
  index("model_usage_session").on(t.sessionId, t.createdAt, t.id),
502
513
  index("model_usage_export").on(t.txid, t.id),
@@ -284,6 +284,7 @@ function environmentOf(row) {
284
284
  ...(row.volumeId !== null ? { volumeId: row.volumeId } : {}),
285
285
  ...(row.imageDigest !== null ? { imageDigest: row.imageDigest } : {}),
286
286
  ...(row.engineImage ? { engineImage: row.engineImage } : {}),
287
+ ...(row.drainedAt !== null ? { drainedAt: row.drainedAt } : {}),
287
288
  },
288
289
  }
289
290
  : {}),
@@ -319,6 +320,7 @@ function podColumns(patch) {
319
320
  volumeId: patch.volumeId,
320
321
  imageDigest: patch.imageDigest,
321
322
  engineImage: patch.engineImage,
323
+ drainedAt: patch.drainedAt,
322
324
  };
323
325
  return Object.fromEntries(Object.entries(columns).filter(([, value]) => value !== undefined));
324
326
  }
@@ -269,6 +269,11 @@ export interface SandboxPodState {
269
269
  version: string;
270
270
  digest?: string;
271
271
  };
272
+ /**
273
+ * When the sandbox was last suspended because its Agent Substrate worker stopped (the sandboxes
274
+ * service's `drainedAt`), so each drain is recorded once (`agent.suspended`, reason `drain`).
275
+ */
276
+ drainedAt?: string;
272
277
  }
273
278
  /** A change to a pod's columns: `null` clears an optional one. */
274
279
  export type SandboxPodPatch = {
@@ -128,6 +128,10 @@ export interface MeteredCall {
128
128
  readonly totalTokens: number;
129
129
  readonly cachedTokens: number;
130
130
  readonly costUsd: number;
131
+ /** The provider calls the gate made for the answer (migration 0047); null on older rows. */
132
+ readonly attempts?: number | null;
133
+ /** Each retry before the answer (migration 0047); null when none. */
134
+ readonly retries?: unknown;
131
135
  }
132
136
  /** From `startedAt` to `endedAt` in whole milliseconds; undefined unless both are times. */
133
137
  export declare function durationBetween(startedAt: string | undefined, endedAt: string | undefined): number | undefined;
@@ -227,6 +231,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
227
231
  gate?: GateTimes | undefined;
228
232
  policy?: ToolPolicyFact | undefined;
229
233
  } | {
234
+ redacted?: undefined;
230
235
  agent: import("@nylorun/core/define").AgentRef;
231
236
  error: {
232
237
  code: string;
@@ -236,7 +241,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
236
241
  retryable?: boolean | undefined;
237
242
  };
238
243
  output?: undefined;
239
- redacted?: undefined;
240
244
  toolName: string;
241
245
  status?: "cancelled" | "completed" | "failed" | undefined;
242
246
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -250,6 +254,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
250
254
  gate?: GateTimes | undefined;
251
255
  policy?: ToolPolicyFact | undefined;
252
256
  } | {
257
+ redacted?: undefined;
253
258
  agent?: undefined;
254
259
  error: {
255
260
  code: string;
@@ -259,7 +264,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
259
264
  retryable?: boolean | undefined;
260
265
  };
261
266
  output?: undefined;
262
- redacted?: undefined;
263
267
  toolName: string;
264
268
  status?: "cancelled" | "completed" | "failed" | undefined;
265
269
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -307,10 +311,10 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
307
311
  gate?: GateTimes | undefined;
308
312
  policy?: ToolPolicyFact | undefined;
309
313
  } | {
314
+ redacted?: undefined;
310
315
  agent: import("@nylorun/core/define").AgentRef;
311
316
  error?: undefined;
312
317
  output: unknown;
313
- redacted?: undefined;
314
318
  toolName: string;
315
319
  status?: "cancelled" | "completed" | "failed" | undefined;
316
320
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -324,10 +328,10 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
324
328
  gate?: GateTimes | undefined;
325
329
  policy?: ToolPolicyFact | undefined;
326
330
  } | {
331
+ redacted?: undefined;
327
332
  agent?: undefined;
328
333
  error?: undefined;
329
334
  output: unknown;
330
- redacted?: undefined;
331
335
  toolName: string;
332
336
  status?: "cancelled" | "completed" | "failed" | undefined;
333
337
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -391,6 +395,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
391
395
  gate?: GateTimes | undefined;
392
396
  policy?: ToolPolicyFact | undefined;
393
397
  } | {
398
+ redacted?: undefined;
394
399
  callId: string;
395
400
  invocationId: string;
396
401
  agent: import("@nylorun/core/define").AgentRef;
@@ -402,7 +407,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
402
407
  retryable?: boolean | undefined;
403
408
  };
404
409
  output?: undefined;
405
- redacted?: undefined;
406
410
  toolName: string;
407
411
  status?: "cancelled" | "completed" | "failed" | undefined;
408
412
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -416,6 +420,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
416
420
  gate?: GateTimes | undefined;
417
421
  policy?: ToolPolicyFact | undefined;
418
422
  } | {
423
+ redacted?: undefined;
419
424
  callId: string;
420
425
  invocationId: string;
421
426
  agent?: undefined;
@@ -427,7 +432,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
427
432
  retryable?: boolean | undefined;
428
433
  };
429
434
  output?: undefined;
430
- redacted?: undefined;
431
435
  toolName: string;
432
436
  status?: "cancelled" | "completed" | "failed" | undefined;
433
437
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -479,12 +483,12 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
479
483
  gate?: GateTimes | undefined;
480
484
  policy?: ToolPolicyFact | undefined;
481
485
  } | {
486
+ redacted?: undefined;
482
487
  callId: string;
483
488
  invocationId: string;
484
489
  agent: import("@nylorun/core/define").AgentRef;
485
490
  error?: undefined;
486
491
  output: unknown;
487
- redacted?: undefined;
488
492
  toolName: string;
489
493
  status?: "cancelled" | "completed" | "failed" | undefined;
490
494
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -498,12 +502,12 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
498
502
  gate?: GateTimes | undefined;
499
503
  policy?: ToolPolicyFact | undefined;
500
504
  } | {
505
+ redacted?: undefined;
501
506
  callId: string;
502
507
  invocationId: string;
503
508
  agent?: undefined;
504
509
  error?: undefined;
505
510
  output: unknown;
506
- redacted?: undefined;
507
511
  toolName: string;
508
512
  status?: "cancelled" | "completed" | "failed" | undefined;
509
513
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -1,5 +1,6 @@
1
1
  import { isModelFailureOutcome } from "@nylorun/core/define";
2
2
  import { HarnessApiError } from "@nylorun/core/harness-api";
3
+ import { attemptFacts } from "../model/retry.js";
3
4
  /** The model's tool call id and the harness invocation, for tool effects. */
4
5
  export function toolIds(context) {
5
6
  const c = context;
@@ -95,6 +96,8 @@ export function modelFailed(request, value) {
95
96
  code: value.code,
96
97
  message: value.message,
97
98
  retryable: value.retryable,
99
+ // How many times the gate called the provider before it gave up, and why again.
100
+ ...attemptFacts(value.attempts, value.retries),
98
101
  ...agentOf(request),
99
102
  };
100
103
  }
@@ -133,6 +136,7 @@ export function modelOutcomeFacts(effectId, window, metered) {
133
136
  },
134
137
  }
135
138
  : {}),
139
+ ...attemptFacts(metered?.attempts, metered?.retries),
136
140
  };
137
141
  }
138
142
  /**
@@ -135,6 +135,8 @@ export function spanOf(row, turn, settings) {
135
135
  .str("gen_ai.agent.id", p.agent?.id)
136
136
  .str("nylorun.effect.id", p.effectId)
137
137
  .int("nylorun.effect.window_ms", p.durationMs)
138
+ // The gate's provider calls for this one (Model Calls §6): more than 1 after a retry.
139
+ .int("nylorun.model.attempts", typeof p.attempts === "number" ? p.attempts : undefined)
138
140
  .str("nylorun.compaction.trigger", row.type === "context.compacted" ? p.trigger : undefined)
139
141
  .str("error.type", row.type === "model.failed" ? p.code : undefined)
140
142
  .str("gen_ai.output.messages", output);
package/dist/version.d.ts CHANGED
@@ -1 +1 @@
1
- export declare const RUNTIME_VERSION = "0.49.2-beta";
1
+ export declare const RUNTIME_VERSION = "0.51.0-beta";
package/dist/version.js CHANGED
@@ -1,3 +1,3 @@
1
1
  // Kept in step with package.json by runtime/server/scripts/check-package.mjs, which runs in npm run check.
2
2
  // release:prepare also rewrites this when it bumps @nylorun/runtime.
3
- export const RUNTIME_VERSION = "0.49.2-beta";
3
+ export const RUNTIME_VERSION = "0.51.0-beta";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@nylorun/runtime",
3
- "version": "0.49.2-beta",
3
+ "version": "0.51.0-beta",
4
4
  "description": "Nylorun Runtime: the agent execution Host on Postgres, Restate and S2, and model providers.",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -61,39 +61,39 @@
61
61
  "dependencies": {
62
62
  "@ag-ui/core": "~1.0.4",
63
63
  "@asteasolutions/zod-to-openapi": "^9.1.0",
64
- "@earendil-works/pi-ai": "1.3.0",
65
- "@hono/node-server": "^2.1.5",
64
+ "@earendil-works/pi-ai": "1.5.2",
65
+ "@hono/node-server": "^2.1.8",
66
66
  "@hono/zod-openapi": "^1.6.3",
67
- "@modelcontextprotocol/sdk": "^1.32.1",
68
- "@nylorun/core": "0.42.0-beta",
69
- "@nylorun/harness": "0.36.2-beta",
70
- "@restatedev/restate-sdk": "1.17.3",
71
- "@restatedev/restate-sdk-clients": "1.17.3",
72
- "@s2-dev/streamstore": "0.28.0",
67
+ "@modelcontextprotocol/sdk": "^1.32.2",
68
+ "@nylorun/core": "0.44.0-beta",
69
+ "@nylorun/harness": "0.36.4-beta",
70
+ "@restatedev/restate-sdk": "1.18.3",
71
+ "@restatedev/restate-sdk-clients": "1.18.3",
72
+ "@s2-dev/streamstore": "0.30.0",
73
73
  "aws4fetch": "1.0.20",
74
74
  "croner": "^9.1.0",
75
75
  "drizzle-orm": "^0.45.4",
76
- "hono": "^4.13.13",
77
- "jose": "^6.2.12",
76
+ "hono": "^4.13.18",
77
+ "jose": "^6.2.17",
78
78
  "just-bash": "3.6.0",
79
79
  "pg": "8.23.1",
80
- "pg-logical-replication": "2.5.0",
80
+ "pg-logical-replication": "2.6.0",
81
81
  "postgres": "^3.4.9",
82
- "ws": "^8.22.0",
82
+ "ws": "^8.22.2",
83
83
  "yaml": "^2.9.1",
84
84
  "zod": "^4.6.7"
85
85
  },
86
86
  "devDependencies": {
87
87
  "@a2a-js/sdk": "~1.3.0",
88
88
  "@ag-ui/client": "1.0.4",
89
- "@scalar/openapi-parser": "^0.29.13",
90
- "@types/node": "^26.6.5",
89
+ "@scalar/openapi-parser": "^0.29.21",
90
+ "@types/node": "^26.6.6",
91
91
  "@types/pg": "^8.23.1",
92
92
  "@types/ws": "^8.18.2",
93
93
  "@typescript/native": "npm:typescript@^7.0.2",
94
94
  "drizzle-kit": "^0.31.11",
95
95
  "esbuild": "^0.28.2",
96
96
  "typescript": "npm:@typescript/typescript6@^6.0.2",
97
- "vitest": "^5.0.4"
97
+ "vitest": "^5.1.0"
98
98
  }
99
99
  }