@nylorun/runtime 0.49.2-beta → 0.50.0-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,23 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.50.0-beta
4
+
5
+ ### Minor Changes
6
+
7
+ - 745ef22: A model call's retries are in its event. `message.assistant` and `model.failed` carry `attempts`, the provider calls the gate made for the call (the first included), and `retries`, one `{ attempt, delayMs, error }` for each retry (the error as the provider worded it, cut to 300 characters, with the provider's credential scrubbed). Before, a call that answered after a retry was one `message.assistant` that showed no sign of it, and a call that failed after its attempts showed only its last error; the retries were only in the gates service's `model_retry` log lines.
8
+
9
+ - `@nylorun/core`: `ModelRetrySchema`, and the optional `attempts` and `retries` on the facts `message.assistant`, `model.failed` and `context.compacted` share. `ModelFailureOutcome` takes optional `attempts` and `retries`.
10
+ - `@nylorun/runtime`: the gate's answer reports them in `evidence.extras`, the usage ledger keeps them (`model_usage.attempts` and `retries`, store migration 0047, nullable, run at start), and the call's event takes them from its row; a failed call's from its failure outcome. A trace's chat span has `nylorun.model.attempts`. A retried call is still metered once, as one call. `/calls/model` is unchanged.
11
+
12
+ ### Patch Changes
13
+
14
+ - Pin core to the tested release.
15
+ - Pin harness to the tested release.
16
+ - Updated dependencies [745ef22]
17
+ - Updated dependencies
18
+ - @nylorun/core@0.43.0-beta
19
+ - @nylorun/harness@0.36.3-beta
20
+
3
21
  ## 0.49.2-beta
4
22
 
5
23
  ### Patch Changes
@@ -3502,6 +3502,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
3502
3502
  cachedTokens: z.ZodOptional<z.ZodNumber>;
3503
3503
  costUsd: z.ZodOptional<z.ZodNumber>;
3504
3504
  }, z.core.$loose>>;
3505
+ attempts: z.ZodOptional<z.ZodNumber>;
3506
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
3507
+ attempt: z.ZodNumber;
3508
+ delayMs: z.ZodNumber;
3509
+ error: z.ZodString;
3510
+ }, z.core.$loose>>>;
3505
3511
  invocationId: z.ZodOptional<z.ZodString>;
3506
3512
  trigger: z.ZodEnum<{
3507
3513
  overflow: "overflow";
@@ -4363,6 +4369,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
4363
4369
  cachedTokens: z.ZodOptional<z.ZodNumber>;
4364
4370
  costUsd: z.ZodOptional<z.ZodNumber>;
4365
4371
  }, z.core.$loose>>;
4372
+ attempts: z.ZodOptional<z.ZodNumber>;
4373
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
4374
+ attempt: z.ZodNumber;
4375
+ delayMs: z.ZodNumber;
4376
+ error: z.ZodString;
4377
+ }, z.core.$loose>>>;
4366
4378
  invocationId: z.ZodString;
4367
4379
  text: z.ZodString;
4368
4380
  model: z.ZodOptional<z.ZodObject<{
@@ -4540,6 +4552,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
4540
4552
  cachedTokens: z.ZodOptional<z.ZodNumber>;
4541
4553
  costUsd: z.ZodOptional<z.ZodNumber>;
4542
4554
  }, z.core.$loose>>;
4555
+ attempts: z.ZodOptional<z.ZodNumber>;
4556
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
4557
+ attempt: z.ZodNumber;
4558
+ delayMs: z.ZodNumber;
4559
+ error: z.ZodString;
4560
+ }, z.core.$loose>>>;
4543
4561
  invocationId: z.ZodOptional<z.ZodString>;
4544
4562
  code: z.ZodString;
4545
4563
  message: z.ZodString;
@@ -6735,6 +6753,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
6735
6753
  cachedTokens: z.ZodOptional<z.ZodNumber>;
6736
6754
  costUsd: z.ZodOptional<z.ZodNumber>;
6737
6755
  }, z.core.$loose>>;
6756
+ attempts: z.ZodOptional<z.ZodNumber>;
6757
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
6758
+ attempt: z.ZodNumber;
6759
+ delayMs: z.ZodNumber;
6760
+ error: z.ZodString;
6761
+ }, z.core.$loose>>>;
6738
6762
  invocationId: z.ZodOptional<z.ZodString>;
6739
6763
  trigger: z.ZodEnum<{
6740
6764
  overflow: "overflow";
@@ -7596,6 +7620,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
7596
7620
  cachedTokens: z.ZodOptional<z.ZodNumber>;
7597
7621
  costUsd: z.ZodOptional<z.ZodNumber>;
7598
7622
  }, z.core.$loose>>;
7623
+ attempts: z.ZodOptional<z.ZodNumber>;
7624
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
7625
+ attempt: z.ZodNumber;
7626
+ delayMs: z.ZodNumber;
7627
+ error: z.ZodString;
7628
+ }, z.core.$loose>>>;
7599
7629
  invocationId: z.ZodString;
7600
7630
  text: z.ZodString;
7601
7631
  model: z.ZodOptional<z.ZodObject<{
@@ -7773,6 +7803,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
7773
7803
  cachedTokens: z.ZodOptional<z.ZodNumber>;
7774
7804
  costUsd: z.ZodOptional<z.ZodNumber>;
7775
7805
  }, z.core.$loose>>;
7806
+ attempts: z.ZodOptional<z.ZodNumber>;
7807
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
7808
+ attempt: z.ZodNumber;
7809
+ delayMs: z.ZodNumber;
7810
+ error: z.ZodString;
7811
+ }, z.core.$loose>>>;
7776
7812
  invocationId: z.ZodOptional<z.ZodString>;
7777
7813
  code: z.ZodString;
7778
7814
  message: z.ZodString;
@@ -9971,6 +10007,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
9971
10007
  cachedTokens: z.ZodOptional<z.ZodNumber>;
9972
10008
  costUsd: z.ZodOptional<z.ZodNumber>;
9973
10009
  }, z.core.$loose>>;
10010
+ attempts: z.ZodOptional<z.ZodNumber>;
10011
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
10012
+ attempt: z.ZodNumber;
10013
+ delayMs: z.ZodNumber;
10014
+ error: z.ZodString;
10015
+ }, z.core.$loose>>>;
9974
10016
  invocationId: z.ZodOptional<z.ZodString>;
9975
10017
  trigger: z.ZodEnum<{
9976
10018
  overflow: "overflow";
@@ -10832,6 +10874,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
10832
10874
  cachedTokens: z.ZodOptional<z.ZodNumber>;
10833
10875
  costUsd: z.ZodOptional<z.ZodNumber>;
10834
10876
  }, z.core.$loose>>;
10877
+ attempts: z.ZodOptional<z.ZodNumber>;
10878
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
10879
+ attempt: z.ZodNumber;
10880
+ delayMs: z.ZodNumber;
10881
+ error: z.ZodString;
10882
+ }, z.core.$loose>>>;
10835
10883
  invocationId: z.ZodString;
10836
10884
  text: z.ZodString;
10837
10885
  model: z.ZodOptional<z.ZodObject<{
@@ -11009,6 +11057,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
11009
11057
  cachedTokens: z.ZodOptional<z.ZodNumber>;
11010
11058
  costUsd: z.ZodOptional<z.ZodNumber>;
11011
11059
  }, z.core.$loose>>;
11060
+ attempts: z.ZodOptional<z.ZodNumber>;
11061
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
11062
+ attempt: z.ZodNumber;
11063
+ delayMs: z.ZodNumber;
11064
+ error: z.ZodString;
11065
+ }, z.core.$loose>>>;
11012
11066
  invocationId: z.ZodOptional<z.ZodString>;
11013
11067
  code: z.ZodString;
11014
11068
  message: z.ZodString;
@@ -13204,6 +13258,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
13204
13258
  cachedTokens: z.ZodOptional<z.ZodNumber>;
13205
13259
  costUsd: z.ZodOptional<z.ZodNumber>;
13206
13260
  }, z.core.$loose>>;
13261
+ attempts: z.ZodOptional<z.ZodNumber>;
13262
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
13263
+ attempt: z.ZodNumber;
13264
+ delayMs: z.ZodNumber;
13265
+ error: z.ZodString;
13266
+ }, z.core.$loose>>>;
13207
13267
  invocationId: z.ZodOptional<z.ZodString>;
13208
13268
  trigger: z.ZodEnum<{
13209
13269
  overflow: "overflow";
@@ -14065,6 +14125,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
14065
14125
  cachedTokens: z.ZodOptional<z.ZodNumber>;
14066
14126
  costUsd: z.ZodOptional<z.ZodNumber>;
14067
14127
  }, z.core.$loose>>;
14128
+ attempts: z.ZodOptional<z.ZodNumber>;
14129
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
14130
+ attempt: z.ZodNumber;
14131
+ delayMs: z.ZodNumber;
14132
+ error: z.ZodString;
14133
+ }, z.core.$loose>>>;
14068
14134
  invocationId: z.ZodString;
14069
14135
  text: z.ZodString;
14070
14136
  model: z.ZodOptional<z.ZodObject<{
@@ -14242,6 +14308,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
14242
14308
  cachedTokens: z.ZodOptional<z.ZodNumber>;
14243
14309
  costUsd: z.ZodOptional<z.ZodNumber>;
14244
14310
  }, z.core.$loose>>;
14311
+ attempts: z.ZodOptional<z.ZodNumber>;
14312
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
14313
+ attempt: z.ZodNumber;
14314
+ delayMs: z.ZodNumber;
14315
+ error: z.ZodString;
14316
+ }, z.core.$loose>>>;
14245
14317
  invocationId: z.ZodOptional<z.ZodString>;
14246
14318
  code: z.ZodString;
14247
14319
  message: z.ZodString;
@@ -17390,6 +17462,12 @@ export declare const HistoryPage: z.ZodObject<{
17390
17462
  cachedTokens: z.ZodOptional<z.ZodNumber>;
17391
17463
  costUsd: z.ZodOptional<z.ZodNumber>;
17392
17464
  }, z.core.$loose>>;
17465
+ attempts: z.ZodOptional<z.ZodNumber>;
17466
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
17467
+ attempt: z.ZodNumber;
17468
+ delayMs: z.ZodNumber;
17469
+ error: z.ZodString;
17470
+ }, z.core.$loose>>>;
17393
17471
  invocationId: z.ZodOptional<z.ZodString>;
17394
17472
  trigger: z.ZodEnum<{
17395
17473
  overflow: "overflow";
@@ -18251,6 +18329,12 @@ export declare const HistoryPage: z.ZodObject<{
18251
18329
  cachedTokens: z.ZodOptional<z.ZodNumber>;
18252
18330
  costUsd: z.ZodOptional<z.ZodNumber>;
18253
18331
  }, z.core.$loose>>;
18332
+ attempts: z.ZodOptional<z.ZodNumber>;
18333
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
18334
+ attempt: z.ZodNumber;
18335
+ delayMs: z.ZodNumber;
18336
+ error: z.ZodString;
18337
+ }, z.core.$loose>>>;
18254
18338
  invocationId: z.ZodString;
18255
18339
  text: z.ZodString;
18256
18340
  model: z.ZodOptional<z.ZodObject<{
@@ -18428,6 +18512,12 @@ export declare const HistoryPage: z.ZodObject<{
18428
18512
  cachedTokens: z.ZodOptional<z.ZodNumber>;
18429
18513
  costUsd: z.ZodOptional<z.ZodNumber>;
18430
18514
  }, z.core.$loose>>;
18515
+ attempts: z.ZodOptional<z.ZodNumber>;
18516
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
18517
+ attempt: z.ZodNumber;
18518
+ delayMs: z.ZodNumber;
18519
+ error: z.ZodString;
18520
+ }, z.core.$loose>>>;
18431
18521
  invocationId: z.ZodOptional<z.ZodString>;
18432
18522
  code: z.ZodString;
18433
18523
  message: z.ZodString;
@@ -20623,6 +20713,12 @@ export declare const HistoryPage: z.ZodObject<{
20623
20713
  cachedTokens: z.ZodOptional<z.ZodNumber>;
20624
20714
  costUsd: z.ZodOptional<z.ZodNumber>;
20625
20715
  }, z.core.$loose>>;
20716
+ attempts: z.ZodOptional<z.ZodNumber>;
20717
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
20718
+ attempt: z.ZodNumber;
20719
+ delayMs: z.ZodNumber;
20720
+ error: z.ZodString;
20721
+ }, z.core.$loose>>>;
20626
20722
  invocationId: z.ZodOptional<z.ZodString>;
20627
20723
  trigger: z.ZodEnum<{
20628
20724
  overflow: "overflow";
@@ -21484,6 +21580,12 @@ export declare const HistoryPage: z.ZodObject<{
21484
21580
  cachedTokens: z.ZodOptional<z.ZodNumber>;
21485
21581
  costUsd: z.ZodOptional<z.ZodNumber>;
21486
21582
  }, z.core.$loose>>;
21583
+ attempts: z.ZodOptional<z.ZodNumber>;
21584
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
21585
+ attempt: z.ZodNumber;
21586
+ delayMs: z.ZodNumber;
21587
+ error: z.ZodString;
21588
+ }, z.core.$loose>>>;
21487
21589
  invocationId: z.ZodString;
21488
21590
  text: z.ZodString;
21489
21591
  model: z.ZodOptional<z.ZodObject<{
@@ -21661,6 +21763,12 @@ export declare const HistoryPage: z.ZodObject<{
21661
21763
  cachedTokens: z.ZodOptional<z.ZodNumber>;
21662
21764
  costUsd: z.ZodOptional<z.ZodNumber>;
21663
21765
  }, z.core.$loose>>;
21766
+ attempts: z.ZodOptional<z.ZodNumber>;
21767
+ retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
21768
+ attempt: z.ZodNumber;
21769
+ delayMs: z.ZodNumber;
21770
+ error: z.ZodString;
21771
+ }, z.core.$loose>>>;
21664
21772
  invocationId: z.ZodOptional<z.ZodString>;
21665
21773
  code: z.ZodString;
21666
21774
  message: z.ZodString;
@@ -11,10 +11,12 @@
11
11
  *
12
12
  * The row also keeps the gate's own timing of the call (R7.1): `now()` before the call and
13
13
  * after it, and the adapter's `firstTokenMs` (`evidence.extras`), so a model call's span is
14
- * what the gate measured.
14
+ * what the gate measured; and the adapter's `attempts` and `retries`, so the call's event says
15
+ * how many times the provider was called for it, and why again.
15
16
  */
16
17
  import { randomUUID } from "node:crypto";
17
18
  import { failure } from "../model/classify.js";
19
+ import { attemptFacts } from "../model/retry.js";
18
20
  export function createMeter(options) {
19
21
  const now = options.now ?? (() => new Date());
20
22
  /** Calls running through this gate, per Tenant and budget scope. */
@@ -143,6 +145,8 @@ export function usageRow(request, outcome, at, timing) {
143
145
  ? { firstTokenMs: Math.round(firstTokenMs) }
144
146
  : {}),
145
147
  ...costSplitColumns(outcome.evidence?.extras?.costSplit),
148
+ // How many times the gate called the provider for this answer, and why again.
149
+ ...attemptFacts(outcome.evidence?.extras?.attempts, outcome.evidence?.extras?.retries),
146
150
  };
147
151
  }
148
152
  /** The ledger's cost-by-kind columns from the adapter's `costSplit`; none unless all four are costs. */
@@ -2,7 +2,7 @@
2
2
  "openapi": "3.2.0",
3
3
  "info": {
4
4
  "title": "Nylorun Management API",
5
- "version": "0.49.2-beta",
5
+ "version": "0.50.0-beta",
6
6
  "x-nylorun-protocol": {
7
7
  "min": 16,
8
8
  "max": 18,
@@ -25,6 +25,8 @@ export function modelCallSettingsOf(limits) {
25
25
  };
26
26
  }
27
27
  const OPENAI_COMPATIBLE_REASONING = /^reasoning(_content|_text)?$/;
28
+ /** How much of a retried attempt's error the outcome keeps. */
29
+ const RETRY_ERROR_CHARS = 300;
28
30
  const emptyUsage = () => ({
29
31
  input: 0,
30
32
  output: 0,
@@ -200,21 +202,35 @@ export function piModel(options = {}) {
200
202
  clearTimeout(timer);
201
203
  }
202
204
  };
203
- const response = await retryModelCall(attempt, { attempts: settings.attempts, baseDelayMs: settings.retryBaseDelayMs }, context.signal, options.onRetry);
205
+ // Each retry, for the outcome (Model Calls §6): the gate's ledger row and the call's event
206
+ // carry how many times the provider was called, and why again.
207
+ const retries = [];
208
+ const response = await retryModelCall(attempt, { attempts: settings.attempts, baseDelayMs: settings.retryBaseDelayMs }, context.signal, (retry) => {
209
+ retries.push({
210
+ attempt: retry.attempt,
211
+ delayMs: retry.delayMs,
212
+ error: String(scrub(retry.errorMessage, secrets)).slice(0, RETRY_ERROR_CHARS),
213
+ });
214
+ options.onRetry?.(retry);
215
+ });
216
+ const tried = { attempts: retries.length + 1, ...(retries.length ? { retries } : {}) };
204
217
  // An abort is never a failure outcome: the host decides what it means.
205
218
  context.signal.throwIfAborted();
206
219
  if (natives.error)
207
220
  return redacted(failure("invalid_request", natives.error, false));
208
221
  if (response.stopReason === "aborted")
209
- return redacted(failure("transient", "The provider stream was aborted.", true));
222
+ return { ...redacted(failure("transient", "The provider stream was aborted.", true)), ...tried };
210
223
  if (response.stopReason === "error") {
211
224
  const outcome = classifyAssistantError(response, selected.contextWindow);
212
- return redacted(outcome.code === "auth"
213
- ? {
214
- ...outcome,
215
- message: `${outcome.message} Update the provider credential in Studio's Model Settings, or run nylorun tenant models set.`,
216
- }
217
- : outcome);
225
+ return {
226
+ ...redacted(outcome.code === "auth"
227
+ ? {
228
+ ...outcome,
229
+ message: `${outcome.message} Update the provider credential in Studio's Model Settings, or run nylorun tenant models set.`,
230
+ }
231
+ : outcome),
232
+ ...tried,
233
+ };
218
234
  }
219
235
  if (response.stopReason === "deferred")
220
236
  return failure("invalid_request", "The provider deferred the response, which the Runtime does not request.", false);
@@ -272,6 +288,9 @@ export function piModel(options = {}) {
272
288
  maxOutputTokens: selected.maxTokens,
273
289
  // From the call to the provider's first stream event (R7.1); absent when none came.
274
290
  ...(firstEventAt !== undefined ? { firstTokenMs: firstEventAt - calledAt } : {}),
291
+ // The provider calls this answer took, and each retry before it: the ledger keeps them.
292
+ attempts: tried.attempts,
293
+ ...(retries.length ? { retries: retries } : {}),
275
294
  },
276
295
  },
277
296
  };
@@ -34,3 +34,17 @@ export interface ModelRetryPolicy {
34
34
  * delay comes back as the last failure marked `aborted`, as pi-ai's `retryAssistantCall` does.
35
35
  */
36
36
  export declare function retryModelCall(produce: () => Promise<AssistantMessage>, policy: ModelRetryPolicy, signal: AbortSignal, onRetry?: (retry: ModelRetry) => void): Promise<AssistantMessage>;
37
+ /** A retry as the ledger and the call's event keep it (Model Calls §6). */
38
+ export interface RecordedRetry {
39
+ readonly attempt: number;
40
+ readonly delayMs: number;
41
+ readonly error: string;
42
+ }
43
+ /** A retry list as the ledger and the events keep it: well-formed entries only, at most 10. */
44
+ export declare function modelRetries(value: unknown): RecordedRetry[];
45
+ /**
46
+ * `attempts` and `retries` for a model call's event (Model Calls §6): how many times the gate
47
+ * called the provider, and each retry: the ledger's columns and the event's fields alike. None
48
+ * unless `attempts` is a positive whole number (a call a Runtime before 0.50 served).
49
+ */
50
+ export declare function attemptFacts(attempts: unknown, retries: unknown): {};
@@ -61,3 +61,37 @@ function wait(ms, signal) {
61
61
  signal.addEventListener("abort", onAbort, { once: true });
62
62
  });
63
63
  }
64
+ /** The most retries a row keeps; `modelAttempts` is 3 unless a Tenant raised it. */
65
+ const MAX_RETRIES = 10;
66
+ /** A retry list as the ledger and the events keep it: well-formed entries only, at most 10. */
67
+ export function modelRetries(value) {
68
+ if (!Array.isArray(value))
69
+ return [];
70
+ const out = [];
71
+ for (const entry of value) {
72
+ if (!entry || typeof entry !== "object")
73
+ continue;
74
+ const { attempt, delayMs, error } = entry;
75
+ if (typeof attempt !== "number" || !Number.isInteger(attempt) || attempt < 2)
76
+ continue;
77
+ out.push({
78
+ attempt,
79
+ delayMs: typeof delayMs === "number" && Number.isFinite(delayMs) && delayMs >= 0 ? Math.round(delayMs) : 0,
80
+ error: typeof error === "string" ? error.slice(0, 300) : "",
81
+ });
82
+ if (out.length === MAX_RETRIES)
83
+ break;
84
+ }
85
+ return out;
86
+ }
87
+ /**
88
+ * `attempts` and `retries` for a model call's event (Model Calls §6): how many times the gate
89
+ * called the provider, and each retry: the ledger's columns and the event's fields alike. None
90
+ * unless `attempts` is a positive whole number (a call a Runtime before 0.50 served).
91
+ */
92
+ export function attemptFacts(attempts, retries) {
93
+ if (typeof attempts !== "number" || !Number.isInteger(attempts) || attempts < 1)
94
+ return {};
95
+ const kept = modelRetries(retries);
96
+ return { attempts, ...(kept.length ? { retries: kept } : {}) };
97
+ }
package/dist/openapi.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "openapi": "3.2.0",
3
3
  "info": {
4
4
  "title": "Nylorun Runtime API",
5
- "version": "0.49.2-beta",
5
+ "version": "0.50.0-beta",
6
6
  "x-nylorun-protocol": {
7
7
  "min": 16,
8
8
  "max": 18,
@@ -13430,6 +13430,35 @@
13430
13430
  ],
13431
13431
  "additionalProperties": {}
13432
13432
  },
13433
+ "attempts": {
13434
+ "type": "integer",
13435
+ "exclusiveMinimum": 0
13436
+ },
13437
+ "retries": {
13438
+ "type": "array",
13439
+ "items": {
13440
+ "type": "object",
13441
+ "properties": {
13442
+ "attempt": {
13443
+ "type": "integer",
13444
+ "minimum": 2
13445
+ },
13446
+ "delayMs": {
13447
+ "type": "integer",
13448
+ "minimum": 0
13449
+ },
13450
+ "error": {
13451
+ "type": "string"
13452
+ }
13453
+ },
13454
+ "required": [
13455
+ "attempt",
13456
+ "delayMs",
13457
+ "error"
13458
+ ],
13459
+ "additionalProperties": {}
13460
+ }
13461
+ },
13433
13462
  "agent": {
13434
13463
  "type": "object",
13435
13464
  "properties": {
@@ -13716,6 +13745,35 @@
13716
13745
  ],
13717
13746
  "additionalProperties": {}
13718
13747
  },
13748
+ "attempts": {
13749
+ "type": "integer",
13750
+ "exclusiveMinimum": 0
13751
+ },
13752
+ "retries": {
13753
+ "type": "array",
13754
+ "items": {
13755
+ "type": "object",
13756
+ "properties": {
13757
+ "attempt": {
13758
+ "type": "integer",
13759
+ "minimum": 2
13760
+ },
13761
+ "delayMs": {
13762
+ "type": "integer",
13763
+ "minimum": 0
13764
+ },
13765
+ "error": {
13766
+ "type": "string"
13767
+ }
13768
+ },
13769
+ "required": [
13770
+ "attempt",
13771
+ "delayMs",
13772
+ "error"
13773
+ ],
13774
+ "additionalProperties": {}
13775
+ }
13776
+ },
13719
13777
  "agent": {
13720
13778
  "type": "object",
13721
13779
  "properties": {
@@ -14007,6 +14065,35 @@
14007
14065
  ],
14008
14066
  "additionalProperties": {}
14009
14067
  },
14068
+ "attempts": {
14069
+ "type": "integer",
14070
+ "exclusiveMinimum": 0
14071
+ },
14072
+ "retries": {
14073
+ "type": "array",
14074
+ "items": {
14075
+ "type": "object",
14076
+ "properties": {
14077
+ "attempt": {
14078
+ "type": "integer",
14079
+ "minimum": 2
14080
+ },
14081
+ "delayMs": {
14082
+ "type": "integer",
14083
+ "minimum": 0
14084
+ },
14085
+ "error": {
14086
+ "type": "string"
14087
+ }
14088
+ },
14089
+ "required": [
14090
+ "attempt",
14091
+ "delayMs",
14092
+ "error"
14093
+ ],
14094
+ "additionalProperties": {}
14095
+ }
14096
+ },
14010
14097
  "agent": {
14011
14098
  "type": "object",
14012
14099
  "properties": {
@@ -0,0 +1,2 @@
1
+ ALTER TABLE "nylorun"."model_usage" ADD COLUMN "attempts" integer;--> statement-breakpoint
2
+ ALTER TABLE "nylorun"."model_usage" ADD COLUMN "retries" json;
@@ -330,6 +330,13 @@
330
330
  "when": 1794240246972,
331
331
  "tag": "0046_artifacts_audience",
332
332
  "breakpoints": true
333
+ },
334
+ {
335
+ "idx": 47,
336
+ "version": "7",
337
+ "when": 1794630124577,
338
+ "tag": "0047_model_usage_attempts",
339
+ "breakpoints": true
333
340
  }
334
341
  ]
335
342
  }
@@ -3587,7 +3587,9 @@ export declare const signingKeys: import("drizzle-orm/pg-core").PgTableWithColum
3587
3587
  * when the provider sent none. `startedAt`, `endedAt` and `firstTokenMs` are the gate's own
3588
3588
  * timing of the call (R7.1, migration 0022); null on rows written before, and `firstTokenMs`
3589
3589
  * when the provider streamed nothing. The `cost*Usd` columns split `costUsd` by token kind, at the
3590
- * prices the adapter applied (migration 0037); null on rows written before.
3590
+ * prices the adapter applied (migration 0037); null on rows written before. `attempts` and
3591
+ * `retries` say how many times the gate called the provider for the answer, and why again
3592
+ * (migration 0047).
3591
3593
  */
3592
3594
  export declare const modelUsage: import("drizzle-orm/pg-core").PgTableWithColumns<{
3593
3595
  name: "model_usage";
@@ -4047,6 +4049,42 @@ export declare const modelUsage: import("drizzle-orm/pg-core").PgTableWithColumn
4047
4049
  identity: undefined;
4048
4050
  generated: undefined;
4049
4051
  }, {}, {}>;
4052
+ attempts: import("drizzle-orm/pg-core").PgColumn<{
4053
+ name: "attempts";
4054
+ tableName: "model_usage";
4055
+ dataType: "number";
4056
+ columnType: "PgInteger";
4057
+ data: number;
4058
+ driverParam: string | number;
4059
+ notNull: false;
4060
+ hasDefault: false;
4061
+ isPrimaryKey: false;
4062
+ isAutoincrement: false;
4063
+ hasRuntimeDefault: false;
4064
+ enumValues: undefined;
4065
+ baseColumn: never;
4066
+ identity: undefined;
4067
+ generated: undefined;
4068
+ }, {}, {}>;
4069
+ retries: import("drizzle-orm/pg-core").PgColumn<{
4070
+ name: "retries";
4071
+ tableName: "model_usage";
4072
+ dataType: "custom";
4073
+ columnType: "PgCustomColumn";
4074
+ data: unknown;
4075
+ driverParam: unknown;
4076
+ notNull: false;
4077
+ hasDefault: false;
4078
+ isPrimaryKey: false;
4079
+ isAutoincrement: false;
4080
+ hasRuntimeDefault: false;
4081
+ enumValues: undefined;
4082
+ baseColumn: never;
4083
+ identity: undefined;
4084
+ generated: undefined;
4085
+ }, {}, {
4086
+ pgColumnBuilderBrand: 'PgCustomColumnBuilderBrand';
4087
+ }>;
4050
4088
  };
4051
4089
  dialect: 'pg';
4052
4090
  }>;
@@ -6707,7 +6745,7 @@ export type EnvironmentRowColumns = typeof agents.$inferSelect;
6707
6745
  export type ArtifactRow = typeof artifacts.$inferSelect;
6708
6746
  export type ArtifactVersionRow = typeof artifactVersions.$inferSelect;
6709
6747
  export type DefinitionFileRow = typeof definitionFiles.$inferSelect;
6710
- export type ModelUsageWrite = Omit<ModelUsageRow, "duplicate" | "txid" | "tokensReported" | "costKnown" | "startedAt" | "endedAt" | "firstTokenMs" | "costInputUsd" | "costCacheReadUsd" | "costCacheWriteUsd" | "costOutputUsd"> & {
6748
+ export type ModelUsageWrite = Omit<ModelUsageRow, "duplicate" | "txid" | "tokensReported" | "costKnown" | "startedAt" | "endedAt" | "firstTokenMs" | "costInputUsd" | "costCacheReadUsd" | "costCacheWriteUsd" | "costOutputUsd" | "attempts" | "retries"> & {
6711
6749
  tokensReported?: boolean | null;
6712
6750
  costKnown?: boolean | null;
6713
6751
  startedAt?: string | null;
@@ -6717,4 +6755,6 @@ export type ModelUsageWrite = Omit<ModelUsageRow, "duplicate" | "txid" | "tokens
6717
6755
  costCacheReadUsd?: number | null;
6718
6756
  costCacheWriteUsd?: number | null;
6719
6757
  costOutputUsd?: number | null;
6758
+ attempts?: number | null;
6759
+ retries?: unknown;
6720
6760
  };
@@ -455,7 +455,9 @@ export const signingKeys = nylorun.table("signing_keys", {
455
455
  * when the provider sent none. `startedAt`, `endedAt` and `firstTokenMs` are the gate's own
456
456
  * timing of the call (R7.1, migration 0022); null on rows written before, and `firstTokenMs`
457
457
  * when the provider streamed nothing. The `cost*Usd` columns split `costUsd` by token kind, at the
458
- * prices the adapter applied (migration 0037); null on rows written before.
458
+ * prices the adapter applied (migration 0037); null on rows written before. `attempts` and
459
+ * `retries` say how many times the gate called the provider for the answer, and why again
460
+ * (migration 0047).
459
461
  */
460
462
  export const modelUsage = nylorun.table("model_usage", {
461
463
  id: textC().primaryKey(),
@@ -497,6 +499,13 @@ export const modelUsage = nylorun.table("model_usage", {
497
499
  costCacheWriteUsd: doublePrecision(),
498
500
  /** What the output cost. */
499
501
  costOutputUsd: doublePrecision(),
502
+ /**
503
+ * The provider calls the gate made for this answer, the first included (migration 0047);
504
+ * null on rows written before.
505
+ */
506
+ attempts: integer(),
507
+ /** Each retry before the answer: `{ attempt, delayMs, error }` (migration 0047); null when none. */
508
+ retries: jsonText(),
500
509
  }, (t) => [
501
510
  index("model_usage_session").on(t.sessionId, t.createdAt, t.id),
502
511
  index("model_usage_export").on(t.txid, t.id),
@@ -128,6 +128,10 @@ export interface MeteredCall {
128
128
  readonly totalTokens: number;
129
129
  readonly cachedTokens: number;
130
130
  readonly costUsd: number;
131
+ /** The provider calls the gate made for the answer (migration 0047); null on older rows. */
132
+ readonly attempts?: number | null;
133
+ /** Each retry before the answer (migration 0047); null when none. */
134
+ readonly retries?: unknown;
131
135
  }
132
136
  /** From `startedAt` to `endedAt` in whole milliseconds; undefined unless both are times. */
133
137
  export declare function durationBetween(startedAt: string | undefined, endedAt: string | undefined): number | undefined;
@@ -227,6 +231,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
227
231
  gate?: GateTimes | undefined;
228
232
  policy?: ToolPolicyFact | undefined;
229
233
  } | {
234
+ redacted?: undefined;
230
235
  agent: import("@nylorun/core/define").AgentRef;
231
236
  error: {
232
237
  code: string;
@@ -236,7 +241,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
236
241
  retryable?: boolean | undefined;
237
242
  };
238
243
  output?: undefined;
239
- redacted?: undefined;
240
244
  toolName: string;
241
245
  status?: "cancelled" | "completed" | "failed" | undefined;
242
246
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -250,6 +254,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
250
254
  gate?: GateTimes | undefined;
251
255
  policy?: ToolPolicyFact | undefined;
252
256
  } | {
257
+ redacted?: undefined;
253
258
  agent?: undefined;
254
259
  error: {
255
260
  code: string;
@@ -259,7 +264,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
259
264
  retryable?: boolean | undefined;
260
265
  };
261
266
  output?: undefined;
262
- redacted?: undefined;
263
267
  toolName: string;
264
268
  status?: "cancelled" | "completed" | "failed" | undefined;
265
269
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -307,10 +311,10 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
307
311
  gate?: GateTimes | undefined;
308
312
  policy?: ToolPolicyFact | undefined;
309
313
  } | {
314
+ redacted?: undefined;
310
315
  agent: import("@nylorun/core/define").AgentRef;
311
316
  error?: undefined;
312
317
  output: unknown;
313
- redacted?: undefined;
314
318
  toolName: string;
315
319
  status?: "cancelled" | "completed" | "failed" | undefined;
316
320
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -324,10 +328,10 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
324
328
  gate?: GateTimes | undefined;
325
329
  policy?: ToolPolicyFact | undefined;
326
330
  } | {
331
+ redacted?: undefined;
327
332
  agent?: undefined;
328
333
  error?: undefined;
329
334
  output: unknown;
330
- redacted?: undefined;
331
335
  toolName: string;
332
336
  status?: "cancelled" | "completed" | "failed" | undefined;
333
337
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -391,6 +395,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
391
395
  gate?: GateTimes | undefined;
392
396
  policy?: ToolPolicyFact | undefined;
393
397
  } | {
398
+ redacted?: undefined;
394
399
  callId: string;
395
400
  invocationId: string;
396
401
  agent: import("@nylorun/core/define").AgentRef;
@@ -402,7 +407,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
402
407
  retryable?: boolean | undefined;
403
408
  };
404
409
  output?: undefined;
405
- redacted?: undefined;
406
410
  toolName: string;
407
411
  status?: "cancelled" | "completed" | "failed" | undefined;
408
412
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -416,6 +420,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
416
420
  gate?: GateTimes | undefined;
417
421
  policy?: ToolPolicyFact | undefined;
418
422
  } | {
423
+ redacted?: undefined;
419
424
  callId: string;
420
425
  invocationId: string;
421
426
  agent?: undefined;
@@ -427,7 +432,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
427
432
  retryable?: boolean | undefined;
428
433
  };
429
434
  output?: undefined;
430
- redacted?: undefined;
431
435
  toolName: string;
432
436
  status?: "cancelled" | "completed" | "failed" | undefined;
433
437
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -479,12 +483,12 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
479
483
  gate?: GateTimes | undefined;
480
484
  policy?: ToolPolicyFact | undefined;
481
485
  } | {
486
+ redacted?: undefined;
482
487
  callId: string;
483
488
  invocationId: string;
484
489
  agent: import("@nylorun/core/define").AgentRef;
485
490
  error?: undefined;
486
491
  output: unknown;
487
- redacted?: undefined;
488
492
  toolName: string;
489
493
  status?: "cancelled" | "completed" | "failed" | undefined;
490
494
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -498,12 +502,12 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
498
502
  gate?: GateTimes | undefined;
499
503
  policy?: ToolPolicyFact | undefined;
500
504
  } | {
505
+ redacted?: undefined;
501
506
  callId: string;
502
507
  invocationId: string;
503
508
  agent?: undefined;
504
509
  error?: undefined;
505
510
  output: unknown;
506
- redacted?: undefined;
507
511
  toolName: string;
508
512
  status?: "cancelled" | "completed" | "failed" | undefined;
509
513
  kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -1,5 +1,6 @@
1
1
  import { isModelFailureOutcome } from "@nylorun/core/define";
2
2
  import { HarnessApiError } from "@nylorun/core/harness-api";
3
+ import { attemptFacts } from "../model/retry.js";
3
4
  /** The model's tool call id and the harness invocation, for tool effects. */
4
5
  export function toolIds(context) {
5
6
  const c = context;
@@ -95,6 +96,8 @@ export function modelFailed(request, value) {
95
96
  code: value.code,
96
97
  message: value.message,
97
98
  retryable: value.retryable,
99
+ // How many times the gate called the provider before it gave up, and why again.
100
+ ...attemptFacts(value.attempts, value.retries),
98
101
  ...agentOf(request),
99
102
  };
100
103
  }
@@ -133,6 +136,7 @@ export function modelOutcomeFacts(effectId, window, metered) {
133
136
  },
134
137
  }
135
138
  : {}),
139
+ ...attemptFacts(metered?.attempts, metered?.retries),
136
140
  };
137
141
  }
138
142
  /**
@@ -135,6 +135,8 @@ export function spanOf(row, turn, settings) {
135
135
  .str("gen_ai.agent.id", p.agent?.id)
136
136
  .str("nylorun.effect.id", p.effectId)
137
137
  .int("nylorun.effect.window_ms", p.durationMs)
138
+ // The gate's provider calls for this one (Model Calls §6): more than 1 after a retry.
139
+ .int("nylorun.model.attempts", typeof p.attempts === "number" ? p.attempts : undefined)
138
140
  .str("nylorun.compaction.trigger", row.type === "context.compacted" ? p.trigger : undefined)
139
141
  .str("error.type", row.type === "model.failed" ? p.code : undefined)
140
142
  .str("gen_ai.output.messages", output);
package/dist/version.d.ts CHANGED
@@ -1 +1 @@
1
- export declare const RUNTIME_VERSION = "0.49.2-beta";
1
+ export declare const RUNTIME_VERSION = "0.50.0-beta";
package/dist/version.js CHANGED
@@ -1,3 +1,3 @@
1
1
  // Kept in step with package.json by runtime/server/scripts/check-package.mjs, which runs in npm run check.
2
2
  // release:prepare also rewrites this when it bumps @nylorun/runtime.
3
- export const RUNTIME_VERSION = "0.49.2-beta";
3
+ export const RUNTIME_VERSION = "0.50.0-beta";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@nylorun/runtime",
3
- "version": "0.49.2-beta",
3
+ "version": "0.50.0-beta",
4
4
  "description": "Nylorun Runtime: the agent execution Host on Postgres, Restate and S2, and model providers.",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -65,8 +65,8 @@
65
65
  "@hono/node-server": "^2.1.5",
66
66
  "@hono/zod-openapi": "^1.6.3",
67
67
  "@modelcontextprotocol/sdk": "^1.32.1",
68
- "@nylorun/core": "0.42.0-beta",
69
- "@nylorun/harness": "0.36.2-beta",
68
+ "@nylorun/core": "0.43.0-beta",
69
+ "@nylorun/harness": "0.36.3-beta",
70
70
  "@restatedev/restate-sdk": "1.17.3",
71
71
  "@restatedev/restate-sdk-clients": "1.17.3",
72
72
  "@s2-dev/streamstore": "0.28.0",