@nylorun/runtime 0.49.1-beta → 0.50.0-beta

package/CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## 0.50.0-beta
+
+### Minor Changes
+
+- 745ef22: A model call's retries are in its event. `message.assistant` and `model.failed` carry `attempts`, the provider calls the gate made for the call (the first included), and `retries`, one `{ attempt, delayMs, error }` for each retry (the error as the provider worded it, cut to 300 characters, with the provider's credential scrubbed). Before, a call that answered after a retry was one `message.assistant` that showed no sign of it, and a call that failed after its attempts showed only its last error; the retries were only in the gates service's `model_retry` log lines.
+
+  - `@nylorun/core`: `ModelRetrySchema`, and the optional `attempts` and `retries` on the facts `message.assistant`, `model.failed` and `context.compacted` share. `ModelFailureOutcome` takes optional `attempts` and `retries`.
+  - `@nylorun/runtime`: the gate's answer reports them in `evidence.extras`, the usage ledger keeps them (`model_usage.attempts` and `retries`, store migration 0047, nullable, run at start), and the call's event takes them from its row; a failed call's from its failure outcome. A trace's chat span has `nylorun.model.attempts`. A retried call is still metered once, as one call. `/calls/model` is unchanged.
+
+### Patch Changes
+
+- Pin core to the tested release.
+- Pin harness to the tested release.
+- Updated dependencies [745ef22]
+- Updated dependencies
+  - @nylorun/core@0.43.0-beta
+  - @nylorun/harness@0.36.3-beta
+
+## 0.49.2-beta
+
+### Patch Changes
+
+- bf9d614: Pods on Agent Substrate (preview):
+
+  - **A Runtime upgrade no longer strands existing pods.** Agent Substrate repoints an actor only to a template with the same volumes, and the engine is an image volume pinned by the Runtime image's digest. So after an upgrade, every existing pod's start failed and was sent again every 2 seconds without end, and only a reset got out, at the cost of the pod's disk. An actor now keeps the engine it was created with. An upgrade keeps the pod, its `/workspace` and `/harness`, and it starts as before. The pod runs its first engine until a new incarnation (a reset). The Runtime's Harness API check at `hello` still guards it.
+  - **A change Substrate can't make to an existing pod fails its start with the reason.** This is any other volume change, such as switching between the image's engine and the Runtime's, or a template Substrate refuses. The sandbox reports `RepointRefused` with why, the Runtime's ready deadline fails the start, and the request is not sent again.
+  - **A crashed actor is reverted before its template changes**, as Substrate requires. Before, a new spec for a crashed pod failed without end.
+
 ## 0.49.1-beta
 
 ### Patch Changes

package/dist/api/components.d.ts

@@ -3502,6 +3502,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
             cachedTokens: z.ZodOptional<z.ZodNumber>;
             costUsd: z.ZodOptional<z.ZodNumber>;
         }, z.core.$loose>>;
+        attempts: z.ZodOptional<z.ZodNumber>;
+        retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            attempt: z.ZodNumber;
+            delayMs: z.ZodNumber;
+            error: z.ZodString;
+        }, z.core.$loose>>>;
         invocationId: z.ZodOptional<z.ZodString>;
         trigger: z.ZodEnum<{
             overflow: "overflow";
@@ -4363,6 +4369,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
             cachedTokens: z.ZodOptional<z.ZodNumber>;
             costUsd: z.ZodOptional<z.ZodNumber>;
         }, z.core.$loose>>;
+        attempts: z.ZodOptional<z.ZodNumber>;
+        retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            attempt: z.ZodNumber;
+            delayMs: z.ZodNumber;
+            error: z.ZodString;
+        }, z.core.$loose>>>;
         invocationId: z.ZodString;
         text: z.ZodString;
         model: z.ZodOptional<z.ZodObject<{
@@ -4540,6 +4552,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
             cachedTokens: z.ZodOptional<z.ZodNumber>;
             costUsd: z.ZodOptional<z.ZodNumber>;
         }, z.core.$loose>>;
+        attempts: z.ZodOptional<z.ZodNumber>;
+        retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            attempt: z.ZodNumber;
+            delayMs: z.ZodNumber;
+            error: z.ZodString;
+        }, z.core.$loose>>>;
         invocationId: z.ZodOptional<z.ZodString>;
         code: z.ZodString;
         message: z.ZodString;
@@ -6735,6 +6753,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
             cachedTokens: z.ZodOptional<z.ZodNumber>;
             costUsd: z.ZodOptional<z.ZodNumber>;
         }, z.core.$loose>>;
+        attempts: z.ZodOptional<z.ZodNumber>;
+        retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            attempt: z.ZodNumber;
+            delayMs: z.ZodNumber;
+            error: z.ZodString;
+        }, z.core.$loose>>>;
         invocationId: z.ZodOptional<z.ZodString>;
         trigger: z.ZodEnum<{
             overflow: "overflow";
@@ -7596,6 +7620,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
             cachedTokens: z.ZodOptional<z.ZodNumber>;
             costUsd: z.ZodOptional<z.ZodNumber>;
         }, z.core.$loose>>;
+        attempts: z.ZodOptional<z.ZodNumber>;
+        retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            attempt: z.ZodNumber;
+            delayMs: z.ZodNumber;
+            error: z.ZodString;
+        }, z.core.$loose>>>;
         invocationId: z.ZodString;
         text: z.ZodString;
         model: z.ZodOptional<z.ZodObject<{
@@ -7773,6 +7803,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
             cachedTokens: z.ZodOptional<z.ZodNumber>;
             costUsd: z.ZodOptional<z.ZodNumber>;
         }, z.core.$loose>>;
+        attempts: z.ZodOptional<z.ZodNumber>;
+        retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            attempt: z.ZodNumber;
+            delayMs: z.ZodNumber;
+            error: z.ZodString;
+        }, z.core.$loose>>>;
         invocationId: z.ZodOptional<z.ZodString>;
         code: z.ZodString;
         message: z.ZodString;
@@ -9971,6 +10007,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
                 cachedTokens: z.ZodOptional<z.ZodNumber>;
                 costUsd: z.ZodOptional<z.ZodNumber>;
             }, z.core.$loose>>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                attempt: z.ZodNumber;
+                delayMs: z.ZodNumber;
+                error: z.ZodString;
+            }, z.core.$loose>>>;
             invocationId: z.ZodOptional<z.ZodString>;
             trigger: z.ZodEnum<{
                 overflow: "overflow";
@@ -10832,6 +10874,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
                 cachedTokens: z.ZodOptional<z.ZodNumber>;
                 costUsd: z.ZodOptional<z.ZodNumber>;
             }, z.core.$loose>>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                attempt: z.ZodNumber;
+                delayMs: z.ZodNumber;
+                error: z.ZodString;
+            }, z.core.$loose>>>;
             invocationId: z.ZodString;
             text: z.ZodString;
             model: z.ZodOptional<z.ZodObject<{
@@ -11009,6 +11057,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
                 cachedTokens: z.ZodOptional<z.ZodNumber>;
                 costUsd: z.ZodOptional<z.ZodNumber>;
             }, z.core.$loose>>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                attempt: z.ZodNumber;
+                delayMs: z.ZodNumber;
+                error: z.ZodString;
+            }, z.core.$loose>>>;
             invocationId: z.ZodOptional<z.ZodString>;
             code: z.ZodString;
             message: z.ZodString;
@@ -13204,6 +13258,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
                 cachedTokens: z.ZodOptional<z.ZodNumber>;
                 costUsd: z.ZodOptional<z.ZodNumber>;
             }, z.core.$loose>>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                attempt: z.ZodNumber;
+                delayMs: z.ZodNumber;
+                error: z.ZodString;
+            }, z.core.$loose>>>;
             invocationId: z.ZodOptional<z.ZodString>;
             trigger: z.ZodEnum<{
                 overflow: "overflow";
@@ -14065,6 +14125,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
                 cachedTokens: z.ZodOptional<z.ZodNumber>;
                 costUsd: z.ZodOptional<z.ZodNumber>;
             }, z.core.$loose>>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                attempt: z.ZodNumber;
+                delayMs: z.ZodNumber;
+                error: z.ZodString;
+            }, z.core.$loose>>>;
             invocationId: z.ZodString;
             text: z.ZodString;
             model: z.ZodOptional<z.ZodObject<{
@@ -14242,6 +14308,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
                 cachedTokens: z.ZodOptional<z.ZodNumber>;
                 costUsd: z.ZodOptional<z.ZodNumber>;
             }, z.core.$loose>>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                attempt: z.ZodNumber;
+                delayMs: z.ZodNumber;
+                error: z.ZodString;
+            }, z.core.$loose>>>;
             invocationId: z.ZodOptional<z.ZodString>;
             code: z.ZodString;
             message: z.ZodString;
@@ -17390,6 +17462,12 @@ export declare const HistoryPage: z.ZodObject<{
                 cachedTokens: z.ZodOptional<z.ZodNumber>;
                 costUsd: z.ZodOptional<z.ZodNumber>;
             }, z.core.$loose>>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                attempt: z.ZodNumber;
+                delayMs: z.ZodNumber;
+                error: z.ZodString;
+            }, z.core.$loose>>>;
             invocationId: z.ZodOptional<z.ZodString>;
             trigger: z.ZodEnum<{
                 overflow: "overflow";
@@ -18251,6 +18329,12 @@ export declare const HistoryPage: z.ZodObject<{
                 cachedTokens: z.ZodOptional<z.ZodNumber>;
                 costUsd: z.ZodOptional<z.ZodNumber>;
             }, z.core.$loose>>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                attempt: z.ZodNumber;
+                delayMs: z.ZodNumber;
+                error: z.ZodString;
+            }, z.core.$loose>>>;
             invocationId: z.ZodString;
             text: z.ZodString;
             model: z.ZodOptional<z.ZodObject<{
@@ -18428,6 +18512,12 @@ export declare const HistoryPage: z.ZodObject<{
                 cachedTokens: z.ZodOptional<z.ZodNumber>;
                 costUsd: z.ZodOptional<z.ZodNumber>;
             }, z.core.$loose>>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                attempt: z.ZodNumber;
+                delayMs: z.ZodNumber;
+                error: z.ZodString;
+            }, z.core.$loose>>>;
             invocationId: z.ZodOptional<z.ZodString>;
             code: z.ZodString;
             message: z.ZodString;
@@ -20623,6 +20713,12 @@ export declare const HistoryPage: z.ZodObject<{
                 cachedTokens: z.ZodOptional<z.ZodNumber>;
                 costUsd: z.ZodOptional<z.ZodNumber>;
             }, z.core.$loose>>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                attempt: z.ZodNumber;
+                delayMs: z.ZodNumber;
+                error: z.ZodString;
+            }, z.core.$loose>>>;
             invocationId: z.ZodOptional<z.ZodString>;
             trigger: z.ZodEnum<{
                 overflow: "overflow";
@@ -21484,6 +21580,12 @@ export declare const HistoryPage: z.ZodObject<{
                 cachedTokens: z.ZodOptional<z.ZodNumber>;
                 costUsd: z.ZodOptional<z.ZodNumber>;
             }, z.core.$loose>>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                attempt: z.ZodNumber;
+                delayMs: z.ZodNumber;
+                error: z.ZodString;
+            }, z.core.$loose>>>;
             invocationId: z.ZodString;
             text: z.ZodString;
             model: z.ZodOptional<z.ZodObject<{
@@ -21661,6 +21763,12 @@ export declare const HistoryPage: z.ZodObject<{
                 cachedTokens: z.ZodOptional<z.ZodNumber>;
                 costUsd: z.ZodOptional<z.ZodNumber>;
             }, z.core.$loose>>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                attempt: z.ZodNumber;
+                delayMs: z.ZodNumber;
+                error: z.ZodString;
+            }, z.core.$loose>>>;
             invocationId: z.ZodOptional<z.ZodString>;
             code: z.ZodString;
             message: z.ZodString;

package/dist/gates/meter.js

@@ -11,10 +11,12 @@
  *
  * The row also keeps the gate's own timing of the call (R7.1): `now()` before the call and
  * after it, and the adapter's `firstTokenMs` (`evidence.extras`), so a model call's span is
- * what the gate measured.
+ * what the gate measured; and the adapter's `attempts` and `retries`, so the call's event says
+ * how many times the provider was called for it, and why again.
  */
 import { randomUUID } from "node:crypto";
 import { failure } from "../model/classify.js";
+import { attemptFacts } from "../model/retry.js";
 export function createMeter(options) {
     const now = options.now ?? (() => new Date());
     /** Calls running through this gate, per Tenant and budget scope. */
@@ -143,6 +145,8 @@ export function usageRow(request, outcome, at, timing) {
             ? { firstTokenMs: Math.round(firstTokenMs) }
             : {}),
         ...costSplitColumns(outcome.evidence?.extras?.costSplit),
+        // How many times the gate called the provider for this answer, and why again.
+        ...attemptFacts(outcome.evidence?.extras?.attempts, outcome.evidence?.extras?.retries),
     };
 }
 /** The ledger's cost-by-kind columns from the adapter's `costSplit`; none unless all four are costs. */

package/dist/management-openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.2.0",
   "info": {
     "title": "Nylorun Management API",
-    "version": "0.49.1-beta",
+    "version": "0.50.0-beta",
     "x-nylorun-protocol": {
       "min": 16,
       "max": 18,

package/dist/model/pi-model.js

@@ -25,6 +25,8 @@ export function modelCallSettingsOf(limits) {
     };
 }
 const OPENAI_COMPATIBLE_REASONING = /^reasoning(_content|_text)?$/;
+/** How much of a retried attempt's error the outcome keeps. */
+const RETRY_ERROR_CHARS = 300;
 const emptyUsage = () => ({
     input: 0,
     output: 0,
@@ -200,21 +202,35 @@ export function piModel(options = {}) {
                 clearTimeout(timer);
             }
         };
-        const response = await retryModelCall(attempt, { attempts: settings.attempts, baseDelayMs: settings.retryBaseDelayMs }, context.signal, options.onRetry);
+        // Each retry, for the outcome (Model Calls §6): the gate's ledger row and the call's event
+        // carry how many times the provider was called, and why again.
+        const retries = [];
+        const response = await retryModelCall(attempt, { attempts: settings.attempts, baseDelayMs: settings.retryBaseDelayMs }, context.signal, (retry) => {
+            retries.push({
+                attempt: retry.attempt,
+                delayMs: retry.delayMs,
+                error: String(scrub(retry.errorMessage, secrets)).slice(0, RETRY_ERROR_CHARS),
+            });
+            options.onRetry?.(retry);
+        });
+        const tried = { attempts: retries.length + 1, ...(retries.length ? { retries } : {}) };
         // An abort is never a failure outcome: the host decides what it means.
         context.signal.throwIfAborted();
         if (natives.error)
             return redacted(failure("invalid_request", natives.error, false));
         if (response.stopReason === "aborted")
-            return redacted(failure("transient", "The provider stream was aborted.", true));
+            return { ...redacted(failure("transient", "The provider stream was aborted.", true)), ...tried };
         if (response.stopReason === "error") {
             const outcome = classifyAssistantError(response, selected.contextWindow);
-            return redacted(outcome.code === "auth"
-                ? {
-                    ...outcome,
-                    message: `${outcome.message} Update the provider credential in Studio's Model Settings, or run nylorun tenant models set.`,
-                }
-                : outcome);
+            return {
+                ...redacted(outcome.code === "auth"
+                    ? {
+                        ...outcome,
+                        message: `${outcome.message} Update the provider credential in Studio's Model Settings, or run nylorun tenant models set.`,
+                    }
+                    : outcome),
+                ...tried,
+            };
         }
         if (response.stopReason === "deferred")
             return failure("invalid_request", "The provider deferred the response, which the Runtime does not request.", false);
@@ -272,6 +288,9 @@ export function piModel(options = {}) {
                     maxOutputTokens: selected.maxTokens,
                     // From the call to the provider's first stream event (R7.1); absent when none came.
                     ...(firstEventAt !== undefined ? { firstTokenMs: firstEventAt - calledAt } : {}),
+                    // The provider calls this answer took, and each retry before it: the ledger keeps them.
+                    attempts: tried.attempts,
+                    ...(retries.length ? { retries: retries } : {}),
                 },
             },
         };

package/dist/model/retry.d.ts

@@ -34,3 +34,17 @@ export interface ModelRetryPolicy {
  * delay comes back as the last failure marked `aborted`, as pi-ai's `retryAssistantCall` does.
  */
 export declare function retryModelCall(produce: () => Promise<AssistantMessage>, policy: ModelRetryPolicy, signal: AbortSignal, onRetry?: (retry: ModelRetry) => void): Promise<AssistantMessage>;
+/** A retry as the ledger and the call's event keep it (Model Calls §6). */
+export interface RecordedRetry {
+    readonly attempt: number;
+    readonly delayMs: number;
+    readonly error: string;
+}
+/** A retry list as the ledger and the events keep it: well-formed entries only, at most 10. */
+export declare function modelRetries(value: unknown): RecordedRetry[];
+/**
+ * `attempts` and `retries` for a model call's event (Model Calls §6): how many times the gate
+ * called the provider, and each retry: the ledger's columns and the event's fields alike. None
+ * unless `attempts` is a positive whole number (a call a Runtime before 0.50 served).
+ */
+export declare function attemptFacts(attempts: unknown, retries: unknown): {};

package/dist/model/retry.js

@@ -61,3 +61,37 @@ function wait(ms, signal) {
         signal.addEventListener("abort", onAbort, { once: true });
     });
 }
+/** The most retries a row keeps; `modelAttempts` is 3 unless a Tenant raised it. */
+const MAX_RETRIES = 10;
+/** A retry list as the ledger and the events keep it: well-formed entries only, at most 10. */
+export function modelRetries(value) {
+    if (!Array.isArray(value))
+        return [];
+    const out = [];
+    for (const entry of value) {
+        if (!entry || typeof entry !== "object")
+            continue;
+        const { attempt, delayMs, error } = entry;
+        if (typeof attempt !== "number" || !Number.isInteger(attempt) || attempt < 2)
+            continue;
+        out.push({
+            attempt,
+            delayMs: typeof delayMs === "number" && Number.isFinite(delayMs) && delayMs >= 0 ? Math.round(delayMs) : 0,
+            error: typeof error === "string" ? error.slice(0, 300) : "",
+        });
+        if (out.length === MAX_RETRIES)
+            break;
+    }
+    return out;
+}
+/**
+ * `attempts` and `retries` for a model call's event (Model Calls §6): how many times the gate
+ * called the provider, and each retry: the ledger's columns and the event's fields alike. None
+ * unless `attempts` is a positive whole number (a call a Runtime before 0.50 served).
+ */
+export function attemptFacts(attempts, retries) {
+    if (typeof attempts !== "number" || !Number.isInteger(attempts) || attempts < 1)
+        return {};
+    const kept = modelRetries(retries);
+    return { attempts, ...(kept.length ? { retries: kept } : {}) };
+}

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.2.0",
   "info": {
     "title": "Nylorun Runtime API",
-    "version": "0.49.1-beta",
+    "version": "0.50.0-beta",
     "x-nylorun-protocol": {
       "min": 16,
       "max": 18,
@@ -13430,6 +13430,35 @@
                 ],
                 "additionalProperties": {}
               },
+              "attempts": {
+                "type": "integer",
+                "exclusiveMinimum": 0
+              },
+              "retries": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "attempt": {
+                      "type": "integer",
+                      "minimum": 2
+                    },
+                    "delayMs": {
+                      "type": "integer",
+                      "minimum": 0
+                    },
+                    "error": {
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "attempt",
+                    "delayMs",
+                    "error"
+                  ],
+                  "additionalProperties": {}
+                }
+              },
               "agent": {
                 "type": "object",
                 "properties": {
@@ -13716,6 +13745,35 @@
                 ],
                 "additionalProperties": {}
               },
+              "attempts": {
+                "type": "integer",
+                "exclusiveMinimum": 0
+              },
+              "retries": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "attempt": {
+                      "type": "integer",
+                      "minimum": 2
+                    },
+                    "delayMs": {
+                      "type": "integer",
+                      "minimum": 0
+                    },
+                    "error": {
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "attempt",
+                    "delayMs",
+                    "error"
+                  ],
+                  "additionalProperties": {}
+                }
+              },
               "agent": {
                 "type": "object",
                 "properties": {
@@ -14007,6 +14065,35 @@
                 ],
                 "additionalProperties": {}
               },
+              "attempts": {
+                "type": "integer",
+                "exclusiveMinimum": 0
+              },
+              "retries": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "attempt": {
+                      "type": "integer",
+                      "minimum": 2
+                    },
+                    "delayMs": {
+                      "type": "integer",
+                      "minimum": 0
+                    },
+                    "error": {
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "attempt",
+                    "delayMs",
+                    "error"
+                  ],
+                  "additionalProperties": {}
+                }
+              },
               "agent": {
                 "type": "object",
                 "properties": {

package/dist/store/postgres/drizzle/0047_model_usage_attempts.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "nylorun"."model_usage" ADD COLUMN "attempts" integer;--> statement-breakpoint
+ALTER TABLE "nylorun"."model_usage" ADD COLUMN "retries" json;

package/dist/store/postgres/drizzle/meta/_journal.json

@@ -330,6 +330,13 @@
       "when": 1794240246972,
       "tag": "0046_artifacts_audience",
       "breakpoints": true
+    },
+    {
+      "idx": 47,
+      "version": "7",
+      "when": 1794630124577,
+      "tag": "0047_model_usage_attempts",
+      "breakpoints": true
     }
   ]
 }

package/dist/store/postgres/schema.d.ts

@@ -3587,7 +3587,9 @@ export declare const signingKeys: import("drizzle-orm/pg-core").PgTableWithColum
  * when the provider sent none. `startedAt`, `endedAt` and `firstTokenMs` are the gate's own
  * timing of the call (R7.1, migration 0022); null on rows written before, and `firstTokenMs`
  * when the provider streamed nothing. The `cost*Usd` columns split `costUsd` by token kind, at the
- * prices the adapter applied (migration 0037); null on rows written before.
+ * prices the adapter applied (migration 0037); null on rows written before. `attempts` and
+ * `retries` say how many times the gate called the provider for the answer, and why again
+ * (migration 0047).
  */
 export declare const modelUsage: import("drizzle-orm/pg-core").PgTableWithColumns<{
     name: "model_usage";
@@ -4047,6 +4049,42 @@ export declare const modelUsage: import("drizzle-orm/pg-core").PgTableWithColumn
             identity: undefined;
             generated: undefined;
         }, {}, {}>;
+        attempts: import("drizzle-orm/pg-core").PgColumn<{
+            name: "attempts";
+            tableName: "model_usage";
+            dataType: "number";
+            columnType: "PgInteger";
+            data: number;
+            driverParam: string | number;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
+        retries: import("drizzle-orm/pg-core").PgColumn<{
+            name: "retries";
+            tableName: "model_usage";
+            dataType: "custom";
+            columnType: "PgCustomColumn";
+            data: unknown;
+            driverParam: unknown;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            pgColumnBuilderBrand: 'PgCustomColumnBuilderBrand';
+        }>;
     };
     dialect: 'pg';
 }>;
@@ -6707,7 +6745,7 @@ export type EnvironmentRowColumns = typeof agents.$inferSelect;
 export type ArtifactRow = typeof artifacts.$inferSelect;
 export type ArtifactVersionRow = typeof artifactVersions.$inferSelect;
 export type DefinitionFileRow = typeof definitionFiles.$inferSelect;
-export type ModelUsageWrite = Omit<ModelUsageRow, "duplicate" | "txid" | "tokensReported" | "costKnown" | "startedAt" | "endedAt" | "firstTokenMs" | "costInputUsd" | "costCacheReadUsd" | "costCacheWriteUsd" | "costOutputUsd"> & {
+export type ModelUsageWrite = Omit<ModelUsageRow, "duplicate" | "txid" | "tokensReported" | "costKnown" | "startedAt" | "endedAt" | "firstTokenMs" | "costInputUsd" | "costCacheReadUsd" | "costCacheWriteUsd" | "costOutputUsd" | "attempts" | "retries"> & {
     tokensReported?: boolean | null;
     costKnown?: boolean | null;
     startedAt?: string | null;
@@ -6717,4 +6755,6 @@ export type ModelUsageWrite = Omit<ModelUsageRow, "duplicate" | "txid" | "tokens
     costCacheReadUsd?: number | null;
     costCacheWriteUsd?: number | null;
     costOutputUsd?: number | null;
+    attempts?: number | null;
+    retries?: unknown;
 };

package/dist/store/postgres/schema.js

@@ -455,7 +455,9 @@ export const signingKeys = nylorun.table("signing_keys", {
  * when the provider sent none. `startedAt`, `endedAt` and `firstTokenMs` are the gate's own
  * timing of the call (R7.1, migration 0022); null on rows written before, and `firstTokenMs`
  * when the provider streamed nothing. The `cost*Usd` columns split `costUsd` by token kind, at the
- * prices the adapter applied (migration 0037); null on rows written before.
+ * prices the adapter applied (migration 0037); null on rows written before. `attempts` and
+ * `retries` say how many times the gate called the provider for the answer, and why again
+ * (migration 0047).
  */
 export const modelUsage = nylorun.table("model_usage", {
     id: textC().primaryKey(),
@@ -497,6 +499,13 @@ export const modelUsage = nylorun.table("model_usage", {
     costCacheWriteUsd: doublePrecision(),
     /** What the output cost. */
     costOutputUsd: doublePrecision(),
+    /**
+     * The provider calls the gate made for this answer, the first included (migration 0047);
+     * null on rows written before.
+     */
+    attempts: integer(),
+    /** Each retry before the answer: `{ attempt, delayMs, error }` (migration 0047); null when none. */
+    retries: jsonText(),
 }, (t) => [
     index("model_usage_session").on(t.sessionId, t.createdAt, t.id),
     index("model_usage_export").on(t.txid, t.id),

package/dist/tenant/transcript.d.ts

@@ -128,6 +128,10 @@ export interface MeteredCall {
     readonly totalTokens: number;
     readonly cachedTokens: number;
     readonly costUsd: number;
+    /** The provider calls the gate made for the answer (migration 0047); null on older rows. */
+    readonly attempts?: number | null;
+    /** Each retry before the answer (migration 0047); null when none. */
+    readonly retries?: unknown;
 }
 /** From `startedAt` to `endedAt` in whole milliseconds; undefined unless both are times. */
 export declare function durationBetween(startedAt: string | undefined, endedAt: string | undefined): number | undefined;
@@ -227,6 +231,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
     gate?: GateTimes | undefined;
     policy?: ToolPolicyFact | undefined;
 } | {
+    redacted?: undefined;
     agent: import("@nylorun/core/define").AgentRef;
     error: {
         code: string;
@@ -236,7 +241,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
         retryable?: boolean | undefined;
     };
     output?: undefined;
-    redacted?: undefined;
     toolName: string;
     status?: "cancelled" | "completed" | "failed" | undefined;
     kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -250,6 +254,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
     gate?: GateTimes | undefined;
     policy?: ToolPolicyFact | undefined;
 } | {
+    redacted?: undefined;
     agent?: undefined;
     error: {
         code: string;
@@ -259,7 +264,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
         retryable?: boolean | undefined;
     };
     output?: undefined;
-    redacted?: undefined;
     toolName: string;
     status?: "cancelled" | "completed" | "failed" | undefined;
     kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -307,10 +311,10 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
     gate?: GateTimes | undefined;
     policy?: ToolPolicyFact | undefined;
 } | {
+    redacted?: undefined;
     agent: import("@nylorun/core/define").AgentRef;
     error?: undefined;
     output: unknown;
-    redacted?: undefined;
     toolName: string;
     status?: "cancelled" | "completed" | "failed" | undefined;
     kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -324,10 +328,10 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
     gate?: GateTimes | undefined;
     policy?: ToolPolicyFact | undefined;
 } | {
+    redacted?: undefined;
     agent?: undefined;
     error?: undefined;
     output: unknown;
-    redacted?: undefined;
     toolName: string;
     status?: "cancelled" | "completed" | "failed" | undefined;
     kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -391,6 +395,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
     gate?: GateTimes | undefined;
     policy?: ToolPolicyFact | undefined;
 } | {
+    redacted?: undefined;
     callId: string;
     invocationId: string;
     agent: import("@nylorun/core/define").AgentRef;
@@ -402,7 +407,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
         retryable?: boolean | undefined;
     };
     output?: undefined;
-    redacted?: undefined;
     toolName: string;
     status?: "cancelled" | "completed" | "failed" | undefined;
     kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -416,6 +420,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
     gate?: GateTimes | undefined;
     policy?: ToolPolicyFact | undefined;
 } | {
+    redacted?: undefined;
     callId: string;
     invocationId: string;
     agent?: undefined;
@@ -427,7 +432,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
         retryable?: boolean | undefined;
     };
     output?: undefined;
-    redacted?: undefined;
     toolName: string;
     status?: "cancelled" | "completed" | "failed" | undefined;
     kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -479,12 +483,12 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
     gate?: GateTimes | undefined;
     policy?: ToolPolicyFact | undefined;
 } | {
+    redacted?: undefined;
     callId: string;
     invocationId: string;
     agent: import("@nylorun/core/define").AgentRef;
     error?: undefined;
     output: unknown;
-    redacted?: undefined;
     toolName: string;
     status?: "cancelled" | "completed" | "failed" | undefined;
     kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
@@ -498,12 +502,12 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
     gate?: GateTimes | undefined;
     policy?: ToolPolicyFact | undefined;
 } | {
+    redacted?: undefined;
     callId: string;
     invocationId: string;
     agent?: undefined;
     error?: undefined;
     output: unknown;
-    redacted?: undefined;
     toolName: string;
     status?: "cancelled" | "completed" | "failed" | undefined;
     kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;

package/dist/tenant/transcript.js

@@ -1,5 +1,6 @@
 import { isModelFailureOutcome } from "@nylorun/core/define";
 import { HarnessApiError } from "@nylorun/core/harness-api";
+import { attemptFacts } from "../model/retry.js";
 /** The model's tool call id and the harness invocation, for tool effects. */
 export function toolIds(context) {
     const c = context;
@@ -95,6 +96,8 @@ export function modelFailed(request, value) {
         code: value.code,
         message: value.message,
         retryable: value.retryable,
+        // How many times the gate called the provider before it gave up, and why again.
+        ...attemptFacts(value.attempts, value.retries),
         ...agentOf(request),
     };
 }
@@ -133,6 +136,7 @@ export function modelOutcomeFacts(effectId, window, metered) {
                 },
             }
             : {}),
+        ...attemptFacts(metered?.attempts, metered?.retries),
     };
 }
 /**

package/dist/traces/map.js

@@ -135,6 +135,8 @@ export function spanOf(row, turn, settings) {
                 .str("gen_ai.agent.id", p.agent?.id)
                 .str("nylorun.effect.id", p.effectId)
                 .int("nylorun.effect.window_ms", p.durationMs)
+                // The gate's provider calls for this one (Model Calls §6): more than 1 after a retry.
+                .int("nylorun.model.attempts", typeof p.attempts === "number" ? p.attempts : undefined)
                 .str("nylorun.compaction.trigger", row.type === "context.compacted" ? p.trigger : undefined)
                 .str("error.type", row.type === "model.failed" ? p.code : undefined)
                 .str("gen_ai.output.messages", output);

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const RUNTIME_VERSION = "0.49.1-beta";
+export declare const RUNTIME_VERSION = "0.50.0-beta";

package/dist/version.js

@@ -1,3 +1,3 @@
 // Kept in step with package.json by runtime/server/scripts/check-package.mjs, which runs in npm run check.
 // release:prepare also rewrites this when it bumps @nylorun/runtime.
-export const RUNTIME_VERSION = "0.49.1-beta";
+export const RUNTIME_VERSION = "0.50.0-beta";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nylorun/runtime",
-  "version": "0.49.1-beta",
+  "version": "0.50.0-beta",
   "description": "Nylorun Runtime: the agent execution Host on Postgres, Restate and S2, and model providers.",
   "type": "module",
   "license": "Apache-2.0",
@@ -65,8 +65,8 @@
     "@hono/node-server": "^2.1.5",
     "@hono/zod-openapi": "^1.6.3",
     "@modelcontextprotocol/sdk": "^1.32.1",
-    "@nylorun/core": "0.42.0-beta",
-    "@nylorun/harness": "0.36.2-beta",
+    "@nylorun/core": "0.43.0-beta",
+    "@nylorun/harness": "0.36.3-beta",
     "@restatedev/restate-sdk": "1.17.3",
     "@restatedev/restate-sdk-clients": "1.17.3",
     "@s2-dev/streamstore": "0.28.0",