@nylorun/runtime 0.49.0-beta → 0.49.2-beta

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 0.49.2-beta
+
+### Patch Changes
+
+- bf9d614: Pods on Agent Substrate (preview):
+
+  - **A Runtime upgrade no longer strands existing pods.** Agent Substrate repoints an actor only to a template with the same volumes, and the engine is an image volume pinned by the Runtime image's digest. So after an upgrade, every existing pod's start failed and was sent again every 2 seconds without end, and only a reset got out, at the cost of the pod's disk. An actor now keeps the engine it was created with. An upgrade keeps the pod, its `/workspace` and `/harness`, and it starts as before. The pod runs its first engine until a new incarnation (a reset). The Runtime's Harness API check at `hello` still guards it.
+  - **A change Substrate can't make to an existing pod fails its start with the reason.** This is any other volume change, such as switching between the image's engine and the Runtime's, or a template Substrate refuses. The sandbox reports `RepointRefused` with why, the Runtime's ready deadline fails the start, and the request is not sent again.
+  - **A crashed actor is reverted before its template changes**, as Substrate requires. Before, a new spec for a crashed pod failed without end.
+
+## 0.49.1-beta
+
+### Patch Changes
+
+- 99a8301: A model call whose stream was cut before its terminal chunk is tried again. Vercel AI Gateway ends a stream with `Upstream stream ended before terminal chunk` when the provider behind it fails after the stream began (a provider's 503, `finish_reason: stream_terminated`). pi-ai's retry list doesn't know that wording, so the call failed `invalid_request` on its first attempt and ended the turn: on beta, six Worker turns of 76 s to 13 minutes were lost this way to one provider's 503. The gate now retries it like any transient error, up to the Tenant's `modelAttempts` (3 by default) with `modelRetryDelayMs` between attempts. A call cut on every attempt fails `transient` (`retryable: true`), not `invalid_request`.
+
+  Retries are visible: the gates service logs a `model_retry` line for each one (session, effect, attempt, delay and the provider's error, cut to 300 characters), and its `model_call` line carries the call's `attempts`.
+
 ## 0.49.0-beta
 
 ### Minor Changes

package/dist/gates/handler.js

@@ -3,7 +3,8 @@
  * Tenant; a call naming another is refused) and calls the model (the fixture model while the
  * Tenant's `model.fixture` is on, R4.3) through `inProcessModelGate`, the same code the loop ran before the gate, so outcomes,
  * retries, classification and redaction are unchanged. The meter records the call's usage in
- * the Tenant's ledger. Logs one line per call, never the prompt, the output or a credential.
+ * the Tenant's ledger. Logs one line per call, with its attempts, and one per retry, never the
+ * prompt, the output or a credential.
  */
 import { classifyThrown } from "../model/classify.js";
 import { callCaller, callFiles, inProcessModelGate, modelCallSettings, serve } from "./in-process.js";
@@ -17,6 +18,8 @@ export function createModelCallHandler(options) {
             const started = Date.now();
             let request = named;
             let outcome;
+            // Attempts the call made, the first included: more than 1 when it was retried.
+            let attempts = 1;
             try {
                 const vault = await vaults.open(named.tenantId);
                 const call = { ...named, tenantId: vault.tenantId };
@@ -29,12 +32,22 @@ export function createModelCallHandler(options) {
                     settings: await modelCallSettings(vault.store),
                     ...(blobs ? { files: callFiles(vault.store, blobs) } : {}),
                     caller: callCaller(vault.store),
+                    onRetry: (_request, retry) => {
+                        attempts = retry.attempt;
+                        logger.info("model_retry", {
+                            ...fields(call),
+                            attempt: retry.attempt,
+                            of: retry.maxAttempts,
+                            delayMs: retry.delayMs,
+                            error: retry.errorMessage.slice(0, 300),
+                        });
+                    },
                 });
                 outcome = await meter.call(vault.store, call, () => serve(vault.store, gate, call, signal));
             }
             catch (error) {
                 if (signal.aborted) {
-                    logger.info("model_call", { ...fields(request), ms: Date.now() - started, outcome: "aborted" });
+                    logger.info("model_call", { ...fields(request), ms: Date.now() - started, attempts, outcome: "aborted" });
                     throw error;
                 }
                 outcome = error instanceof GateRefusal ? error.outcome : classifyThrown(error);
@@ -42,6 +55,7 @@ export function createModelCallHandler(options) {
             logger.info("model_call", {
                 ...fields(request),
                 ms: Date.now() - started,
+                attempts,
                 outcome: typeof outcome === "object" && outcome !== null && "kind" in outcome
                     ? outcome.code
                     : "ok",

package/dist/gates/in-process.d.ts

@@ -12,6 +12,7 @@ import type { ModelGate, ModelGateRequest } from "./model-gate.js";
 import { type FileResolver } from "../artifacts/files.js";
 import type { BlobStore } from "../blob/index.js";
 import type { ModelCaller } from "../model/call-headers.js";
+import type { ModelRetry } from "../model/retry.js";
 export interface InProcessModelGateOptions {
     /** The Tenant's home: `piModel` redacts secrets it finds there from failure messages. */
     readonly root?: string;
@@ -24,6 +25,8 @@ export interface InProcessModelGateOptions {
     readonly files?: (request: ModelGateRequest) => FileResolver;
     /** Reads who makes a call, for a custom endpoint's per-call headers (#493). */
     readonly caller?: (request: ModelGateRequest) => Promise<ModelCaller | undefined>;
+    /** Hears each retry of a call (`piModel`'s `onRetry`): the gates service logs it. */
+    readonly onRetry?: (request: ModelGateRequest, retry: ModelRetry) => void;
 }
 /**
  * The files a call names, from the Tenant's artifacts, limited to the call's session: in

package/dist/gates/in-process.js

@@ -85,6 +85,7 @@ export function inProcessModelGate(options) {
                 ...(options.settings ? { settings: options.settings } : {}),
                 ...(options.files ? { files: options.files(request) } : {}),
                 ...(options.caller ? { caller: () => options.caller(request) } : {}),
+                ...(options.onRetry ? { onRetry: (retry) => options.onRetry(request, retry) } : {}),
             });
             return adapter(request.call, {
                 // The Runtime never journals the provider request (harness `durable.ts`), and

package/dist/management-openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.2.0",
   "info": {
     "title": "Nylorun Management API",
-    "version": "0.49.0-beta",
+    "version": "0.49.2-beta",
     "x-nylorun-protocol": {
       "min": 16,
       "max": 18,

package/dist/model/classify.js

@@ -1,4 +1,5 @@
 import { isContextOverflow, isRetryableAssistantError, } from "@earendil-works/pi-ai";
+import { STREAM_CUT } from "./retry.js";
 const RATE_LIMITED = /\b429\b|rate.?limit|too many requests/i;
 const QUOTA = /insufficient_quota|quota exceeded|out of budget|billing|usage limit/i;
 const OVERLOADED = /overloaded|\b529\b|\b503\b|service.?unavailable|high demand/i;
@@ -22,6 +23,10 @@ export function classifyAssistantError(message, contextWindow) {
         return failure("overloaded", text, true);
     if (TIMEOUT.test(text))
         return failure("timeout", text, true);
+    // A stream cut before its last chunk (a provider failing behind a gateway): nothing of the
+    // step ran, so it is the provider's transient failure, not the request's.
+    if (STREAM_CUT.test(text))
+        return failure("transient", text, true);
     if (CONTENT_POLICY.test(text))
         return failure("content_policy", text, false);
     if (AUTH.test(text))

package/dist/model/pi-model.d.ts

@@ -3,6 +3,7 @@ import type { RuntimeModelAdapter } from "../contracts.js";
 import { type RuntimeLimits } from "@nylorun/core/contracts";
 import { type FileResolver } from "../artifacts/files.js";
 import type { HostModelSecret } from "../vault/service.js";
+import { type ModelRetry } from "./retry.js";
 import { type ModelCaller } from "./call-headers.js";
 export interface ModelPreview {
     readonly invocationId: string;
@@ -45,6 +46,8 @@ export interface PiModelOptions {
      * come from it. Without it, a call sends none of them.
      */
     readonly caller?: () => Promise<ModelCaller | undefined>;
+    /** Hears each retry of a call before its delay: the gate logs it. Never changes the call. */
+    readonly onRetry?: (retry: ModelRetry) => void;
 }
 /**
  * Node model adapter. Local provider configuration is read only when invoked.

package/dist/model/pi-model.js

@@ -1,10 +1,11 @@
-import { retryAssistantCall, } from "@earendil-works/pi-ai";
+import {} from "@earendil-works/pi-ai";
 import { RUNTIME_LIMIT_DEFAULTS } from "@nylorun/core/contracts";
 import { mediaReferenceOf } from "@nylorun/core/define";
 import { FileUnavailableError, } from "../artifacts/files.js";
 import { essence, inputLabel, isModelImage, modelInputOf } from "@nylorun/core/contracts";
 import { scrub } from "../redact.js";
 import { classifyAssistantError, failure } from "./classify.js";
+import { retryModelCall } from "./retry.js";
 import { addedInputsOf, describeModel, selectionOf } from "./descriptor.js";
 import { modelsFor } from "./models.js";
 import { callHeaders } from "./call-headers.js";
@@ -199,11 +200,7 @@ export function piModel(options = {}) {
                 clearTimeout(timer);
             }
         };
-        const response = await retryAssistantCall(attempt, {
-            enabled: settings.attempts > 1,
-            maxRetries: settings.attempts - 1,
-            baseDelayMs: settings.retryBaseDelayMs,
-        }, context.signal);
+        const response = await retryModelCall(attempt, { attempts: settings.attempts, baseDelayMs: settings.retryBaseDelayMs }, context.signal, options.onRetry);
         // An abort is never a failure outcome: the host decides what it means.
         context.signal.throwIfAborted();
         if (natives.error)

package/dist/model/retry.d.ts

@@ -0,0 +1,36 @@
+/**
+ * One model call's attempts (Model Calls §6): pi-ai's transient errors, plus a stream the provider
+ * or a gateway cut before its last chunk, are tried again with pi-ai's backoff, up to the Tenant's
+ * `modelAttempts`. Each retry is reported, so a call that needed one is visible in the gate's log.
+ */
+import { type AssistantMessage } from "@earendil-works/pi-ai";
+/**
+ * A stream that ended before its terminal chunk: Vercel AI Gateway's `Upstream stream ended before
+ * terminal chunk` (its `finish_reason: stream_terminated`) when the provider behind it fails after
+ * the stream began. Nothing of the step ran, so it is safe to call again; pi-ai's own list doesn't
+ * know this wording.
+ */
+export declare const STREAM_CUT: RegExp;
+/** True for a failed message worth another attempt. */
+export declare function isRetryableModelError(message: AssistantMessage): boolean;
+/** One retry, as `onRetry` hears it: before the delay is waited. */
+export interface ModelRetry {
+    /** The attempt about to start, the first counted as 1. */
+    readonly attempt: number;
+    readonly maxAttempts: number;
+    readonly delayMs: number;
+    /** The failed attempt's error, as the provider or pi-ai worded it. */
+    readonly errorMessage: string;
+}
+export interface ModelRetryPolicy {
+    /** Attempts, the first included. 1 means no retry. */
+    readonly attempts: number;
+    /** The delay before the second attempt; it doubles each time (pi-ai's `retryDelayMs`). */
+    readonly baseDelayMs: number;
+}
+/**
+ * Runs `produce` until it answers, fails in a way no retry changes, or uses up its attempts.
+ * An abort is never retried: an aborted message comes back as it is, and an abort during the
+ * delay comes back as the last failure marked `aborted`, as pi-ai's `retryAssistantCall` does.
+ */
+export declare function retryModelCall(produce: () => Promise<AssistantMessage>, policy: ModelRetryPolicy, signal: AbortSignal, onRetry?: (retry: ModelRetry) => void): Promise<AssistantMessage>;

package/dist/model/retry.js

@@ -0,0 +1,63 @@
+/**
+ * One model call's attempts (Model Calls §6): pi-ai's transient errors, plus a stream the provider
+ * or a gateway cut before its last chunk, are tried again with pi-ai's backoff, up to the Tenant's
+ * `modelAttempts`. Each retry is reported, so a call that needed one is visible in the gate's log.
+ */
+import { isRetryableAssistantError, retryDelayMs, } from "@earendil-works/pi-ai";
+/**
+ * A stream that ended before its terminal chunk: Vercel AI Gateway's `Upstream stream ended before
+ * terminal chunk` (its `finish_reason: stream_terminated`) when the provider behind it fails after
+ * the stream began. Nothing of the step ran, so it is safe to call again; pi-ai's own list doesn't
+ * know this wording.
+ */
+export const STREAM_CUT = /stream ended before (?:its |the |a )?terminal chunk|upstream stream ended|stream[_ ]terminated/i;
+/** True for a failed message worth another attempt. */
+export function isRetryableModelError(message) {
+    if (message.stopReason !== "error")
+        return false;
+    if (isRetryableAssistantError(message))
+        return true;
+    return STREAM_CUT.test(message.errorMessage ?? "");
+}
+/**
+ * Runs `produce` until it answers, fails in a way no retry changes, or uses up its attempts.
+ * An abort is never retried: an aborted message comes back as it is, and an abort during the
+ * delay comes back as the last failure marked `aborted`, as pi-ai's `retryAssistantCall` does.
+ */
+export async function retryModelCall(produce, policy, signal, onRetry) {
+    const maxAttempts = Math.max(1, policy.attempts);
+    for (let attempt = 1;; attempt += 1) {
+        const response = await produce();
+        if (response.stopReason !== "error")
+            return response;
+        if (attempt >= maxAttempts || !isRetryableModelError(response))
+            return response;
+        const delayMs = retryDelayMs({ baseDelayMs: policy.baseDelayMs }, attempt);
+        try {
+            onRetry?.({ attempt: attempt + 1, maxAttempts, delayMs, errorMessage: response.errorMessage ?? "Unknown error" });
+        }
+        catch {
+            /* Reporting never changes the call. */
+        }
+        if (!(await wait(delayMs, signal))) {
+            const { errorMessage: _errorMessage, ...rest } = response;
+            return { ...rest, stopReason: "aborted" };
+        }
+    }
+}
+/** Resolves true after `ms`, or false as soon as `signal` aborts. */
+function wait(ms, signal) {
+    if (signal.aborted)
+        return Promise.resolve(false);
+    return new Promise((resolve) => {
+        const timer = setTimeout(() => {
+            signal.removeEventListener("abort", onAbort);
+            resolve(true);
+        }, ms);
+        const onAbort = () => {
+            clearTimeout(timer);
+            resolve(false);
+        };
+        signal.addEventListener("abort", onAbort, { once: true });
+    });
+}

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.2.0",
   "info": {
     "title": "Nylorun Runtime API",
-    "version": "0.49.0-beta",
+    "version": "0.49.2-beta",
     "x-nylorun-protocol": {
       "min": 16,
       "max": 18,

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const RUNTIME_VERSION = "0.49.0-beta";
+export declare const RUNTIME_VERSION = "0.49.2-beta";

package/dist/version.js

@@ -1,3 +1,3 @@
 // Kept in step with package.json by runtime/server/scripts/check-package.mjs, which runs in npm run check.
 // release:prepare also rewrites this when it bumps @nylorun/runtime.
-export const RUNTIME_VERSION = "0.49.0-beta";
+export const RUNTIME_VERSION = "0.49.2-beta";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nylorun/runtime",
-  "version": "0.49.0-beta",
+  "version": "0.49.2-beta",
   "description": "Nylorun Runtime: the agent execution Host on Postgres, Restate and S2, and model providers.",
   "type": "module",
   "license": "Apache-2.0",