@boardwalk-labs/runner 0.2.13 → 0.2.15

package/dist/runtime/direct_inference.d.ts

@@ -23,7 +23,7 @@ export interface DirectInferenceDeps {
 }
 /** One model turn, straight to the org's endpoint. Returns the ChatTurn + canonical model ref;
  *  BYO carries no platform cost (costMicros stays 0 — BYO is never metered). */
-export declare function streamDirectTurn(deps: DirectInferenceDeps, entry: ByoInferenceProvider, req: DirectTurnRequest, onDelta: ((text: string) => void) | undefined, 
+export declare function streamDirectTurn(deps: DirectInferenceDeps, entry: ByoInferenceProvider, req: DirectTurnRequest, onDelta: ((text: string) => void) | undefined, onReasoningDelta: ((text: string) => void) | undefined, 
 /** Register the resolved key with the CURRENT leaf's engine redactor, before the model call.
  *  The key resolves mid-leaf (after the leaf's redactor snapshot is seeded), so without this a
  *  provider that echoes the Authorization header in an error body would leak it into that

package/dist/runtime/direct_inference.js

@@ -53,7 +53,7 @@ export function directProviderFor(registry, provider) {
 }
 /** One model turn, straight to the org's endpoint. Returns the ChatTurn + canonical model ref;
  *  BYO carries no platform cost (costMicros stays 0 — BYO is never metered). */
-export async function streamDirectTurn(deps, entry, req, onDelta, 
+export async function streamDirectTurn(deps, entry, req, onDelta, onReasoningDelta, 
 /** Register the resolved key with the CURRENT leaf's engine redactor, before the model call.
  *  The key resolves mid-leaf (after the leaf's redactor snapshot is seeded), so without this a
  *  provider that echoes the Authorization header in an error body would leak it into that
@@ -77,6 +77,7 @@ registerSecret) {
     const io = {
         ...(deps.fetchImpl !== undefined ? { fetchImpl: deps.fetchImpl } : {}),
         ...(onDelta !== undefined ? { onDelta } : {}),
+        ...(onReasoningDelta !== undefined ? { onReasoningDelta } : {}),
     };
     log.debug("direct_turn", { provider: entry.name, source: entry.source, model: req.model });
     const turn = entry.source === "anthropic"

package/dist/runtime/leaf_executor.js

@@ -218,7 +218,7 @@ export class EngineLeafExecutor {
                 messages: req.messages,
                 tools: req.tools,
                 ...(req.reasoning !== undefined ? { reasoning: req.reasoning } : {}),
-            }, providerIo.onDelta, 
+            }, providerIo.onDelta, providerIo.onReasoningDelta, 
             // Register the org's key with THIS leaf's redactor before the model call, so an error
             // body echoing it is scrubbed from the leaf's run events (not just the terminal error).
             (value) => redactor.add("byo-key", value));
@@ -237,6 +237,9 @@ export class EngineLeafExecutor {
             if (frame.kind === "delta") {
                 providerIo.onDelta?.(frame.text);
             }
+            else if (frame.kind === "reasoning") {
+                providerIo.onReasoningDelta?.(frame.text);
+            }
             else if (frame.kind === "result") {
                 // contextTokens (when the broker knows the served model's window) lets the engine's loop
                 // size compaction against the real window instead of a conservative default.
@@ -331,6 +334,8 @@ export function toRunEventBody(body, identity) {
             return { kind: "text_delta", blockId: body.blockId, text: body.text };
         case "text_end":
             return { kind: "text_end", blockId: body.blockId };
+        case "reasoning_delta":
+            return { kind: "reasoning_delta", text: body.text };
         case "tool_call_start":
             return { kind: "tool_call_start", toolCallId: body.toolCallId, toolName: body.toolName };
         case "tool_call_input_complete":

package/dist/runtime/runner_control_client.d.ts

@@ -208,8 +208,24 @@ export declare class RunnerControlClient {
      * Backs {@link InferenceProxyTransport} (inference_transport.ts) — the model swap is invisible to
      * the engine loop, which keeps the runner provider-agnostic (the broker owns model invocation). A
      * non-200 (a failure BEFORE the stream began) throws the broker's already-classified message.
+     *
+     * Resilience (added because a bare drop here surfaced as a run-fatal `PROVIDER_ERROR: terminated`):
+     * a transient failure is retried WHILE it is still safe to re-POST — i.e. before any CONTENT frame
+     * has been relayed. Three cases:
+     *   - the POST never returned a response (connection refused/reset during an api-server rollover),
+     *   - a load-balancer {@link INFERENCE_RETRYABLE_STATUSES} (no healthy / gateway timeout) before the
+     *     stream began — NOT 502, which the broker uses for a real upstream model error,
+     *   - the body dropped MID-stream (undici `terminated` / socket close) but only `ping` heartbeats
+     *     had been relayed so far (the observed failure: a huge-context turn streamed only pings for
+     *     ~120s during time-to-first-token, then the connection dropped).
+     * Once a `delta`/`reasoning`/`result`/`error` frame has been yielded, the model has already
+     * produced output (or the turn finished), so a re-POST would duplicate it: the drop surfaces as
+     * before. Re-POST is billing-safe — the broker aborts the abandoned turn on our disconnect and
+     * returns before it meters (no `result` frame ⇒ no usage). The body is a reusable string.
      */
     streamInference(req: InferenceProxyRequest): AsyncGenerator<InferenceFrame, void, undefined>;
+    /** Log + wait one backoff step before re-POSTing a transient inference failure (see streamInference). */
+    private backoffInferenceRetry;
     private url;
     private headers;
 }

package/dist/runtime/runner_control_client.js

@@ -20,6 +20,14 @@ const log = createLogger("RunnerControlClient");
  * handler error could duplicate a side effect for no healing value.
  */
 const RETRYABLE_STATUSES = new Set([502, 503, 504]);
+/**
+ * Transient statuses worth retrying on the STREAMING inference POST — a narrower set than
+ * {@link RETRYABLE_STATUSES}. 502 is EXCLUDED: on `/inference` the broker uses 502 for a real
+ * upstream model error (a clean, customer-facing message), so retrying it just re-hits a guaranteed
+ * failure. 503/504 are only ever the load balancer during a rollover (no healthy / draining target,
+ * gateway timeout) — safe to re-POST since the stream hasn't begun. See {@link streamInference}.
+ */
+const INFERENCE_RETRYABLE_STATUSES = new Set([503, 504]);
 /** Default backoff (ms) between attempts — ~17.5s of spread, sized to ride out the target-group
  *  rotation window of an api-server rolling deploy (the observed killer: guests died hard
  *  mid-suspend/finalize during TWO deploys, 2026-07-13, and only crash-reclaim saved the runs). */
@@ -473,20 +481,85 @@ export class RunnerControlClient {
      * Backs {@link InferenceProxyTransport} (inference_transport.ts) — the model swap is invisible to
      * the engine loop, which keeps the runner provider-agnostic (the broker owns model invocation). A
      * non-200 (a failure BEFORE the stream began) throws the broker's already-classified message.
+     *
+     * Resilience (added because a bare drop here surfaced as a run-fatal `PROVIDER_ERROR: terminated`):
+     * a transient failure is retried WHILE it is still safe to re-POST — i.e. before any CONTENT frame
+     * has been relayed. Three cases:
+     *   - the POST never returned a response (connection refused/reset during an api-server rollover),
+     *   - a load-balancer {@link INFERENCE_RETRYABLE_STATUSES} (no healthy / gateway timeout) before the
+     *     stream began — NOT 502, which the broker uses for a real upstream model error,
+     *   - the body dropped MID-stream (undici `terminated` / socket close) but only `ping` heartbeats
+     *     had been relayed so far (the observed failure: a huge-context turn streamed only pings for
+     *     ~120s during time-to-first-token, then the connection dropped).
+     * Once a `delta`/`reasoning`/`result`/`error` frame has been yielded, the model has already
+     * produced output (or the turn finished), so a re-POST would duplicate it: the drop surfaces as
+     * before. Re-POST is billing-safe — the broker aborts the abandoned turn on our disconnect and
+     * returns before it meters (no `result` frame ⇒ no usage). The body is a reusable string.
      */
     async *streamInference(req) {
-        const res = await this.fetchImpl(this.url("inference"), {
-            method: "POST",
-            headers: { ...this.headers(true), accept: INFERENCE_NDJSON_CONTENT_TYPE },
-            body: serializeInferenceRequest(req),
-        });
-        if (res.status !== 200 || res.body === null) {
-            throw await inferenceHttpError(res);
-        }
-        for await (const line of readNdjsonLines(res.body)) {
-            yield parseInferenceFrame(line);
+        const body = serializeInferenceRequest(req);
+        for (let attempt = 0;; attempt += 1) {
+            const last = attempt >= this.retryDelaysMs.length;
+            let res;
+            try {
+                res = await this.fetchImpl(this.url("inference"), {
+                    method: "POST",
+                    headers: { ...this.headers(true), accept: INFERENCE_NDJSON_CONTENT_TYPE },
+                    body,
+                });
+            }
+            catch (err) {
+                // No response at all — nothing streamed, so a re-POST is always safe.
+                if (last)
+                    throw err;
+                await this.backoffInferenceRetry(attempt, "connect", err);
+                continue;
+            }
+            if (res.status !== 200 || res.body === null) {
+                // A load-balancer transient (target draining / gateway timeout) before the stream began —
+                // safe to re-POST. Any other non-200 is the broker's real, already-classified answer
+                // (incl. a 502 upstream model error): surface it as the run's customer-facing error.
+                if (!last && res.body !== null && INFERENCE_RETRYABLE_STATUSES.has(res.status)) {
+                    await res.body.cancel().catch(() => { });
+                    await this.backoffInferenceRetry(attempt, "status", res.status);
+                    continue;
+                }
+                throw await inferenceHttpError(res);
+            }
+            // Relay the stream. A mid-stream transport drop throws out of readNdjsonLines; retry it only
+            // while nothing but heartbeats has gone out (see the doc-comment). `ping` frames carry no
+            // model output, so a drop during the pre-first-token wait stays safely retryable.
+            let sawContent = false;
+            try {
+                for await (const line of readNdjsonLines(res.body)) {
+                    const frame = parseInferenceFrame(line);
+                    if (frame.kind !== "ping")
+                        sawContent = true;
+                    yield frame;
+                }
+                return; // the stream ended cleanly
+            }
+            catch (err) {
+                if (last || sawContent)
+                    throw err;
+                await res.body.cancel().catch(() => { });
+                await this.backoffInferenceRetry(attempt, "stream", err);
+                continue;
+            }
         }
     }
+    /** Log + wait one backoff step before re-POSTing a transient inference failure (see streamInference). */
+    async backoffInferenceRetry(attempt, reason, detail) {
+        log.warn("broker_inference_retry", {
+            reason,
+            ...(reason === "status"
+                ? { status: detail }
+                : { error: detail instanceof Error ? detail.name : "unknown" }),
+            attempt: attempt + 1,
+            delayMs: this.retryDelaysMs[attempt],
+        });
+        await sleep(this.retryDelaysMs[attempt] ?? 0);
+    }
     url(suffix) {
         return `${this.base}/runner/v1/runs/${encodeURIComponent(this.cfg.runId)}/${suffix}`;
     }

package/dist/runtime/wire/inference_proxy.d.ts

@@ -36,6 +36,9 @@ export interface ProxyError {
 export type InferenceFrame = {
     kind: "delta";
     text: string;
+} | {
+    kind: "reasoning";
+    text: string;
 } | {
     kind: "result";
     turn: ChatTurn;
@@ -58,6 +61,9 @@ export declare function serializeInferenceRequest(req: InferenceProxyRequest): s
 export declare function parseInferenceRequest(body: string): InferenceProxyRequest;
 /** Serialize one streamed text delta as a single NDJSON line (trailing "\n" included). */
 export declare function serializeDeltaFrame(text: string): string;
+/** Serialize one streamed reasoning/thinking delta as a single NDJSON line. Separate from a `delta`
+ *  frame so the worker routes it to the thinking trace, never into the assistant answer. */
+export declare function serializeReasoningFrame(text: string): string;
 /** Serialize the single terminal turn result as one NDJSON line. `costMicros` is the turn's EXACT
  *  upstream cost (the managed provider's per-request cost × 1e6) on the managed lane — 0 for BYO or when unavailable.
  *  The worker feeds it to the budget guardrail so `max_usd` tracks real spend, not a token estimate. */

package/dist/runtime/wire/inference_proxy.js

@@ -78,6 +78,11 @@ export function parseInferenceRequest(body) {
 export function serializeDeltaFrame(text) {
     return `${JSON.stringify({ t: "delta", text })}\n`;
 }
+/** Serialize one streamed reasoning/thinking delta as a single NDJSON line. Separate from a `delta`
+ *  frame so the worker routes it to the thinking trace, never into the assistant answer. */
+export function serializeReasoningFrame(text) {
+    return `${JSON.stringify({ t: "reasoning", text })}\n`;
+}
 /** Serialize the single terminal turn result as one NDJSON line. `costMicros` is the turn's EXACT
  *  upstream cost (the managed provider's per-request cost × 1e6) on the managed lane — 0 for BYO or when unavailable.
  *  The worker feeds it to the budget guardrail so `max_usd` tracks real spend, not a token estimate. */
@@ -110,6 +115,8 @@ export function parseInferenceFrame(line) {
     switch (rec.t) {
         case "delta":
             return { kind: "delta", text: typeof rec.text === "string" ? rec.text : "" };
+        case "reasoning":
+            return { kind: "reasoning", text: typeof rec.text === "string" ? rec.text : "" };
         case "result":
             return {
                 kind: "result",
@@ -129,7 +136,11 @@ export function parseInferenceFrame(line) {
         case "ping":
             return { kind: "ping" };
         default:
-            throw new Error(`Unknown inference frame kind: ${String(rec.t)}`);
+            // Lenient consumer: a well-formed frame whose kind this runner predates must not crash the
+            // stream — a newer broker may emit frames an older runner can't name (this is how `reasoning`
+            // itself reaches a fleet mid-rollout). Treat it as a no-op heartbeat, which the leaf loop
+            // already ignores. Only a genuinely malformed line (caught above) is fatal.
+            return { kind: "ping" };
     }
 }
 /** Narrow a parsed `result` frame's turn back to the engine's `ChatTurn`, failing closed on shape —

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boardwalk-labs/runner",
-  "version": "0.2.13",
+  "version": "0.2.15",
   "description": "Boardwalk self-hosted runner: execute cloud-scheduled runs on your own machines. Currently: the canonical runner contract types.",
   "license": "Apache-2.0",
   "repository": {
@@ -53,7 +53,7 @@
     "prebuild": "prebuildify --napi --strip -t 24.0.0"
   },
   "dependencies": {
-    "@boardwalk-labs/engine": "0.2.11",
+    "@boardwalk-labs/engine": "0.2.12",
     "@boardwalk-labs/workflow": "0.2.5",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "node-gyp-build": "^4.8.4",