@boardwalk-labs/runner 0.2.14 → 0.2.16

package/README.md

@@ -31,7 +31,9 @@ boardwalk-runner start --url https://api.boardwalk.sh --pool default
 ```
 
 `start` polls for runs targeting `runs_on: { kind: "self-hosted" }` and executes one at a
-time; run more daemons (or machines) for concurrency. `Ctrl-C` drains: the current run
+time; run more daemons (or machines) for concurrency. Each run executes in a container by default
+(Docker or Podman required); pass `--host` to run it directly on the machine instead. `Ctrl-C`
+drains: the current run
 finishes, nothing new is claimed. Useful flags: `--once` (execute one run, then exit),
 `--verbose` (debug-level daemon logs), `--debug` (also debug-logs inside each run process),
 `--work-dir`, `--identity-dir`. Behind a corporate proxy, launch with `NODE_USE_ENV_PROXY=1`
@@ -58,7 +60,7 @@ runs every non-browser workflow exactly as before.
 
 ## Security model
 
-This part of the contract is settled, even though the client isn't built yet:
+This part of the contract is settled:
 
 - The runner **never receives broad credentials**: a single-purpose registration token to join,
   a standing runner token that can only poll/claim/heartbeat, and a short-lived **run-scoped**
@@ -87,7 +89,7 @@ pnpm lint && pnpm typecheck && pnpm build
 - [`sdk`](https://github.com/boardwalk-labs/sdk) — `@boardwalk-labs/workflow`, the TypeScript API a workflow program imports
 - [`cli`](https://github.com/boardwalk-labs/cli) — `boardwalk`: scaffold, validate, run locally, deploy
 - [`examples`](https://github.com/boardwalk-labs/examples) — copyable workflow templates (`boardwalk init --template`)
-- [`plugins`](https://github.com/boardwalk-labs/plugins) — skills + MCP server for Claude Code, Codex, Cursor, OpenClaw, OpenCode
+- [`plugins`](https://github.com/boardwalk-labs/plugins) — coding-agent skills (Claude Code, Codex, Cursor, OpenClaw, OpenCode) + a control-plane MCP server
 - [`runner-images`](https://github.com/boardwalk-labs/runner-images) — reproducible base images hosted runners execute in
 
 Hosted platform and docs: [boardwalk.sh](https://boardwalk.sh).

package/dist/runtime/leaf_executor.js

@@ -83,6 +83,11 @@ export class EngineLeafExecutor {
         // io (children get their own via `forChild`), and the loop is strictly streamModel→reportUsage per
         // turn, so a single slot never races. Null ⇒ no upstream cost for the next turn (BYO / unavailable).
         let pendingCostMicros = null;
+        // The turn currently streaming, tracked so `streamModel` can attribute a `turn_reset` to it when
+        // the broker restarts a dropped model call mid-stream. Set on `startTurn` and every `emit` (the
+        // engine stamps both with the turn's id); within a leaf, turns are strictly sequential, so this is
+        // never stale for the turn actually streaming.
+        let currentTurnId = undefined;
         return {
             identity,
             redactor,
@@ -113,16 +118,24 @@ export class EngineLeafExecutor {
                 throwIfAborted(signal);
                 return this.streamModel(req, providerIo, signal, redactor, (costMicros) => {
                     pendingCostMicros = (pendingCostMicros ?? 0) + costMicros;
+                }, 
+                // The broker restarted a dropped model call after content had already streamed: void the
+                // turn's emitted text/reasoning so a viewer re-renders from the restart, not the concatenation.
+                () => {
+                    if (currentTurnId !== undefined)
+                        sink.emit({ kind: "turn_reset" }, currentTurnId);
                 });
             },
             // turn_started rides a NEW stride block (the supervisor opens it); subsequent leaf events ride
             // the same block. The shared emitter owns the run-global cursor.
             startTurn: (turnId) => {
+                currentTurnId = turnId;
                 sink.beginTurn(turnId, { kind: "turn_started", ...identity });
             },
             // Engine LeafEventBody → the platform's v1 RunEventBody. The platform already adopted the SDK
             // v1 wire format, and the engine emits the same kinds, so this is a near-identity mapping.
             emit: (turnId, body) => {
+                currentTurnId = turnId;
                 sink.emit(toRunEventBody(body, identity), turnId);
             },
             // Usage flows to the budget authority after EVERY model call. Feed the run-level meter and, if
@@ -202,7 +215,7 @@ export class EngineLeafExecutor {
     /** POST one model turn to the broker's `/inference` and adapt its NDJSON stream into the engine's
      *  ModelTurnResult: each `delta` frame drives providerIo.onDelta; the terminal `result` frame is
      *  the turn. An `error` frame throws (the broker already classified it). An abort mid-stream throws. */
-    async streamModel(req, providerIo, signal, redactor, onCost) {
+    async streamModel(req, providerIo, signal, redactor, onCost, onReset) {
         // Runner-direct BYO (D7): the org's own endpoint + key, called with the same engine adapters
         // the broker uses. No platform cost (BYO is never metered) — onCost stays untouched.
         // A direct turn needs an explicit model (an omitted model means the managed auto lane,
@@ -240,6 +253,12 @@ export class EngineLeafExecutor {
             else if (frame.kind === "reasoning") {
                 providerIo.onReasoningDelta?.(frame.text);
             }
+            else if (frame.kind === "reset") {
+                // The broker recovered a transient mid-stream drop and is restarting the turn: everything
+                // relayed above is void. Signal the viewer to discard the turn's streamed text/reasoning; the
+                // authoritative turn still comes from the single terminal `result` frame below.
+                onReset?.();
+            }
             else if (frame.kind === "result") {
                 // contextTokens (when the broker knows the served model's window) lets the engine's loop
                 // size compaction against the real window instead of a conservative default.

package/dist/runtime/runner_control_client.d.ts

@@ -208,8 +208,24 @@ export declare class RunnerControlClient {
      * Backs {@link InferenceProxyTransport} (inference_transport.ts) — the model swap is invisible to
      * the engine loop, which keeps the runner provider-agnostic (the broker owns model invocation). A
      * non-200 (a failure BEFORE the stream began) throws the broker's already-classified message.
+     *
+     * Resilience (added because a bare drop here surfaced as a run-fatal `PROVIDER_ERROR: terminated`):
+     * a transient failure is retried WHILE it is still safe to re-POST — i.e. before any CONTENT frame
+     * has been relayed. Three cases:
+     *   - the POST never returned a response (connection refused/reset during an api-server rollover),
+     *   - a load-balancer {@link INFERENCE_RETRYABLE_STATUSES} (no healthy / gateway timeout) before the
+     *     stream began — NOT 502, which the broker uses for a real upstream model error,
+     *   - the body dropped MID-stream (undici `terminated` / socket close) but only `ping` heartbeats
+     *     had been relayed so far (the observed failure: a huge-context turn streamed only pings for
+     *     ~120s during time-to-first-token, then the connection dropped).
+     * Once a `delta`/`reasoning`/`result`/`error` frame has been yielded, the model has already
+     * produced output (or the turn finished), so a re-POST would duplicate it: the drop surfaces as
+     * before. Re-POST is billing-safe — the broker aborts the abandoned turn on our disconnect and
+     * returns before it meters (no `result` frame ⇒ no usage). The body is a reusable string.
      */
     streamInference(req: InferenceProxyRequest): AsyncGenerator<InferenceFrame, void, undefined>;
+    /** Log + wait one backoff step before re-POSTing a transient inference failure (see streamInference). */
+    private backoffInferenceRetry;
     private url;
     private headers;
 }

package/dist/runtime/runner_control_client.js

@@ -20,6 +20,14 @@ const log = createLogger("RunnerControlClient");
  * handler error could duplicate a side effect for no healing value.
  */
 const RETRYABLE_STATUSES = new Set([502, 503, 504]);
+/**
+ * Transient statuses worth retrying on the STREAMING inference POST — a narrower set than
+ * {@link RETRYABLE_STATUSES}. 502 is EXCLUDED: on `/inference` the broker uses 502 for a real
+ * upstream model error (a clean, customer-facing message), so retrying it just re-hits a guaranteed
+ * failure. 503/504 are only ever the load balancer during a rollover (no healthy / draining target,
+ * gateway timeout) — safe to re-POST since the stream hasn't begun. See {@link streamInference}.
+ */
+const INFERENCE_RETRYABLE_STATUSES = new Set([503, 504]);
 /** Default backoff (ms) between attempts — ~17.5s of spread, sized to ride out the target-group
  *  rotation window of an api-server rolling deploy (the observed killer: guests died hard
  *  mid-suspend/finalize during TWO deploys, 2026-07-13, and only crash-reclaim saved the runs). */
@@ -473,20 +481,85 @@ export class RunnerControlClient {
      * Backs {@link InferenceProxyTransport} (inference_transport.ts) — the model swap is invisible to
      * the engine loop, which keeps the runner provider-agnostic (the broker owns model invocation). A
      * non-200 (a failure BEFORE the stream began) throws the broker's already-classified message.
+     *
+     * Resilience (added because a bare drop here surfaced as a run-fatal `PROVIDER_ERROR: terminated`):
+     * a transient failure is retried WHILE it is still safe to re-POST — i.e. before any CONTENT frame
+     * has been relayed. Three cases:
+     *   - the POST never returned a response (connection refused/reset during an api-server rollover),
+     *   - a load-balancer {@link INFERENCE_RETRYABLE_STATUSES} (no healthy / gateway timeout) before the
+     *     stream began — NOT 502, which the broker uses for a real upstream model error,
+     *   - the body dropped MID-stream (undici `terminated` / socket close) but only `ping` heartbeats
+     *     had been relayed so far (the observed failure: a huge-context turn streamed only pings for
+     *     ~120s during time-to-first-token, then the connection dropped).
+     * Once a `delta`/`reasoning`/`result`/`error` frame has been yielded, the model has already
+     * produced output (or the turn finished), so a re-POST would duplicate it: the drop surfaces as
+     * before. Re-POST is billing-safe — the broker aborts the abandoned turn on our disconnect and
+     * returns before it meters (no `result` frame ⇒ no usage). The body is a reusable string.
      */
     async *streamInference(req) {
-        const res = await this.fetchImpl(this.url("inference"), {
-            method: "POST",
-            headers: { ...this.headers(true), accept: INFERENCE_NDJSON_CONTENT_TYPE },
-            body: serializeInferenceRequest(req),
-        });
-        if (res.status !== 200 || res.body === null) {
-            throw await inferenceHttpError(res);
-        }
-        for await (const line of readNdjsonLines(res.body)) {
-            yield parseInferenceFrame(line);
+        const body = serializeInferenceRequest(req);
+        for (let attempt = 0;; attempt += 1) {
+            const last = attempt >= this.retryDelaysMs.length;
+            let res;
+            try {
+                res = await this.fetchImpl(this.url("inference"), {
+                    method: "POST",
+                    headers: { ...this.headers(true), accept: INFERENCE_NDJSON_CONTENT_TYPE },
+                    body,
+                });
+            }
+            catch (err) {
+                // No response at all — nothing streamed, so a re-POST is always safe.
+                if (last)
+                    throw err;
+                await this.backoffInferenceRetry(attempt, "connect", err);
+                continue;
+            }
+            if (res.status !== 200 || res.body === null) {
+                // A load-balancer transient (target draining / gateway timeout) before the stream began —
+                // safe to re-POST. Any other non-200 is the broker's real, already-classified answer
+                // (incl. a 502 upstream model error): surface it as the run's customer-facing error.
+                if (!last && res.body !== null && INFERENCE_RETRYABLE_STATUSES.has(res.status)) {
+                    await res.body.cancel().catch(() => { });
+                    await this.backoffInferenceRetry(attempt, "status", res.status);
+                    continue;
+                }
+                throw await inferenceHttpError(res);
+            }
+            // Relay the stream. A mid-stream transport drop throws out of readNdjsonLines; retry it only
+            // while nothing but heartbeats has gone out (see the doc-comment). `ping` frames carry no
+            // model output, so a drop during the pre-first-token wait stays safely retryable.
+            let sawContent = false;
+            try {
+                for await (const line of readNdjsonLines(res.body)) {
+                    const frame = parseInferenceFrame(line);
+                    if (frame.kind !== "ping")
+                        sawContent = true;
+                    yield frame;
+                }
+                return; // the stream ended cleanly
+            }
+            catch (err) {
+                if (last || sawContent)
+                    throw err;
+                await res.body.cancel().catch(() => { });
+                await this.backoffInferenceRetry(attempt, "stream", err);
+                continue;
+            }
         }
     }
+    /** Log + wait one backoff step before re-POSTing a transient inference failure (see streamInference). */
+    async backoffInferenceRetry(attempt, reason, detail) {
+        log.warn("broker_inference_retry", {
+            reason,
+            ...(reason === "status"
+                ? { status: detail }
+                : { error: detail instanceof Error ? detail.name : "unknown" }),
+            attempt: attempt + 1,
+            delayMs: this.retryDelaysMs[attempt],
+        });
+        await sleep(this.retryDelaysMs[attempt] ?? 0);
+    }
     url(suffix) {
         return `${this.base}/runner/v1/runs/${encodeURIComponent(this.cfg.runId)}/${suffix}`;
     }

package/dist/runtime/wire/inference_proxy.d.ts

@@ -32,13 +32,17 @@ export interface ProxyError {
 }
 /** A parsed response frame. `ping` is a no-payload heartbeat the broker emits during a long model
  *  turn to keep the connection producing bytes (so idle/body timeouts don't sever it); the worker
- *  ignores it. */
+ *  ignores it. `reset` voids every delta/reasoning relayed so far for the in-progress turn — the
+ *  broker restarts a turn after a transient mid-stream drop; the worker signals the viewer to discard
+ *  the turn's streamed output (the authoritative turn still arrives in the single `result`). */
 export type InferenceFrame = {
     kind: "delta";
     text: string;
 } | {
     kind: "reasoning";
     text: string;
+} | {
+    kind: "reset";
 } | {
     kind: "result";
     turn: ChatTurn;
@@ -70,6 +74,10 @@ export declare function serializeReasoningFrame(text: string): string;
 export declare function serializeResultFrame(turn: ChatTurn, modelRef: string, costMicros?: number, contextTokens?: number): string;
 /** Serialize a terminal error as a single NDJSON line. */
 export declare function serializeErrorFrame(error: ProxyError): string;
+/** Serialize a `reset` frame (no payload): the broker restarted a turn after a transient mid-stream
+ *  drop, so every delta/reasoning relayed so far is void. The broker is the producer; this mirror
+ *  exists so runner tests can construct the frame. */
+export declare function serializeResetFrame(): string;
 /** Serialize a heartbeat (no payload) as a single NDJSON line. The broker emits these on an interval
  *  during a long model turn so the worker↔broker connection keeps producing bytes — neither side's
  *  idle/body timeout fires while the model is generating but not yet streaming text. */

package/dist/runtime/wire/inference_proxy.js

@@ -100,6 +100,12 @@ export function serializeResultFrame(turn, modelRef, costMicros = 0, contextToke
 export function serializeErrorFrame(error) {
     return `${JSON.stringify({ t: "error", error })}\n`;
 }
+/** Serialize a `reset` frame (no payload): the broker restarted a turn after a transient mid-stream
+ *  drop, so every delta/reasoning relayed so far is void. The broker is the producer; this mirror
+ *  exists so runner tests can construct the frame. */
+export function serializeResetFrame() {
+    return `${JSON.stringify({ t: "reset" })}\n`;
+}
 /** Serialize a heartbeat (no payload) as a single NDJSON line. The broker emits these on an interval
  *  during a long model turn so the worker↔broker connection keeps producing bytes — neither side's
  *  idle/body timeout fires while the model is generating but not yet streaming text. */
@@ -133,6 +139,8 @@ export function parseInferenceFrame(line) {
             };
         case "error":
             return { kind: "error", error: toProxyError(rec.error) };
+        case "reset":
+            return { kind: "reset" };
         case "ping":
             return { kind: "ping" };
         default:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boardwalk-labs/runner",
-  "version": "0.2.14",
+  "version": "0.2.16",
   "description": "Boardwalk self-hosted runner: execute cloud-scheduled runs on your own machines. Currently: the canonical runner contract types.",
   "license": "Apache-2.0",
   "repository": {
@@ -54,7 +54,7 @@
   },
   "dependencies": {
     "@boardwalk-labs/engine": "0.2.12",
-    "@boardwalk-labs/workflow": "0.2.5",
+    "@boardwalk-labs/workflow": "0.2.6",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "node-gyp-build": "^4.8.4",
     "tar": "^7.5.16",