@boardwalk-labs/runner 0.1.18 → 0.1.19

package/dist/runtime/index.js

@@ -326,6 +326,15 @@ export function assembleWorkerDeps(runtime) {
                 }
                 return Promise.resolve(runApiKey);
             },
+            // OIDC id-token for cloud federation: minted per call by the broker with the CURRENT run
+            // token (so post-resume calls just work — no captured value to swap on wake). Recorded in
+            // the redactor like every run credential; the broker 403s (naming permissions.id_token)
+            // when the pinned manifest doesn't grant it.
+            idToken: async (audience) => {
+                const { token } = await broker.requestOidcToken(audience);
+                redactor.record(token);
+                return token;
+            },
         };
         const host = new WorkerWorkflowHost({
             leaf,

package/dist/runtime/runner_control_client.d.ts

@@ -17,6 +17,9 @@ export interface RunnerControlClientConfig {
     controlTimeoutMs?: number;
     /** Per-call ceiling for bulk artifact/workspace transfers (default 5 min). */
     bulkTimeoutMs?: number;
+    /** Backoff schedule for transient-failure retries (length = extra attempts after the first;
+     *  see {@link RETRYABLE_STATUSES}). Injectable for tests; [] disables retries. */
+    retryDelaysMs?: number[];
 }
 /** The pinned program's download reference (the worker fetches + verifies + extracts it). */
 export interface BrokerProgram {
@@ -53,6 +56,7 @@ export declare class RunnerControlClient {
     private runToken;
     private readonly controlTimeoutMs;
     private readonly bulkTimeoutMs;
+    private readonly retryDelaysMs;
     constructor(cfg: RunnerControlClientConfig);
     /** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
     swapRunToken(token: string): void;
@@ -69,6 +73,16 @@ export declare class RunnerControlClient {
     /** Bulk transfers (artifact + workspace up/download over presigned S3) — a much larger ceiling
      *  than a control call, but still bounded so a dead socket can't hang the run. */
     private bulkFetch;
+    /**
+     * One attempt per entry in the backoff schedule (+1): retry thrown network failures (connection
+     * reset/refused mid-rollover, our own per-attempt timeout on a dead socket) and the
+     * load-balancer's {@link RETRYABLE_STATUSES}. Safe to re-send because every caller's body is a
+     * reusable string/byte-array (the streaming inference call bypasses this entirely), and the
+     * broker's mutating endpoints are idempotent per worker/identifier (journal seq, usage
+     * identifier, lease per workerId). Before this, ONE blip during an api-server deploy rollover
+     * crashed the worker hard mid-suspend/finalize and only crash-reclaim recovered the run.
+     */
+    private retryingFetch;
     /** Claim the run's lease. Returns the run on success, or null when it isn't claimable (409 —
      *  another worker has it, or it isn't pending), which the worker treats as "claim lost". */
     claim(workerId: string, leaseSeconds: number): Promise<{

package/dist/runtime/runner_control_client.js

@@ -7,11 +7,24 @@
 //
 // Thin + injectable (fetch is overridable) so it's unit-tested without a live server. Status mapping
 // mirrors the broker handlers: claim 409 ⇒ "claim lost" (null), version 404 ⇒ missing (null); any
-// other non-success status throws so the worker fails loud (→ restart, lease reclaimed).
+// other non-success status throws so the worker fails loud (→ restart, lease reclaimed). TRANSIENT
+// failures (thrown network errors, LB 502/503/504 — a control-plane deploy rollover) are retried
+// with backoff first, so a blip heals in place instead of crashing the run.
 import { createLogger } from "./support/index.js";
 import { journalLookupSchema, } from "./suspension.js";
 import { INFERENCE_NDJSON_CONTENT_TYPE, parseInferenceFrame, serializeInferenceRequest, } from "./wire/inference_proxy.js";
 const log = createLogger("RunnerControlClient");
+/**
+ * Transient statuses worth retrying: the load-balancer answers during a control-plane deploy
+ * rollover (no healthy target / draining target / gateway timeout). Anything else — including a
+ * 500 — is treated as a real answer from a live handler and surfaces immediately: retrying a
+ * handler error could duplicate a side effect for no healing value.
+ */
+const RETRYABLE_STATUSES = new Set([502, 503, 504]);
+/** Default backoff (ms) between attempts — ~17.5s of spread, sized to ride out the target-group
+ *  rotation window of an api-server rolling deploy (the observed killer: guests died hard
+ *  mid-suspend/finalize during TWO deploys, 2026-07-13, and only crash-reclaim saved the runs). */
+const DEFAULT_RETRY_DELAYS_MS = [500, 2_000, 5_000, 10_000];
 export class RunnerControlClient {
     cfg;
     base;
@@ -21,6 +34,7 @@ export class RunnerControlClient {
     runToken;
     controlTimeoutMs;
     bulkTimeoutMs;
+    retryDelaysMs;
     constructor(cfg) {
         this.cfg = cfg;
         this.base = cfg.baseUrl.replace(/\/+$/, "");
@@ -28,6 +42,7 @@ export class RunnerControlClient {
         this.runToken = cfg.runToken;
         this.controlTimeoutMs = cfg.controlTimeoutMs ?? 30_000;
         this.bulkTimeoutMs = cfg.bulkTimeoutMs ?? 300_000;
+        this.retryDelaysMs = cfg.retryDelaysMs ?? DEFAULT_RETRY_DELAYS_MS;
     }
     /** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
     swapRunToken(token) {
@@ -43,12 +58,51 @@ export class RunnerControlClient {
      * streaming inference call is the ONE exception (long-lived NDJSON) and bypasses this.
      */
     controlFetch(url, init) {
-        return this.fetchImpl(url, { ...init, signal: AbortSignal.timeout(this.controlTimeoutMs) });
+        return this.retryingFetch(url, init, this.controlTimeoutMs);
     }
     /** Bulk transfers (artifact + workspace up/download over presigned S3) — a much larger ceiling
      *  than a control call, but still bounded so a dead socket can't hang the run. */
     bulkFetch(url, init) {
-        return this.fetchImpl(url, { ...init, signal: AbortSignal.timeout(this.bulkTimeoutMs) });
+        return this.retryingFetch(url, init, this.bulkTimeoutMs);
+    }
+    /**
+     * One attempt per entry in the backoff schedule (+1): retry thrown network failures (connection
+     * reset/refused mid-rollover, our own per-attempt timeout on a dead socket) and the
+     * load-balancer's {@link RETRYABLE_STATUSES}. Safe to re-send because every caller's body is a
+     * reusable string/byte-array (the streaming inference call bypasses this entirely), and the
+     * broker's mutating endpoints are idempotent per worker/identifier (journal seq, usage
+     * identifier, lease per workerId). Before this, ONE blip during an api-server deploy rollover
+     * crashed the worker hard mid-suspend/finalize and only crash-reclaim recovered the run.
+     */
+    async retryingFetch(url, init, timeoutMs) {
+        for (let attempt = 0;; attempt += 1) {
+            const last = attempt >= this.retryDelaysMs.length;
+            try {
+                const res = await this.fetchImpl(url, {
+                    ...init,
+                    signal: AbortSignal.timeout(timeoutMs),
+                });
+                if (last || !RETRYABLE_STATUSES.has(res.status))
+                    return res;
+                // Drop the unused body so the socket is released before the retry.
+                await res.body?.cancel().catch(() => { });
+                log.warn("broker_call_retry", {
+                    status: res.status,
+                    attempt: attempt + 1,
+                    delayMs: this.retryDelaysMs[attempt],
+                });
+            }
+            catch (err) {
+                if (last)
+                    throw err;
+                log.warn("broker_call_retry", {
+                    error: err instanceof Error ? err.name : "unknown",
+                    attempt: attempt + 1,
+                    delayMs: this.retryDelaysMs[attempt],
+                });
+            }
+            await sleep(this.retryDelaysMs[attempt] ?? 0);
+        }
     }
     /** Claim the run's lease. Returns the run on success, or null when it isn't claimable (409 —
      *  another worker has it, or it isn't pending), which the worker treats as "claim lost". */
@@ -493,6 +547,9 @@ export class RunnerControlClient {
         return h;
     }
 }
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
 /** Yield complete newline-delimited lines from a streaming response body (NDJSON inference frames),
  *  buffering partial chunks and flushing any trailing line. Blank lines are skipped. */
 async function* readNdjsonLines(body) {

package/dist/runtime/workflow_host.d.ts

@@ -93,6 +93,11 @@ export interface RuntimeContext {
     apiUrl: string;
     /** A short-lived, manifest-scoped bearer for the public API / MCP / CLI. */
     apiToken(): Promise<string>;
+    /** A short-lived OIDC id-token asserting this run's identity for `audience`, for federation into
+     *  the org's OWN cloud (AWS `AssumeRoleWithWebIdentity` / GCP / Azure). Minted per call by the
+     *  broker (gated server-side on `permissions.id_token: "write"`) with the CURRENT run token, so it
+     *  needs no swap handling across suspend/resume — unlike the captured `apiToken` bearer. */
+    idToken(audience: string): Promise<string>;
 }
 export interface WorkerWorkflowHostDeps {
     leaf: LeafExecutor;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boardwalk-labs/runner",
-  "version": "0.1.18",
+  "version": "0.1.19",
   "description": "Boardwalk self-hosted runner: execute cloud-scheduled runs on your own machines. Currently: the canonical runner contract types.",
   "license": "Apache-2.0",
   "repository": {
@@ -54,7 +54,7 @@
   },
   "dependencies": {
     "@boardwalk-labs/engine": "^0.1.34",
-    "@boardwalk-labs/workflow": "^0.1.25",
+    "@boardwalk-labs/workflow": "^0.1.28",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "node-gyp-build": "^4.8.4",
     "tar": "^7.5.16",