@boardwalk-labs/runner 0.1.17 → 0.1.19

package/dist/runtime/browser_session_backend.js

@@ -162,7 +162,12 @@ export function makeGuestBrowserBackend(cfg) {
                 mcp.once("error", (err) => {
                     log.error("browser_mcp_spawn_error", { error: err.message });
                 });
-                await waitForHttp(`http://127.0.0.1:${String(mcpPort)}/mcp`, cfg.readyTimeoutMs);
+                // Probe the EXACT url the program's client will connect to (the `localhost` form), not a
+                // `127.0.0.1` stand-in: if `localhost` doesn't resolve in the guest, connectSessionMcp would
+                // otherwise throw a bare `TypeError: fetch failed` while this readiness check passed on a
+                // different address (masking the fault). Probing mcpUrl turns that into a diagnosable
+                // `browser_backend_timeout`. (The guest ships /etc/hosts mapping localhost → 127.0.0.1.)
+                await waitForHttp(mcpUrl, cfg.readyTimeoutMs);
                 let killed = false;
                 return {
                     cdpUrl,

package/dist/runtime/index.js

@@ -49,6 +49,7 @@ import { RunnerControlClient } from "./runner_control_client.js";
 import { BrokerChildDispatcher } from "./broker_child_dispatcher.js";
 import { BrokerEventPublisher } from "./broker_event_publisher.js";
 import { createProgramLogSink } from "./program_log_capture.js";
+import { makeRunLogFileSink } from "./run_log_file_sink.js";
 import { BrokerArtifactStore } from "./broker_artifact_store.js";
 import { buildBrokerToolHost } from "./broker_tool_host.js";
 import { CreditWatcher } from "./credit_watcher.js";
@@ -130,7 +131,18 @@ export function assembleWorkerDeps(runtime) {
     // declared output, and every agent turn stamp through this emitter, so cursors are run-globally
     // monotonic with no separate program band. Frames publish as `{cursor, event}` rows via the
     // broker telemetry path. The claim wrapper below bumps it past a previous session's frames.
-    const runEvents = new WorkerRunEventEmitter({ runId: runtime.runId, publisher: eventPublisher });
+    // Optional on-screen mirror: append the (already-redacted) event stream to a local file an xterm in
+    // the ambient desktop tails, so the live-view/recording shows the run working. Opt-in via the env the
+    // desktop guest image sets (BOARDWALK_RUN_LOG_FILE); absent everywhere else, so no cost off-desktop.
+    const runLogFilePath = process.env.BOARDWALK_RUN_LOG_FILE?.trim();
+    const runLogLocalSink = runLogFilePath !== undefined && runLogFilePath !== ""
+        ? makeRunLogFileSink(runLogFilePath)
+        : undefined;
+    const runEvents = new WorkerRunEventEmitter({
+        runId: runtime.runId,
+        publisher: eventPublisher,
+        ...(runLogLocalSink !== undefined ? { localSink: runLogLocalSink } : {}),
+    });
     const phaseTracker = new PhaseTracker({ sink: runEvents });
     // The run's public-API credential (was BOARDWALK_API_KEY). It is NO LONGER in process.env — the
     // bootstrap captured + scrubbed it (capturePlatformContext) so the agent's bash / subprocesses
@@ -314,6 +326,15 @@ export function assembleWorkerDeps(runtime) {
                 }
                 return Promise.resolve(runApiKey);
             },
+            // OIDC id-token for cloud federation: minted per call by the broker with the CURRENT run
+            // token (so post-resume calls just work — no captured value to swap on wake). Recorded in
+            // the redactor like every run credential; the broker 403s (naming permissions.id_token)
+            // when the pinned manifest doesn't grant it.
+            idToken: async (audience) => {
+                const { token } = await broker.requestOidcToken(audience);
+                redactor.record(token);
+                return token;
+            },
         };
         const host = new WorkerWorkflowHost({
             leaf,

package/dist/runtime/run_event_emitter.d.ts

@@ -6,11 +6,16 @@ export interface RunEventEmitterOptions {
     resumeAfterCursor?: number;
     /** Injected clock for deterministic tests. Defaults to Date.now. */
     now?: () => number;
+    /** Optional local mirror of every (already-redacted) event — used to surface the run's output in an
+     *  on-screen terminal in the ambient desktop (docs/SCREEN_CAPTURE.md). Best-effort: a sink throw never
+     *  affects the run or the broker publish. Only fed events that already passed the run's redactor. */
+    localSink?: (event: RunEvent) => void;
 }
 export declare class WorkerRunEventEmitter {
     private readonly runId;
     private readonly publisher;
     private readonly now;
+    private readonly localSink;
     private turn;
     private seq;
     constructor(opts: RunEventEmitterOptions);

package/dist/runtime/run_event_emitter.js

@@ -20,12 +20,14 @@ export class WorkerRunEventEmitter {
     runId;
     publisher;
     now;
+    localSink;
     turn;
     seq;
     constructor(opts) {
         this.runId = opts.runId;
         this.publisher = opts.publisher;
         this.now = opts.now ?? Date.now;
+        this.localSink = opts.localSink;
         const max = opts.resumeAfterCursor ?? 0;
         this.turn = max > 0 ? Math.floor(max / TURN_CURSOR_STRIDE) + 1 : 0;
         this.seq = 0;
@@ -44,6 +46,14 @@ export class WorkerRunEventEmitter {
         this.publisher
             .publish(`run:${this.runId}`, JSON.stringify({ cursor, event }))
             .catch(() => undefined); // best-effort; the publisher logs its own failures
+        if (this.localSink !== undefined) {
+            try {
+                this.localSink(event); // best-effort on-screen mirror; must never affect the run
+            }
+            catch {
+                /* ignore */
+            }
+        }
         return event;
     }
     /** Restart resume: jump past the previous session's frames (claim carries the store max). */

package/dist/runtime/run_log_file_sink.d.ts

@@ -0,0 +1,9 @@
+import type { RunEvent } from "./agent/events.js";
+/** One readable, ANSI-colored line for a run event, or null to skip it (streaming text/reasoning
+ *  deltas are buffered by the sink, not formatted here). Reads only a few fields off the SDK's typed
+ *  event and degrades gracefully on anything unexpected. */
+export declare function formatRunEventLine(event: RunEvent): string | null;
+/** Build the {@link WorkerRunEventEmitter} local sink: append formatted lines to `path`, buffering
+ *  streaming agent text/reasoning into whole lines. Best-effort — a broken stream (full disk, closed
+ *  pipe) is swallowed and disables the sink; it never throws into the run. */
+export declare function makeRunLogFileSink(path: string): (event: RunEvent) => void;

package/dist/runtime/run_log_file_sink.js

@@ -0,0 +1,106 @@
+// On-screen run-output mirror. Formats the run's (already-redacted) event stream into readable,
+// ANSI-colored lines appended to a local file, which an xterm in the ambient desktop tails so the
+// live-view / recording shows the run working (docs/SCREEN_CAPTURE.md). Gated on
+// BOARDWALK_RUN_LOG_FILE — the desktop guest image sets it. This ONLY ever sees events that already
+// passed the run's redactor (the same stream the web run-detail renders), so the file is safe to
+// capture in a durable recording.
+import { createWriteStream } from "node:fs";
+function oneLine(s, n) {
+    const t = s.replace(/\s+/g, " ").trim();
+    return t.length > n ? t.slice(0, n - 1) + "…" : t;
+}
+function safeJson(v) {
+    try {
+        return typeof v === "string" ? v : JSON.stringify(v);
+    }
+    catch {
+        return String(v);
+    }
+}
+/** One readable, ANSI-colored line for a run event, or null to skip it (streaming text/reasoning
+ *  deltas are buffered by the sink, not formatted here). Reads only a few fields off the SDK's typed
+ *  event and degrades gracefully on anything unexpected. */
+export function formatRunEventLine(event) {
+    const e = event;
+    const k = typeof e.kind === "string" ? e.kind : "";
+    const s = (v) => (typeof v === "string" ? v : "");
+    switch (k) {
+        case "run_status":
+            return `\x1b[1;32m● ${s(e.status)}\x1b[0m`;
+        case "phase":
+            return `\x1b[1;36m▸ ${s(e.name)}\x1b[0m`;
+        case "program_output": {
+            const t = s(e.text).replace(/\s+$/, "");
+            return t ? `  ${t}` : null;
+        }
+        case "turn_started":
+            return `\x1b[35m· agent ${s(e.agentName) || s(e.agentId)}\x1b[0m`;
+        case "tool_call_start":
+            return `  \x1b[33m⚙ ${s(e.toolName)}\x1b[0m`;
+        case "tool_output_delta": {
+            const t = s(e.text).replace(/\s+$/, "");
+            return t ? `    \x1b[90m${oneLine(t, 200)}\x1b[0m` : null;
+        }
+        case "tool_call_result":
+            return `  \x1b[32m✓ done\x1b[0m`;
+        case "tool_call_error":
+            return `  \x1b[31m✗ tool error\x1b[0m`;
+        case "output":
+            return `\x1b[1;32m✔ output  ${oneLine(safeJson(e.value), 300)}\x1b[0m`;
+        case "human_input_requested":
+            return `\x1b[1;33m⏸ awaiting human input\x1b[0m`;
+        case "human_input_resolved":
+            return `\x1b[33m▶ input received\x1b[0m`;
+        case "suspended":
+            return `\x1b[90m⏸ suspended\x1b[0m`;
+        case "resumed":
+            return `\x1b[90m▶ resumed\x1b[0m`;
+        case "egress_denied":
+            return `  \x1b[31m⛔ egress denied\x1b[0m`;
+        default:
+            return null; // turn_ended, text_*, tool_call_input_*, tool_call_executing — noise for the terminal
+    }
+}
+/** Build the {@link WorkerRunEventEmitter} local sink: append formatted lines to `path`, buffering
+ *  streaming agent text/reasoning into whole lines. Best-effort — a broken stream (full disk, closed
+ *  pipe) is swallowed and disables the sink; it never throws into the run. */
+export function makeRunLogFileSink(path) {
+    let stream = null;
+    try {
+        stream = createWriteStream(path, { flags: "a" });
+        stream.on("error", () => {
+            stream = null;
+        });
+        stream.write("\n\x1b[1;36m── boardwalk run ──\x1b[0m\n");
+    }
+    catch {
+        stream = null;
+    }
+    let textBuf = "";
+    const flushText = () => {
+        if (textBuf.trim() !== "" && stream !== null) {
+            stream.write(`  \x1b[37m${oneLine(textBuf, 2000)}\x1b[0m\n`);
+        }
+        textBuf = "";
+    };
+    return (event) => {
+        if (stream === null)
+            return;
+        const e = event;
+        const k = typeof e.kind === "string" ? e.kind : "";
+        if (k === "text_delta" || k === "reasoning_delta") {
+            textBuf += typeof e.text === "string" ? e.text : "";
+            return;
+        }
+        if (k === "text_start")
+            return;
+        if (k === "text_end") {
+            flushText();
+            return;
+        }
+        flushText(); // a non-text event ends any in-flight text line
+        const line = formatRunEventLine(event);
+        if (line !== null)
+            stream.write(line + "\n");
+    };
+}

package/dist/runtime/runner_control_client.d.ts

@@ -17,6 +17,9 @@ export interface RunnerControlClientConfig {
     controlTimeoutMs?: number;
     /** Per-call ceiling for bulk artifact/workspace transfers (default 5 min). */
     bulkTimeoutMs?: number;
+    /** Backoff schedule for transient-failure retries (length = extra attempts after the first;
+     *  see {@link RETRYABLE_STATUSES}). Injectable for tests; [] disables retries. */
+    retryDelaysMs?: number[];
 }
 /** The pinned program's download reference (the worker fetches + verifies + extracts it). */
 export interface BrokerProgram {
@@ -53,6 +56,7 @@ export declare class RunnerControlClient {
     private runToken;
     private readonly controlTimeoutMs;
     private readonly bulkTimeoutMs;
+    private readonly retryDelaysMs;
     constructor(cfg: RunnerControlClientConfig);
     /** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
     swapRunToken(token: string): void;
@@ -69,6 +73,16 @@ export declare class RunnerControlClient {
     /** Bulk transfers (artifact + workspace up/download over presigned S3) — a much larger ceiling
      *  than a control call, but still bounded so a dead socket can't hang the run. */
     private bulkFetch;
+    /**
+     * One attempt per entry in the backoff schedule (+1): retry thrown network failures (connection
+     * reset/refused mid-rollover, our own per-attempt timeout on a dead socket) and the
+     * load-balancer's {@link RETRYABLE_STATUSES}. Safe to re-send because every caller's body is a
+     * reusable string/byte-array (the streaming inference call bypasses this entirely), and the
+     * broker's mutating endpoints are idempotent per worker/identifier (journal seq, usage
+     * identifier, lease per workerId). Before this, ONE blip during an api-server deploy rollover
+     * crashed the worker hard mid-suspend/finalize and only crash-reclaim recovered the run.
+     */
+    private retryingFetch;
     /** Claim the run's lease. Returns the run on success, or null when it isn't claimable (409 —
      *  another worker has it, or it isn't pending), which the worker treats as "claim lost". */
     claim(workerId: string, leaseSeconds: number): Promise<{

package/dist/runtime/runner_control_client.js

@@ -7,11 +7,24 @@
 //
 // Thin + injectable (fetch is overridable) so it's unit-tested without a live server. Status mapping
 // mirrors the broker handlers: claim 409 ⇒ "claim lost" (null), version 404 ⇒ missing (null); any
-// other non-success status throws so the worker fails loud (→ restart, lease reclaimed).
+// other non-success status throws so the worker fails loud (→ restart, lease reclaimed). TRANSIENT
+// failures (thrown network errors, LB 502/503/504 — a control-plane deploy rollover) are retried
+// with backoff first, so a blip heals in place instead of crashing the run.
 import { createLogger } from "./support/index.js";
 import { journalLookupSchema, } from "./suspension.js";
 import { INFERENCE_NDJSON_CONTENT_TYPE, parseInferenceFrame, serializeInferenceRequest, } from "./wire/inference_proxy.js";
 const log = createLogger("RunnerControlClient");
+/**
+ * Transient statuses worth retrying: the load-balancer answers during a control-plane deploy
+ * rollover (no healthy target / draining target / gateway timeout). Anything else — including a
+ * 500 — is treated as a real answer from a live handler and surfaces immediately: retrying a
+ * handler error could duplicate a side effect for no healing value.
+ */
+const RETRYABLE_STATUSES = new Set([502, 503, 504]);
+/** Default backoff (ms) between attempts — ~17.5s of spread, sized to ride out the target-group
+ *  rotation window of an api-server rolling deploy (the observed killer: guests died hard
+ *  mid-suspend/finalize during TWO deploys, 2026-07-13, and only crash-reclaim saved the runs). */
+const DEFAULT_RETRY_DELAYS_MS = [500, 2_000, 5_000, 10_000];
 export class RunnerControlClient {
     cfg;
     base;
@@ -21,6 +34,7 @@ export class RunnerControlClient {
     runToken;
     controlTimeoutMs;
     bulkTimeoutMs;
+    retryDelaysMs;
     constructor(cfg) {
         this.cfg = cfg;
         this.base = cfg.baseUrl.replace(/\/+$/, "");
@@ -28,6 +42,7 @@ export class RunnerControlClient {
         this.runToken = cfg.runToken;
         this.controlTimeoutMs = cfg.controlTimeoutMs ?? 30_000;
         this.bulkTimeoutMs = cfg.bulkTimeoutMs ?? 300_000;
+        this.retryDelaysMs = cfg.retryDelaysMs ?? DEFAULT_RETRY_DELAYS_MS;
     }
     /** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
     swapRunToken(token) {
@@ -43,12 +58,51 @@ export class RunnerControlClient {
      * streaming inference call is the ONE exception (long-lived NDJSON) and bypasses this.
      */
     controlFetch(url, init) {
-        return this.fetchImpl(url, { ...init, signal: AbortSignal.timeout(this.controlTimeoutMs) });
+        return this.retryingFetch(url, init, this.controlTimeoutMs);
     }
     /** Bulk transfers (artifact + workspace up/download over presigned S3) — a much larger ceiling
      *  than a control call, but still bounded so a dead socket can't hang the run. */
     bulkFetch(url, init) {
-        return this.fetchImpl(url, { ...init, signal: AbortSignal.timeout(this.bulkTimeoutMs) });
+        return this.retryingFetch(url, init, this.bulkTimeoutMs);
+    }
+    /**
+     * One attempt per entry in the backoff schedule (+1): retry thrown network failures (connection
+     * reset/refused mid-rollover, our own per-attempt timeout on a dead socket) and the
+     * load-balancer's {@link RETRYABLE_STATUSES}. Safe to re-send because every caller's body is a
+     * reusable string/byte-array (the streaming inference call bypasses this entirely), and the
+     * broker's mutating endpoints are idempotent per worker/identifier (journal seq, usage
+     * identifier, lease per workerId). Before this, ONE blip during an api-server deploy rollover
+     * crashed the worker hard mid-suspend/finalize and only crash-reclaim recovered the run.
+     */
+    async retryingFetch(url, init, timeoutMs) {
+        for (let attempt = 0;; attempt += 1) {
+            const last = attempt >= this.retryDelaysMs.length;
+            try {
+                const res = await this.fetchImpl(url, {
+                    ...init,
+                    signal: AbortSignal.timeout(timeoutMs),
+                });
+                if (last || !RETRYABLE_STATUSES.has(res.status))
+                    return res;
+                // Drop the unused body so the socket is released before the retry.
+                await res.body?.cancel().catch(() => { });
+                log.warn("broker_call_retry", {
+                    status: res.status,
+                    attempt: attempt + 1,
+                    delayMs: this.retryDelaysMs[attempt],
+                });
+            }
+            catch (err) {
+                if (last)
+                    throw err;
+                log.warn("broker_call_retry", {
+                    error: err instanceof Error ? err.name : "unknown",
+                    attempt: attempt + 1,
+                    delayMs: this.retryDelaysMs[attempt],
+                });
+            }
+            await sleep(this.retryDelaysMs[attempt] ?? 0);
+        }
     }
     /** Claim the run's lease. Returns the run on success, or null when it isn't claimable (409 —
      *  another worker has it, or it isn't pending), which the worker treats as "claim lost". */
@@ -493,6 +547,9 @@ export class RunnerControlClient {
         return h;
     }
 }
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
 /** Yield complete newline-delimited lines from a streaming response body (NDJSON inference frames),
  *  buffering partial chunks and flushing any trailing line. Blank lines are skipped. */
 async function* readNdjsonLines(body) {

package/dist/runtime/workflow_host.d.ts

@@ -93,6 +93,11 @@ export interface RuntimeContext {
     apiUrl: string;
     /** A short-lived, manifest-scoped bearer for the public API / MCP / CLI. */
     apiToken(): Promise<string>;
+    /** A short-lived OIDC id-token asserting this run's identity for `audience`, for federation into
+     *  the org's OWN cloud (AWS `AssumeRoleWithWebIdentity` / GCP / Azure). Minted per call by the
+     *  broker (gated server-side on `permissions.id_token: "write"`) with the CURRENT run token, so it
+     *  needs no swap handling across suspend/resume — unlike the captured `apiToken` bearer. */
+    idToken(audience: string): Promise<string>;
 }
 export interface WorkerWorkflowHostDeps {
     leaf: LeafExecutor;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boardwalk-labs/runner",
-  "version": "0.1.17",
+  "version": "0.1.19",
   "description": "Boardwalk self-hosted runner: execute cloud-scheduled runs on your own machines. Currently: the canonical runner contract types.",
   "license": "Apache-2.0",
   "repository": {
@@ -54,7 +54,7 @@
   },
   "dependencies": {
     "@boardwalk-labs/engine": "^0.1.34",
-    "@boardwalk-labs/workflow": "^0.1.25",
+    "@boardwalk-labs/workflow": "^0.1.28",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "node-gyp-build": "^4.8.4",
     "tar": "^7.5.16",