@boardwalk-labs/runner 0.1.13 → 0.1.16

package/dist/runtime/broker_artifact_store.js

@@ -27,6 +27,10 @@ export class BrokerArtifactStore {
             name: input.name,
             contentType: input.contentType,
             sizeBytes: bytes.length,
+            // Thread metadata to presign too (not only commit) so the broker can apply quota exemption +
+            // the retention tag at presign time — needed for large recording segments (a near-full org's
+            // recording must not be rejected by the storage ceiling).
+            ...(input.metadata !== undefined ? { metadata: input.metadata } : {}),
         });
         await this.broker.uploadBytes(presign.uploadUrl, presign.uploadHeaders, bytes);
         return this.broker.commitArtifact({

package/dist/runtime/index.d.ts

@@ -3,6 +3,7 @@ import type { Run } from "./wire/run.js";
 import { type IdentityRelay } from "./identity_relay.js";
 import type { ByoInferenceProvider } from "../contract.js";
 import { type BrowserBackend } from "./browser_session.js";
+import { type CaptureBackend } from "./screen_capture.js";
 import { type ProgramWorkerDeps } from "./program_worker.js";
 /** Constructed primitives the pure assembly consumes (real ones in main(), fakes in tests). The
  *  runner holds NO platform credential — only the per-run control-plane handle (run token + broker
@@ -37,6 +38,9 @@ export interface WorkerRuntime {
      *  (Chromium + a pre-installed Playwright MCP + an X display; gated by BOARDWALK_BROWSER_TIER).
      *  When absent, `computer.openBrowser()` fails with a clear "not available on this runner image". */
     browserBackend?: BrowserBackend;
+    /** Screen-capture backend (session recording + live-view frames), present ONLY when the runner IMAGE
+     *  ships the desktop stack (ffmpeg + an X display) and recording isn't disabled. Absent ⇒ no capture. */
+    captureBackend?: CaptureBackend;
 }
 /** Durable events are deferred (durable event storage) — live fan-out via the broker only. */
 /** Synthesize the run's AuthContext. The run was already authorized at trigger time; the

package/dist/runtime/index.js

@@ -41,6 +41,8 @@ import { reseedUserspaceCsprng } from "./uniqueness_reseed.js";
 import { WorkerWorkflowHost } from "./workflow_host.js";
 import { BrowserSessionManager } from "./browser_session.js";
 import { loadGuestBrowserConfig, makeGuestBrowserBackend, connectSessionMcp, } from "./browser_session_backend.js";
+import { ScreenCapture } from "./screen_capture.js";
+import { loadCaptureConfig, makeCaptureBackend } from "./screen_capture_backend.js";
 import { runProgramWorker, } from "./program_worker.js";
 import { RunnerControlClient } from "./runner_control_client.js";
 import { BrokerChildDispatcher } from "./broker_child_dispatcher.js";
@@ -196,6 +198,23 @@ export function assembleWorkerDeps(runtime) {
                 nextId: () => newId(),
             })
             : undefined;
+        // Screen capture (session recording + live-view). Present only when the image ships the desktop
+        // stack (runtime.captureBackend set) AND the workflow hasn't opted out (`recording: false` in the
+        // manifest — SCREEN_CAPTURE §4.5; default on). Segments upload through the SAME broker artifact store
+        // as a `recording-segment` artifact (quota-exempt, 30-day retention — the broker applies that
+        // server-side off the metadata kind); live frames push up the broker's live-view channel while a
+        // viewer watches. (Live-view rides the same capture, so opting out of recording also stops live-view.)
+        const capture = runtime.captureBackend !== undefined && manifest.recording !== false
+            ? new ScreenCapture({
+                backend: runtime.captureBackend,
+                writeArtifact: (name, contentType, base64, metadata) => artifactStore
+                    .write({ name, contentType, body: base64, encoding: "base64", metadata })
+                    .then((res) => ({ id: res.id })),
+                publishLiveFrames: (frames) => broker.publishLiveView(frames),
+                liveViewWanted: () => broker.liveViewWanted(),
+                now: () => Date.now(),
+            })
+            : undefined;
         // Backend for the engine's host-backed built-in tools (webfetch/web_search/artifacts): web_search
         // + artifact read-back go through the broker (no Tavily/S3 creds here); webfetch is an in-process
         // fetch already gated by the worker's egress proxy.
@@ -362,6 +381,9 @@ export function assembleWorkerDeps(runtime) {
                     freezeWallMs = Date.now();
                     await activeFlusher?.flushNow();
                     await workspaceStore?.persist();
+                    // The recorder never spans a snapshot: finalize + upload the in-flight segment before the
+                    // freeze (SCREEN_CAPTURE §4.3). Bounded internally, so a slow upload delays, not blocks, it.
+                    await capture?.stopAndFlush();
                 },
                 // On wake: reseed the userspace CSPRNG (clause 3 — a suspend snapshot restored more than
                 // once, e.g. a re-dispatch retry, would otherwise repeat its post-wake `crypto.*` draws),
@@ -378,6 +400,8 @@ export function assembleWorkerDeps(runtime) {
                         redactor.record(wake.api_token);
                     }
                     activeFlusher?.excludeIdle(wake.wall_clock_ms - freezeWallMs);
+                    // Resume capture in a fresh segment epoch (a suspend/resume boundary is a segment boundary).
+                    void capture?.startFresh();
                 },
             });
         }
@@ -407,6 +431,8 @@ export function assembleWorkerDeps(runtime) {
             // The orchestrator reaps every still-open browser session on terminal (kill Chromium + its
             // Playwright MCP) so no browser process leaks past the run.
             ...(browserSessions !== undefined ? { browserSessions } : {}),
+            // The orchestrator starts capture before the program runs and flushes it on terminal.
+            ...(capture !== undefined ? { capture } : {}),
         });
     };
     return {
@@ -618,6 +644,10 @@ export async function main() {
     // absent ⇒ `computer.openBrowser()` fails clearly. Read here so env-reading stays out of the pure wiring.
     const guestBrowserConfig = loadGuestBrowserConfig(process.env);
     const browserBackend = guestBrowserConfig !== null ? makeGuestBrowserBackend(guestBrowserConfig) : undefined;
+    // Screen capture (recording + live-view): present only when the image ships the desktop stack
+    // (ffmpeg + a display) and recording isn't disabled. Read here so env-reading stays out of the wiring.
+    const captureConfig = loadCaptureConfig(process.env);
+    const captureBackend = captureConfig !== null ? makeCaptureBackend(captureConfig) : undefined;
     const deps = assembleWorkerDeps({
         workerId: process.env.WORKER_ID ?? `worker-${runId}`,
         workspaceRoot: process.env.WORKSPACE_ROOT ?? "/workspace",
@@ -627,6 +657,7 @@ export async function main() {
         ...(byoProviders.length > 0 ? { byoProviders } : {}),
         ...(freezeRelay !== undefined ? { freezeRelay } : {}),
         ...(browserBackend !== undefined ? { browserBackend } : {}),
+        ...(captureBackend !== undefined ? { captureBackend } : {}),
     });
     // The only thing to drain is the batched telemetry buffer — the runner opens no database, cache, or queue.
     const cleanup = async () => {

package/dist/runtime/program_worker.d.ts

@@ -103,6 +103,13 @@ export type ProgramHostBuilder = (run: Run, manifest: WorkflowManifest, signal:
     browserSessions?: {
         closeAll(): Promise<void>;
     };
+    /** Session recording + live-view capture (docs/SCREEN_CAPTURE.md). The orchestrator starts it before
+     *  the program runs and flushes it on EVERY terminal path. Best-effort; absent without the desktop
+     *  stack. */
+    capture?: {
+        start(): Promise<void>;
+        stopAndFlush(): Promise<void>;
+    };
 }>;
 /** Handle to a running per-session loop (metering or credit watch); `stop()` ends + drains it. */
 export interface RunSessionHandle {

package/dist/runtime/program_worker.js

@@ -112,6 +112,7 @@ export async function runProgramWorker(runId, deps) {
     }
     const { host, redactor, workspace, phases, activity, setProgramDir, lsp, suspendSignal } = built;
     const browserSessions = built.browserSessions;
+    const capture = built.capture;
     // Guarantee the /workspace sandbox dir exists for EVERY run (persist or not) so a program can write
     // to /workspace without a defensive mkdir. Runs before hydrate (whose extract targets the dir).
     // Best-effort — the image pre-creates the dir; a failure here would resurface at the program's write.
@@ -130,6 +131,16 @@ export async function runProgramWorker(runId, deps) {
     // self-hosted, or on a first run). Best-effort — never fails the run.
     if (workspace !== undefined)
         await workspace.hydrate();
+    // Start desktop capture (recording + live-view) once the run has identity + a workspace, before the
+    // program runs. Best-effort — a capture failure must never fail the run.
+    if (capture !== undefined) {
+        await capture.start().catch((err) => {
+            log.warn("screen_capture_start_failed", {
+                runId,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        });
+    }
     // Token metering is PER-LEAF: each agent() leaf reports its own tokens + model to the broker, which
     // decides `billed_by_boardwalk` per model + meters usage to the platform (see leaf_executor `meterUsage`). A
     // workflow has no run-level model, so there is no run-level token metering here.
@@ -219,6 +230,16 @@ export async function runProgramWorker(runId, deps) {
                 });
             });
         }
+        // Stop capture + flush the final recording segment on EVERY terminal path, before the workspace
+        // snapshot. Best-effort — a capture failure must not mask the run's outcome.
+        if (capture !== undefined) {
+            await capture.stopAndFlush().catch((err) => {
+                log.warn("screen_capture_flush_failed", {
+                    runId,
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            });
+        }
         // Snapshot the final /workspace so the workflow's NEXT run hydrates it. Best-effort.
         if (workspace !== undefined) {
             await workspace.persist();

package/dist/runtime/runner_control_client.d.ts

@@ -156,6 +156,13 @@ export declare class RunnerControlClient {
     /** Publish a batch of live agent-event frames (the SSE live-tail source) — the broker publishes
      *  them to the run's Redis channel server-side, so the runner holds no Redis credential. */
     publishTelemetry(frames: string[]): Promise<void>;
+    /** Push a batch of encoded desktop frames (base64 JPEG) for the live-view surface — the broker
+     *  republishes them to the run's live-view channel server-side (never durably stored; the session
+     *  recording is the durable copy). See docs/SCREEN_CAPTURE.md §5. */
+    publishLiveView(frames: string[]): Promise<void>;
+    /** Is a browser currently watching this run's live-view? The capture loop polls this so it only
+     *  captures + pushes frames while someone is attached (capture costs guest CPU + metered egress). */
+    liveViewWanted(): Promise<boolean>;
     /** Store a run artifact through the broker (which holds the S3 credential + neutralizes the served
      *  content type server-side). Returns the catalog id + a signed download URL. */
     writeArtifact(input: ArtifactWriteInput): Promise<ArtifactWriteResult>;

package/dist/runtime/runner_control_client.js

@@ -283,6 +283,30 @@ export class RunnerControlClient {
         if (res.status !== 204)
             throw await brokerError(res, "telemetry");
     }
+    /** Push a batch of encoded desktop frames (base64 JPEG) for the live-view surface — the broker
+     *  republishes them to the run's live-view channel server-side (never durably stored; the session
+     *  recording is the durable copy). See docs/SCREEN_CAPTURE.md §5. */
+    async publishLiveView(frames) {
+        const res = await this.controlFetch(this.url("liveview"), {
+            method: "POST",
+            headers: this.headers(true),
+            body: JSON.stringify({ frames }),
+        });
+        if (res.status !== 204)
+            throw await brokerError(res, "liveview");
+    }
+    /** Is a browser currently watching this run's live-view? The capture loop polls this so it only
+     *  captures + pushes frames while someone is attached (capture costs guest CPU + metered egress). */
+    async liveViewWanted() {
+        const res = await this.controlFetch(this.url("liveview/wanted"), {
+            method: "GET",
+            headers: this.headers(false),
+        });
+        if (res.status !== 200)
+            throw await brokerError(res, "liveview/wanted");
+        const body = (await res.json());
+        return body.wanted === true;
+    }
     /** Store a run artifact through the broker (which holds the S3 credential + neutralizes the served
      *  content type server-side). Returns the catalog id + a signed download URL. */
     async writeArtifact(input) {

package/dist/runtime/screen_capture.d.ts

@@ -0,0 +1,74 @@
+/** A completed recording segment surfaced by the backend (a rolled or finalized-on-stop MP4 file). */
+export interface CaptureSegment {
+    /** Read the segment's bytes as base64 (for the artifact upload). */
+    read: () => Promise<string>;
+    /** Delete the segment file after it's uploaded (bulk bytes stay off the snapshot + guest disk). */
+    discard: () => Promise<void>;
+    /** Wall-clock bounds of the captured span (best-effort, the backend's clock). */
+    startedAtMs: number;
+    endedAtMs: number;
+}
+/** A running capture: emits completed segments and exposes the latest live frame. */
+export interface CaptureSession {
+    /** Register the completed-segment callback (fires per rolled segment + the final one on stop). */
+    onSegment: (cb: (segment: CaptureSegment) => void) => void;
+    /** The most recent live frame (base64 JPEG), or null before the first frame is written. */
+    latestFrame: () => Promise<string | null>;
+    /** Stop ffmpeg, finalize + emit the last in-flight segment, then resolve. */
+    stop: () => Promise<void>;
+}
+/** The guest-coupled half: starts ffmpeg on the display. Production impl in screen_capture_backend.ts. */
+export interface CaptureBackend {
+    /** Pixel dimensions of the captured display — stamped into segment metadata. */
+    readonly width: number;
+    readonly height: number;
+    /** Interval (ms) between live-frame pushes while a viewer is attached. */
+    readonly liveFrameIntervalMs: number;
+    /** How often (ms) to poll the broker for whether a viewer is attached. */
+    readonly wantedPollIntervalMs: number;
+    start: () => Promise<CaptureSession>;
+}
+/** Upload a recording segment as a run artifact (the broker holds the S3 credential). */
+export type SegmentArtifactWriter = (name: string, contentType: string, base64: string, metadata: Record<string, unknown>) => Promise<{
+    id: string;
+}>;
+export interface ScreenCaptureDeps {
+    backend: CaptureBackend;
+    /** Store a completed segment as a `recording-segment` artifact. */
+    writeArtifact: SegmentArtifactWriter;
+    /** Push encoded live frames to the broker's live-view channel. */
+    publishLiveFrames: (frames: string[]) => Promise<void>;
+    /** Whether a browser is currently watching (gates the live push loop). */
+    liveViewWanted: () => Promise<boolean>;
+    /** Injected clock (deterministic in tests). */
+    now: () => number;
+    /** Bounds `stopAndFlush()` so a hung upload can't stall a suspend indefinitely. Default 20s. */
+    flushTimeoutMs?: number;
+}
+export declare class ScreenCapture {
+    private readonly deps;
+    private session;
+    /** Monotonic across the whole run, so segments stay contiguous across suspend/resume epochs. */
+    private segmentIndex;
+    /** Serializes segment uploads so `stopAndFlush()` can await the whole in-flight tail. */
+    private uploadTail;
+    private liveLoop;
+    constructor(deps: ScreenCaptureDeps);
+    /** Begin capturing. Idempotent-safe: a second call while running is a no-op. */
+    start(): Promise<void>;
+    /** Post-wake: start a fresh capture (new segment files), keeping the monotonic segment index. */
+    startFresh(): Promise<void>;
+    /**
+     * Pre-freeze / terminal: stop capture, finalize + upload the in-flight segment, and drain the upload
+     * tail — so the recorder never spans a snapshot and the last committed segment is always playable.
+     * Bounded by `flushTimeoutMs` so a stuck upload delays (never blocks) the suspend. Safe to call with
+     * no active session.
+     */
+    stopAndFlush(): Promise<void>;
+    /** Chain a segment upload onto the tail (serialized, best-effort). */
+    private enqueueSegment;
+    private uploadSegment;
+    /** Lazy live-view push: poll whether a viewer is attached; while attached, push the latest frame at
+     *  the backend's cadence. Fully best-effort — a failed poll/push never disturbs the run. */
+    private startLiveLoop;
+}

package/dist/runtime/screen_capture.js

@@ -0,0 +1,152 @@
+// Screen capture — the runtime backing for session recording + the desktop live-view feed
+// (docs/SCREEN_CAPTURE.md §4, §5). ONE ffmpeg reads the guest display `:0` and produces two sinks:
+//
+//   1. Rolling fragmented-MP4 SEGMENTS → uploaded as `recording-segment` artifacts (the durable,
+//      scrub-able recording / DVR). Each segment is a standalone playable MP4.
+//   2. A low-fps LIVE frame (base64 JPEG) → pushed up the broker's live-view channel WHILE a viewer is
+//      attached (lazy: the capture polls `liveViewWanted`, so a run with nobody watching pays nothing
+//      beyond the always-on recording).
+//
+// The process spawn + segment file management live behind the injected `CaptureBackend` seam
+// (screen_capture_backend.ts, guest-coupled: ffmpeg + fs), so this orchestration — segment upload
+// sequencing, the lazy live loop, and the freeze-flush contract — is unit-tested without a guest.
+//
+// Suspend/resume (North Star): the recorder NEVER spans a snapshot. `stopAndFlush()` runs in the
+// worker's pre-freeze hook (finalize + upload the in-flight segment, then the VM freezes); `startFresh()`
+// runs post-wake (a new segment epoch). So a suspend/resume boundary is always a segment boundary, and
+// no encoder ever wakes to a wall clock days ahead of its stream.
+import { createLogger } from "./support/index.js";
+const log = createLogger("screen_capture");
+const DEFAULT_FLUSH_TIMEOUT_MS = 20_000;
+export class ScreenCapture {
+    deps;
+    session = null;
+    /** Monotonic across the whole run, so segments stay contiguous across suspend/resume epochs. */
+    segmentIndex = 0;
+    /** Serializes segment uploads so `stopAndFlush()` can await the whole in-flight tail. */
+    uploadTail = Promise.resolve();
+    liveLoop = null;
+    constructor(deps) {
+        this.deps = deps;
+    }
+    /** Begin capturing. Idempotent-safe: a second call while running is a no-op. */
+    async start() {
+        if (this.session !== null)
+            return;
+        const session = await this.deps.backend.start();
+        this.session = session;
+        session.onSegment((segment) => this.enqueueSegment(segment));
+        this.startLiveLoop(session);
+    }
+    /** Post-wake: start a fresh capture (new segment files), keeping the monotonic segment index. */
+    async startFresh() {
+        await this.start();
+    }
+    /**
+     * Pre-freeze / terminal: stop capture, finalize + upload the in-flight segment, and drain the upload
+     * tail — so the recorder never spans a snapshot and the last committed segment is always playable.
+     * Bounded by `flushTimeoutMs` so a stuck upload delays (never blocks) the suspend. Safe to call with
+     * no active session.
+     */
+    async stopAndFlush() {
+        const session = this.session;
+        if (session === null)
+            return;
+        this.session = null;
+        this.liveLoop?.stop();
+        this.liveLoop = null;
+        const flush = (async () => {
+            try {
+                await session.stop(); // finalizes the last segment → enqueued via onSegment
+            }
+            catch (err) {
+                log.warn("screen_capture_stop_failed", { error: errMsg(err) });
+            }
+            await this.uploadTail; // drain all queued segment uploads (incl. the final one)
+        })();
+        const timeoutMs = this.deps.flushTimeoutMs ?? DEFAULT_FLUSH_TIMEOUT_MS;
+        let timer;
+        const bound = new Promise((resolve) => {
+            timer = setTimeout(() => {
+                log.warn("screen_capture_flush_timeout", { timeoutMs });
+                resolve();
+            }, timeoutMs);
+        });
+        try {
+            await Promise.race([flush, bound]);
+        }
+        finally {
+            if (timer !== undefined)
+                clearTimeout(timer);
+        }
+    }
+    /** Chain a segment upload onto the tail (serialized, best-effort). */
+    enqueueSegment(segment) {
+        const index = this.segmentIndex++;
+        this.uploadTail = this.uploadTail.then(() => this.uploadSegment(segment, index));
+    }
+    async uploadSegment(segment, index) {
+        try {
+            const base64 = await segment.read();
+            await this.deps.writeArtifact(`recording-${String(index).padStart(5, "0")}.mp4`, "video/mp4", base64, {
+                kind: "recording-segment",
+                segment_index: index,
+                wall_start: segment.startedAtMs,
+                wall_end: segment.endedAtMs,
+                width: this.deps.backend.width,
+                height: this.deps.backend.height,
+            });
+        }
+        catch (err) {
+            // Recording is observability, not a ledger: a dropped segment must never fail the run.
+            log.warn("recording_segment_upload_failed", { index, error: errMsg(err) });
+        }
+        finally {
+            await segment.discard().catch(() => undefined);
+        }
+    }
+    /** Lazy live-view push: poll whether a viewer is attached; while attached, push the latest frame at
+     *  the backend's cadence. Fully best-effort — a failed poll/push never disturbs the run. */
+    startLiveLoop(session) {
+        let stopped = false;
+        let wanted = false;
+        let sincePollMs = Number.POSITIVE_INFINITY; // force a poll on the first tick
+        const { liveFrameIntervalMs, wantedPollIntervalMs } = this.deps.backend;
+        const tick = async () => {
+            if (stopped)
+                return;
+            try {
+                if (sincePollMs >= wantedPollIntervalMs) {
+                    wanted = await this.deps.liveViewWanted();
+                    sincePollMs = 0;
+                }
+                if (wanted) {
+                    const frame = await session.latestFrame();
+                    if (frame !== null && frame.length > 0) {
+                        await this.deps.publishLiveFrames([frame]);
+                    }
+                }
+            }
+            catch (err) {
+                log.debug("live_view_tick_failed", { error: errMsg(err) });
+            }
+            sincePollMs += liveFrameIntervalMs;
+            if (!stopped) {
+                timer = setTimeout(() => void tick(), liveFrameIntervalMs);
+                timer.unref();
+            }
+        };
+        let timer = setTimeout(() => void tick(), liveFrameIntervalMs);
+        timer.unref();
+        this.liveLoop = {
+            stop: () => {
+                stopped = true;
+                if (timer !== undefined)
+                    clearTimeout(timer);
+            },
+        };
+    }
+}
+function errMsg(err) {
+    return err instanceof Error ? err.message : String(err);
+}

package/dist/runtime/screen_capture_backend.d.ts

@@ -0,0 +1,19 @@
+import type { CaptureBackend } from "./screen_capture.js";
+export interface CaptureConfig {
+    /** X display to grab (DISPLAY, default ":0"). */
+    display: string;
+    /** Recording capture frame rate (fps). */
+    fps: number;
+    /** Screen dimensions (must match the ambient desktop — SCREEN_CAPTURE §1.3). */
+    width: number;
+    height: number;
+    /** Recording segment roll interval (seconds). A suspend/resume also forces a boundary. */
+    segmentSeconds: number;
+    /** Live-view push cadence (ms) — how often the latest frame is pushed while a viewer is attached. */
+    liveFrameIntervalMs: number;
+    /** How often (ms) the capture polls the broker for viewer presence. */
+    wantedPollIntervalMs: number;
+}
+/** Read the capture config from env, or null when the desktop stack is absent or recording is off. */
+export declare function loadCaptureConfig(env: NodeJS.ProcessEnv): CaptureConfig | null;
+export declare function makeCaptureBackend(cfg: CaptureConfig): CaptureBackend;

package/dist/runtime/screen_capture_backend.js

@@ -0,0 +1,212 @@
+// The guest-coupled half of screen capture (screen_capture.ts's `CaptureBackend`): one ffmpeg reading
+// the X display `:0` with two outputs — rolling MP4 segments (recording) and a single low-fps JPEG that
+// is continuously overwritten (the live-view frame). See docs/SCREEN_CAPTURE.md §4.
+//
+// Only runs where the runner IMAGE ships the desktop stack (Xvfb + ffmpeg), gated by
+// BOARDWALK_BROWSER_TIER=1 (the same "desktop present" signal the browser tier uses) + a
+// BOARDWALK_RECORDING_ENABLED kill switch (default on). Off on Fargate / self-hosted images with no
+// display, where `loadCaptureConfig` returns null and no capture is constructed.
+//
+// This layer is validated by a local ffmpeg smoke + the substrate E2E (it needs a real X display), the
+// same way browser_session_backend.ts is — the unit tests cover screen_capture.ts's pure orchestration.
+import { spawn } from "node:child_process";
+import { mkdtemp, readdir, readFile, rm, unlink } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { browserTierEnabled } from "./browser_session_backend.js";
+import { createLogger } from "./support/index.js";
+const log = createLogger("screen_capture_backend");
+const SEGMENT_PREFIX = "rec-";
+const LIVE_FRAME_FILE = "live.jpg";
+/** JPEG end-of-image marker — a complete frame ends with these bytes; used to skip a torn read. */
+const JPEG_EOI = Buffer.from([0xff, 0xd9]);
+function intFromEnv(raw, fallback) {
+    const n = Number(raw);
+    return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
+}
+/** Read the capture config from env, or null when the desktop stack is absent or recording is off. */
+export function loadCaptureConfig(env) {
+    // The desktop-present signal is the browser tier flag; without a display there is nothing to grab.
+    if (!browserTierEnabled(env))
+        return null;
+    // Kill switch (default on): BOARDWALK_RECORDING_ENABLED=0 disables recording + live-view capture.
+    if (env.BOARDWALK_RECORDING_ENABLED === "0")
+        return null;
+    return {
+        display: env.DISPLAY?.trim() || ":0",
+        fps: intFromEnv(env.BOARDWALK_RECORDING_FPS, 6),
+        width: intFromEnv(env.BOARDWALK_SCREEN_WIDTH, 1280),
+        height: intFromEnv(env.BOARDWALK_SCREEN_HEIGHT, 800),
+        segmentSeconds: intFromEnv(env.BOARDWALK_RECORDING_SEGMENT_SECONDS, 240),
+        liveFrameIntervalMs: intFromEnv(env.BOARDWALK_LIVEVIEW_FRAME_INTERVAL_MS, 1000),
+        wantedPollIntervalMs: intFromEnv(env.BOARDWALK_LIVEVIEW_WANTED_POLL_MS, 3000),
+    };
+}
+/** ffmpeg args: one x11grab input, two outputs (segmented MP4 + a single overwritten JPEG). */
+function ffmpegArgs(cfg, dir) {
+    const liveFps = Math.max(1, Math.round(1000 / cfg.liveFrameIntervalMs));
+    return [
+        "-y",
+        "-loglevel",
+        "error",
+        "-f",
+        "x11grab",
+        "-framerate",
+        String(cfg.fps),
+        "-video_size",
+        `${String(cfg.width)}x${String(cfg.height)}`,
+        "-i",
+        cfg.display,
+        // Recording: H.264 MP4 segments, each a standalone playable file (docs/SCREEN_CAPTURE.md §4.2).
+        "-map",
+        "0:v",
+        "-c:v",
+        "libx264",
+        "-preset",
+        "veryfast",
+        "-crf",
+        "28",
+        "-pix_fmt",
+        "yuv420p",
+        "-g",
+        String(cfg.fps * 2),
+        "-f",
+        "segment",
+        "-segment_time",
+        String(cfg.segmentSeconds),
+        "-reset_timestamps",
+        "1",
+        "-segment_format",
+        "mp4",
+        join(dir, `${SEGMENT_PREFIX}%05d.mp4`),
+        // Live-view: a single low-fps JPEG, continuously overwritten (the latest frame).
+        "-map",
+        "0:v",
+        "-r",
+        String(liveFps),
+        "-q:v",
+        "6",
+        "-update",
+        "1",
+        join(dir, LIVE_FRAME_FILE),
+    ];
+}
+/** The recording-segment index from a `rec-00007.mp4` filename, or null if it doesn't match. */
+function segmentIndexOf(name) {
+    const m = /^rec-(\d+)\.mp4$/.exec(name);
+    if (m === null || m[1] === undefined)
+        return null;
+    const n = Number(m[1]);
+    return Number.isFinite(n) ? n : null;
+}
+export function makeCaptureBackend(cfg) {
+    return {
+        width: cfg.width,
+        height: cfg.height,
+        liveFrameIntervalMs: cfg.liveFrameIntervalMs,
+        wantedPollIntervalMs: cfg.wantedPollIntervalMs,
+        async start() {
+            const dir = await mkdtemp(join(tmpdir(), "bw-capture-"));
+            const proc = spawn("ffmpeg", ffmpegArgs(cfg, dir), {
+                stdio: ["ignore", "ignore", "inherit"],
+            });
+            proc.once("error", (err) => log.error("ffmpeg_spawn_error", { error: err.message }));
+            let onSegmentCb = null;
+            const emitted = new Set();
+            // A ffmpeg segment is COMPLETE once the next-index file exists (or once ffmpeg exits). Poll the
+            // dir and emit every rec file below the current highest index that hasn't been emitted yet.
+            let lastEmitAtMs = Date.now();
+            const makeSegment = (index) => {
+                const path = join(dir, `${SEGMENT_PREFIX}${String(index).padStart(5, "0")}.mp4`);
+                const startedAtMs = lastEmitAtMs;
+                const endedAtMs = Date.now();
+                lastEmitAtMs = endedAtMs;
+                return {
+                    startedAtMs,
+                    endedAtMs,
+                    read: async () => (await readFile(path)).toString("base64"),
+                    discard: () => unlink(path).catch(() => undefined),
+                };
+            };
+            const sweep = async (includeHighest) => {
+                let names;
+                try {
+                    names = await readdir(dir);
+                }
+                catch {
+                    return;
+                }
+                const indices = names
+                    .map(segmentIndexOf)
+                    .filter((n) => n !== null)
+                    .sort((a, b) => a - b);
+                if (indices.length === 0)
+                    return;
+                const highest = indices[indices.length - 1] ?? 0;
+                for (const index of indices) {
+                    // While running, the highest-index file is still being written — hold it back until the next
+                    // one appears (or until stop, when `includeHighest` releases it).
+                    if (!includeHighest && index === highest)
+                        continue;
+                    if (emitted.has(index))
+                        continue;
+                    emitted.add(index);
+                    onSegmentCb?.(makeSegment(index));
+                }
+            };
+            const poll = setInterval(() => void sweep(false), 2000);
+            poll.unref();
+            return {
+                onSegment(cb) {
+                    onSegmentCb = cb;
+                },
+                async latestFrame() {
+                    try {
+                        const bytes = await readFile(join(dir, LIVE_FRAME_FILE));
+                        // Skip a torn read (ffmpeg mid-overwrite) — a complete JPEG ends with the EOI marker.
+                        if (bytes.length < 2 || !bytes.subarray(-2).equals(JPEG_EOI))
+                            return null;
+                        return bytes.toString("base64");
+                    }
+                    catch {
+                        return null;
+                    }
+                },
+                async stop() {
+                    clearInterval(poll);
+                    await stopFfmpeg(proc);
+                    // ffmpeg finalized the in-flight segment on exit — release every remaining segment, then
+                    // clean up the live frame + the scratch dir (segment files are discarded post-upload).
+                    await sweep(true);
+                    await rm(join(dir, LIVE_FRAME_FILE)).catch(() => undefined);
+                    // The dir itself is removed after uploads discard the segment files; a best-effort rm here
+                    // clears anything left (empty dir / an un-uploaded torn segment) without blocking.
+                    setTimeout(() => void rm(dir, { recursive: true, force: true }).catch(() => undefined), 30_000).unref();
+                },
+            };
+        },
+    };
+}
+/** SIGINT ffmpeg so it flushes the current segment's moov atom, then wait (bounded) for it to exit. */
+async function stopFfmpeg(proc) {
+    if (proc.exitCode !== null || proc.signalCode !== null)
+        return;
+    await new Promise((resolve) => {
+        const done = () => {
+            clearTimeout(kill);
+            resolve();
+        };
+        proc.once("exit", done);
+        proc.kill("SIGINT");
+        // If ffmpeg doesn't exit promptly, SIGKILL it (the last segment may be lost — acceptable).
+        const kill = setTimeout(() => {
+            try {
+                proc.kill("SIGKILL");
+            }
+            catch {
+                // already gone
+            }
+        }, 5000);
+        kill.unref();
+    });
+}

package/dist/runtime/tools/artifacts.d.ts

@@ -106,6 +106,10 @@ export interface ArtifactPresignInput {
     name: string;
     contentType: string;
     sizeBytes: number;
+    /** Passed through so the broker can decide storage-quota exemption + the retention tag at presign
+     *  time (e.g. session-recording segments are quota-exempt and get a shorter retention). The catalog
+     *  row's metadata is still set from the commit call; this is only for the presign-time decisions. */
+    metadata?: Record<string, unknown>;
 }
 /** The broker's presign response: where + how to PUT the bytes, and the chosen `s3Key` the worker
  *  echoes back at commit. No catalog id / download URL yet — the row doesn't exist until the upload

package/dist/runtime/wire/inference_proxy.d.ts

@@ -47,8 +47,9 @@ export type InferenceFrame = {
 } | {
     kind: "ping";
 };
-/** Serialize the request body. (Binary message content — multimodal image/doc bytes — is not yet
- *  supported on this path; v0 agent flows are text + JSON tool results.) */
+/** Serialize the request body. Multimodal image content rides transparently: message content parts
+ *  (incl. `{ type: "image" }`) are plain JSON here — the broker's engine adapters render them per
+ *  provider (@boardwalk-labs/engine ≥ 0.1.32). */
 export declare function serializeInferenceRequest(req: InferenceProxyRequest): string;
 /** Parse + minimally validate the request body (the runner is semi-trusted; fail closed on shape).
  *  The conversation is shape-trusted past the array check — it was built by the engine loop on the

package/dist/runtime/wire/inference_proxy.js

@@ -31,8 +31,9 @@ export function runnerInferencePath(runId) {
 /** Response content type — newline-delimited JSON frames. */
 export const INFERENCE_NDJSON_CONTENT_TYPE = "application/x-ndjson";
 // ---- request serialize / parse ----
-/** Serialize the request body. (Binary message content — multimodal image/doc bytes — is not yet
- *  supported on this path; v0 agent flows are text + JSON tool results.) */
+/** Serialize the request body. Multimodal image content rides transparently: message content parts
+ *  (incl. `{ type: "image" }`) are plain JSON here — the broker's engine adapters render them per
+ *  provider (@boardwalk-labs/engine ≥ 0.1.32). */
 export function serializeInferenceRequest(req) {
     return JSON.stringify(req);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boardwalk-labs/runner",
-  "version": "0.1.13",
+  "version": "0.1.16",
   "description": "Boardwalk self-hosted runner: execute cloud-scheduled runs on your own machines. Currently: the canonical runner contract types.",
   "license": "Apache-2.0",
   "repository": {
@@ -53,8 +53,8 @@
     "prebuild": "prebuildify --napi --strip -t 24.0.0"
   },
   "dependencies": {
-    "@boardwalk-labs/engine": "^0.1.31",
-    "@boardwalk-labs/workflow": "^0.1.23",
+    "@boardwalk-labs/engine": "^0.1.34",
+    "@boardwalk-labs/workflow": "^0.1.25",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "node-gyp-build": "^4.8.4",
     "tar": "^7.5.16",