@boardwalk-labs/runner 0.1.2

package/dist/runtime/index.js

@@ -0,0 +1,525 @@
+// Worker composition root (docs/WORKFLOW_RUNTIME.md + docs/RUNNER_BROKER.md). The Boardwalk-hosted
+// worker container runs `node dist/fargate/worker/index.js`; the dispatcher launches it per run with
+// RUN_ID + the per-run control-plane handle. This file assembles real implementations behind every
+// seam `runProgramWorker(runId, deps)` injects, then runs the one workflow program to terminal.
+//
+// BROKERED-ONLY (security review #1): the runner is UNTRUSTED and holds NO platform credential — only
+// its short-lived run token. EVERY privileged seam goes through the Runner Control API (the broker):
+//   - agent()         → EngineLeafExecutor: the OSS engine's loop (runAgentLeaf) over a broker-backed
+//                       LeafIo — the model turn streams through /inference (invoked server-side).
+//   - sleep()         → WorkerWorkflowHost in-process hold (hold-and-pay; no checkpoint/exit).
+//   - secrets.get()   → broker /secrets/resolve (allowlist enforced server-side).
+//   - workflows.call()→ broker /children (durable child run, hold + poll).
+//   - events.emit()   → broker /events (fan-out server-side).
+//   - artifacts / web_search → broker /artifacts + /tools/web_search (no S3 / Tavily creds here).
+//   - lifecycle / usage / telemetry → broker /claim,/version,/finalize,/usage,/telemetry.
+// There is no DB / Redis / Stripe / Bedrock client on the runner — so its task role is near-zero and
+// the metadata-endpoint escape has nothing to steal. The legacy direct (pre-broker) path is removed.
+//
+// Split: `assembleWorkerDeps(runtime)` is pure wiring (unit-tested with a fake fetch); `main()` is the
+// bootstrap shell (read the control-plane env, install signal handlers).
+//
+// Per-session loops wired here (all brokered): a UsageFlusher meters token deltas (§10.7 →
+// /usage/tokens), a CreditWatcher polls funding (§15 → /credit, aborting the run on exhaustion), and a
+// CancelWatcher polls for a user cancel (§6 → /cancel, aborting the run when the user cancels — the
+// brokered worker holds no Redis, so it can't receive the api-server's cancel publish directly).
+//
+// Documented v0 deferral (a clear seam): durable events — a no-op AgentEventStore (live Redis fan-out
+// via the broker only) until the run_step_events table lands.
+import { createLogger, newId, AppError, ErrorCode, DEFAULT_LEASE_MS } from "./support/index.js";
+import { LspService } from "@boardwalk-labs/engine/core";
+import { BudgetMeter } from "./agent/budget.js";
+import { WorkerRunEventEmitter } from "./run_event_emitter.js";
+import { BUDGET_GUARDRAIL_RATE } from "./agent/model_rates.js";
+import { SecretRedactor } from "./agent/secret_redactor.js";
+import { RecordingSecretResolver } from "./recording_secret_resolver.js";
+import { EngineLeafExecutor } from "./leaf_executor.js";
+import { parseByoProviders } from "./direct_inference.js";
+import { WorkerWorkflowHost } from "./workflow_host.js";
+import { runProgramWorker, } from "./program_worker.js";
+import { RunnerControlClient } from "./runner_control_client.js";
+import { BrokerChildDispatcher } from "./broker_child_dispatcher.js";
+import { BrokerEventPublisher } from "./broker_event_publisher.js";
+import { createProgramLogSink } from "./program_log_capture.js";
+import { BrokerArtifactStore } from "./broker_artifact_store.js";
+import { buildBrokerToolHost } from "./broker_tool_host.js";
+import { CreditWatcher } from "./credit_watcher.js";
+import { CancelWatcher } from "./cancel_watcher.js";
+import { LeaseRenewer } from "./lease_renewer.js";
+import { RuntimeFlusher } from "./runtime_flusher.js";
+import { WorkspaceStore, TarWorkspaceArchiver, NodeWorkspaceFs } from "./workspace_store.js";
+import { PhaseTracker } from "./phase_tracker.js";
+import { mkdir } from "node:fs/promises";
+import { join } from "node:path";
+const log = createLogger("worker_entrypoint");
+/** Durable events are deferred (run_step_events table) — live fan-out via the broker only. */
+/** Synthesize the run's AuthContext. The run was already authorized at trigger time; the
+ *  worker acts on the org's behalf, so it carries the org + an owner role. Tool-level
+ *  boundaries (the broker's server-side manifest allowlist) are the real guard.
+ *
+ *  source='workflow' (NOT 'session_jwt'): the program must never perform SESSION_JWT_ONLY
+ *  credential mutations (§12.11), so a tool that ever exposed such a service is denied by
+ *  construction regardless of the owner role. */
+export function workerAuthContext(run) {
+    const userId = run.actor.type === "user" ? run.actor.user_id : `workflow:${run.workflowId}`;
+    return { userId, orgId: run.orgId, role: "owner", source: "workflow" };
+}
+/**
+ * Sequencer for per-call token-metering identifiers, `<runId>:<sessionId>:<leafIndex>:<turnSeq>`.
+ *
+ * The worker's `meterUsage` hook fires once per model TURN (the engine's leaf loop reports usage
+ * after every model call), but the display aggregate is idempotent on this identifier
+ * (UsageEventRepository.recordAndAggregate → `onConflictDoNothing`). A key that didn't vary per call
+ * (it was `<runId>:<sessionId>:<leafIndex>`) collapsed every turn after a leaf's first into a dropped
+ * duplicate — so `runs.tokens_in/out` reflected only each leaf's FIRST turn and under-reported
+ * multi-turn leaves by orders of magnitude, even as the BudgetMeter (which accumulates every turn)
+ * still tripped on the real spend. Bumping a per-session counter gives every turn its own idempotent
+ * event. Restart safety rides the session id (a fresh `meteringSessionId` per worker session) — a
+ * restart genuinely re-spends inference, so it must re-meter rather than dedupe against a prior run.
+ */
+export function tokenMeterIdentifiers(runId, sessionId) {
+    let seq = 0;
+    return (leafIndex) => `${runId}:${sessionId}:${String(leafIndex)}:${String(seq++)}`;
+}
+/** Pure wiring: build the full ProgramWorkerDeps from the control-plane handle. The runner reaches
+ *  every privileged seam through the broker over its run token — no DB / Redis / Stripe / model creds. */
+export function assembleWorkerDeps(runtime) {
+    const broker = new RunnerControlClient({
+        baseUrl: runtime.controlPlane.baseUrl,
+        runToken: runtime.controlPlane.runToken,
+        runId: runtime.runId,
+    });
+    log.info("runner_control_enabled", { runId: runtime.runId });
+    // The replay frontier (the highest journaled seq at claim) drives SILENT REPLAY (docs/SUSPENSION.md):
+    // a resumed run suppresses observability for seams it already ran. Captured at claim, read by
+    // buildHost (which runs right after claim) when constructing the per-run host.
+    let replayFrontier = 0;
+    // Live agent-event stream → /telemetry (broker publishes to Redis server-side). Inference,
+    // web_search, artifacts, and the model call are all brokered — no Redis / model creds / Tavily / S3.
+    const eventPublisher = new BrokerEventPublisher({
+        send: (frames) => broker.publishTelemetry(frames),
+    });
+    // The run's ONE event envelope counter (v1 wire format, engine parity): phases, program logs,
+    // declared output, and every agent turn stamp through this emitter, so cursors are run-globally
+    // monotonic with no separate program band. Frames publish as `{cursor, event}` rows via the
+    // broker telemetry path. The claim wrapper below bumps it past a previous session's frames.
+    const runEvents = new WorkerRunEventEmitter({ runId: runtime.runId, publisher: eventPublisher });
+    const phaseTracker = new PhaseTracker({ sink: runEvents });
+    // The run's public-API credential (was BOARDWALK_API_KEY). It is NO LONGER in process.env — the
+    // bootstrap captured + scrubbed it (capturePlatformContext) so the agent's bash / subprocesses
+    // can't read it. The program reaches it ONLY via `runtime.apiToken()` (built below); we still
+    // record it in each run's redactor so a value the program threads into a tool can't echo back out
+    // of a tool result. Absent in the dev no-signing-key path. (docs/RUN_ENV_AND_CREDS.md)
+    const runApiKey = runtime.controlPlane.apiToken;
+    // The broker (Runner Control API) is hosted on the api-server, so the public API shares its
+    // origin; `runtime.apiUrl` exposes it and the program appends `/v1` or `/mcp/v1`.
+    const apiUrl = publicApiOrigin(runtime.controlPlane.baseUrl);
+    // Per-run host: agent() leaf + sleep-hold + secrets + children + events, all brokered. `signal`
+    // carries cooperative cancellation (credit exhaustion) into every host hook.
+    const buildHost = (run, manifest, signal) => {
+        const budget = new BudgetMeter({
+            ...(manifest.budget !== undefined ? { budget: manifest.budget } : {}),
+            // Fallback budget-cap rate. A managed turn caps on the broker's EXACT upstream cost (forwarded on
+            // the inference result frame → BudgetMeter `realCostUsd`); this representative sonnet-class rate
+            // applies only to a turn with no upstream cost (BYO / unavailable). `max_usd` is a guardrail, not
+            // the bill — that's per-leaf cost pass-through at the broker.
+            rate: BUDGET_GUARDRAIL_RATE,
+            startedAt: Date.now(),
+            // deadline_seconds is WALL-CLOCK from the run's ORIGINAL start (incl. suspended idle), so a run
+            // resumed past its deadline trips on its first cap check. Falls back to now on the first session
+            // (startedAt is being set this session anyway).
+            deadlineStartedAt: run.startedAt ?? Date.now(),
+        });
+        // Per-session id for per-turn metering identifiers (`<runId>:<sessionId>:<leafIndex>:<turnSeq>`) —
+        // a fresh value per worker session keeps a resumed run's events from colliding with a prior
+        // session's (a restart genuinely re-spends, so it must re-meter).
+        const meteringSessionId = newId();
+        // Mint a unique identifier per metering call. `meterUsage` fires once per model TURN, so the key
+        // must vary per call: the display aggregate is idempotent on it (see {@link tokenMeterIdentifiers}).
+        const nextMeterIdentifier = tokenMeterIdentifiers(run.id, meteringSessionId);
+        // Per-run secret boundary: every value resolved (program `secrets.get` OR a tool's
+        // ctx.secrets.resolve) is recorded into one shared redactor; the leaf seeds a fresh engine
+        // Redactor from it so the loop scrubs those values out of all model-bound content. MASTER_SPEC §12.
+        const redactor = new SecretRedactor();
+        // Record the run's own API key so a prompt-injected agent can't echo it back to the model.
+        if (runApiKey !== undefined && runApiKey !== "")
+            redactor.record(runApiKey);
+        const secretResolver = new RecordingSecretResolver({ resolve: (ref) => broker.resolveSecret(ref.name) }, redactor);
+        // The extracted program dir isn't known until the runner unpacks the artifact (mid-run); the leaf
+        // reads it through this holder, which `setProgramDir` (returned below → runner `onExtracted`) fills.
+        let programDir = null;
+        // Broker-backed artifact store (shared by the program's artifacts.write hook AND the engine's
+        // host-backed `artifacts` tool). Presigned for large bodies; the runner holds no S3 creds.
+        const artifactStore = new BrokerArtifactStore(broker);
+        // Backend for the engine's host-backed built-in tools (webfetch/web_search/artifacts): web_search
+        // + artifact read-back go through the broker (no Tavily/S3 creds here); webfetch is an in-process
+        // fetch already gated by the worker's egress proxy.
+        const toolHost = buildBrokerToolHost({ broker, artifacts: artifactStore });
+        // ONE engine-native LSP per RUN (not per leaf): the language server must persist across the run's
+        // edits/leaves so it stays warm. The engine reads `capabilities.lspService` to light up the
+        // `diagnostics` tool + diagnostics-after-edit; it spawns `typescript-language-server` in the worker
+        // and degrades gracefully when that binary isn't in the image (a separate runner-image change adds
+        // it). Workspace-rooted at the same `/workspace` the leaf + tools use. Closed on the run's teardown
+        // (returned below as `lsp` → program_worker's finally) so no language-server process leaks.
+        const lspService = new LspService({ workspaceDir: runtime.workspaceRoot });
+        const leaf = new EngineLeafExecutor({
+            inference: broker,
+            // Direct BYO (D7): the claim's registry + the run's RECORDING resolver, so a provider key
+            // registers with the redactor the moment it resolves.
+            ...(runtime.byoProviders !== undefined && runtime.byoProviders.length > 0
+                ? {
+                    byo: {
+                        registry: runtime.byoProviders,
+                        resolveSecret: (name) => secretResolver.resolve({ name }),
+                    },
+                }
+                : {}),
+            budget,
+            redactor,
+            toolHost,
+            lspService,
+            workspaceRoot: runtime.workspaceRoot,
+            // Resolve this run's deployed `skills/` dir for `agent({ skills })` — the engine reads
+            // `<skillsDir>/<name>.md`, so it's the `skills` subdir of the extracted program tree (filled
+            // once the artifact extracts). Null until then / when no program dir is known.
+            skillsDir: () => (programDir === null ? null : join(programDir, "skills")),
+            // The extracted program tree IS the workflow PACKAGE root on hosted runs: the deploy artifact
+            // ships the bundled `AGENTS.md` at its root (and `skills/` beside it), so the engine reads
+            // `<programDir>/AGENTS.md` for the bundled tier. Same holder the `skillsDir` thunk reads — null
+            // until the artifact extracts. The empty `/workspace` and this dir are SEPARATE dirs on hosted,
+            // exactly the two-tier case the engine's AGENTS.md loader is built for.
+            programDir: () => programDir,
+            // Per-leaf, per-model token metering — reported THROUGH the broker (the worker holds no Stripe
+            // credential); the broker decides `billed_by_boardwalk` per model + meters to Stripe (BYO models
+            // no-op there). Fire-and-forget: a metering hiccup must never fail the run.
+            meterUsage: ({ model, inputTokens, outputTokens, cachedReadTokens, cachedWriteTokens, leafIndex, }) => {
+                void broker
+                    .meterTokens({
+                    inputTokens,
+                    outputTokens,
+                    model,
+                    identifier: nextMeterIdentifier(leafIndex),
+                    ...(cachedReadTokens === undefined ? {} : { cachedReadTokens }),
+                    ...(cachedWriteTokens === undefined ? {} : { cachedWriteTokens }),
+                })
+                    .catch((err) => {
+                    log.warn("leaf_usage_meter_failed", {
+                        runId: run.id,
+                        error: err instanceof Error ? err.message : String(err),
+                    });
+                });
+            },
+            // Turn numbering is owned by the shared emitter (beginTurn opens the stride block); the leaf
+            // index stays only in metering identifiers. `identity` names the leaf on its turn frames.
+            makeEventSink: (_leafIndex, _identity) => runEvents,
+            // Broker an OAuth bearer for an `agent({ mcp })` server (called reactively on a 401). The token
+            // comes from the org's connection vault via the Runner Control API — never stored on the worker.
+            brokerMcpToken: (serverUrl, invalidateToken) => broker.mcpToken(serverUrl, invalidateToken),
+        });
+        // Per-workflow persistent /workspace (§5): only when the manifest opts in. The BROKER additionally
+        // gates on hosted (self-hosted runs get null URLs), and snapshots are keyed per-workflow + scoped
+        // by the run token — so even the untrusted in-process program can't reach another tenant's data.
+        const workspaceStore = manifest.workspace?.persist === true
+            ? new WorkspaceStore({
+                broker,
+                archiver: new TarWorkspaceArchiver(),
+                fs: new NodeWorkspaceFs(),
+                workspaceRoot: runtime.workspaceRoot,
+            })
+            : undefined;
+        // Durable suspension (docs/SUSPENSION.md): the host raises a suspend OUT OF BAND (onSuspend +
+        // a never-resolving seam), so a program's own try/catch can't swallow it; the program runner
+        // races `suspendSignal` against the body. One deferred per run — the first seam to suspend wins.
+        let resolveSuspend = () => undefined;
+        const suspendSignal = new Promise((resolve) => {
+            resolveSuspend = resolve;
+        });
+        // The run's identity + on-demand public-API bearer, surfaced to the program via `import { runtime }`.
+        // ids come from the claimed run; the bearer is the captured (scrubbed-from-env) api token, already
+        // recorded in this run's redactor above — so threading it into an MCP header keeps it out of LLM
+        // context. Throws clearly when a run was provisioned no api token (dev no-signing-key path).
+        const runtimeContext = {
+            runId: run.id,
+            workflowId: run.workflowId,
+            orgId: run.orgId,
+            apiUrl,
+            apiToken: () => {
+                if (runApiKey === undefined || runApiKey === "") {
+                    return Promise.reject(new AppError(ErrorCode.FORBIDDEN, "runtime.apiToken() is unavailable: this run was not provisioned a public-API token"));
+                }
+                return Promise.resolve(runApiKey);
+            },
+        };
+        const host = new WorkerWorkflowHost({
+            leaf,
+            runtime: runtimeContext,
+            // workflows.call → broker /children (resolve + callable_by gate server-side), bound to THIS run.
+            children: new BrokerChildDispatcher({ client: broker }),
+            // The durable-seam journal (memoization + replay) over the broker; the replay frontier drives
+            // silent replay; onSuspend resolves the deferred the program runner races.
+            journal: broker.journalSeam(),
+            replayFrontier,
+            onSuspend: (signal) => {
+                resolveSuspend(signal);
+            },
+            // The program's secrets.get(name) → the run's fail-closed RECORDING resolver.
+            secrets: { get: (name) => secretResolver.resolve({ name }) },
+            // events.emit → broker /events (fan-out server-side).
+            // artifacts.write → broker artifact store (presigned; the runner holds no S3 creds). Text ships
+            // as utf8, raw bytes as base64; the stored artifact's signed URL surfaces as ArtifactRef.url.
+            writeArtifact: (name, contentType, body, metadata) => artifactStore
+                .write({
+                name,
+                contentType,
+                body: typeof body === "string" ? body : Buffer.from(body).toString("base64"),
+                encoding: typeof body === "string" ? "utf8" : "base64",
+                ...(metadata !== undefined ? { metadata } : {}),
+            })
+                .then((res) => ({ id: res.id, name: res.name, url: res.signedUrl })),
+            // Cooperative cancellation: every host hook honors this (credit exhaustion aborts it).
+            signal,
+            // Snapshot the workspace before a long sleep (crash-during-hold recovery). The byte count
+            // is only logged by the store itself.
+            ...(workspaceStore !== undefined
+                ? {
+                    onBeforeSleep: async () => {
+                        await workspaceStore.persist();
+                    },
+                }
+                : {}),
+            phases: phaseTracker,
+        });
+        // Hand the redactor back too: the worker scrubs a terminal error's message with it (review #5);
+        // workspace (when opted in) hydrates at start + persists at terminal.
+        return Promise.resolve({
+            host,
+            redactor,
+            phases: phaseTracker,
+            // The orchestrator closes the per-run LSP on every terminal path (close() is idempotent + never
+            // throws) so no language-server process leaks past the run.
+            lsp: lspService,
+            // The orchestrator emits the program's declared output onto the wire (`output` kind).
+            activity: { output: (value) => void runEvents.emit({ kind: "output", value }) },
+            // Filled by the runner once the artifact extracts; the leaf's `skillsDir` thunk reads it.
+            setProgramDir: (dir) => {
+                programDir = dir;
+            },
+            // Resolves when a host seam suspends the run; the program runner races it against the body.
+            suspendSignal,
+            ...(workspaceStore !== undefined ? { workspace: workspaceStore } : {}),
+        });
+    };
+    return {
+        runs: {
+            // The broker owns the lease duration server-side; convert the absolute lease back to seconds.
+            claimForWorker: async (_runId, workerId, leaseUntil, nowMs) => {
+                const claimed = await broker.claim(workerId, Math.max(1, Math.round((leaseUntil - nowMs) / 1000)));
+                if (claimed === null)
+                    return null;
+                // Order this session's frames after a previous (crashed) session's, then announce the
+                // lifecycle transition the claim IS — the wire's `running` frame.
+                runEvents.resumeAfter(claimed.lastEventCursor);
+                // The replay frontier for a resumed run (0 on a fresh run); buildHost reads it next.
+                replayFrontier = claimed.lastJournalSeq;
+                runEvents.emit({ kind: "run_status", status: "running" });
+                return claimed.run;
+            },
+        },
+        versions: {
+            getById: async () => {
+                const v = await broker.getVersion();
+                return v === null ? null : { manifest: v.manifest, program: v.program };
+            },
+        },
+        // Download the pinned program artifact from the broker's presigned URL; the worker verifies its
+        // digest (program_worker) before extracting + importing it.
+        fetchProgram: async (downloadUrl) => {
+            const bytes = await broker.downloadBytes(downloadUrl);
+            if (bytes === null)
+                throw new Error("program artifact download returned no body");
+            return bytes;
+        },
+        // Extract the verified artifact tarball via system `tar` (same impl as workspace snapshots).
+        extractArchive: (tgzPath, destDir) => new TarWorkspaceArchiver().extract(tgzPath, destDir),
+        // Guarantee /workspace exists before the program runs (override-safe; the image also pre-creates
+        // it). Lets authors write to /workspace without a defensive mkdir — see WorkflowMeta.workspace.
+        ensureWorkspace: async () => {
+            await mkdir(runtime.workspaceRoot, { recursive: true });
+        },
+        finalizer: {
+            finalize: (_id, status, output) => broker.finalize(status, output, runtime.workerId),
+        },
+        // Durable suspension (docs/SUSPENSION.md): persist the wake condition through the broker (journal
+        // entry + a HITL request row, or the wake time for a long sleep) and release the lease — no
+        // finalize. A wake (an answer or a timer) re-dispatches the run.
+        suspender: {
+            suspend: (signal, workerId) => broker.suspend(signal, workerId),
+        },
+        // Runtime metering (§15): flush runtime as periodic deltas (+ a terminal tail) through the broker,
+        // idempotent per flush, so a long/perpetual run bills as it burns and the credit watcher sees it —
+        // not a single charge at terminal (which a never-terminating run never reached). A fresh per-session
+        // id keeps a restarted run's sessions distinct in the idempotency key (distinct ids sum).
+        startRuntimeFlush: ({ run, startedAtMs }) => {
+            const flusher = new RuntimeFlusher({
+                runId: run.id,
+                sessionId: newId(),
+                startedAtMs,
+                // Bill per vCPU-second: scale wall-clock by the task's vCPUs ($0.05/vCPU-min). At the 1-vCPU
+                // default this is a no-op; a 4-vCPU `large` run books 4× the seconds it holds the task for.
+                vcpus: runtime.vcpus,
+                now: Date.now,
+                report: (deltaSeconds, identifier) => broker.reportUsage(deltaSeconds, identifier),
+            });
+            flusher.start();
+            return { stop: () => flusher.stop(), flushFinal: () => flusher.flushFinal() };
+        },
+        buildHost,
+        // Mid-run credit watching (§15): a CreditWatcher polls the broker's GET /credit on a timer; when
+        // the org runs out, onExhausted aborts the run (the orchestrator wires it to AbortController).
+        startCreditWatch: ({ run, onExhausted }) => {
+            const watcher = new CreditWatcher({
+                runId: run.id,
+                isFunded: () => broker.checkCredit(),
+                onExhausted,
+            });
+            watcher.start();
+            return { stop: () => watcher.stop() };
+        },
+        // Mid-run user-cancel watching (§6): a CancelWatcher polls the broker's GET /cancel on a timer;
+        // when the user cancels (the run is flipped to `cancelling`), onCancelled aborts the run. This is
+        // how the brokered (Redis-less) worker learns of a cancel — the api-server's Redis publish can't
+        // reach it.
+        startCancelWatch: ({ run, onCancelled }) => {
+            const watcher = new CancelWatcher({
+                runId: run.id,
+                isCancelled: () => broker.checkCancelled(),
+                onCancelled,
+            });
+            watcher.start();
+            return { stop: () => watcher.stop() };
+        },
+        // Lease renewal (§6): a LeaseRenewer extends the lease through the broker's POST /renew on a timer
+        // (well under the lease), so a run longer than the 5-min lease isn't reclaimed mid-flight by the
+        // recovery sweep. `renew` re-extends with the SAME workerId that claimed it; a null result (the
+        // run is no longer ours) fires onLost → the run aborts `lease_lost` without finalizing.
+        startLeaseRenew: ({ run, onLost }) => {
+            const watcher = new LeaseRenewer({
+                runId: run.id,
+                renew: async () => (await broker.renewLease(runtime.workerId, Math.round(DEFAULT_LEASE_MS / 1000))) !== null,
+                onLost,
+            });
+            watcher.start();
+            return { stop: () => watcher.stop() };
+        },
+        // Surface the program's console.* output as `program_output` run-events (the log channel),
+        // so a plain console.log program shows up in the run's live tail.
+        onProgramLog: createProgramLogSink({ sink: runEvents }),
+        workerId: runtime.workerId,
+        // Drain the live-tail telemetry buffer before the worker exits (it batches frames).
+        flushTelemetry: () => eventPublisher.close(),
+    };
+}
+// ---- Bootstrap (real container only) -------------------------------------------------
+/**
+ * The per-run platform values the dispatcher injects as container env (ecs_run_task_client.ts). They
+ * are captured into private worker state at bootstrap and DELETED from `process.env` before any user
+ * program / agent leaf / subprocess can run — the run token + API token are credentials, and nothing
+ * untrusted run code touches should inherit them (docs/RUN_ENV_AND_CREDS.md). The user owns the rest
+ * of `process.env` outright.
+ */
+export const PLATFORM_ENV_KEYS = [
+    "RUN_ID",
+    "BOARDWALK_CONTROL_PLANE_URL",
+    "BOARDWALK_RUN_TOKEN",
+    "BOARDWALK_API_KEY",
+    "BOARDWALK_TASK_CPU_UNITS",
+];
+/** The public-API origin a run uses for raw API / MCP / CLI calls. The broker (Runner Control API)
+ *  is hosted on the api-server, so the public API shares its origin; the program appends `/v1` or
+ *  `/mcp/v1`. Falls back to the broker URL unchanged if it can't be parsed. */
+export function publicApiOrigin(controlPlaneBaseUrl) {
+    try {
+        return new URL(controlPlaneBaseUrl).origin;
+    }
+    catch {
+        return controlPlaneBaseUrl;
+    }
+}
+/**
+ * Read the dispatcher-injected platform context from `env`, then DELETE every {@link PLATFORM_ENV_KEYS}
+ * key from it so the user program, the agent leaf, and any subprocess inherit none of them. Mutates
+ * `env` (called once on `process.env` at the very top of `main`, before any run code can read it).
+ */
+export function capturePlatformContext(env) {
+    const read = (name) => {
+        const value = env[name];
+        if (value === undefined || value.trim().length === 0) {
+            throw new Error(`Worker missing required env var ${name}`);
+        }
+        return value.trim();
+    };
+    // The dispatcher always injects the control-plane handle (docs/RUNNER_BROKER.md). The runner is
+    // brokered-only — there is no DB/Redis/Stripe fallback — so a missing handle is a hard config error.
+    const runId = read("RUN_ID");
+    const apiToken = env.BOARDWALK_API_KEY?.trim();
+    const controlPlane = {
+        baseUrl: read("BOARDWALK_CONTROL_PLANE_URL"),
+        runToken: read("BOARDWALK_RUN_TOKEN"),
+        ...(apiToken !== undefined && apiToken.length > 0 ? { apiToken } : {}),
+    };
+    // The dispatcher injects the resolved Fargate size as BOARDWALK_TASK_CPU_UNITS (1024 = 1 vCPU). Fall
+    // back to 1 vCPU when absent (legacy task / self-hosted) so runtime bills at least wall-clock.
+    const cpuUnits = Number(env.BOARDWALK_TASK_CPU_UNITS);
+    const vcpus = Number.isFinite(cpuUnits) && cpuUnits > 0 ? cpuUnits / 1024 : 1;
+    // Scrub LAST — after every read — so the credentials live only in the returned (private) object.
+    // (`delete` via Reflect, not `env[key] = undefined`: assigning undefined to process.env coerces to
+    // the string "undefined", leaving the key readable; deletion actually removes it.)
+    for (const key of PLATFORM_ENV_KEYS)
+        Reflect.deleteProperty(env, key);
+    return { runId, controlPlane, vcpus };
+}
+export async function main() {
+    // Capture the platform context into private state and remove it from process.env BEFORE anything
+    // else — so no user program / agent tool / subprocess we later spawn can read the run token or
+    // API token (docs/RUN_ENV_AND_CREDS.md). WORKER_ID / WORKSPACE_ROOT are non-secret infra knobs.
+    const platform = capturePlatformContext(process.env);
+    const runId = platform.runId;
+    const byoProviders = parseByoProviders(process.env.BOARDWALK_BYO_PROVIDERS);
+    Reflect.deleteProperty(process.env, "BOARDWALK_BYO_PROVIDERS");
+    const deps = assembleWorkerDeps({
+        workerId: process.env.WORKER_ID ?? `worker-${runId}`,
+        workspaceRoot: process.env.WORKSPACE_ROOT ?? "/workspace",
+        runId,
+        controlPlane: platform.controlPlane,
+        vcpus: platform.vcpus,
+        ...(byoProviders.length > 0 ? { byoProviders } : {}),
+    });
+    // The only thing to drain is the batched telemetry buffer — the runner opens no DB/Redis/SQS.
+    const cleanup = async () => {
+        await deps.flushTelemetry?.().catch(() => undefined);
+    };
+    // SIGTERM: ECS is stopping the task. Hold-and-pay has no mid-run checkpoint; we exit and let the
+    // lease expire → the scheduler-sweep reclaims it and a fresh worker RESTARTS the run from the
+    // top (Lambda/GHA semantics; durable children re-attach via idempotency). Crash-safe by design.
+    let shuttingDown = false;
+    for (const signal of ["SIGTERM", "SIGINT"]) {
+        process.on(signal, () => {
+            if (shuttingDown)
+                return;
+            shuttingDown = true;
+            log.warn("worker_signal_exit", { runId, signal });
+            void cleanup().finally(() => process.exit(0));
+        });
+    }
+    try {
+        const outcome = await runProgramWorker(runId, deps);
+        log.info("worker_finished", { runId, outcome: outcome.kind });
+    }
+    catch (err) {
+        log.error("worker_crashed", { runId, error: err instanceof Error ? err.message : String(err) });
+        await cleanup();
+        process.exit(1);
+    }
+    await cleanup();
+    process.exit(0);
+}

package/dist/runtime/inference_transport.d.ts

@@ -0,0 +1,5 @@
+import type { InferenceFrame, InferenceProxyRequest } from "./wire/inference_proxy.js";
+/** Streams one model turn through the broker as `InferenceFrame`s (RunnerControlClient satisfies it). */
+export interface InferenceProxyTransport {
+    streamInference(req: InferenceProxyRequest): AsyncIterable<InferenceFrame>;
+}

package/dist/runtime/inference_transport.js

@@ -0,0 +1,9 @@
+// The broker inference transport the engine-backed leaf streams a model turn through
+// (docs/RUNNER_BROKER.md §4 — "Inference (class 2)").
+//
+// Under the Runner Credential Broker the runner invokes NO model directly: it holds neither the
+// managed-inference key nor any BYO provider key. So the `agent()` leaf's `LeafIo.streamModel`
+// forwards the neutral conversation here, and this transport POSTs it to the broker's `/inference`
+// endpoint; the broker resolves the real model server-side and relays its stream back as the wire's
+// `InferenceFrame`s (delta / result / error). `RunnerControlClient.streamInference` satisfies it.
+export {};

package/dist/runtime/leaf_executor.d.ts

@@ -0,0 +1,104 @@
+import { type LeafResume, type ToolHost, type LspService, type McpTokenResult } from "@boardwalk-labs/engine/core";
+import type { AgentOptions } from "@boardwalk-labs/workflow/runtime";
+import type { BudgetMeter } from "./agent/budget.js";
+import type { SecretRedactor } from "./agent/secret_redactor.js";
+import type { AgentIdentity, TurnEventSink } from "./agent/events.js";
+import type { InferenceProxyTransport } from "./inference_transport.js";
+import { type DirectInferenceDeps } from "./direct_inference.js";
+import type { LeafExecutor } from "./workflow_host.js";
+/** Builds the per-leaf event sink (`TurnEventSink`) for the leaf's `leafIndex`-numbered turn. */
+export type EventSinkFactory = (leafIndex: number, identity: AgentIdentity) => TurnEventSink;
+/** Per-leaf, per-model token metering input (fire-and-forget → the broker). */
+export interface MeterUsageInput {
+    model: string;
+    inputTokens: number;
+    outputTokens: number;
+    /** Cache-served input tokens for this leaf (display-only annotation; omitted when absent). */
+    cachedReadTokens?: number;
+    cachedWriteTokens?: number;
+    leafIndex: number;
+}
+export interface EngineLeafExecutorDeps {
+    /** Streams one model turn through the broker (RunnerControlClient satisfies it). */
+    inference: InferenceProxyTransport;
+    /** Runner-direct BYO inference (docs/SELF_HOSTED_RUNNERS.md D7): when a per-`agent()` provider
+     *  matches this registry (key-based HTTP sources only), the turn goes STRAIGHT to the org's
+     *  endpoint with the engine adapters — the managed lane and bedrock stay brokered. Omitted ⇒
+     *  everything brokered (legacy dispatch without a registry). */
+    byo?: DirectInferenceDeps;
+    /** RUN-level meter, shared across every leaf so caps + token totals span the whole run. The engine
+     *  reports usage after EVERY model call via `reportUsage`; we feed it here and throw on a cap
+     *  breach so the loop terminates mid-flight (the budget authority). */
+    budget: BudgetMeter;
+    /** RUN-level redactor (shared with the secret resolver): every resolved secret value is recorded
+     *  here. We seed a fresh engine `Redactor` from it per leaf so the loop scrubs known values out of
+     *  the prompt, tool args/results, and transcript before they reach the model (MASTER_SPEC §12). */
+    redactor: SecretRedactor;
+    /** Builds this leaf's event sink; `leafIndex` (1-based) is only a metering identifier — the sink
+     *  owns the run-global cursor. `identity` names the leaf on its turn frames. */
+    makeEventSink: EventSinkFactory;
+    /** The run's persistent `/workspace` root — memory dirs (`agent({ memory })`) are relative to it. */
+    workspaceRoot: string;
+    /** Resolves the directory holding this run's bundled files (the extracted program tree, where a
+     *  skill lives at `skills/<name>.md`). Known only once the artifact is extracted (mid-run), so it's
+     *  a thunk. Null / omitted ⇒ a leaf that names `skills` fails loud. */
+    skillsDir?: () => string | null;
+    /** Resolves the run's workflow PACKAGE root — the extracted program tree, whose ROOT holds the
+     *  author's standing instructions (`<programDir>/AGENTS.md`) and `skills/` beside them. The engine
+     *  reads `capabilities.programDir` for the BUNDLED `AGENTS.md` tier, read by every agent() before
+     *  any AGENTS.md the run cloned into its workspace. A thunk for the same reason `skillsDir` is (the
+     *  dir is known only once the artifact extracts); it is in fact the parent of `skillsDir`. Null /
+     *  omitted ⇒ no bundled tier (only the workspace AGENTS.md applies). */
+    programDir?: () => string | null;
+    /** Per-leaf token metering seam (fire-and-forget). Reports THIS leaf's tokens + its model to the
+     *  broker, which decides `billed_by_boardwalk` per model + meters to Stripe. Omitted in tests. */
+    meterUsage?: (input: MeterUsageInput) => void;
+    /** Backend for the engine's host-backed built-in tools (`webfetch` / `web_search` / `artifacts`):
+     *  set as the leaf's `capabilities.host` so the engine registers them. Broker-backed on hosted runs
+     *  (BrokerToolHost). Omitted ⇒ those three tools are simply absent (the engine never registers a
+     *  host-backed tool whose hook the host doesn't provide). */
+    toolHost?: ToolHost;
+    /** Engine-native LSP for the `diagnostics` tool + diagnostics-after-edit. Set as the leaf's
+     *  `capabilities.lspService`. Constructed ONCE per RUN (not per leaf) so the language server stays
+     *  warm across the run's edits/leaves, and closed on the run's teardown. Omitted ⇒ the `diagnostics`
+     *  tool and after-edit diagnostics are best-effort-skipped (the correct degradation). */
+    lspService?: LspService;
+    /** Brokers a short-lived OAuth bearer for a hosted MCP server. The engine calls this REACTIVELY —
+     *  only after a server answers 401 to the static `headers` — so static-bearer servers never reach
+     *  it. Routes to the Runner Control API's `mcp/token` vend endpoint (the OAuth token state lives in
+     *  the control-plane vault, never on the worker). Omitted ⇒ no OAuth brokering: a server that needs
+     *  a token gets `{ accessToken: null }` and the leaf fails loud with a clear hint (the correct
+     *  degradation — static-bearer and no-auth servers still work). */
+    brokerMcpToken?: (serverUrl: string, invalidateToken?: string) => Promise<McpTokenResult>;
+}
+/**
+ * Per-run leaf executor. The worker constructs one bound to the run + run-level budget, and wires it
+ * as the `LeafExecutor` on the run's WorkflowHost, so every `agent()` the program calls runs here
+ * through the engine's loop.
+ */
+export declare class EngineLeafExecutor implements LeafExecutor {
+    private readonly deps;
+    private leafCount;
+    constructor(deps: EngineLeafExecutorDeps);
+    run(prompt: string, opts: AgentOptions | undefined, signal?: AbortSignal, resume?: LeafResume): Promise<unknown>;
+    /** Assemble the broker-backed `LeafIo` the engine loop drives for one leaf call. */
+    private buildLeafIo;
+    /** POST one model turn to the broker's `/inference` and adapt its NDJSON stream into the engine's
+     *  ModelTurnResult: each `delta` frame drives providerIo.onDelta; the terminal `result` frame is
+     *  the turn. An `error` frame throws (the broker already classified it). An abort mid-stream throws. */
+    private streamModel;
+}
+/** The MCP server refs an `agent({ mcp })` call may name, derived from the SDK's `AgentOptions`. */
+type HostedMcpServerRef = NonNullable<AgentOptions["mcp"]>[number];
+/**
+ * Gate the MCP servers a hosted leaf may use: only the `http` transport (no arbitrary `stdio`
+ * processes on the worker) and a parseable URL. Throws a clear VALIDATION_FAILED for the first
+ * offending ref so the leaf fails at its boundary, before the engine connects out.
+ *
+ * Reachability of the host is NOT gated here: a hosted run's egress is OPEN by default (the forward
+ * proxy allows all public destinations; a workflow restricts it via manifest.egress), and the proxy
+ * is the single enforcement point — a server blocked by a restrictive egress fails at the proxy when
+ * the engine connects, not via an MCP-specific allowlist that would be stricter than the platform.
+ */
+export declare function assertHostedMcpAllowed(refs: readonly HostedMcpServerRef[] | undefined): void;
+export {};