@boardwalk-labs/runner 0.1.10 → 0.1.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/binding.gyp +12 -0
  2. package/dist/runtime/agent/budget.d.ts +5 -5
  3. package/dist/runtime/agent/budget.js +8 -8
  4. package/dist/runtime/agent/model_rates.js +4 -5
  5. package/dist/runtime/agent/secret_redactor.js +1 -1
  6. package/dist/runtime/cancel_watcher.js +1 -1
  7. package/dist/runtime/credit_watcher.d.ts +1 -1
  8. package/dist/runtime/credit_watcher.js +5 -5
  9. package/dist/runtime/freeze_coordinator.d.ts +124 -0
  10. package/dist/runtime/freeze_coordinator.js +276 -0
  11. package/dist/runtime/identity_relay.d.ts +45 -2
  12. package/dist/runtime/identity_relay.js +137 -14
  13. package/dist/runtime/index.d.ts +11 -6
  14. package/dist/runtime/index.js +119 -39
  15. package/dist/runtime/leaf_executor.d.ts +2 -2
  16. package/dist/runtime/program_runner.d.ts +1 -1
  17. package/dist/runtime/program_runner.js +1 -1
  18. package/dist/runtime/program_worker.d.ts +3 -3
  19. package/dist/runtime/program_worker.js +7 -7
  20. package/dist/runtime/recording_secret_resolver.js +1 -1
  21. package/dist/runtime/run_abort.js +3 -3
  22. package/dist/runtime/runner_control_client.d.ts +34 -4
  23. package/dist/runtime/runner_control_client.js +85 -33
  24. package/dist/runtime/runtime_flusher.d.ts +9 -0
  25. package/dist/runtime/runtime_flusher.js +17 -3
  26. package/dist/runtime/support/index.js +5 -6
  27. package/dist/runtime/tools/artifacts.js +1 -1
  28. package/dist/runtime/tools/types.d.ts +5 -5
  29. package/dist/runtime/tools/types.js +7 -7
  30. package/dist/runtime/tools/web_search.js +5 -6
  31. package/dist/runtime/uniqueness_reseed.d.ts +7 -0
  32. package/dist/runtime/uniqueness_reseed.js +65 -0
  33. package/dist/runtime/wire/artifact_storage.js +2 -2
  34. package/dist/runtime/wire/inference_proxy.d.ts +1 -1
  35. package/dist/runtime/wire/inference_proxy.js +2 -2
  36. package/dist/runtime/workflow_host.d.ts +50 -1
  37. package/dist/runtime/workflow_host.js +185 -31
  38. package/native/reseed.c +52 -0
  39. package/package.json +12 -5
  40. package/prebuilds/linux-x64/@boardwalk-labs+runner.node +0 -0
@@ -7,14 +7,23 @@
7
7
  // AF_UNIX socketpair; init tells us its fd via `BOARDWALK_IDENTITY_RELAY_FD`. Wire: one JSON
8
8
  // object per LF-terminated line —
9
9
  //
10
- // {"type":"worker_ready"} worker → init the pre-identity park is reached
10
+ // {"type":"worker_ready","payload":{...}} worker → init the pre-identity park is reached
11
+ // (payload: optional version diagnostics,
12
+ // forwarded verbatim by init)
11
13
  // {"type":"identity","payload":{...}} init → worker the run identity (see schema below)
12
14
  // {"type":"identity_accepted"} worker → init captured; init acks its host
15
+ // {"type":"suspend_request","payload":{...}} worker → init a quiescent seam asks to be frozen
16
+ // {"type":"suspend_abort","payload":{...}} init → worker snapshot failed; the seam holds instead
17
+ // {"type":"wake","payload":{...}} init → worker resolve the frozen seam (fresh tokens
18
+ // + the wake value, verbatim)
19
+ // {"type":"wake_accepted"} worker → init tokens swapped, seam resolved; init acks
13
20
  //
14
- // The payload carries exactly the platform env contract (snake_case) plus the resolved user
15
- // env; `applyIdentityToEnv` maps it onto `process.env` so `capturePlatformContext` runs
21
+ // The identity payload carries exactly the platform env contract (snake_case) plus the resolved
22
+ // user env; `applyIdentityToEnv` maps it onto `process.env` so `capturePlatformContext` runs
16
23
  // UNCHANGED afterward — same fields, same capture-and-delete discipline, two transports.
17
- // The stream stays open after identity: later platform phases speak suspend/wake over it.
24
+ // The stream stays open after identity: `openChannel` attaches the suspend/wake consumer
25
+ // (the FreezeCoordinator drives the choreography above it).
26
+ import { createRequire } from "node:module";
18
27
  import { Socket } from "node:net";
19
28
  import { z } from "zod";
20
29
  import { createLogger } from "./support/index.js";
@@ -22,6 +31,11 @@ const log = createLogger("identity_relay");
22
31
  /** Env var naming the inherited relay fd. Set by the guest init; absent everywhere else
23
32
  * (Fargate, self-hosted daemon), where the worker env-boots as always. */
24
33
  export const RELAY_FD_ENV = "BOARDWALK_IDENTITY_RELAY_FD";
34
+ /** Max bytes of one relay line — the wire protocol's 32 MiB frame cap, mirrored in-guest.
35
+ * An oversized line costs the connection (LF framing cannot resynchronize past it), which
36
+ * for this relay means a hard bootstrap failure. Measured in UTF-16 code units, which for
37
+ * this ASCII-JSON wire is the byte count; the cap is a guard, not exact accounting. */
38
+ export const MAX_RELAY_LINE_BYTES = 32 * 1024 * 1024;
25
39
  /** The identity payload — the env contract's fields, as JSON instead of env vars. */
26
40
  export const relayIdentitySchema = z.object({
27
41
  run_id: z.string().min(1),
@@ -74,30 +88,73 @@ export function applyIdentityToEnv(identity, env) {
74
88
  env.BOARDWALK_BYO_PROVIDERS = JSON.stringify(identity.byo_providers);
75
89
  }
76
90
  }
91
+ /** Collect the worker_ready diagnostics. Version lookups are best-effort: the runner's own
92
+ * package.json sits two levels above this module in both the src and published dist
93
+ * layouts; the SDK's is reachable only if its exports expose ./package.json. */
94
+ export function workerDiagnostics() {
95
+ const require = createRequire(import.meta.url);
96
+ const versionOf = (spec) => {
97
+ try {
98
+ const pkg = require(spec);
99
+ return typeof pkg.version === "string" ? pkg.version : undefined;
100
+ }
101
+ catch {
102
+ return undefined;
103
+ }
104
+ };
105
+ const worker = versionOf("../../package.json");
106
+ const sdk = versionOf("@boardwalk-labs/workflow/package.json");
107
+ return {
108
+ ...(worker !== undefined ? { worker_version: worker } : {}),
109
+ node_version: process.version,
110
+ ...(sdk !== undefined ? { sdk_version: sdk } : {}),
111
+ };
112
+ }
77
113
  /** One end of the init↔worker relay. Wraps any Duplex so tests run over in-memory streams. */
78
114
  export class IdentityRelay {
79
115
  stream;
80
116
  buffer = "";
81
117
  closed = false;
118
+ failure = null;
82
119
  wake = null;
120
+ onData;
83
121
  constructor(stream) {
84
122
  this.stream = stream;
85
123
  // One persistent listener for the relay's lifetime: a flowing stream DROPS chunks
86
124
  // emitted while nobody listens, so attach-per-read would lose lines between reads.
87
- stream.on("data", (chunk) => {
125
+ this.onData = (chunk) => {
88
126
  this.buffer += typeof chunk === "string" ? chunk : chunk.toString("utf8");
127
+ if (this.buffer.length > MAX_RELAY_LINE_BYTES && !this.buffer.includes("\n")) {
128
+ // The unterminated tail can never become a legal line — fail now instead of
129
+ // buffering without bound (the wire cap, mirrored; init is trusted, so this is a
130
+ // bug or corruption, not an attack to survive).
131
+ this.fail(new Error(`identity relay line exceeds ${MAX_RELAY_LINE_BYTES} bytes`));
132
+ return;
133
+ }
89
134
  this.wake?.();
90
- });
91
- const onClose = () => {
135
+ };
136
+ stream.on("data", this.onData);
137
+ stream.on("end", () => {
92
138
  this.closed = true;
93
139
  this.wake?.();
94
- };
95
- stream.on("end", onClose);
96
- stream.on("error", onClose);
140
+ });
141
+ stream.on("error", (err) => {
142
+ // Keep the error's detail — "closed" alone hides why the socket died.
143
+ this.fail(err);
144
+ });
97
145
  }
98
- /** Announce the pre-identity park. Init forwards this as the base-snapshot gate. */
99
- announceReady() {
100
- this.writeLine({ type: "worker_ready" });
146
+ fail(err) {
147
+ this.failure ??= err;
148
+ this.closed = true;
149
+ this.stream.destroy();
150
+ this.wake?.();
151
+ }
152
+ /** Announce the pre-identity park (with the diagnostics payload when provided).
153
+ * Init forwards this as the base-snapshot gate. */
154
+ announceReady(diagnostics) {
155
+ this.writeLine(diagnostics === undefined
156
+ ? { type: "worker_ready" }
157
+ : { type: "worker_ready", payload: diagnostics });
101
158
  }
102
159
  /**
103
160
  * Block until init relays the run identity. THIS is the point the base snapshot freezes:
@@ -137,6 +194,64 @@ export class IdentityRelay {
137
194
  acceptIdentity() {
138
195
  this.writeLine({ type: "identity_accepted" });
139
196
  }
197
+ /**
198
+ * Attach the post-identity consumer: the suspend/wake channel (the same fd, the same JSON
199
+ * lines). From here the relay dispatches init→worker messages to the handlers —
200
+ * `wake` (resolve a frozen seam) and `suspend_abort` (the seam falls back to holding) —
201
+ * and the returned channel sends the worker→init halves (`suspend_request`,
202
+ * `wake_accepted`). Unknown types are ignored (forward-compatible); malformed lines are
203
+ * logged and skipped, exactly as pre-identity. Call once, after {@link acceptIdentity}.
204
+ */
205
+ openChannel(handlers) {
206
+ void this.pumpChannel(handlers);
207
+ return {
208
+ sendSuspendRequest: (payload) => {
209
+ this.writeLine({ type: "suspend_request", payload });
210
+ },
211
+ sendWakeAccepted: () => {
212
+ this.writeLine({ type: "wake_accepted" });
213
+ },
214
+ };
215
+ }
216
+ async pumpChannel(handlers) {
217
+ for (;;) {
218
+ let line;
219
+ try {
220
+ line = await this.readLine();
221
+ }
222
+ catch (err) {
223
+ // The relay dying mid-run means init is gone — the guest is being torn down; the
224
+ // worker's own exit follows. Log, don't throw into nowhere.
225
+ log.warn("relay_channel_closed", {
226
+ error: err instanceof Error ? err.message : String(err),
227
+ });
228
+ return;
229
+ }
230
+ let parsed;
231
+ try {
232
+ parsed = JSON.parse(line);
233
+ }
234
+ catch {
235
+ log.warn("relay_malformed_line_skipped", { length: line.length });
236
+ continue;
237
+ }
238
+ const message = relayMessageSchema.safeParse(parsed);
239
+ if (!message.success) {
240
+ log.warn("relay_malformed_message_skipped", {});
241
+ continue;
242
+ }
243
+ switch (message.data.type) {
244
+ case "wake":
245
+ handlers.onWake(message.data.payload);
246
+ break;
247
+ case "suspend_abort":
248
+ handlers.onSuspendAbort(message.data.payload);
249
+ break;
250
+ default:
251
+ log.warn("relay_unexpected_type_ignored", { type: message.data.type });
252
+ }
253
+ }
254
+ }
140
255
  writeLine(message) {
141
256
  this.stream.write(JSON.stringify(message) + "\n");
142
257
  }
@@ -146,7 +261,15 @@ export class IdentityRelay {
146
261
  if (newline >= 0) {
147
262
  const line = this.buffer.slice(0, newline);
148
263
  this.buffer = this.buffer.slice(newline + 1);
149
- return line;
264
+ if (line.length > MAX_RELAY_LINE_BYTES) {
265
+ this.fail(new Error(`identity relay line exceeds ${MAX_RELAY_LINE_BYTES} bytes`));
266
+ }
267
+ else {
268
+ return line;
269
+ }
270
+ }
271
+ if (this.failure !== null) {
272
+ throw this.failure;
150
273
  }
151
274
  if (this.closed) {
152
275
  throw new Error("identity relay closed before the run identity arrived");
@@ -1,12 +1,13 @@
1
1
  import type { AuthContext } from "./support/index.js";
2
2
  import type { Run } from "./wire/run.js";
3
+ import { type IdentityRelay } from "./identity_relay.js";
3
4
  import type { ByoInferenceProvider } from "../contract.js";
4
5
  import { type ProgramWorkerDeps } from "./program_worker.js";
5
6
  /** Constructed primitives the pure assembly consumes (real ones in main(), fakes in tests). The
6
7
  * runner holds NO platform credential — only the per-run control-plane handle (run token + broker
7
8
  * URL); everything else is reached through the Runner Control API (the Runner Credential Broker model). */
8
9
  export interface WorkerRuntime {
9
- /** Stable worker identity stamped onto the lease (ECS task arn / hostname / run-derived). */
10
+ /** Stable worker identity stamped onto the lease (task ARN / hostname / run-derived). */
10
11
  workerId: string;
11
12
  /** Sandbox workspace root for filesystem/shell/git tools. */
12
13
  workspaceRoot: string;
@@ -27,14 +28,18 @@ export interface WorkerRuntime {
27
28
  /** vCPUs provisioned for this task (the dispatcher's resolved machine size ÷ 1024 cpu units). Runtime
28
29
  * is billed per vCPU-SECOND, so the RuntimeFlusher scales wall-clock by this. Defaults to 1. */
29
30
  vcpus: number;
31
+ /** The in-guest identity relay, present ONLY on the snapshot-based microVM substrate (relay-mode
32
+ * boot). When set, the worker suspends by FREEZING — the FreezeCoordinator parks seams over this
33
+ * relay's suspend/wake channel — instead of the exit-and-replay path. */
34
+ freezeRelay?: IdentityRelay;
30
35
  }
31
- /** Durable events are deferred (run_step_events table) — live fan-out via the broker only. */
36
+ /** Durable events are deferred (durable event storage) — live fan-out via the broker only. */
32
37
  /** Synthesize the run's AuthContext. The run was already authorized at trigger time; the
33
38
  * worker acts on the org's behalf, so it carries the org + an owner role. Tool-level
34
39
  * boundaries (the broker's server-side manifest allowlist) are the real guard.
35
40
  *
36
41
  * source='workflow' (NOT 'session_jwt'): the program must never perform SESSION_JWT_ONLY
37
- * credential mutations (§12.11), so a tool that ever exposed such a service is denied by
42
+ * credential mutations, so a tool that ever exposed such a service is denied by
38
43
  * construction regardless of the owner role. */
39
44
  export declare function workerAuthContext(run: Run): AuthContext;
40
45
  /**
@@ -52,10 +57,10 @@ export declare function workerAuthContext(run: Run): AuthContext;
52
57
  */
53
58
  export declare function tokenMeterIdentifiers(runId: string, sessionId: string): (leafIndex: number) => string;
54
59
  /** Pure wiring: build the full ProgramWorkerDeps from the control-plane handle. The runner reaches
55
- * every privileged seam through the broker over its run token — no DB / Redis / Stripe / model creds. */
60
+ * every privileged seam through the broker over its run token — no database, cache, billing, or model creds. */
56
61
  export declare function assembleWorkerDeps(runtime: WorkerRuntime): ProgramWorkerDeps;
57
62
  /**
58
- * The per-run platform values the dispatcher injects as container env (ecs_run_task_client.ts). They
63
+ * The per-run platform values the dispatcher injects as container env. They
59
64
  * are captured into private worker state at bootstrap and DELETED from `process.env` before any user
60
65
  * program / agent leaf / subprocess can run — the run token + API token are credentials, and nothing
61
66
  * untrusted run code touches should inherit them (the run env/credential rules). The user owns the rest
@@ -73,7 +78,7 @@ export interface PlatformContext {
73
78
  vcpus: number;
74
79
  }
75
80
  /** The public-API origin a run uses for raw API / MCP / CLI calls. The broker (Runner Control API)
76
- * is hosted on the api-server, so the public API shares its origin; the program appends `/v1` or
81
+ * shares an origin with the public API, so the program appends `/v1` or
77
82
  * `/mcp/v1`. Falls back to the broker URL unchanged if it can't be parsed. */
78
83
  export declare function publicApiOrigin(controlPlaneBaseUrl: string): string;
79
84
  /**
@@ -1,5 +1,5 @@
1
- // Worker composition root (the workflow runtime design). The Boardwalk-hosted
2
- // worker container runs `node dist/fargate/worker/index.js`; the dispatcher launches it per run with
1
+ // Worker composition root (the workflow runtime design). The hosted worker container runs the
2
+ // compiled worker entrypoint; the dispatcher launches it per run with
3
3
  // RUN_ID + the per-run control-plane handle. This file assembles real implementations behind every
4
4
  // seam `runProgramWorker(runId, deps)` injects, then runs the one workflow program to terminal.
5
5
  //
@@ -11,21 +11,21 @@
11
11
  // - secrets.get() → broker /secrets/resolve (allowlist enforced server-side).
12
12
  // - workflows.call()→ broker /children (durable child run, hold + poll).
13
13
  // - events.emit() → broker /events (fan-out server-side).
14
- // - artifacts / web_search → broker /artifacts + /tools/web_search (no S3 / Tavily creds here).
14
+ // - artifacts / web_search → broker /artifacts + /tools/web_search (no storage or search-provider creds here).
15
15
  // - lifecycle / usage / telemetry → broker /claim,/version,/finalize,/usage,/telemetry.
16
- // There is no DB / Redis / Stripe / Bedrock client on the runner — so its task role is near-zero and
17
- // the metadata-endpoint escape has nothing to steal. The legacy direct (pre-broker) path is removed.
16
+ // There is no database, cache, billing, or model-provider client on the runner — so its task role is
17
+ // near-zero and the metadata-endpoint escape has nothing to steal. The legacy direct (pre-broker) path is removed.
18
18
  //
19
19
  // Split: `assembleWorkerDeps(runtime)` is pure wiring (unit-tested with a fake fetch); `main()` is the
20
20
  // bootstrap shell (read the control-plane env, install signal handlers).
21
21
  //
22
- // Per-session loops wired here (all brokered): a UsageFlusher meters token deltas (§10.7
23
- // /usage/tokens), a CreditWatcher polls funding (§15 → /credit, aborting the run on exhaustion), and a
24
- // CancelWatcher polls for a user cancel (§6 → /cancel, aborting the run when the user cancels — the
25
- // brokered worker holds no Redis, so it can't receive the api-server's cancel publish directly).
22
+ // Per-session loops wired here (all brokered): a UsageFlusher meters token deltas ( /usage/tokens), a
23
+ // CreditWatcher polls funding (→ /credit, aborting the run on exhaustion), and a CancelWatcher polls
24
+ // for a user cancel (→ /cancel, aborting the run when the user cancels — the brokered worker holds no
25
+ // cache client, so it can't receive the platform's cancel publish directly).
26
26
  //
27
- // Documented v0 deferral (a clear seam): durable events — a no-op AgentEventStore (live Redis fan-out
28
- // via the broker only) until the run_step_events table lands.
27
+ // Documented v0 deferral (a clear seam): durable events — a no-op AgentEventStore (live fan-out
28
+ // via the broker only) until durable event storage lands.
29
29
  import { createLogger, newId, AppError, ErrorCode, DEFAULT_LEASE_MS } from "./support/index.js";
30
30
  import { LspService } from "@boardwalk-labs/engine/core";
31
31
  import { BudgetMeter } from "./agent/budget.js";
@@ -35,7 +35,9 @@ import { SecretRedactor } from "./agent/secret_redactor.js";
35
35
  import { RecordingSecretResolver } from "./recording_secret_resolver.js";
36
36
  import { EngineLeafExecutor } from "./leaf_executor.js";
37
37
  import { parseByoProviders } from "./direct_inference.js";
38
- import { applyIdentityToEnv, connectIdentityRelayFd, relayFdFromEnv } from "./identity_relay.js";
38
+ import { applyIdentityToEnv, connectIdentityRelayFd, relayFdFromEnv, workerDiagnostics, } from "./identity_relay.js";
39
+ import { FreezeCoordinator } from "./freeze_coordinator.js";
40
+ import { reseedUserspaceCsprng } from "./uniqueness_reseed.js";
39
41
  import { WorkerWorkflowHost } from "./workflow_host.js";
40
42
  import { runProgramWorker, } from "./program_worker.js";
41
43
  import { RunnerControlClient } from "./runner_control_client.js";
@@ -53,13 +55,13 @@ import { PhaseTracker } from "./phase_tracker.js";
53
55
  import { mkdir } from "node:fs/promises";
54
56
  import { join } from "node:path";
55
57
  const log = createLogger("worker_entrypoint");
56
- /** Durable events are deferred (run_step_events table) — live fan-out via the broker only. */
58
+ /** Durable events are deferred (durable event storage) — live fan-out via the broker only. */
57
59
  /** Synthesize the run's AuthContext. The run was already authorized at trigger time; the
58
60
  * worker acts on the org's behalf, so it carries the org + an owner role. Tool-level
59
61
  * boundaries (the broker's server-side manifest allowlist) are the real guard.
60
62
  *
61
63
  * source='workflow' (NOT 'session_jwt'): the program must never perform SESSION_JWT_ONLY
62
- * credential mutations (§12.11), so a tool that ever exposed such a service is denied by
64
+ * credential mutations, so a tool that ever exposed such a service is denied by
63
65
  * construction regardless of the owner role. */
64
66
  export function workerAuthContext(run) {
65
67
  const userId = run.actor.type === "user" ? run.actor.user_id : `workflow:${run.workflowId}`;
@@ -83,7 +85,7 @@ export function tokenMeterIdentifiers(runId, sessionId) {
83
85
  return (leafIndex) => `${runId}:${sessionId}:${String(leafIndex)}:${String(seq++)}`;
84
86
  }
85
87
  /** Pure wiring: build the full ProgramWorkerDeps from the control-plane handle. The runner reaches
86
- * every privileged seam through the broker over its run token — no DB / Redis / Stripe / model creds. */
88
+ * every privileged seam through the broker over its run token — no database, cache, billing, or model creds. */
87
89
  export function assembleWorkerDeps(runtime) {
88
90
  const broker = new RunnerControlClient({
89
91
  baseUrl: runtime.controlPlane.baseUrl,
@@ -95,8 +97,27 @@ export function assembleWorkerDeps(runtime) {
95
97
  // a resumed run suppresses observability for seams it already ran. Captured at claim, read by
96
98
  // buildHost (which runs right after claim) when constructing the per-run host.
97
99
  let replayFrontier = 0;
98
- // Live agent-event stream /telemetry (broker publishes to Redis server-side). Inference,
99
- // web_search, artifacts, and the model call are all brokered no Redis / model creds / Tavily / S3.
100
+ // Snapshot-substrate suspension (relay-mode boot only): one coordinator for the process, wired to
101
+ // the relay's suspend/wake channel now; its per-run hooks (token swap, meter rebase, workspace
102
+ // persist) late-bind in buildHost/startRuntimeFlush once those objects exist. The circular
103
+ // channel↔handler reference resolves through the thunks.
104
+ let freeze;
105
+ let activeFlusher = null;
106
+ if (runtime.freezeRelay !== undefined) {
107
+ const channel = runtime.freezeRelay.openChannel({
108
+ onWake: (payload) => {
109
+ freeze?.onWake(payload);
110
+ },
111
+ onSuspendAbort: (payload) => {
112
+ freeze?.onSuspendAbort(payload);
113
+ },
114
+ });
115
+ freeze = new FreezeCoordinator({ channel });
116
+ log.info("freeze_mode_enabled", { runId: runtime.runId });
117
+ }
118
+ // Live agent-event stream → /telemetry (broker fans out server-side). Inference,
119
+ // web_search, artifacts, and the model call are all brokered — no cache client, model creds,
120
+ // search-provider creds, or object-storage creds here.
100
121
  const eventPublisher = new BrokerEventPublisher({
101
122
  send: (frames) => broker.publishTelemetry(frames),
102
123
  });
@@ -110,10 +131,11 @@ export function assembleWorkerDeps(runtime) {
110
131
  // bootstrap captured + scrubbed it (capturePlatformContext) so the agent's bash / subprocesses
111
132
  // can't read it. The program reaches it ONLY via `runtime.apiToken()` (built below); we still
112
133
  // record it in each run's redactor so a value the program threads into a tool can't echo back out
113
- // of a tool result. Absent in the dev no-signing-key path. (the run env/credential rules)
114
- const runApiKey = runtime.controlPlane.apiToken;
115
- // The broker (Runner Control API) is hosted on the api-server, so the public API shares its
116
- // origin; `runtime.apiUrl` exposes it and the program appends `/v1` or `/mcp/v1`.
134
+ // of a tool result. Absent in the dev no-signing-key path.
135
+ // MUTABLE: on the snapshot substrate a wake carries a fresh token (the frozen one expired).
136
+ let runApiKey = runtime.controlPlane.apiToken;
137
+ // The broker (Runner Control API) shares an origin with the public API;
138
+ // `runtime.apiUrl` exposes it and the program appends `/v1` or `/mcp/v1`.
117
139
  const apiUrl = publicApiOrigin(runtime.controlPlane.baseUrl);
118
140
  // Per-run host: agent() leaf + sleep-hold + secrets + children + events, all brokered. `signal`
119
141
  // carries cooperative cancellation (credit exhaustion) into every host hook.
@@ -123,7 +145,7 @@ export function assembleWorkerDeps(runtime) {
123
145
  // Fallback budget-cap rate. A managed turn caps on the broker's EXACT upstream cost (forwarded on
124
146
  // the inference result frame → BudgetMeter `realCostUsd`); this representative sonnet-class rate
125
147
  // applies only to a turn with no upstream cost (BYO / unavailable). `max_usd` is a guardrail, not
126
- // the bill — that's per-leaf cost pass-through at the broker.
148
+ // the bill — the platform meters the actual per-leaf usage.
127
149
  rate: BUDGET_GUARDRAIL_RATE,
128
150
  startedAt: Date.now(),
129
151
  // deadline_seconds is WALL-CLOCK from the run's ORIGINAL start (incl. suspended idle), so a run
@@ -140,7 +162,7 @@ export function assembleWorkerDeps(runtime) {
140
162
  const nextMeterIdentifier = tokenMeterIdentifiers(run.id, meteringSessionId);
141
163
  // Per-run secret boundary: every value resolved (program `secrets.get` OR a tool's
142
164
  // ctx.secrets.resolve) is recorded into one shared redactor; the leaf seeds a fresh engine
143
- // Redactor from it so the loop scrubs those values out of all model-bound content. the platform spec
165
+ // Redactor from it so the loop scrubs those values out of all model-bound content.
144
166
  const redactor = new SecretRedactor();
145
167
  // Record the run's own API key so a prompt-injected agent can't echo it back to the model.
146
168
  if (runApiKey !== undefined && runApiKey !== "")
@@ -190,9 +212,9 @@ export function assembleWorkerDeps(runtime) {
190
212
  // until the artifact extracts. The empty `/workspace` and this dir are SEPARATE dirs on hosted,
191
213
  // exactly the two-tier case the engine's AGENTS.md loader is built for.
192
214
  programDir: () => programDir,
193
- // Per-leaf, per-model token metering — reported THROUGH the broker (the worker holds no Stripe
194
- // credential); the broker decides `billed_by_boardwalk` per model + meters to Stripe (BYO models
195
- // no-op there). Fire-and-forget: a metering hiccup must never fail the run.
215
+ // Per-leaf, per-model token metering — reported THROUGH the broker (the worker holds no billing
216
+ // credential); the broker decides `billed_by_boardwalk` per model + meters usage to the platform
217
+ // (BYO models no-op there). Fire-and-forget: a metering hiccup must never fail the run.
196
218
  meterUsage: ({ model, inputTokens, outputTokens, cachedReadTokens, cachedWriteTokens, leafIndex, }) => {
197
219
  void broker
198
220
  .meterTokens({
@@ -217,7 +239,7 @@ export function assembleWorkerDeps(runtime) {
217
239
  // comes from the org's connection vault via the Runner Control API — never stored on the worker.
218
240
  brokerMcpToken: (serverUrl, invalidateToken) => broker.mcpToken(serverUrl, invalidateToken),
219
241
  });
220
- // Per-workflow persistent /workspace (§5): only when the manifest opts in. The BROKER additionally
242
+ // Per-workflow persistent /workspace: only when the manifest opts in. The BROKER additionally
221
243
  // gates on hosted (self-hosted runs get null URLs), and snapshots are keyed per-workflow + scoped
222
244
  // by the run token — so even the untrusted in-process program can't reach another tenant's data.
223
245
  const workspaceStore = manifest.workspace?.persist === true
@@ -288,8 +310,52 @@ export function assembleWorkerDeps(runtime) {
288
310
  },
289
311
  }
290
312
  : {}),
313
+ // Snapshot-substrate suspension: seams freeze in place under the quiescence gate instead of
314
+ // raising onSuspend (which stays wired as the transitional/local path's fallback).
315
+ ...(freeze !== undefined ? { freeze } : {}),
316
+ // Register-without-release for held HITL gates — only on the
317
+ // freeze substrate, backed by the broker's inputs endpoints.
318
+ ...(freeze !== undefined
319
+ ? {
320
+ heldInput: {
321
+ register: (seq, gate) => broker.registerInput(seq, gate),
322
+ poll: (seq) => broker.pollInputAnswers(seq),
323
+ },
324
+ }
325
+ : {}),
291
326
  phases: phaseTracker,
292
327
  });
328
+ // Late-bind the coordinator's per-run hooks now that the run-scoped objects exist.
329
+ if (freeze !== undefined) {
330
+ const coordinator = freeze;
331
+ let freezeWallMs = 0;
332
+ coordinator.setHooks({
333
+ // At quiescence, immediately before the freeze: book the runtime tail (suspended time must
334
+ // never appear billed) and persist the workspace (crash-during-suspension
335
+ // recovery parity with the sleep-hold path). The wall stamp anchors the idle rebase below.
336
+ onBeforeFreeze: async () => {
337
+ freezeWallMs = Date.now();
338
+ await activeFlusher?.flushNow();
339
+ await workspaceStore?.persist();
340
+ },
341
+ // On wake: reseed the userspace CSPRNG (clause 3 — a suspend snapshot restored more than
342
+ // once, e.g. a re-dispatch retry, would otherwise repeat its post-wake `crypto.*` draws),
343
+ // swap the fresh tokens onto the broker client + the program-facing apiToken() (recording
344
+ // the new values in the run's redactor, same discipline as boot), and exclude the frozen
345
+ // window from billed runtime using the wake's authoritative wall clock. The reseed runs
346
+ // BEFORE the seam resolves, so no woken author code draws from the stale (pre-suspend) DRBG.
347
+ onAfterWake: (wake) => {
348
+ reseedUserspaceCsprng();
349
+ broker.swapRunToken(wake.run_token);
350
+ redactor.record(wake.run_token);
351
+ if (wake.api_token !== undefined && wake.api_token !== "") {
352
+ runApiKey = wake.api_token;
353
+ redactor.record(wake.api_token);
354
+ }
355
+ activeFlusher?.excludeIdle(wake.wall_clock_ms - freezeWallMs);
356
+ },
357
+ });
358
+ }
293
359
  // Hand the redactor back too: the worker scrubs a terminal error's message with it;
294
360
  // workspace (when opted in) hydrates at start + persists at terminal.
295
361
  return Promise.resolve({
@@ -361,7 +427,7 @@ export function assembleWorkerDeps(runtime) {
361
427
  suspender: {
362
428
  suspend: (signal, workerId) => broker.suspend(signal, workerId),
363
429
  },
364
- // Runtime metering (§15): flush runtime as periodic deltas (+ a terminal tail) through the broker,
430
+ // Runtime metering: flush runtime as periodic deltas (+ a terminal tail) through the broker,
365
431
  // idempotent per flush, so a long/perpetual run bills as it burns and the credit watcher sees it —
366
432
  // not a single charge at terminal (which a never-terminating run never reached). A fresh per-session
367
433
  // id keeps a restarted run's sessions distinct in the idempotency key (distinct ids sum).
@@ -376,11 +442,14 @@ export function assembleWorkerDeps(runtime) {
376
442
  now: Date.now,
377
443
  report: (deltaSeconds, identifier) => broker.reportUsage(deltaSeconds, identifier),
378
444
  });
445
+ // The freeze coordinator's hooks flush this before a snapshot + rebase it past the frozen
446
+ // window on wake (suspended time must never appear as billed runtime).
447
+ activeFlusher = flusher;
379
448
  flusher.start();
380
449
  return { stop: () => flusher.stop(), flushFinal: () => flusher.flushFinal() };
381
450
  },
382
451
  buildHost,
383
- // Mid-run credit watching (§15): a CreditWatcher polls the broker's GET /credit on a timer; when
452
+ // Mid-run credit watching: a CreditWatcher polls the broker's GET /credit on a timer; when
384
453
  // the org runs out, onExhausted aborts the run (the orchestrator wires it to AbortController).
385
454
  startCreditWatch: ({ run, onExhausted }) => {
386
455
  const watcher = new CreditWatcher({
@@ -391,10 +460,10 @@ export function assembleWorkerDeps(runtime) {
391
460
  watcher.start();
392
461
  return { stop: () => watcher.stop() };
393
462
  },
394
- // Mid-run user-cancel watching (§6): a CancelWatcher polls the broker's GET /cancel on a timer;
463
+ // Mid-run user-cancel watching: a CancelWatcher polls the broker's GET /cancel on a timer;
395
464
  // when the user cancels (the run is flipped to `cancelling`), onCancelled aborts the run. This is
396
- // how the brokered (Redis-less) worker learns of a cancel — the api-server's Redis publish can't
397
- // reach it.
465
+ // how the brokered (cache-less) worker learns of a cancel — the platform's cancel publish can't
466
+ // reach it directly.
398
467
  startCancelWatch: ({ run, onCancelled }) => {
399
468
  const watcher = new CancelWatcher({
400
469
  runId: run.id,
@@ -404,7 +473,7 @@ export function assembleWorkerDeps(runtime) {
404
473
  watcher.start();
405
474
  return { stop: () => watcher.stop() };
406
475
  },
407
- // Lease renewal (§6): a LeaseRenewer extends the lease through the broker's POST /renew on a timer
476
+ // Lease renewal: a LeaseRenewer extends the lease through the broker's POST /renew on a timer
408
477
  // (well under the lease), so a run longer than the 5-min lease isn't reclaimed mid-flight by the
409
478
  // recovery sweep. `renew` re-extends with the SAME workerId that claimed it; a null result (the
410
479
  // run is no longer ours) fires onLost → the run aborts `lease_lost` without finalizing.
@@ -427,7 +496,7 @@ export function assembleWorkerDeps(runtime) {
427
496
  }
428
497
  // ---- Bootstrap (real container only) -------------------------------------------------
429
498
  /**
430
- * The per-run platform values the dispatcher injects as container env (ecs_run_task_client.ts). They
499
+ * The per-run platform values the dispatcher injects as container env. They
431
500
  * are captured into private worker state at bootstrap and DELETED from `process.env` before any user
432
501
  * program / agent leaf / subprocess can run — the run token + API token are credentials, and nothing
433
502
  * untrusted run code touches should inherit them (the run env/credential rules). The user owns the rest
@@ -441,7 +510,7 @@ export const PLATFORM_ENV_KEYS = [
441
510
  "BOARDWALK_TASK_CPU_UNITS",
442
511
  ];
443
512
  /** The public-API origin a run uses for raw API / MCP / CLI calls. The broker (Runner Control API)
444
- * is hosted on the api-server, so the public API shares its origin; the program appends `/v1` or
513
+ * shares an origin with the public API, so the program appends `/v1` or
445
514
  * `/mcp/v1`. Falls back to the broker URL unchanged if it can't be parsed. */
446
515
  export function publicApiOrigin(controlPlaneBaseUrl) {
447
516
  try {
@@ -465,7 +534,7 @@ export function capturePlatformContext(env) {
465
534
  return value.trim();
466
535
  };
467
536
  // The dispatcher always injects the control-plane handle (the Runner Credential Broker model). The runner is
468
- // brokered-only — there is no DB/Redis/Stripe fallback — so a missing handle is a hard config error.
537
+ // brokered-only — there is no database, cache, or billing fallback — so a missing handle is a hard config error.
469
538
  const runId = read("RUN_ID");
470
539
  const apiToken = env.BOARDWALK_API_KEY?.trim();
471
540
  const controlPlane = {
@@ -492,12 +561,22 @@ export async function main() {
492
561
  // daemon) the fd is absent and the worker env-boots exactly as before. The post-restore
493
562
  // uniqueness reseed will hook in at this boundary, before any run code executes.
494
563
  const relayFd = relayFdFromEnv(process.env);
564
+ let freezeRelay;
495
565
  if (relayFd !== null) {
496
566
  const relay = connectIdentityRelayFd(relayFd);
497
- relay.announceReady();
567
+ relay.announceReady(workerDiagnostics());
498
568
  const identity = await relay.awaitIdentity();
499
569
  applyIdentityToEnv(identity, process.env);
570
+ // Clause 3 (SNAPSHOT_UNIQUENESS_CONTRACT): this run was restored from the SHARED base
571
+ // snapshot, so its OpenSSL DRBG is identical to every other run's. Reseed it from the
572
+ // (VMGenID-diverged) OS entropy BEFORE `acceptIdentity` releases the worker into the brokered
573
+ // lifecycle — so no `crypto.*` draw by the SDK, the agent, or author code can collide across
574
+ // clones. Every wake reseeds again (the after-wake hook).
575
+ reseedUserspaceCsprng();
500
576
  relay.acceptIdentity();
577
+ // The relay now becomes the suspend/wake channel: assembleWorkerDeps opens it into the
578
+ // FreezeCoordinator, and the host's suspending seams freeze in place instead of exiting.
579
+ freezeRelay = relay;
501
580
  }
502
581
  // Capture the platform context into private state and remove it from process.env BEFORE anything
503
582
  // else — so no user program / agent tool / subprocess we later spawn can read the run token or
@@ -513,12 +592,13 @@ export async function main() {
513
592
  controlPlane: platform.controlPlane,
514
593
  vcpus: platform.vcpus,
515
594
  ...(byoProviders.length > 0 ? { byoProviders } : {}),
595
+ ...(freezeRelay !== undefined ? { freezeRelay } : {}),
516
596
  });
517
- // The only thing to drain is the batched telemetry buffer — the runner opens no DB/Redis/SQS.
597
+ // The only thing to drain is the batched telemetry buffer — the runner opens no database, cache, or queue.
518
598
  const cleanup = async () => {
519
599
  await deps.flushTelemetry?.().catch(() => undefined);
520
600
  };
521
- // SIGTERM: ECS is stopping the task. Hold-and-pay has no mid-run checkpoint; we exit and let the
601
+ // SIGTERM: the orchestrator is stopping the task. Hold-and-pay has no mid-run checkpoint; we exit and let the
522
602
  // lease expire → the scheduler-sweep reclaims it and a fresh worker RESTARTS the run from the
523
603
  // top (Lambda/GHA semantics; durable children re-attach via idempotency). Crash-safe by design.
524
604
  let shuttingDown = false;
@@ -32,7 +32,7 @@ export interface EngineLeafExecutorDeps {
32
32
  budget: BudgetMeter;
33
33
  /** RUN-level redactor (shared with the secret resolver): every resolved secret value is recorded
34
34
  * here. We seed a fresh engine `Redactor` from it per leaf so the loop scrubs known values out of
35
- * the prompt, tool args/results, and transcript before they reach the model (the platform spec). */
35
+ * the prompt, tool args/results, and transcript before they reach the model. */
36
36
  redactor: SecretRedactor;
37
37
  /** Builds this leaf's event sink; `leafIndex` (1-based) is only a metering identifier — the sink
38
38
  * owns the run-global cursor. `identity` names the leaf on its turn frames. */
@@ -51,7 +51,7 @@ export interface EngineLeafExecutorDeps {
51
51
  * omitted ⇒ no bundled tier (only the workspace AGENTS.md applies). */
52
52
  programDir?: () => string | null;
53
53
  /** Per-leaf token metering seam (fire-and-forget). Reports THIS leaf's tokens + its model to the
54
- * broker, which decides `billed_by_boardwalk` per model + meters to Stripe. Omitted in tests. */
54
+ * broker, which decides `billed_by_boardwalk` per model + meters usage to the platform. Omitted in tests. */
55
55
  meterUsage?: (input: MeterUsageInput) => void;
56
56
  /** Backend for the engine's host-backed built-in tools (`webfetch` / `web_search` / `artifacts`):
57
57
  * set as the leaf's `capabilities.host` so the engine registers them. Broker-backed on hosted runs
@@ -53,7 +53,7 @@ export interface ProgramRunnerDeps {
53
53
  * Scrubs known secret values out of a string (the run's `SecretRedactor.redactText`). Applied to a
54
54
  * top-level throw's message before it is logged AND before it is returned to the worker — a program
55
55
  * that resolves a secret and then throws it in an error message must NOT land that secret raw in the
56
- * logs or the finalized run output (the platform spec). Defaults to identity (tests/local).
56
+ * logs or the finalized run output. Defaults to identity (tests/local).
57
57
  */
58
58
  redactText?: (text: string) => string;
59
59
  /**
@@ -1,6 +1,6 @@
1
1
  // WorkflowProgramRunner — executes a workflow program (the JS-body model, the workflow runtime design).
2
2
  //
3
- // A run is the execution of a built program ARTIFACT (§3.9): the worker is handed the VERIFIED tarball
3
+ // A run is the execution of a built program ARTIFACT: the worker is handed the VERIFIED tarball
4
4
  // (its sha256 already checked against the pinned digest by the orchestrator) plus the entry module
5
5
  // name. This module is the mechanism: it installs the host adapter + trigger payload onto the
6
6
  // `@boardwalk-labs/workflow` singleton, extracts the artifact into a unique temp dir under a