@boardwalk-labs/runner 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,119 @@
1
+ import { z } from "zod";
2
+ import type { RelayChannel } from "./identity_relay.js";
3
+ import type { SuspendSignal } from "./suspension.js";
4
+ /** First retry delay after a suspend_abort (doubles per attempt, capped below). */
5
+ export declare const ABORT_RETRY_INITIAL_MS = 30000;
6
+ export declare const ABORT_RETRY_MAX_MS: number;
7
+ /** The wake value inside a wake injection — `kind` echoes the parked seam's reason; the
8
+ * per-kind fields are opaque to the coordinator and interpreted by the seam. */
9
+ export declare const wakeValueSchema: z.ZodObject<{
10
+ kind: z.ZodEnum<{
11
+ sleep: "sleep";
12
+ human_input: "human_input";
13
+ workflow_call: "workflow_call";
14
+ }>;
15
+ answers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
16
+ child: z.ZodOptional<z.ZodObject<{
17
+ run_id: z.ZodString;
18
+ status: z.ZodString;
19
+ output: z.ZodUnknown;
20
+ }, z.core.$strip>>;
21
+ }, z.core.$strip>;
22
+ export type WakeValue = z.infer<typeof wakeValueSchema>;
23
+ /** The wake-injection payload as init relays it (snake_case, the platform env contract's
24
+ * sibling): fresh tokens — the frozen ones expired while suspended — plus the
25
+ * authoritative wall clock (the guest's own clock was stopped) and the wake value. */
26
+ export declare const wakePayloadSchema: z.ZodObject<{
27
+ run_token: z.ZodString;
28
+ api_token: z.ZodOptional<z.ZodString>;
29
+ wall_clock_ms: z.ZodNumber;
30
+ wake: z.ZodObject<{
31
+ kind: z.ZodEnum<{
32
+ sleep: "sleep";
33
+ human_input: "human_input";
34
+ workflow_call: "workflow_call";
35
+ }>;
36
+ answers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
37
+ child: z.ZodOptional<z.ZodObject<{
38
+ run_id: z.ZodString;
39
+ status: z.ZodString;
40
+ output: z.ZodUnknown;
41
+ }, z.core.$strip>>;
42
+ }, z.core.$strip>;
43
+ }, z.core.$strip>;
44
+ export type WakePayload = z.infer<typeof wakePayloadSchema>;
45
+ /** What a suspending wait resolves to: the wake (the normal path), or — for a `sleep` seam
46
+ * only — the abort that tells it to hold its remainder in-process (other reasons retry the
47
+ * freeze internally and never surface an abort). */
48
+ export type FreezeOutcome = {
49
+ kind: "wake";
50
+ wake: WakeValue;
51
+ } | {
52
+ kind: "aborted";
53
+ reason: string;
54
+ };
55
+ export interface FreezeCoordinatorHooks {
56
+ /** Runs at quiescence, immediately before the freeze is requested: flush billable runtime
57
+ * (suspended time must never appear billed) and persist the workspace. Its own failure
58
+ * aborts THIS freeze attempt (the seam holds/retries) — never the run. */
59
+ onBeforeFreeze?: () => Promise<void>;
60
+ /** Runs when a wake lands, before the seam resolves: swap the run/api tokens onto the
61
+ * broker client and rebase the runtime meter past the frozen window. */
62
+ onAfterWake?: (wake: WakePayload) => void | Promise<void>;
63
+ }
64
+ export interface FreezeCoordinatorDeps {
65
+ channel: RelayChannel;
66
+ now?: () => number;
67
+ /** Injected delay (tests). Defaults to a real timer. */
68
+ delay?: (ms: number) => Promise<void>;
69
+ }
70
+ export declare class FreezeCoordinator {
71
+ private readonly deps;
72
+ private readonly now;
73
+ private readonly delay;
74
+ private hooks;
75
+ /** Non-suspending runtime work in flight (the quiescence gate's count). */
76
+ private inFlight;
77
+ private quiescenceWaiters;
78
+ /** True from freeze request until wake/abort — new runtime work queues behind it. */
79
+ private freezePending;
80
+ private gateWaiters;
81
+ /** The parked suspending wait (at most one — waits serialize). */
82
+ private parked;
83
+ /** Serializes suspending waits. */
84
+ private chain;
85
+ constructor(deps: FreezeCoordinatorDeps);
86
+ /** Late-bound per-run hooks (the flusher/broker/redactor exist only once a run is claimed). */
87
+ setHooks(hooks: FreezeCoordinatorHooks): void;
88
+ /**
89
+ * Run one unit of non-suspending runtime work under the gate: queue while a freeze is
90
+ * pending (the work never starts, so nothing can be torn by the pause), then count it
91
+ * in-flight so a suspending wait holds until it drains. The gate check and the count
92
+ * increment share one synchronous segment — a freeze requested in between cannot miss us.
93
+ */
94
+ trackWork<T>(fn: () => Promise<T>): Promise<T>;
95
+ /** A suspending seam's own wait is NOT "work" — it decrements around its park so the gate
96
+ * sees true quiescence. (The host wraps every hook in trackWork; the suspending seams
97
+ * call these around their freeze wait.) */
98
+ beginWork(): void;
99
+ endWork(): void;
100
+ /**
101
+ * The suspending wait: hold until quiescence, run the pre-freeze hook, request the
102
+ * freeze, and park. The next thing this promise sees is a wake (possibly epochs later,
103
+ * through a restored heap) or a suspend_abort. Returns the wake, or — for `sleep` only —
104
+ * the abort (the seam then holds in-process); other reasons retry the freeze after a
105
+ * backoff. Concurrent suspending waits serialize through an internal chain.
106
+ */
107
+ suspendingWait(signal: SuspendSignal): Promise<FreezeOutcome>;
108
+ /** Relay handler: a wake injection landed. Validates, runs the after-wake hook (token
109
+ * swap + meter rebase), confirms to init, and resolves the parked seam. A wake with no
110
+ * parked seam is a duplicate delivery — re-confirm (idempotent), never crash. */
111
+ onWake(payload: unknown): void;
112
+ /** Relay handler: the snapshot attempt failed; the parked seam falls back to holding. */
113
+ onSuspendAbort(payload: unknown): void;
114
+ private awaitQuiescence;
115
+ private releaseGate;
116
+ /** The host-readable wake summary on the wire (logs/metrics/placement) — the opaque
117
+ * broker_signal beside it carries the full condition. */
118
+ private wakeConditionOf;
119
+ }
@@ -0,0 +1,262 @@
1
+ // FreezeCoordinator — the runtime's half of snapshot suspension on the microVM substrate.
2
+ //
3
+ // On this substrate the worker process NEVER exits to suspend: a suspending seam (a long
4
+ // sleep, a human-input gate, a child-run wait) BLOCKS on a real promise, the platform
5
+ // freezes the whole VM into a snapshot, and a later restore resolves that same promise with
6
+ // the wake value — the heap is the literal heap, nothing replays. This coordinator owns the
7
+ // two policies that make freezing safe and predictable:
8
+ //
9
+ // 1. THE QUIESCENCE GATE: never freeze while non-suspending runtime work (an agent leaf, a
10
+ // tool call, an artifact write) is in flight — a snapshot must not capture a live
11
+ // platform stream that would be dead on restore. A suspending wait created during such
12
+ // work HOLDS until the work drains; new runtime work that arrives while a freeze is
13
+ // pending QUEUES (it never started, so nothing is torn) and runs after the wake.
14
+ // 2. SNAPSHOT-FIRST FALLBACK: a `suspend_abort` from the platform (snapshot/store/broker
15
+ // failure) means the seam falls back to HOLDING — a sleep waits in-process for its
16
+ // remainder; a human-input or child wait retries the freeze after a backoff (the host
17
+ // may throttle repeated attempts).
18
+ //
19
+ // Concurrent suspending waits SERIALIZE: the first to reach quiescence freezes with its own
20
+ // wake condition; after its wake, the next takes its turn. (A compound wake condition —
21
+ // one freeze covering several waits — is a future optimization, not a correctness need:
22
+ // the heap survives every cycle.)
23
+ //
24
+ // The wire below this is the guest init's relay (identity_relay `openChannel`); the policy
25
+ // above it is decided platform-side. The coordinator never interprets the wake VALUE — it
26
+ // validates the envelope and hands `wake` to the seam that parked.
27
+ import { z } from "zod";
28
+ import { createLogger } from "./support/index.js";
29
+ const log = createLogger("FreezeCoordinator");
30
+ /** First retry delay after a suspend_abort (doubles per attempt, capped below). */
31
+ export const ABORT_RETRY_INITIAL_MS = 30_000;
32
+ export const ABORT_RETRY_MAX_MS = 5 * 60_000;
33
+ /** The wake value inside a wake injection — `kind` echoes the parked seam's reason; the
34
+ * per-kind fields are opaque to the coordinator and interpreted by the seam. */
35
+ export const wakeValueSchema = z.object({
36
+ kind: z.enum(["sleep", "human_input", "workflow_call"]),
37
+ /** human_input: EVERY gate this suspension raised, keyed by gate key / tool-call id. */
38
+ answers: z.record(z.string(), z.unknown()).optional(),
39
+ /** workflow_call: the finalized child. */
40
+ child: z
41
+ .object({
42
+ run_id: z.string().min(1),
43
+ status: z.string().min(1),
44
+ output: z.unknown(),
45
+ })
46
+ .optional(),
47
+ });
48
+ /** The wake-injection payload as init relays it (snake_case, the platform env contract's
49
+ * sibling): fresh tokens — the frozen ones expired while suspended — plus the
50
+ * authoritative wall clock (the guest's own clock was stopped) and the wake value. */
51
+ export const wakePayloadSchema = z.object({
52
+ run_token: z.string().min(1),
53
+ api_token: z.string().optional(),
54
+ wall_clock_ms: z.number(),
55
+ wake: wakeValueSchema,
56
+ });
57
+ const suspendAbortSchema = z.object({ reason: z.string().optional() });
58
+ export class FreezeCoordinator {
59
+ deps;
60
+ now;
61
+ delay;
62
+ hooks = {};
63
+ /** Non-suspending runtime work in flight (the quiescence gate's count). */
64
+ inFlight = 0;
65
+ quiescenceWaiters = [];
66
+ /** True from freeze request until wake/abort — new runtime work queues behind it. */
67
+ freezePending = false;
68
+ gateWaiters = [];
69
+ /** The parked suspending wait (at most one — waits serialize). */
70
+ parked = null;
71
+ /** Serializes suspending waits. */
72
+ chain = Promise.resolve();
73
+ constructor(deps) {
74
+ this.deps = deps;
75
+ this.now = deps.now ?? Date.now;
76
+ // A pending retry delay legitimately keeps the process alive — the run isn't done.
77
+ this.delay =
78
+ deps.delay ??
79
+ ((ms) => new Promise((resolve) => {
80
+ setTimeout(resolve, ms);
81
+ }));
82
+ }
83
+ /** Late-bound per-run hooks (the flusher/broker/redactor exist only once a run is claimed). */
84
+ setHooks(hooks) {
85
+ this.hooks = hooks;
86
+ }
87
+ /**
88
+ * Run one unit of non-suspending runtime work under the gate: queue while a freeze is
89
+ * pending (the work never starts, so nothing can be torn by the pause), then count it
90
+ * in-flight so a suspending wait holds until it drains. The gate check and the count
91
+ * increment share one synchronous segment — a freeze requested in between cannot miss us.
92
+ */
93
+ async trackWork(fn) {
94
+ while (this.freezePending) {
95
+ await new Promise((resolve) => {
96
+ this.gateWaiters.push(resolve);
97
+ });
98
+ }
99
+ this.beginWork();
100
+ try {
101
+ return await fn();
102
+ }
103
+ finally {
104
+ this.endWork();
105
+ }
106
+ }
107
+ /** A suspending seam's own wait is NOT "work" — it decrements around its park so the gate
108
+ * sees true quiescence. (The host wraps every hook in trackWork; the suspending seams
109
+ * call these around their freeze wait.) */
110
+ beginWork() {
111
+ this.inFlight += 1;
112
+ }
113
+ endWork() {
114
+ this.inFlight -= 1;
115
+ if (this.inFlight === 0) {
116
+ const waiters = this.quiescenceWaiters;
117
+ this.quiescenceWaiters = [];
118
+ for (const w of waiters)
119
+ w();
120
+ }
121
+ }
122
+ /**
123
+ * The suspending wait: hold until quiescence, run the pre-freeze hook, request the
124
+ * freeze, and park. The next thing this promise sees is a wake (possibly epochs later,
125
+ * through a restored heap) or a suspend_abort. Returns the wake, or — for `sleep` only —
126
+ * the abort (the seam then holds in-process); other reasons retry the freeze after a
127
+ * backoff. Concurrent suspending waits serialize through an internal chain.
128
+ */
129
+ suspendingWait(signal) {
130
+ const turn = this.chain;
131
+ let release = () => undefined;
132
+ this.chain = new Promise((resolve) => {
133
+ release = resolve;
134
+ });
135
+ return (async () => {
136
+ await turn;
137
+ try {
138
+ let backoff = ABORT_RETRY_INITIAL_MS;
139
+ for (;;) {
140
+ await this.awaitQuiescence();
141
+ try {
142
+ await this.hooks.onBeforeFreeze?.();
143
+ }
144
+ catch (err) {
145
+ // A failed pre-freeze flush must not strand the seam: treat it like an abort.
146
+ log.warn("freeze_prepare_failed", {
147
+ error: err instanceof Error ? err.message : String(err),
148
+ });
149
+ if (signal.reason === "sleep")
150
+ return { kind: "aborted", reason: "prepare_failed" };
151
+ await this.delay(backoff);
152
+ backoff = Math.min(backoff * 2, ABORT_RETRY_MAX_MS);
153
+ continue;
154
+ }
155
+ this.freezePending = true;
156
+ const outcome = await new Promise((resolve) => {
157
+ this.parked = resolve;
158
+ this.deps.channel.sendSuspendRequest({
159
+ reason: signal.reason,
160
+ wake: this.wakeConditionOf(signal),
161
+ broker_signal: signal,
162
+ });
163
+ // ← the VM freezes while this promise is pending; the wake resolves it with the
164
+ // heap (and this very closure) restored.
165
+ });
166
+ this.freezePending = false;
167
+ this.releaseGate();
168
+ if (outcome.kind === "wake")
169
+ return outcome;
170
+ if (signal.reason === "sleep")
171
+ return outcome;
172
+ log.warn("suspend_aborted_retrying", {
173
+ reason: outcome.reason,
174
+ seamReason: signal.reason,
175
+ backoffMs: backoff,
176
+ });
177
+ await this.delay(backoff);
178
+ backoff = Math.min(backoff * 2, ABORT_RETRY_MAX_MS);
179
+ }
180
+ }
181
+ finally {
182
+ release();
183
+ }
184
+ })();
185
+ }
186
+ /** Relay handler: a wake injection landed. Validates, runs the after-wake hook (token
187
+ * swap + meter rebase), confirms to init, and resolves the parked seam. A wake with no
188
+ * parked seam is a duplicate delivery — re-confirm (idempotent), never crash. */
189
+ onWake(payload) {
190
+ const parsed = wakePayloadSchema.safeParse(payload);
191
+ if (!parsed.success) {
192
+ // Unanswerable garbage: init will retry, time out, and the platform's crash path owns
193
+ // recovery. Log loudly — this is a control-plane/init bug, not author code.
194
+ log.error("wake_payload_invalid", { issues: parsed.error.message });
195
+ return;
196
+ }
197
+ if (this.parked === null) {
198
+ log.warn("duplicate_wake_ignored", {});
199
+ this.deps.channel.sendWakeAccepted();
200
+ return;
201
+ }
202
+ const resolve = this.parked;
203
+ this.parked = null;
204
+ void (async () => {
205
+ try {
206
+ await this.hooks.onAfterWake?.(parsed.data);
207
+ }
208
+ catch (err) {
209
+ // Token swap / meter rebase failing is survivable only loudly — the run continues
210
+ // on the old token and fails fast if it truly expired.
211
+ log.error("after_wake_hook_failed", {
212
+ error: err instanceof Error ? err.message : String(err),
213
+ });
214
+ }
215
+ this.deps.channel.sendWakeAccepted();
216
+ resolve({ kind: "wake", wake: parsed.data.wake });
217
+ })();
218
+ }
219
+ /** Relay handler: the snapshot attempt failed; the parked seam falls back to holding. */
220
+ onSuspendAbort(payload) {
221
+ const parsed = suspendAbortSchema.safeParse(payload);
222
+ const reason = parsed.success ? (parsed.data.reason ?? "unknown") : "unknown";
223
+ if (this.parked === null) {
224
+ log.warn("suspend_abort_without_parked_seam", { reason });
225
+ return;
226
+ }
227
+ const resolve = this.parked;
228
+ this.parked = null;
229
+ resolve({ kind: "aborted", reason });
230
+ }
231
+ awaitQuiescence() {
232
+ if (this.inFlight === 0)
233
+ return Promise.resolve();
234
+ return new Promise((resolve) => {
235
+ this.quiescenceWaiters.push(resolve);
236
+ });
237
+ }
238
+ releaseGate() {
239
+ const waiters = this.gateWaiters;
240
+ this.gateWaiters = [];
241
+ for (const w of waiters)
242
+ w();
243
+ }
244
+ /** The host-readable wake summary on the wire (logs/metrics/placement) — the opaque
245
+ * broker_signal beside it carries the full condition. */
246
+ wakeConditionOf(signal) {
247
+ switch (signal.reason) {
248
+ case "sleep":
249
+ return { kind: "sleep", wake_at_ms: this.now() + (signal.durationMs ?? 0) };
250
+ case "human_input":
251
+ return {
252
+ kind: "human_input",
253
+ ...(signal.humanInput !== undefined ? { request_keys: [signal.humanInput.key] } : {}),
254
+ };
255
+ case "workflow_call":
256
+ return {
257
+ kind: "workflow_call",
258
+ ...(signal.childRunId !== undefined ? { child_run_id: signal.childRunId } : {}),
259
+ };
260
+ }
261
+ }
262
+ }
@@ -0,0 +1,97 @@
1
+ import type { Duplex } from "node:stream";
2
+ import { z } from "zod";
3
+ /** Env var naming the inherited relay fd. Set by the guest init; absent everywhere else
4
+ * (Fargate, self-hosted daemon), where the worker env-boots as always. */
5
+ export declare const RELAY_FD_ENV = "BOARDWALK_IDENTITY_RELAY_FD";
6
+ /** Max bytes of one relay line — the wire protocol's 32 MiB frame cap, mirrored in-guest.
7
+ * An oversized line costs the connection (LF framing cannot resynchronize past it), which
8
+ * for this relay means a hard bootstrap failure. Measured in UTF-16 code units, which for
9
+ * this ASCII-JSON wire is the byte count; the cap is a guard, not exact accounting. */
10
+ export declare const MAX_RELAY_LINE_BYTES: number;
11
+ /** The identity payload — the env contract's fields, as JSON instead of env vars. */
12
+ export declare const relayIdentitySchema: z.ZodObject<{
13
+ run_id: z.ZodString;
14
+ control_plane_url: z.ZodString;
15
+ run_token: z.ZodString;
16
+ api_token: z.ZodOptional<z.ZodString>;
17
+ task_cpu_units: z.ZodOptional<z.ZodNumber>;
18
+ byo_providers: z.ZodOptional<z.ZodUnknown>;
19
+ env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
20
+ }, z.core.$strip>;
21
+ export type RelayIdentity = z.infer<typeof relayIdentitySchema>;
22
+ /** The init→worker halves of the post-identity relay: `wake` resolves a frozen suspending
23
+ * seam (payload: the wake-inject body, verbatim), `suspend_abort` tells a parked seam the
24
+ * snapshot attempt failed and it should fall back to holding. */
25
+ export interface RelayChannelHandlers {
26
+ onWake(payload: unknown): void;
27
+ onSuspendAbort(payload: unknown): void;
28
+ }
29
+ /** The worker→init halves: ask to be frozen (payload: the suspend-request body — reason,
30
+ * wake summary, opaque broker signal), and confirm a wake landed (init then acks its host). */
31
+ export interface RelayChannel {
32
+ sendSuspendRequest(payload: unknown): void;
33
+ sendWakeAccepted(): void;
34
+ }
35
+ /**
36
+ * Read the relay fd from `env`, deleting the key (bootstrap-only plumbing; run code has no
37
+ * business seeing it). Returns null when unset — the normal env-boot path.
38
+ */
39
+ export declare function relayFdFromEnv(env: NodeJS.ProcessEnv): number | null;
40
+ /**
41
+ * Map a relayed identity onto `env` for `capturePlatformContext`. User env lands FIRST and
42
+ * the platform keys LAST, so nothing user-supplied can shadow a platform value.
43
+ */
44
+ export declare function applyIdentityToEnv(identity: RelayIdentity, env: NodeJS.ProcessEnv): void;
45
+ /** The worker_ready diagnostics — supplied by the worker (init cannot know these),
46
+ * forwarded verbatim by the guest init. Best-effort, never load-bearing. */
47
+ export interface WorkerDiagnostics {
48
+ worker_version?: string;
49
+ node_version: string;
50
+ sdk_version?: string;
51
+ }
52
+ /** Collect the worker_ready diagnostics. Version lookups are best-effort: the runner's own
53
+ * package.json sits two levels above this module in both the src and published dist
54
+ * layouts; the SDK's is reachable only if its exports expose ./package.json. */
55
+ export declare function workerDiagnostics(): WorkerDiagnostics;
56
+ /** One end of the init↔worker relay. Wraps any Duplex so tests run over in-memory streams. */
57
+ export declare class IdentityRelay {
58
+ private readonly stream;
59
+ private buffer;
60
+ private closed;
61
+ private failure;
62
+ private wake;
63
+ private readonly onData;
64
+ constructor(stream: Duplex);
65
+ private fail;
66
+ /** Announce the pre-identity park (with the diagnostics payload when provided).
67
+ * Init forwards this as the base-snapshot gate. */
68
+ announceReady(diagnostics?: WorkerDiagnostics): void;
69
+ /**
70
+ * Block until init relays the run identity. THIS is the point the base snapshot freezes:
71
+ * everything before it is generic warm-up shared by every run; everything after belongs
72
+ * to one run. Unknown message types are ignored (forward-compatible), malformed lines are
73
+ * logged and skipped (init is trusted; a torn line must not kill PID 1's only worker), but
74
+ * a malformed IDENTITY payload is a hard error — same as a missing env var today.
75
+ */
76
+ awaitIdentity(): Promise<RelayIdentity>;
77
+ /** Confirm capture. Init acks its host only after this arrives. */
78
+ acceptIdentity(): void;
79
+ /**
80
+ * Attach the post-identity consumer: the suspend/wake channel (the same fd, the same JSON
81
+ * lines). From here the relay dispatches init→worker messages to the handlers —
82
+ * `wake` (resolve a frozen seam) and `suspend_abort` (the seam falls back to holding) —
83
+ * and the returned channel sends the worker→init halves (`suspend_request`,
84
+ * `wake_accepted`). Unknown types are ignored (forward-compatible); malformed lines are
85
+ * logged and skipped, exactly as pre-identity. Call once, after {@link acceptIdentity}.
86
+ */
87
+ openChannel(handlers: RelayChannelHandlers): RelayChannel;
88
+ private pumpChannel;
89
+ private writeLine;
90
+ private readLine;
91
+ }
92
+ /**
93
+ * Open the relay over the inherited fd. Wrapping the fd in a net.Socket also marks it
94
+ * close-on-exec (libuv does this on adoption), so subprocesses the run later spawns cannot
95
+ * inherit the relay and speak to init.
96
+ */
97
+ export declare function connectIdentityRelayFd(fd: number): IdentityRelay;
@@ -0,0 +1,291 @@
1
+ // Identity relay — the microVM bootstrap boundary.
2
+ //
3
+ // On Boardwalk's snapshot-based microVM substrate the worker starts BEFORE it has a run: the
4
+ // guest init (PID 1) spawns it warm (Node up, this code loaded, parked pre-identity), the
5
+ // platform snapshots the whole VM, and every run restores from that snapshot and INJECTS the
6
+ // run identity over an in-guest relay instead of container env. The relay is an inherited
7
+ // AF_UNIX socketpair; init tells us its fd via `BOARDWALK_IDENTITY_RELAY_FD`. Wire: one JSON
8
+ // object per LF-terminated line —
9
+ //
10
+ // {"type":"worker_ready","payload":{...}} worker → init the pre-identity park is reached
11
+ // (payload: optional version diagnostics,
12
+ // forwarded verbatim by init)
13
+ // {"type":"identity","payload":{...}} init → worker the run identity (see schema below)
14
+ // {"type":"identity_accepted"} worker → init captured; init acks its host
15
+ // {"type":"suspend_request","payload":{...}} worker → init a quiescent seam asks to be frozen
16
+ // {"type":"suspend_abort","payload":{...}} init → worker snapshot failed; the seam holds instead
17
+ // {"type":"wake","payload":{...}} init → worker resolve the frozen seam (fresh tokens
18
+ // + the wake value, verbatim)
19
+ // {"type":"wake_accepted"} worker → init tokens swapped, seam resolved; init acks
20
+ //
21
+ // The identity payload carries exactly the platform env contract (snake_case) plus the resolved
22
+ // user env; `applyIdentityToEnv` maps it onto `process.env` so `capturePlatformContext` runs
23
+ // UNCHANGED afterward — same fields, same capture-and-delete discipline, two transports.
24
+ // The stream stays open after identity: `openChannel` attaches the suspend/wake consumer
25
+ // (the FreezeCoordinator drives the choreography above it).
26
+ import { createRequire } from "node:module";
27
+ import { Socket } from "node:net";
28
+ import { z } from "zod";
29
+ import { createLogger } from "./support/index.js";
30
+ const log = createLogger("identity_relay");
31
+ /** Env var naming the inherited relay fd. Set by the guest init; absent everywhere else
32
+ * (Fargate, self-hosted daemon), where the worker env-boots as always. */
33
+ export const RELAY_FD_ENV = "BOARDWALK_IDENTITY_RELAY_FD";
34
+ /** Max bytes of one relay line — the wire protocol's 32 MiB frame cap, mirrored in-guest.
35
+ * An oversized line costs the connection (LF framing cannot resynchronize past it), which
36
+ * for this relay means a hard bootstrap failure. Measured in UTF-16 code units, which for
37
+ * this ASCII-JSON wire is the byte count; the cap is a guard, not exact accounting. */
38
+ export const MAX_RELAY_LINE_BYTES = 32 * 1024 * 1024;
39
+ /** The identity payload — the env contract's fields, as JSON instead of env vars. */
40
+ export const relayIdentitySchema = z.object({
41
+ run_id: z.string().min(1),
42
+ control_plane_url: z.string().min(1),
43
+ run_token: z.string().min(1),
44
+ api_token: z.string().optional(),
45
+ task_cpu_units: z.number().int().positive().optional(),
46
+ /** The BYO inference provider registry, verbatim (stringified into BOARDWALK_BYO_PROVIDERS). */
47
+ byo_providers: z.unknown().optional(),
48
+ /** Resolved NON-secret user env. Platform keys always win over these (applied last). */
49
+ env: z.record(z.string(), z.string()).optional(),
50
+ });
51
+ const relayMessageSchema = z.object({
52
+ type: z.string(),
53
+ payload: z.unknown().optional(),
54
+ });
55
+ /**
56
+ * Read the relay fd from `env`, deleting the key (bootstrap-only plumbing; run code has no
57
+ * business seeing it). Returns null when unset — the normal env-boot path.
58
+ */
59
+ export function relayFdFromEnv(env) {
60
+ const raw = env[RELAY_FD_ENV];
61
+ Reflect.deleteProperty(env, RELAY_FD_ENV);
62
+ if (raw === undefined || raw.trim().length === 0)
63
+ return null;
64
+ const fd = Number(raw);
65
+ if (!Number.isInteger(fd) || fd < 3) {
66
+ throw new Error(`${RELAY_FD_ENV} must name an inherited fd (>= 3), got ${raw}`);
67
+ }
68
+ return fd;
69
+ }
70
+ /**
71
+ * Map a relayed identity onto `env` for `capturePlatformContext`. User env lands FIRST and
72
+ * the platform keys LAST, so nothing user-supplied can shadow a platform value.
73
+ */
74
+ export function applyIdentityToEnv(identity, env) {
75
+ for (const [key, value] of Object.entries(identity.env ?? {})) {
76
+ env[key] = value;
77
+ }
78
+ env.RUN_ID = identity.run_id;
79
+ env.BOARDWALK_CONTROL_PLANE_URL = identity.control_plane_url;
80
+ env.BOARDWALK_RUN_TOKEN = identity.run_token;
81
+ if (identity.api_token !== undefined && identity.api_token.length > 0) {
82
+ env.BOARDWALK_API_KEY = identity.api_token;
83
+ }
84
+ if (identity.task_cpu_units !== undefined) {
85
+ env.BOARDWALK_TASK_CPU_UNITS = String(identity.task_cpu_units);
86
+ }
87
+ if (identity.byo_providers !== undefined) {
88
+ env.BOARDWALK_BYO_PROVIDERS = JSON.stringify(identity.byo_providers);
89
+ }
90
+ }
91
+ /** Collect the worker_ready diagnostics. Version lookups are best-effort: the runner's own
92
+ * package.json sits two levels above this module in both the src and published dist
93
+ * layouts; the SDK's is reachable only if its exports expose ./package.json. */
94
+ export function workerDiagnostics() {
95
+ const require = createRequire(import.meta.url);
96
+ const versionOf = (spec) => {
97
+ try {
98
+ const pkg = require(spec);
99
+ return typeof pkg.version === "string" ? pkg.version : undefined;
100
+ }
101
+ catch {
102
+ return undefined;
103
+ }
104
+ };
105
+ const worker = versionOf("../../package.json");
106
+ const sdk = versionOf("@boardwalk-labs/workflow/package.json");
107
+ return {
108
+ ...(worker !== undefined ? { worker_version: worker } : {}),
109
+ node_version: process.version,
110
+ ...(sdk !== undefined ? { sdk_version: sdk } : {}),
111
+ };
112
+ }
113
+ /** One end of the init↔worker relay. Wraps any Duplex so tests run over in-memory streams. */
114
+ export class IdentityRelay {
115
+ stream;
116
+ buffer = "";
117
+ closed = false;
118
+ failure = null;
119
+ wake = null;
120
+ onData;
121
+ constructor(stream) {
122
+ this.stream = stream;
123
+ // One persistent listener for the relay's lifetime: a flowing stream DROPS chunks
124
+ // emitted while nobody listens, so attach-per-read would lose lines between reads.
125
+ this.onData = (chunk) => {
126
+ this.buffer += typeof chunk === "string" ? chunk : chunk.toString("utf8");
127
+ if (this.buffer.length > MAX_RELAY_LINE_BYTES && !this.buffer.includes("\n")) {
128
+ // The unterminated tail can never become a legal line — fail now instead of
129
+ // buffering without bound (the wire cap, mirrored; init is trusted, so this is a
130
+ // bug or corruption, not an attack to survive).
131
+ this.fail(new Error(`identity relay line exceeds ${MAX_RELAY_LINE_BYTES} bytes`));
132
+ return;
133
+ }
134
+ this.wake?.();
135
+ };
136
+ stream.on("data", this.onData);
137
+ stream.on("end", () => {
138
+ this.closed = true;
139
+ this.wake?.();
140
+ });
141
+ stream.on("error", (err) => {
142
+ // Keep the error's detail — "closed" alone hides why the socket died.
143
+ this.fail(err);
144
+ });
145
+ }
146
+ fail(err) {
147
+ this.failure ??= err;
148
+ this.closed = true;
149
+ this.stream.destroy();
150
+ this.wake?.();
151
+ }
152
+ /** Announce the pre-identity park (with the diagnostics payload when provided).
153
+ * Init forwards this as the base-snapshot gate. */
154
+ announceReady(diagnostics) {
155
+ this.writeLine(diagnostics === undefined
156
+ ? { type: "worker_ready" }
157
+ : { type: "worker_ready", payload: diagnostics });
158
+ }
159
+ /**
160
+ * Block until init relays the run identity. THIS is the point the base snapshot freezes:
161
+ * everything before it is generic warm-up shared by every run; everything after belongs
162
+ * to one run. Unknown message types are ignored (forward-compatible), malformed lines are
163
+ * logged and skipped (init is trusted; a torn line must not kill PID 1's only worker), but
164
+ * a malformed IDENTITY payload is a hard error — same as a missing env var today.
165
+ */
166
+ async awaitIdentity() {
167
+ for (;;) {
168
+ const line = await this.readLine();
169
+ let parsed;
170
+ try {
171
+ parsed = JSON.parse(line);
172
+ }
173
+ catch {
174
+ log.warn("relay_malformed_line_skipped", { length: line.length });
175
+ continue;
176
+ }
177
+ const message = relayMessageSchema.safeParse(parsed);
178
+ if (!message.success) {
179
+ log.warn("relay_malformed_message_skipped", {});
180
+ continue;
181
+ }
182
+ if (message.data.type !== "identity") {
183
+ log.warn("relay_unexpected_type_ignored", { type: message.data.type });
184
+ continue;
185
+ }
186
+ const identity = relayIdentitySchema.safeParse(message.data.payload);
187
+ if (!identity.success) {
188
+ throw new Error(`Relayed identity payload is invalid: ${identity.error.message}`);
189
+ }
190
+ return identity.data;
191
+ }
192
+ }
193
+ /** Confirm capture. Init acks its host only after this arrives. */
194
+ acceptIdentity() {
195
+ this.writeLine({ type: "identity_accepted" });
196
+ }
197
+ /**
198
+ * Attach the post-identity consumer: the suspend/wake channel (the same fd, the same JSON
199
+ * lines). From here the relay dispatches init→worker messages to the handlers —
200
+ * `wake` (resolve a frozen seam) and `suspend_abort` (the seam falls back to holding) —
201
+ * and the returned channel sends the worker→init halves (`suspend_request`,
202
+ * `wake_accepted`). Unknown types are ignored (forward-compatible); malformed lines are
203
+ * logged and skipped, exactly as pre-identity. Call once, after {@link acceptIdentity}.
204
+ */
205
+ openChannel(handlers) {
206
+ void this.pumpChannel(handlers);
207
+ return {
208
+ sendSuspendRequest: (payload) => {
209
+ this.writeLine({ type: "suspend_request", payload });
210
+ },
211
+ sendWakeAccepted: () => {
212
+ this.writeLine({ type: "wake_accepted" });
213
+ },
214
+ };
215
+ }
216
+ async pumpChannel(handlers) {
217
+ for (;;) {
218
+ let line;
219
+ try {
220
+ line = await this.readLine();
221
+ }
222
+ catch (err) {
223
+ // The relay dying mid-run means init is gone — the guest is being torn down; the
224
+ // worker's own exit follows. Log, don't throw into nowhere.
225
+ log.warn("relay_channel_closed", {
226
+ error: err instanceof Error ? err.message : String(err),
227
+ });
228
+ return;
229
+ }
230
+ let parsed;
231
+ try {
232
+ parsed = JSON.parse(line);
233
+ }
234
+ catch {
235
+ log.warn("relay_malformed_line_skipped", { length: line.length });
236
+ continue;
237
+ }
238
+ const message = relayMessageSchema.safeParse(parsed);
239
+ if (!message.success) {
240
+ log.warn("relay_malformed_message_skipped", {});
241
+ continue;
242
+ }
243
+ switch (message.data.type) {
244
+ case "wake":
245
+ handlers.onWake(message.data.payload);
246
+ break;
247
+ case "suspend_abort":
248
+ handlers.onSuspendAbort(message.data.payload);
249
+ break;
250
+ default:
251
+ log.warn("relay_unexpected_type_ignored", { type: message.data.type });
252
+ }
253
+ }
254
+ }
255
+ writeLine(message) {
256
+ this.stream.write(JSON.stringify(message) + "\n");
257
+ }
258
+ async readLine() {
259
+ for (;;) {
260
+ const newline = this.buffer.indexOf("\n");
261
+ if (newline >= 0) {
262
+ const line = this.buffer.slice(0, newline);
263
+ this.buffer = this.buffer.slice(newline + 1);
264
+ if (line.length > MAX_RELAY_LINE_BYTES) {
265
+ this.fail(new Error(`identity relay line exceeds ${MAX_RELAY_LINE_BYTES} bytes`));
266
+ }
267
+ else {
268
+ return line;
269
+ }
270
+ }
271
+ if (this.failure !== null) {
272
+ throw this.failure;
273
+ }
274
+ if (this.closed) {
275
+ throw new Error("identity relay closed before the run identity arrived");
276
+ }
277
+ await new Promise((resolve) => {
278
+ this.wake = resolve;
279
+ });
280
+ this.wake = null;
281
+ }
282
+ }
283
+ }
284
+ /**
285
+ * Open the relay over the inherited fd. Wrapping the fd in a net.Socket also marks it
286
+ * close-on-exec (libuv does this on adoption), so subprocesses the run later spawns cannot
287
+ * inherit the relay and speak to init.
288
+ */
289
+ export function connectIdentityRelayFd(fd) {
290
+ return new IdentityRelay(new Socket({ fd, readable: true, writable: true }));
291
+ }
@@ -1,5 +1,6 @@
1
1
  import type { AuthContext } from "./support/index.js";
2
2
  import type { Run } from "./wire/run.js";
3
+ import { type IdentityRelay } from "./identity_relay.js";
3
4
  import type { ByoInferenceProvider } from "../contract.js";
4
5
  import { type ProgramWorkerDeps } from "./program_worker.js";
5
6
  /** Constructed primitives the pure assembly consumes (real ones in main(), fakes in tests). The
@@ -27,6 +28,10 @@ export interface WorkerRuntime {
27
28
  /** vCPUs provisioned for this task (the dispatcher's resolved machine size ÷ 1024 cpu units). Runtime
28
29
  * is billed per vCPU-SECOND, so the RuntimeFlusher scales wall-clock by this. Defaults to 1. */
29
30
  vcpus: number;
31
+ /** The in-guest identity relay, present ONLY on the snapshot-based microVM substrate (relay-mode
32
+ * boot). When set, the worker suspends by FREEZING — the FreezeCoordinator parks seams over this
33
+ * relay's suspend/wake channel — instead of the exit-and-replay path. */
34
+ freezeRelay?: IdentityRelay;
30
35
  }
31
36
  /** Durable events are deferred (run_step_events table) — live fan-out via the broker only. */
32
37
  /** Synthesize the run's AuthContext. The run was already authorized at trigger time; the
@@ -35,6 +35,8 @@ import { SecretRedactor } from "./agent/secret_redactor.js";
35
35
  import { RecordingSecretResolver } from "./recording_secret_resolver.js";
36
36
  import { EngineLeafExecutor } from "./leaf_executor.js";
37
37
  import { parseByoProviders } from "./direct_inference.js";
38
+ import { applyIdentityToEnv, connectIdentityRelayFd, relayFdFromEnv, workerDiagnostics, } from "./identity_relay.js";
39
+ import { FreezeCoordinator } from "./freeze_coordinator.js";
38
40
  import { WorkerWorkflowHost } from "./workflow_host.js";
39
41
  import { runProgramWorker, } from "./program_worker.js";
40
42
  import { RunnerControlClient } from "./runner_control_client.js";
@@ -94,6 +96,24 @@ export function assembleWorkerDeps(runtime) {
94
96
  // a resumed run suppresses observability for seams it already ran. Captured at claim, read by
95
97
  // buildHost (which runs right after claim) when constructing the per-run host.
96
98
  let replayFrontier = 0;
99
+ // Snapshot-substrate suspension (relay-mode boot only): one coordinator for the process, wired to
100
+ // the relay's suspend/wake channel now; its per-run hooks (token swap, meter rebase, workspace
101
+ // persist) late-bind in buildHost/startRuntimeFlush once those objects exist. The circular
102
+ // channel↔handler reference resolves through the thunks.
103
+ let freeze;
104
+ let activeFlusher = null;
105
+ if (runtime.freezeRelay !== undefined) {
106
+ const channel = runtime.freezeRelay.openChannel({
107
+ onWake: (payload) => {
108
+ freeze?.onWake(payload);
109
+ },
110
+ onSuspendAbort: (payload) => {
111
+ freeze?.onSuspendAbort(payload);
112
+ },
113
+ });
114
+ freeze = new FreezeCoordinator({ channel });
115
+ log.info("freeze_mode_enabled", { runId: runtime.runId });
116
+ }
97
117
  // Live agent-event stream → /telemetry (broker publishes to Redis server-side). Inference,
98
118
  // web_search, artifacts, and the model call are all brokered — no Redis / model creds / Tavily / S3.
99
119
  const eventPublisher = new BrokerEventPublisher({
@@ -110,7 +130,8 @@ export function assembleWorkerDeps(runtime) {
110
130
  // can't read it. The program reaches it ONLY via `runtime.apiToken()` (built below); we still
111
131
  // record it in each run's redactor so a value the program threads into a tool can't echo back out
112
132
  // of a tool result. Absent in the dev no-signing-key path. (the run env/credential rules)
113
- const runApiKey = runtime.controlPlane.apiToken;
133
+ // MUTABLE: on the snapshot substrate a wake carries a fresh token (the frozen one expired).
134
+ let runApiKey = runtime.controlPlane.apiToken;
114
135
  // The broker (Runner Control API) is hosted on the api-server, so the public API shares its
115
136
  // origin; `runtime.apiUrl` exposes it and the program appends `/v1` or `/mcp/v1`.
116
137
  const apiUrl = publicApiOrigin(runtime.controlPlane.baseUrl);
@@ -287,8 +308,39 @@ export function assembleWorkerDeps(runtime) {
287
308
  },
288
309
  }
289
310
  : {}),
311
+ // Snapshot-substrate suspension: seams freeze in place under the quiescence gate instead of
312
+ // raising onSuspend (which stays wired as the transitional/local path's fallback).
313
+ ...(freeze !== undefined ? { freeze } : {}),
290
314
  phases: phaseTracker,
291
315
  });
316
+ // Late-bind the coordinator's per-run hooks now that the run-scoped objects exist.
317
+ if (freeze !== undefined) {
318
+ const coordinator = freeze;
319
+ let freezeWallMs = 0;
320
+ coordinator.setHooks({
321
+ // At quiescence, immediately before the freeze: book the runtime tail (suspended time must
322
+ // never appear billed — SUSPEND_POLICY) and persist the workspace (crash-during-suspension
323
+ // recovery parity with the sleep-hold path). The wall stamp anchors the idle rebase below.
324
+ onBeforeFreeze: async () => {
325
+ freezeWallMs = Date.now();
326
+ await activeFlusher?.flushNow();
327
+ await workspaceStore?.persist();
328
+ },
329
+ // On wake: swap the fresh tokens onto the broker client + the program-facing apiToken()
330
+ // (recording the new values in the run's redactor, same discipline as boot), and exclude
331
+ // the frozen window from billed runtime using the wake's authoritative wall clock (the
332
+ // guest's own clock was stopped).
333
+ onAfterWake: (wake) => {
334
+ broker.swapRunToken(wake.run_token);
335
+ redactor.record(wake.run_token);
336
+ if (wake.api_token !== undefined && wake.api_token !== "") {
337
+ runApiKey = wake.api_token;
338
+ redactor.record(wake.api_token);
339
+ }
340
+ activeFlusher?.excludeIdle(wake.wall_clock_ms - freezeWallMs);
341
+ },
342
+ });
343
+ }
292
344
  // Hand the redactor back too: the worker scrubs a terminal error's message with it;
293
345
  // workspace (when opted in) hydrates at start + persists at terminal.
294
346
  return Promise.resolve({
@@ -375,6 +427,9 @@ export function assembleWorkerDeps(runtime) {
375
427
  now: Date.now,
376
428
  report: (deltaSeconds, identifier) => broker.reportUsage(deltaSeconds, identifier),
377
429
  });
430
+ // The freeze coordinator's hooks flush this before a snapshot + rebase it past the frozen
431
+ // window on wake (suspended time must never appear as billed runtime).
432
+ activeFlusher = flusher;
378
433
  flusher.start();
379
434
  return { stop: () => flusher.stop(), flushFinal: () => flusher.flushFinal() };
380
435
  },
@@ -484,6 +539,24 @@ export function capturePlatformContext(env) {
484
539
  return { runId, controlPlane, vcpus };
485
540
  }
486
541
  export async function main() {
542
+ // Relay-mode bootstrap (the snapshot-based microVM substrate): when the guest init handed
543
+ // us an identity relay fd, park here — warm, generic, pre-identity; this await is what the
544
+ // base snapshot freezes — until the run's identity arrives, then map it onto process.env
545
+ // so the capture below is transport-agnostic. Everywhere else (Fargate, self-hosted
546
+ // daemon) the fd is absent and the worker env-boots exactly as before. The post-restore
547
+ // uniqueness reseed will hook in at this boundary, before any run code executes.
548
+ const relayFd = relayFdFromEnv(process.env);
549
+ let freezeRelay;
550
+ if (relayFd !== null) {
551
+ const relay = connectIdentityRelayFd(relayFd);
552
+ relay.announceReady(workerDiagnostics());
553
+ const identity = await relay.awaitIdentity();
554
+ applyIdentityToEnv(identity, process.env);
555
+ relay.acceptIdentity();
556
+ // The relay now becomes the suspend/wake channel: assembleWorkerDeps opens it into the
557
+ // FreezeCoordinator, and the host's suspending seams freeze in place instead of exiting.
558
+ freezeRelay = relay;
559
+ }
487
560
  // Capture the platform context into private state and remove it from process.env BEFORE anything
488
561
  // else — so no user program / agent tool / subprocess we later spawn can read the run token or
489
562
  // API token (the run env/credential rules). WORKER_ID / WORKSPACE_ROOT are non-secret infra knobs.
@@ -498,6 +571,7 @@ export async function main() {
498
571
  controlPlane: platform.controlPlane,
499
572
  vcpus: platform.vcpus,
500
573
  ...(byoProviders.length > 0 ? { byoProviders } : {}),
574
+ ...(freezeRelay !== undefined ? { freezeRelay } : {}),
501
575
  });
502
576
  // The only thing to drain is the batched telemetry buffer — the runner opens no DB/Redis/SQS.
503
577
  const cleanup = async () => {
@@ -44,7 +44,12 @@ export declare class RunnerControlClient {
44
44
  private readonly cfg;
45
45
  private readonly base;
46
46
  private readonly fetchImpl;
47
+ /** The live bearer. Mutable: on the snapshot substrate a wake carries a FRESH run token (the
48
+ * frozen one expired while suspended) and the worker swaps it at runtime. */
49
+ private runToken;
47
50
  constructor(cfg: RunnerControlClientConfig);
51
+ /** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
52
+ swapRunToken(token: string): void;
48
53
  /** Claim the run's lease. Returns the run on success, or null when it isn't claimable (409 —
49
54
  * another worker has it, or it isn't pending), which the worker treats as "claim lost". */
50
55
  claim(workerId: string, leaseSeconds: number): Promise<{
@@ -16,10 +16,18 @@ export class RunnerControlClient {
16
16
  cfg;
17
17
  base;
18
18
  fetchImpl;
19
+ /** The live bearer. Mutable: on the snapshot substrate a wake carries a FRESH run token (the
20
+ * frozen one expired while suspended) and the worker swaps it at runtime. */
21
+ runToken;
19
22
  constructor(cfg) {
20
23
  this.cfg = cfg;
21
24
  this.base = cfg.baseUrl.replace(/\/+$/, "");
22
25
  this.fetchImpl = cfg.fetchImpl ?? fetch;
26
+ this.runToken = cfg.runToken;
27
+ }
28
+ /** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
29
+ swapRunToken(token) {
30
+ this.runToken = token;
23
31
  }
24
32
  /** Claim the run's lease. Returns the run on success, or null when it isn't claimable (409 —
25
33
  * another worker has it, or it isn't pending), which the worker treats as "claim lost". */
@@ -409,7 +417,7 @@ export class RunnerControlClient {
409
417
  }
410
418
  headers(json) {
411
419
  const h = {
412
- authorization: `Bearer ${this.cfg.runToken}`,
420
+ authorization: `Bearer ${this.runToken}`,
413
421
  accept: "application/json",
414
422
  };
415
423
  if (json)
@@ -26,7 +26,16 @@ export declare class RuntimeFlusher {
26
26
  private flushedSeconds;
27
27
  /** Per-flush sequence — advances only on a successful flush, so a retry reuses the same id. */
28
28
  private seq;
29
+ /** Wall-clock ms excluded from billing — frozen (suspended) time on the snapshot substrate. */
30
+ private excludedMs;
29
31
  constructor(deps: RuntimeFlusherDeps);
32
+ /** Book everything unbilled right now (the pre-freeze flush: suspended time must never appear as
33
+ * billed runtime, so the tail is booked BEFORE the snapshot — SUSPEND_POLICY billing rule). */
34
+ flushNow(): Promise<void>;
35
+ /** Exclude `ms` of wall-clock from billing — the frozen window on a wake (computed from the wake's
36
+ * authoritative wall clock, since the guest's own clock was stopped). Never lets elapsed go
37
+ * negative. */
38
+ excludeIdle(ms: number): void;
30
39
  /** Begin periodic delta flushing. */
31
40
  start(): void;
32
41
  /** Stop the periodic timer and drain any in-flight flush. Does NOT book the final tail — call
@@ -30,9 +30,23 @@ export class RuntimeFlusher {
30
30
  flushedSeconds = 0;
31
31
  /** Per-flush sequence — advances only on a successful flush, so a retry reuses the same id. */
32
32
  seq = 0;
33
+ /** Wall-clock ms excluded from billing — frozen (suspended) time on the snapshot substrate. */
34
+ excludedMs = 0;
33
35
  constructor(deps) {
34
36
  this.deps = deps;
35
37
  }
38
+ /** Book everything unbilled right now (the pre-freeze flush: suspended time must never appear as
39
+ * billed runtime, so the tail is booked BEFORE the snapshot — SUSPEND_POLICY billing rule). */
40
+ async flushNow() {
41
+ await this.flush(false);
42
+ }
43
+ /** Exclude `ms` of wall-clock from billing — the frozen window on a wake (computed from the wake's
44
+ * authoritative wall clock, since the guest's own clock was stopped). Never lets elapsed go
45
+ * negative. */
46
+ excludeIdle(ms) {
47
+ if (ms > 0)
48
+ this.excludedMs += ms;
49
+ }
36
50
  /** Begin periodic delta flushing. */
37
51
  start() {
38
52
  if (this.timer !== null)
@@ -67,7 +81,7 @@ export class RuntimeFlusher {
67
81
  // vCPU-seconds = wall-clock seconds × vCPUs (billed per vCPU-second). Rounding the cumulative
68
82
  // product (not per-delta) keeps the booked total aligned with wall-clock×vcpus over many flushes.
69
83
  const vcpus = this.deps.vcpus ?? 1;
70
- const total = Math.max(0, Math.round(((this.deps.now() - this.deps.startedAtMs) / 1000) * vcpus));
84
+ const total = Math.max(0, Math.round(((this.deps.now() - this.deps.startedAtMs - this.excludedMs) / 1000) * vcpus));
71
85
  const delta = total - this.flushedSeconds;
72
86
  if (delta < 1)
73
87
  return;
@@ -1,6 +1,7 @@
1
1
  import type { WorkflowHost, AgentOptions, ArtifactBody, ArtifactRef, CallOptions, HumanInputOptions, HumanInputResult, PhaseOptions, SleepArg } from "@boardwalk-labs/workflow/runtime";
2
2
  import { type LeafResume } from "@boardwalk-labs/engine/core";
3
3
  import { type JournalSeam, type SuspendSignal } from "./suspension.js";
4
+ import type { FreezeCoordinator } from "./freeze_coordinator.js";
4
5
  /** Parse a `humanInput({ timeout })` string (`"48h"`, `"30m"`, `"90s"`, `"7d"`) to milliseconds, or
5
6
  * null when absent/unparseable (the gate then waits indefinitely). */
6
7
  export declare function parseTimeoutMs(timeout: string | undefined): number | null;
@@ -126,6 +127,14 @@ export interface WorkerWorkflowHostDeps {
126
127
  * rejects with {@link SuspendError} instead (the local/test path).
127
128
  */
128
129
  onSuspend?: (signal: SuspendSignal) => void;
130
+ /**
131
+ * Snapshot-substrate suspension (the microVM freeze model): when present, a suspending seam
132
+ * BLOCKS on the coordinator instead of raising `onSuspend` — the platform freezes the whole VM
133
+ * and a wake resolves the seam in place, heap intact (no exit, no journal replay). Every hook
134
+ * also runs under the coordinator's quiescence gate: a freeze never captures a live platform
135
+ * stream, and work arriving while a freeze is pending queues until the wake.
136
+ */
137
+ freeze?: FreezeCoordinator;
129
138
  /**
130
139
  * The highest journaled seq at claim (0 on a fresh run). While re-running seams up to this
131
140
  * frontier on a resume, the host is REPLAYING and observability (phase markers, program logs) is
@@ -152,8 +161,16 @@ export declare class WorkerWorkflowHost implements WorkflowHost {
152
161
  private suspend;
153
162
  /** Run `fn` only if the run isn't aborted; otherwise REJECT (never throw synchronously) with the
154
163
  * signal's RunAbortedError — every Promise-returning hook funnels through this so callers always
155
- * get a rejected promise on abort, not a sync throw. */
164
+ * get a rejected promise on abort, not a sync throw. On the snapshot substrate the body also runs
165
+ * under the freeze coordinator's quiescence gate (see {@link WorkerWorkflowHostDeps.freeze}). */
156
166
  private guarded;
167
+ /** A suspending seam's freeze wait: step out of the "work" count around the park (the wait itself
168
+ * is what the gate waits FOR, not work that blocks it), then rejoin on resume. */
169
+ private freezeWait;
170
+ /** The wake's answer for one gate key. A wake whose value is missing the parked gate means the
171
+ * control plane and the snapshot disagree about what this run was waiting for — a platform bug,
172
+ * failed loudly, never a retry. */
173
+ private gateAnswer;
157
174
  setPhase(name: string, opts: PhaseOptions | undefined): void;
158
175
  agent(prompt: string, opts: AgentOptions | undefined): Promise<unknown>;
159
176
  /** The `agent()` durable seam: a journal hit returns the memoized result (never re-runs the LLM); a
@@ -97,7 +97,8 @@ export class WorkerWorkflowHost {
97
97
  }
98
98
  /** Run `fn` only if the run isn't aborted; otherwise REJECT (never throw synchronously) with the
99
99
  * signal's RunAbortedError — every Promise-returning hook funnels through this so callers always
100
- * get a rejected promise on abort, not a sync throw. */
100
+ * get a rejected promise on abort, not a sync throw. On the snapshot substrate the body also runs
101
+ * under the freeze coordinator's quiescence gate (see {@link WorkerWorkflowHostDeps.freeze}). */
101
102
  guarded(fn) {
102
103
  try {
103
104
  throwIfAborted(this.deps.signal);
@@ -106,10 +107,35 @@ export class WorkerWorkflowHost {
106
107
  return Promise.reject(err instanceof Error ? err : new Error(String(err)));
107
108
  }
108
109
  const phases = this.deps.phases;
109
- if (phases === undefined)
110
- return fn();
111
- const phaseId = phases.capture();
112
- return phases.runInPhase(phaseId, fn);
110
+ // Capture the phase at CALL time — BEFORE any gate queueing — so a hook that queued across a
111
+ // freeze still lands in the phase its call site was in.
112
+ const phaseId = phases?.capture() ?? null;
113
+ const body = phases === undefined ? fn : () => phases.runInPhase(phaseId, fn);
114
+ const freeze = this.deps.freeze;
115
+ if (freeze === undefined)
116
+ return body();
117
+ return freeze.trackWork(body);
118
+ }
119
+ /** A suspending seam's freeze wait: step out of the "work" count around the park (the wait itself
120
+ * is what the gate waits FOR, not work that blocks it), then rejoin on resume. */
121
+ async freezeWait(freeze, signal) {
122
+ freeze.endWork();
123
+ try {
124
+ return await freeze.suspendingWait(signal);
125
+ }
126
+ finally {
127
+ freeze.beginWork();
128
+ }
129
+ }
130
+ /** The wake's answer for one gate key. A wake whose value is missing the parked gate means the
131
+ * control plane and the snapshot disagree about what this run was waiting for — a platform bug,
132
+ * failed loudly, never a retry. */
133
+ gateAnswer(wake, key) {
134
+ const answer = wake.answers?.[key];
135
+ if (wake.kind !== "human_input" || answer === undefined) {
136
+ throw new AppError(ErrorCode.INTERNAL_ERROR, `Wake value does not answer the parked gate "${key}" (got kind "${wake.kind}")`, { kind: "wake_mismatch", key });
137
+ }
138
+ return answer;
113
139
  }
114
140
  setPhase(name, opts) {
115
141
  throwIfAborted(this.deps.signal);
@@ -146,21 +172,23 @@ export class WorkerWorkflowHost {
146
172
  resume = leafResumeSchema.parse(existing.result);
147
173
  }
148
174
  }
149
- try {
150
- const result = await this.deps.leaf.run(prompt, opts, this.deps.signal, resume);
151
- await this.deps.journal?.put({
152
- seq,
153
- kind: "agent",
154
- fingerprint,
155
- label: prompt.slice(0, 120),
156
- result,
157
- });
158
- return result;
159
- }
160
- catch (err) {
161
- if (err instanceof LeafParked) {
162
- // The model paused for a person: suspend with the leaf's checkpoint + the gate.
163
- return this.suspend({
175
+ for (;;) {
176
+ try {
177
+ const result = await this.deps.leaf.run(prompt, opts, this.deps.signal, resume);
178
+ await this.deps.journal?.put({
179
+ seq,
180
+ kind: "agent",
181
+ fingerprint,
182
+ label: prompt.slice(0, 120),
183
+ result,
184
+ });
185
+ return result;
186
+ }
187
+ catch (err) {
188
+ if (!(err instanceof LeafParked))
189
+ throw err;
190
+ // The model paused for a person (the in-leaf `human_input` tool).
191
+ const signal = {
164
192
  reason: "human_input",
165
193
  seq,
166
194
  fingerprint,
@@ -170,9 +198,25 @@ export class WorkerWorkflowHost {
170
198
  prompt: err.request.prompt,
171
199
  inputSpec: err.request.inputSpec,
172
200
  },
173
- });
201
+ };
202
+ const freeze = this.deps.freeze;
203
+ if (freeze === undefined)
204
+ return this.suspend(signal);
205
+ // Snapshot substrate: freeze in place; the wake carries the answers and the leaf
206
+ // re-enters from its checkpoint — heap intact, no journal round-trip. A leaf may park
207
+ // again on a later turn, so this loops.
208
+ if (err.checkpoint === undefined) {
209
+ throw new AppError(ErrorCode.INTERNAL_ERROR, "Parked agent leaf has no checkpoint to resume from", { kind: "leaf_parked_no_checkpoint", seq });
210
+ }
211
+ const outcome = await this.freezeWait(freeze, signal);
212
+ if (outcome.kind !== "wake") {
213
+ throw new AppError(ErrorCode.INTERNAL_ERROR, "Suspend aborted surfaced to a human-input seam (the coordinator retries these)", { kind: "unexpected_abort", seq });
214
+ }
215
+ // Validate OUR gate is answered, but hand the leaf the whole batch (spec: a wake
216
+ // carries every answer the suspension raised, keyed by tool-call id).
217
+ this.gateAnswer(outcome.wake, err.request.toolCallId);
218
+ resume = { checkpoint: err.checkpoint, answers: { ...(outcome.wake.answers ?? {}) } };
174
219
  }
175
- throw err;
176
220
  }
177
221
  }
178
222
  /** Program-level `humanInput()`: suspend the run on a gate, resume with the validated answer. The
@@ -197,7 +241,7 @@ export class WorkerWorkflowHost {
197
241
  }
198
242
  }
199
243
  const expiresAt = this.timeoutExpiry(opts.timeout);
200
- return this.suspend({
244
+ const signal = {
201
245
  reason: "human_input",
202
246
  seq,
203
247
  fingerprint,
@@ -209,7 +253,17 @@ export class WorkerWorkflowHost {
209
253
  // A timeout only matters with a wake to fire at; carry onTimeout alongside expiresAt.
210
254
  ...(expiresAt !== null ? { expiresAt, onTimeout: opts.onTimeout ?? "fail" } : {}),
211
255
  },
212
- });
256
+ };
257
+ const freeze = this.deps.freeze;
258
+ if (freeze !== undefined) {
259
+ // Snapshot substrate: park in place; the wake carries the validated answer.
260
+ const outcome = await this.freezeWait(freeze, signal);
261
+ if (outcome.kind !== "wake") {
262
+ throw new AppError(ErrorCode.INTERNAL_ERROR, "Suspend aborted surfaced to a human-input seam (the coordinator retries these)", { kind: "unexpected_abort", seq });
263
+ }
264
+ return normalizeHumanInputResult(this.gateAnswer(outcome.wake, key));
265
+ }
266
+ return this.suspend(signal);
213
267
  }
214
268
  /** Absolute wake time for a `humanInput({ timeout })`, or null when there is none / it's unparseable. */
215
269
  timeoutExpiry(timeout) {
@@ -290,13 +344,30 @@ export class WorkerWorkflowHost {
290
344
  if (child.status === "failed" || child.status === "cancelled") {
291
345
  throw new AppError(ErrorCode.VALIDATION_FAILED, `Called workflow "${slug}" ${child.status} (run ${child.childRunId})`, { childRunId: child.childRunId, status: child.status });
292
346
  }
293
- // Still running → suspend (release the task); the child's finalize wakes us.
294
- return this.suspend({
347
+ // Still running → suspend; the child's finalize wakes us.
348
+ const signal = {
295
349
  reason: "workflow_call",
296
350
  seq,
297
351
  fingerprint,
298
352
  childRunId: child.childRunId,
299
- });
353
+ };
354
+ const freeze = this.deps.freeze;
355
+ if (freeze !== undefined) {
356
+ // Snapshot substrate: park in place; the wake carries the finalized child directly (no
357
+ // journal write for the wake-derived value — the heap holds it and nothing replays).
358
+ const outcome = await this.freezeWait(freeze, signal);
359
+ if (outcome.kind !== "wake") {
360
+ throw new AppError(ErrorCode.INTERNAL_ERROR, "Suspend aborted surfaced to a child-wait seam (the coordinator retries these)", { kind: "unexpected_abort", seq });
361
+ }
362
+ const woken = outcome.wake.child;
363
+ if (outcome.wake.kind !== "workflow_call" || woken === undefined) {
364
+ throw new AppError(ErrorCode.INTERNAL_ERROR, `Wake value does not carry the awaited child run (got kind "${outcome.wake.kind}")`, { kind: "wake_mismatch", childRunId: child.childRunId });
365
+ }
366
+ if (woken.status === "completed")
367
+ return woken.output;
368
+ throw new AppError(ErrorCode.VALIDATION_FAILED, `Called workflow "${slug}" ${woken.status} (run ${woken.run_id})`, { childRunId: woken.run_id, status: woken.status });
369
+ }
370
+ return this.suspend(signal);
300
371
  }
301
372
  /** Fire-and-forget trigger of another workflow; resolves to the new run's id (no hold/poll). */
302
373
  runWorkflow(slug, input, opts) {
@@ -355,9 +426,32 @@ export class WorkerWorkflowHost {
355
426
  const holdMs = Math.max(0, ms);
356
427
  if (holdMs === 0)
357
428
  return;
358
- // Long wait: SUSPEND (release the task). The broker records the (resolved) sleep journal entry +
359
- // the wake time transactionally, so on wake this seam replays past it (the journal hit above).
360
- // Only when a journal is wired without it there is no resume, so fall back to holding.
429
+ const freeze = this.deps.freeze;
430
+ if (holdMs >= SUSPEND_THRESHOLD_MS && freeze !== undefined) {
431
+ // Snapshot substrate: freeze in place; the wake (when the sleep is due) resolves this very
432
+ // await, heap intact. An abort falls back to holding the REMAINDER in-process — the
433
+ // two-ways rule: snapshot or hold, never replay.
434
+ const requestedAt = this.now();
435
+ const outcome = await this.freezeWait(freeze, {
436
+ reason: "sleep",
437
+ seq,
438
+ fingerprint,
439
+ durationMs: holdMs,
440
+ });
441
+ if (outcome.kind === "wake")
442
+ return;
443
+ const remaining = holdMs - (this.now() - requestedAt);
444
+ if (remaining <= 0)
445
+ return;
446
+ if (this.deps.onBeforeSleep !== undefined)
447
+ await this.deps.onBeforeSleep();
448
+ await this.sleeper.hold(remaining, this.deps.signal);
449
+ return;
450
+ }
451
+ // Long wait (transitional substrate): SUSPEND (release the task). The broker records the
452
+ // (resolved) sleep journal entry + the wake time transactionally, so on wake this seam replays
453
+ // past it (the journal hit above). Only when a journal is wired — without it there is no
454
+ // resume, so fall back to holding.
361
455
  if (holdMs >= SUSPEND_THRESHOLD_MS && this.deps.journal !== undefined) {
362
456
  return this.suspend({ reason: "sleep", seq, fingerprint, durationMs: holdMs });
363
457
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@boardwalk-labs/runner",
3
- "version": "0.1.9",
3
+ "version": "0.1.11",
4
4
  "description": "Boardwalk self-hosted runner: execute cloud-scheduled runs on your own machines. Currently: the canonical runner contract types.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {