@boardwalk-labs/runner 0.1.10 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/runtime/freeze_coordinator.d.ts +119 -0
- package/dist/runtime/freeze_coordinator.js +262 -0
- package/dist/runtime/identity_relay.d.ts +45 -2
- package/dist/runtime/identity_relay.js +137 -14
- package/dist/runtime/index.d.ts +5 -0
- package/dist/runtime/index.js +62 -3
- package/dist/runtime/runner_control_client.d.ts +5 -0
- package/dist/runtime/runner_control_client.js +9 -1
- package/dist/runtime/runtime_flusher.d.ts +9 -0
- package/dist/runtime/runtime_flusher.js +15 -1
- package/dist/runtime/workflow_host.d.ts +18 -1
- package/dist/runtime/workflow_host.js +124 -30
- package/package.json +1 -1
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { RelayChannel } from "./identity_relay.js";
|
|
3
|
+
import type { SuspendSignal } from "./suspension.js";
|
|
4
|
+
/** First retry delay after a suspend_abort (doubles per attempt, capped below). */
|
|
5
|
+
export declare const ABORT_RETRY_INITIAL_MS = 30000;
|
|
6
|
+
export declare const ABORT_RETRY_MAX_MS: number;
|
|
7
|
+
/** The wake value inside a wake injection — `kind` echoes the parked seam's reason; the
|
|
8
|
+
* per-kind fields are opaque to the coordinator and interpreted by the seam. */
|
|
9
|
+
export declare const wakeValueSchema: z.ZodObject<{
|
|
10
|
+
kind: z.ZodEnum<{
|
|
11
|
+
sleep: "sleep";
|
|
12
|
+
human_input: "human_input";
|
|
13
|
+
workflow_call: "workflow_call";
|
|
14
|
+
}>;
|
|
15
|
+
answers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
16
|
+
child: z.ZodOptional<z.ZodObject<{
|
|
17
|
+
run_id: z.ZodString;
|
|
18
|
+
status: z.ZodString;
|
|
19
|
+
output: z.ZodUnknown;
|
|
20
|
+
}, z.core.$strip>>;
|
|
21
|
+
}, z.core.$strip>;
|
|
22
|
+
export type WakeValue = z.infer<typeof wakeValueSchema>;
|
|
23
|
+
/** The wake-injection payload as init relays it (snake_case, the platform env contract's
|
|
24
|
+
* sibling): fresh tokens — the frozen ones expired while suspended — plus the
|
|
25
|
+
* authoritative wall clock (the guest's own clock was stopped) and the wake value. */
|
|
26
|
+
export declare const wakePayloadSchema: z.ZodObject<{
|
|
27
|
+
run_token: z.ZodString;
|
|
28
|
+
api_token: z.ZodOptional<z.ZodString>;
|
|
29
|
+
wall_clock_ms: z.ZodNumber;
|
|
30
|
+
wake: z.ZodObject<{
|
|
31
|
+
kind: z.ZodEnum<{
|
|
32
|
+
sleep: "sleep";
|
|
33
|
+
human_input: "human_input";
|
|
34
|
+
workflow_call: "workflow_call";
|
|
35
|
+
}>;
|
|
36
|
+
answers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
37
|
+
child: z.ZodOptional<z.ZodObject<{
|
|
38
|
+
run_id: z.ZodString;
|
|
39
|
+
status: z.ZodString;
|
|
40
|
+
output: z.ZodUnknown;
|
|
41
|
+
}, z.core.$strip>>;
|
|
42
|
+
}, z.core.$strip>;
|
|
43
|
+
}, z.core.$strip>;
|
|
44
|
+
export type WakePayload = z.infer<typeof wakePayloadSchema>;
|
|
45
|
+
/** What a suspending wait resolves to: the wake (the normal path), or — for a `sleep` seam
|
|
46
|
+
* only — the abort that tells it to hold its remainder in-process (other reasons retry the
|
|
47
|
+
* freeze internally and never surface an abort). */
|
|
48
|
+
export type FreezeOutcome = {
|
|
49
|
+
kind: "wake";
|
|
50
|
+
wake: WakeValue;
|
|
51
|
+
} | {
|
|
52
|
+
kind: "aborted";
|
|
53
|
+
reason: string;
|
|
54
|
+
};
|
|
55
|
+
export interface FreezeCoordinatorHooks {
|
|
56
|
+
/** Runs at quiescence, immediately before the freeze is requested: flush billable runtime
|
|
57
|
+
* (suspended time must never appear billed) and persist the workspace. Its own failure
|
|
58
|
+
* aborts THIS freeze attempt (the seam holds/retries) — never the run. */
|
|
59
|
+
onBeforeFreeze?: () => Promise<void>;
|
|
60
|
+
/** Runs when a wake lands, before the seam resolves: swap the run/api tokens onto the
|
|
61
|
+
* broker client and rebase the runtime meter past the frozen window. */
|
|
62
|
+
onAfterWake?: (wake: WakePayload) => void | Promise<void>;
|
|
63
|
+
}
|
|
64
|
+
export interface FreezeCoordinatorDeps {
|
|
65
|
+
channel: RelayChannel;
|
|
66
|
+
now?: () => number;
|
|
67
|
+
/** Injected delay (tests). Defaults to a real timer. */
|
|
68
|
+
delay?: (ms: number) => Promise<void>;
|
|
69
|
+
}
|
|
70
|
+
export declare class FreezeCoordinator {
|
|
71
|
+
private readonly deps;
|
|
72
|
+
private readonly now;
|
|
73
|
+
private readonly delay;
|
|
74
|
+
private hooks;
|
|
75
|
+
/** Non-suspending runtime work in flight (the quiescence gate's count). */
|
|
76
|
+
private inFlight;
|
|
77
|
+
private quiescenceWaiters;
|
|
78
|
+
/** True from freeze request until wake/abort — new runtime work queues behind it. */
|
|
79
|
+
private freezePending;
|
|
80
|
+
private gateWaiters;
|
|
81
|
+
/** The parked suspending wait (at most one — waits serialize). */
|
|
82
|
+
private parked;
|
|
83
|
+
/** Serializes suspending waits. */
|
|
84
|
+
private chain;
|
|
85
|
+
constructor(deps: FreezeCoordinatorDeps);
|
|
86
|
+
/** Late-bound per-run hooks (the flusher/broker/redactor exist only once a run is claimed). */
|
|
87
|
+
setHooks(hooks: FreezeCoordinatorHooks): void;
|
|
88
|
+
/**
|
|
89
|
+
* Run one unit of non-suspending runtime work under the gate: queue while a freeze is
|
|
90
|
+
* pending (the work never starts, so nothing can be torn by the pause), then count it
|
|
91
|
+
* in-flight so a suspending wait holds until it drains. The gate check and the count
|
|
92
|
+
* increment share one synchronous segment — a freeze requested in between cannot miss us.
|
|
93
|
+
*/
|
|
94
|
+
trackWork<T>(fn: () => Promise<T>): Promise<T>;
|
|
95
|
+
/** A suspending seam's own wait is NOT "work" — it decrements around its park so the gate
|
|
96
|
+
* sees true quiescence. (The host wraps every hook in trackWork; the suspending seams
|
|
97
|
+
* call these around their freeze wait.) */
|
|
98
|
+
beginWork(): void;
|
|
99
|
+
endWork(): void;
|
|
100
|
+
/**
|
|
101
|
+
* The suspending wait: hold until quiescence, run the pre-freeze hook, request the
|
|
102
|
+
* freeze, and park. The next thing this promise sees is a wake (possibly epochs later,
|
|
103
|
+
* through a restored heap) or a suspend_abort. Returns the wake, or — for `sleep` only —
|
|
104
|
+
* the abort (the seam then holds in-process); other reasons retry the freeze after a
|
|
105
|
+
* backoff. Concurrent suspending waits serialize through an internal chain.
|
|
106
|
+
*/
|
|
107
|
+
suspendingWait(signal: SuspendSignal): Promise<FreezeOutcome>;
|
|
108
|
+
/** Relay handler: a wake injection landed. Validates, runs the after-wake hook (token
|
|
109
|
+
* swap + meter rebase), confirms to init, and resolves the parked seam. A wake with no
|
|
110
|
+
* parked seam is a duplicate delivery — re-confirm (idempotent), never crash. */
|
|
111
|
+
onWake(payload: unknown): void;
|
|
112
|
+
/** Relay handler: the snapshot attempt failed; the parked seam falls back to holding. */
|
|
113
|
+
onSuspendAbort(payload: unknown): void;
|
|
114
|
+
private awaitQuiescence;
|
|
115
|
+
private releaseGate;
|
|
116
|
+
/** The host-readable wake summary on the wire (logs/metrics/placement) — the opaque
|
|
117
|
+
* broker_signal beside it carries the full condition. */
|
|
118
|
+
private wakeConditionOf;
|
|
119
|
+
}
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
// FreezeCoordinator — the runtime's half of snapshot suspension on the microVM substrate.
|
|
2
|
+
//
|
|
3
|
+
// On this substrate the worker process NEVER exits to suspend: a suspending seam (a long
|
|
4
|
+
// sleep, a human-input gate, a child-run wait) BLOCKS on a real promise, the platform
|
|
5
|
+
// freezes the whole VM into a snapshot, and a later restore resolves that same promise with
|
|
6
|
+
// the wake value — the heap is the literal heap, nothing replays. This coordinator owns the
|
|
7
|
+
// two policies that make freezing safe and predictable:
|
|
8
|
+
//
|
|
9
|
+
// 1. THE QUIESCENCE GATE: never freeze while non-suspending runtime work (an agent leaf, a
|
|
10
|
+
// tool call, an artifact write) is in flight — a snapshot must not capture a live
|
|
11
|
+
// platform stream that would be dead on restore. A suspending wait created during such
|
|
12
|
+
// work HOLDS until the work drains; new runtime work that arrives while a freeze is
|
|
13
|
+
// pending QUEUES (it never started, so nothing is torn) and runs after the wake.
|
|
14
|
+
// 2. SNAPSHOT-FIRST FALLBACK: a `suspend_abort` from the platform (snapshot/store/broker
|
|
15
|
+
// failure) means the seam falls back to HOLDING — a sleep waits in-process for its
|
|
16
|
+
// remainder; a human-input or child wait retries the freeze after a backoff (the host
|
|
17
|
+
// may throttle repeated attempts).
|
|
18
|
+
//
|
|
19
|
+
// Concurrent suspending waits SERIALIZE: the first to reach quiescence freezes with its own
|
|
20
|
+
// wake condition; after its wake, the next takes its turn. (A compound wake condition —
|
|
21
|
+
// one freeze covering several waits — is a future optimization, not a correctness need:
|
|
22
|
+
// the heap survives every cycle.)
|
|
23
|
+
//
|
|
24
|
+
// The wire below this is the guest init's relay (identity_relay `openChannel`); the policy
|
|
25
|
+
// above it is decided platform-side. The coordinator never interprets the wake VALUE — it
|
|
26
|
+
// validates the envelope and hands `wake` to the seam that parked.
|
|
27
|
+
import { z } from "zod";
|
|
28
|
+
import { createLogger } from "./support/index.js";
|
|
29
|
+
const log = createLogger("FreezeCoordinator");
|
|
30
|
+
/** First retry delay after a suspend_abort (doubles per attempt, capped below). */
|
|
31
|
+
export const ABORT_RETRY_INITIAL_MS = 30_000;
|
|
32
|
+
export const ABORT_RETRY_MAX_MS = 5 * 60_000;
|
|
33
|
+
/** The wake value inside a wake injection — `kind` echoes the parked seam's reason; the
|
|
34
|
+
* per-kind fields are opaque to the coordinator and interpreted by the seam. */
|
|
35
|
+
export const wakeValueSchema = z.object({
|
|
36
|
+
kind: z.enum(["sleep", "human_input", "workflow_call"]),
|
|
37
|
+
/** human_input: EVERY gate this suspension raised, keyed by gate key / tool-call id. */
|
|
38
|
+
answers: z.record(z.string(), z.unknown()).optional(),
|
|
39
|
+
/** workflow_call: the finalized child. */
|
|
40
|
+
child: z
|
|
41
|
+
.object({
|
|
42
|
+
run_id: z.string().min(1),
|
|
43
|
+
status: z.string().min(1),
|
|
44
|
+
output: z.unknown(),
|
|
45
|
+
})
|
|
46
|
+
.optional(),
|
|
47
|
+
});
|
|
48
|
+
/** The wake-injection payload as init relays it (snake_case, the platform env contract's
|
|
49
|
+
* sibling): fresh tokens — the frozen ones expired while suspended — plus the
|
|
50
|
+
* authoritative wall clock (the guest's own clock was stopped) and the wake value. */
|
|
51
|
+
export const wakePayloadSchema = z.object({
|
|
52
|
+
run_token: z.string().min(1),
|
|
53
|
+
api_token: z.string().optional(),
|
|
54
|
+
wall_clock_ms: z.number(),
|
|
55
|
+
wake: wakeValueSchema,
|
|
56
|
+
});
|
|
57
|
+
const suspendAbortSchema = z.object({ reason: z.string().optional() });
|
|
58
|
+
export class FreezeCoordinator {
|
|
59
|
+
deps;
|
|
60
|
+
now;
|
|
61
|
+
delay;
|
|
62
|
+
hooks = {};
|
|
63
|
+
/** Non-suspending runtime work in flight (the quiescence gate's count). */
|
|
64
|
+
inFlight = 0;
|
|
65
|
+
quiescenceWaiters = [];
|
|
66
|
+
/** True from freeze request until wake/abort — new runtime work queues behind it. */
|
|
67
|
+
freezePending = false;
|
|
68
|
+
gateWaiters = [];
|
|
69
|
+
/** The parked suspending wait (at most one — waits serialize). */
|
|
70
|
+
parked = null;
|
|
71
|
+
/** Serializes suspending waits. */
|
|
72
|
+
chain = Promise.resolve();
|
|
73
|
+
constructor(deps) {
|
|
74
|
+
this.deps = deps;
|
|
75
|
+
this.now = deps.now ?? Date.now;
|
|
76
|
+
// A pending retry delay legitimately keeps the process alive — the run isn't done.
|
|
77
|
+
this.delay =
|
|
78
|
+
deps.delay ??
|
|
79
|
+
((ms) => new Promise((resolve) => {
|
|
80
|
+
setTimeout(resolve, ms);
|
|
81
|
+
}));
|
|
82
|
+
}
|
|
83
|
+
/** Late-bound per-run hooks (the flusher/broker/redactor exist only once a run is claimed). */
|
|
84
|
+
setHooks(hooks) {
|
|
85
|
+
this.hooks = hooks;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Run one unit of non-suspending runtime work under the gate: queue while a freeze is
|
|
89
|
+
* pending (the work never starts, so nothing can be torn by the pause), then count it
|
|
90
|
+
* in-flight so a suspending wait holds until it drains. The gate check and the count
|
|
91
|
+
* increment share one synchronous segment — a freeze requested in between cannot miss us.
|
|
92
|
+
*/
|
|
93
|
+
async trackWork(fn) {
|
|
94
|
+
while (this.freezePending) {
|
|
95
|
+
await new Promise((resolve) => {
|
|
96
|
+
this.gateWaiters.push(resolve);
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
this.beginWork();
|
|
100
|
+
try {
|
|
101
|
+
return await fn();
|
|
102
|
+
}
|
|
103
|
+
finally {
|
|
104
|
+
this.endWork();
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/** A suspending seam's own wait is NOT "work" — it decrements around its park so the gate
|
|
108
|
+
* sees true quiescence. (The host wraps every hook in trackWork; the suspending seams
|
|
109
|
+
* call these around their freeze wait.) */
|
|
110
|
+
beginWork() {
|
|
111
|
+
this.inFlight += 1;
|
|
112
|
+
}
|
|
113
|
+
endWork() {
|
|
114
|
+
this.inFlight -= 1;
|
|
115
|
+
if (this.inFlight === 0) {
|
|
116
|
+
const waiters = this.quiescenceWaiters;
|
|
117
|
+
this.quiescenceWaiters = [];
|
|
118
|
+
for (const w of waiters)
|
|
119
|
+
w();
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* The suspending wait: hold until quiescence, run the pre-freeze hook, request the
|
|
124
|
+
* freeze, and park. The next thing this promise sees is a wake (possibly epochs later,
|
|
125
|
+
* through a restored heap) or a suspend_abort. Returns the wake, or — for `sleep` only —
|
|
126
|
+
* the abort (the seam then holds in-process); other reasons retry the freeze after a
|
|
127
|
+
* backoff. Concurrent suspending waits serialize through an internal chain.
|
|
128
|
+
*/
|
|
129
|
+
suspendingWait(signal) {
|
|
130
|
+
const turn = this.chain;
|
|
131
|
+
let release = () => undefined;
|
|
132
|
+
this.chain = new Promise((resolve) => {
|
|
133
|
+
release = resolve;
|
|
134
|
+
});
|
|
135
|
+
return (async () => {
|
|
136
|
+
await turn;
|
|
137
|
+
try {
|
|
138
|
+
let backoff = ABORT_RETRY_INITIAL_MS;
|
|
139
|
+
for (;;) {
|
|
140
|
+
await this.awaitQuiescence();
|
|
141
|
+
try {
|
|
142
|
+
await this.hooks.onBeforeFreeze?.();
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
// A failed pre-freeze flush must not strand the seam: treat it like an abort.
|
|
146
|
+
log.warn("freeze_prepare_failed", {
|
|
147
|
+
error: err instanceof Error ? err.message : String(err),
|
|
148
|
+
});
|
|
149
|
+
if (signal.reason === "sleep")
|
|
150
|
+
return { kind: "aborted", reason: "prepare_failed" };
|
|
151
|
+
await this.delay(backoff);
|
|
152
|
+
backoff = Math.min(backoff * 2, ABORT_RETRY_MAX_MS);
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
this.freezePending = true;
|
|
156
|
+
const outcome = await new Promise((resolve) => {
|
|
157
|
+
this.parked = resolve;
|
|
158
|
+
this.deps.channel.sendSuspendRequest({
|
|
159
|
+
reason: signal.reason,
|
|
160
|
+
wake: this.wakeConditionOf(signal),
|
|
161
|
+
broker_signal: signal,
|
|
162
|
+
});
|
|
163
|
+
// ← the VM freezes while this promise is pending; the wake resolves it with the
|
|
164
|
+
// heap (and this very closure) restored.
|
|
165
|
+
});
|
|
166
|
+
this.freezePending = false;
|
|
167
|
+
this.releaseGate();
|
|
168
|
+
if (outcome.kind === "wake")
|
|
169
|
+
return outcome;
|
|
170
|
+
if (signal.reason === "sleep")
|
|
171
|
+
return outcome;
|
|
172
|
+
log.warn("suspend_aborted_retrying", {
|
|
173
|
+
reason: outcome.reason,
|
|
174
|
+
seamReason: signal.reason,
|
|
175
|
+
backoffMs: backoff,
|
|
176
|
+
});
|
|
177
|
+
await this.delay(backoff);
|
|
178
|
+
backoff = Math.min(backoff * 2, ABORT_RETRY_MAX_MS);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
finally {
|
|
182
|
+
release();
|
|
183
|
+
}
|
|
184
|
+
})();
|
|
185
|
+
}
|
|
186
|
+
/** Relay handler: a wake injection landed. Validates, runs the after-wake hook (token
|
|
187
|
+
* swap + meter rebase), confirms to init, and resolves the parked seam. A wake with no
|
|
188
|
+
* parked seam is a duplicate delivery — re-confirm (idempotent), never crash. */
|
|
189
|
+
onWake(payload) {
|
|
190
|
+
const parsed = wakePayloadSchema.safeParse(payload);
|
|
191
|
+
if (!parsed.success) {
|
|
192
|
+
// Unanswerable garbage: init will retry, time out, and the platform's crash path owns
|
|
193
|
+
// recovery. Log loudly — this is a control-plane/init bug, not author code.
|
|
194
|
+
log.error("wake_payload_invalid", { issues: parsed.error.message });
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
if (this.parked === null) {
|
|
198
|
+
log.warn("duplicate_wake_ignored", {});
|
|
199
|
+
this.deps.channel.sendWakeAccepted();
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
const resolve = this.parked;
|
|
203
|
+
this.parked = null;
|
|
204
|
+
void (async () => {
|
|
205
|
+
try {
|
|
206
|
+
await this.hooks.onAfterWake?.(parsed.data);
|
|
207
|
+
}
|
|
208
|
+
catch (err) {
|
|
209
|
+
// Token swap / meter rebase failing is survivable only loudly — the run continues
|
|
210
|
+
// on the old token and fails fast if it truly expired.
|
|
211
|
+
log.error("after_wake_hook_failed", {
|
|
212
|
+
error: err instanceof Error ? err.message : String(err),
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
this.deps.channel.sendWakeAccepted();
|
|
216
|
+
resolve({ kind: "wake", wake: parsed.data.wake });
|
|
217
|
+
})();
|
|
218
|
+
}
|
|
219
|
+
/** Relay handler: the snapshot attempt failed; the parked seam falls back to holding. */
|
|
220
|
+
onSuspendAbort(payload) {
|
|
221
|
+
const parsed = suspendAbortSchema.safeParse(payload);
|
|
222
|
+
const reason = parsed.success ? (parsed.data.reason ?? "unknown") : "unknown";
|
|
223
|
+
if (this.parked === null) {
|
|
224
|
+
log.warn("suspend_abort_without_parked_seam", { reason });
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
const resolve = this.parked;
|
|
228
|
+
this.parked = null;
|
|
229
|
+
resolve({ kind: "aborted", reason });
|
|
230
|
+
}
|
|
231
|
+
awaitQuiescence() {
|
|
232
|
+
if (this.inFlight === 0)
|
|
233
|
+
return Promise.resolve();
|
|
234
|
+
return new Promise((resolve) => {
|
|
235
|
+
this.quiescenceWaiters.push(resolve);
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
releaseGate() {
|
|
239
|
+
const waiters = this.gateWaiters;
|
|
240
|
+
this.gateWaiters = [];
|
|
241
|
+
for (const w of waiters)
|
|
242
|
+
w();
|
|
243
|
+
}
|
|
244
|
+
/** The host-readable wake summary on the wire (logs/metrics/placement) — the opaque
|
|
245
|
+
* broker_signal beside it carries the full condition. */
|
|
246
|
+
wakeConditionOf(signal) {
|
|
247
|
+
switch (signal.reason) {
|
|
248
|
+
case "sleep":
|
|
249
|
+
return { kind: "sleep", wake_at_ms: this.now() + (signal.durationMs ?? 0) };
|
|
250
|
+
case "human_input":
|
|
251
|
+
return {
|
|
252
|
+
kind: "human_input",
|
|
253
|
+
...(signal.humanInput !== undefined ? { request_keys: [signal.humanInput.key] } : {}),
|
|
254
|
+
};
|
|
255
|
+
case "workflow_call":
|
|
256
|
+
return {
|
|
257
|
+
kind: "workflow_call",
|
|
258
|
+
...(signal.childRunId !== undefined ? { child_run_id: signal.childRunId } : {}),
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
@@ -3,6 +3,11 @@ import { z } from "zod";
|
|
|
3
3
|
/** Env var naming the inherited relay fd. Set by the guest init; absent everywhere else
|
|
4
4
|
* (Fargate, self-hosted daemon), where the worker env-boots as always. */
|
|
5
5
|
export declare const RELAY_FD_ENV = "BOARDWALK_IDENTITY_RELAY_FD";
|
|
6
|
+
/** Max bytes of one relay line — the wire protocol's 32 MiB frame cap, mirrored in-guest.
|
|
7
|
+
* An oversized line costs the connection (LF framing cannot resynchronize past it), which
|
|
8
|
+
* for this relay means a hard bootstrap failure. Measured in UTF-16 code units, which for
|
|
9
|
+
* this ASCII-JSON wire is the byte count; the cap is a guard, not exact accounting. */
|
|
10
|
+
export declare const MAX_RELAY_LINE_BYTES: number;
|
|
6
11
|
/** The identity payload — the env contract's fields, as JSON instead of env vars. */
|
|
7
12
|
export declare const relayIdentitySchema: z.ZodObject<{
|
|
8
13
|
run_id: z.ZodString;
|
|
@@ -14,6 +19,19 @@ export declare const relayIdentitySchema: z.ZodObject<{
|
|
|
14
19
|
env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
15
20
|
}, z.core.$strip>;
|
|
16
21
|
export type RelayIdentity = z.infer<typeof relayIdentitySchema>;
|
|
22
|
+
/** The init→worker halves of the post-identity relay: `wake` resolves a frozen suspending
|
|
23
|
+
* seam (payload: the wake-inject body, verbatim), `suspend_abort` tells a parked seam the
|
|
24
|
+
* snapshot attempt failed and it should fall back to holding. */
|
|
25
|
+
export interface RelayChannelHandlers {
|
|
26
|
+
onWake(payload: unknown): void;
|
|
27
|
+
onSuspendAbort(payload: unknown): void;
|
|
28
|
+
}
|
|
29
|
+
/** The worker→init halves: ask to be frozen (payload: the suspend-request body — reason,
|
|
30
|
+
* wake summary, opaque broker signal), and confirm a wake landed (init then acks its host). */
|
|
31
|
+
export interface RelayChannel {
|
|
32
|
+
sendSuspendRequest(payload: unknown): void;
|
|
33
|
+
sendWakeAccepted(): void;
|
|
34
|
+
}
|
|
17
35
|
/**
|
|
18
36
|
* Read the relay fd from `env`, deleting the key (bootstrap-only plumbing; run code has no
|
|
19
37
|
* business seeing it). Returns null when unset — the normal env-boot path.
|
|
@@ -24,15 +42,30 @@ export declare function relayFdFromEnv(env: NodeJS.ProcessEnv): number | null;
|
|
|
24
42
|
* the platform keys LAST, so nothing user-supplied can shadow a platform value.
|
|
25
43
|
*/
|
|
26
44
|
export declare function applyIdentityToEnv(identity: RelayIdentity, env: NodeJS.ProcessEnv): void;
|
|
45
|
+
/** The worker_ready diagnostics — supplied by the worker (init cannot know these),
|
|
46
|
+
* forwarded verbatim by the guest init. Best-effort, never load-bearing. */
|
|
47
|
+
export interface WorkerDiagnostics {
|
|
48
|
+
worker_version?: string;
|
|
49
|
+
node_version: string;
|
|
50
|
+
sdk_version?: string;
|
|
51
|
+
}
|
|
52
|
+
/** Collect the worker_ready diagnostics. Version lookups are best-effort: the runner's own
|
|
53
|
+
* package.json sits two levels above this module in both the src and published dist
|
|
54
|
+
* layouts; the SDK's is reachable only if its exports expose ./package.json. */
|
|
55
|
+
export declare function workerDiagnostics(): WorkerDiagnostics;
|
|
27
56
|
/** One end of the init↔worker relay. Wraps any Duplex so tests run over in-memory streams. */
|
|
28
57
|
export declare class IdentityRelay {
|
|
29
58
|
private readonly stream;
|
|
30
59
|
private buffer;
|
|
31
60
|
private closed;
|
|
61
|
+
private failure;
|
|
32
62
|
private wake;
|
|
63
|
+
private readonly onData;
|
|
33
64
|
constructor(stream: Duplex);
|
|
34
|
-
|
|
35
|
-
|
|
65
|
+
private fail;
|
|
66
|
+
/** Announce the pre-identity park (with the diagnostics payload when provided).
|
|
67
|
+
* Init forwards this as the base-snapshot gate. */
|
|
68
|
+
announceReady(diagnostics?: WorkerDiagnostics): void;
|
|
36
69
|
/**
|
|
37
70
|
* Block until init relays the run identity. THIS is the point the base snapshot freezes:
|
|
38
71
|
* everything before it is generic warm-up shared by every run; everything after belongs
|
|
@@ -43,6 +76,16 @@ export declare class IdentityRelay {
|
|
|
43
76
|
awaitIdentity(): Promise<RelayIdentity>;
|
|
44
77
|
/** Confirm capture. Init acks its host only after this arrives. */
|
|
45
78
|
acceptIdentity(): void;
|
|
79
|
+
/**
|
|
80
|
+
* Attach the post-identity consumer: the suspend/wake channel (the same fd, the same JSON
|
|
81
|
+
* lines). From here the relay dispatches init→worker messages to the handlers —
|
|
82
|
+
* `wake` (resolve a frozen seam) and `suspend_abort` (the seam falls back to holding) —
|
|
83
|
+
* and the returned channel sends the worker→init halves (`suspend_request`,
|
|
84
|
+
* `wake_accepted`). Unknown types are ignored (forward-compatible); malformed lines are
|
|
85
|
+
* logged and skipped, exactly as pre-identity. Call once, after {@link acceptIdentity}.
|
|
86
|
+
*/
|
|
87
|
+
openChannel(handlers: RelayChannelHandlers): RelayChannel;
|
|
88
|
+
private pumpChannel;
|
|
46
89
|
private writeLine;
|
|
47
90
|
private readLine;
|
|
48
91
|
}
|
|
@@ -7,14 +7,23 @@
|
|
|
7
7
|
// AF_UNIX socketpair; init tells us its fd via `BOARDWALK_IDENTITY_RELAY_FD`. Wire: one JSON
|
|
8
8
|
// object per LF-terminated line —
|
|
9
9
|
//
|
|
10
|
-
// {"type":"worker_ready"}
|
|
10
|
+
// {"type":"worker_ready","payload":{...}} worker → init the pre-identity park is reached
|
|
11
|
+
// (payload: optional version diagnostics,
|
|
12
|
+
// forwarded verbatim by init)
|
|
11
13
|
// {"type":"identity","payload":{...}} init → worker the run identity (see schema below)
|
|
12
14
|
// {"type":"identity_accepted"} worker → init captured; init acks its host
|
|
15
|
+
// {"type":"suspend_request","payload":{...}} worker → init a quiescent seam asks to be frozen
|
|
16
|
+
// {"type":"suspend_abort","payload":{...}} init → worker snapshot failed; the seam holds instead
|
|
17
|
+
// {"type":"wake","payload":{...}} init → worker resolve the frozen seam (fresh tokens
|
|
18
|
+
// + the wake value, verbatim)
|
|
19
|
+
// {"type":"wake_accepted"} worker → init tokens swapped, seam resolved; init acks
|
|
13
20
|
//
|
|
14
|
-
// The payload carries exactly the platform env contract (snake_case) plus the resolved
|
|
15
|
-
// env; `applyIdentityToEnv` maps it onto `process.env` so `capturePlatformContext` runs
|
|
21
|
+
// The identity payload carries exactly the platform env contract (snake_case) plus the resolved
|
|
22
|
+
// user env; `applyIdentityToEnv` maps it onto `process.env` so `capturePlatformContext` runs
|
|
16
23
|
// UNCHANGED afterward — same fields, same capture-and-delete discipline, two transports.
|
|
17
|
-
// The stream stays open after identity:
|
|
24
|
+
// The stream stays open after identity: `openChannel` attaches the suspend/wake consumer
|
|
25
|
+
// (the FreezeCoordinator drives the choreography above it).
|
|
26
|
+
import { createRequire } from "node:module";
|
|
18
27
|
import { Socket } from "node:net";
|
|
19
28
|
import { z } from "zod";
|
|
20
29
|
import { createLogger } from "./support/index.js";
|
|
@@ -22,6 +31,11 @@ const log = createLogger("identity_relay");
|
|
|
22
31
|
/** Env var naming the inherited relay fd. Set by the guest init; absent everywhere else
|
|
23
32
|
* (Fargate, self-hosted daemon), where the worker env-boots as always. */
|
|
24
33
|
export const RELAY_FD_ENV = "BOARDWALK_IDENTITY_RELAY_FD";
|
|
34
|
+
/** Max bytes of one relay line — the wire protocol's 32 MiB frame cap, mirrored in-guest.
|
|
35
|
+
* An oversized line costs the connection (LF framing cannot resynchronize past it), which
|
|
36
|
+
* for this relay means a hard bootstrap failure. Measured in UTF-16 code units, which for
|
|
37
|
+
* this ASCII-JSON wire is the byte count; the cap is a guard, not exact accounting. */
|
|
38
|
+
export const MAX_RELAY_LINE_BYTES = 32 * 1024 * 1024;
|
|
25
39
|
/** The identity payload — the env contract's fields, as JSON instead of env vars. */
|
|
26
40
|
export const relayIdentitySchema = z.object({
|
|
27
41
|
run_id: z.string().min(1),
|
|
@@ -74,30 +88,73 @@ export function applyIdentityToEnv(identity, env) {
|
|
|
74
88
|
env.BOARDWALK_BYO_PROVIDERS = JSON.stringify(identity.byo_providers);
|
|
75
89
|
}
|
|
76
90
|
}
|
|
91
|
+
/** Collect the worker_ready diagnostics. Version lookups are best-effort: the runner's own
|
|
92
|
+
* package.json sits two levels above this module in both the src and published dist
|
|
93
|
+
* layouts; the SDK's is reachable only if its exports expose ./package.json. */
|
|
94
|
+
export function workerDiagnostics() {
|
|
95
|
+
const require = createRequire(import.meta.url);
|
|
96
|
+
const versionOf = (spec) => {
|
|
97
|
+
try {
|
|
98
|
+
const pkg = require(spec);
|
|
99
|
+
return typeof pkg.version === "string" ? pkg.version : undefined;
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
return undefined;
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
const worker = versionOf("../../package.json");
|
|
106
|
+
const sdk = versionOf("@boardwalk-labs/workflow/package.json");
|
|
107
|
+
return {
|
|
108
|
+
...(worker !== undefined ? { worker_version: worker } : {}),
|
|
109
|
+
node_version: process.version,
|
|
110
|
+
...(sdk !== undefined ? { sdk_version: sdk } : {}),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
77
113
|
/** One end of the init↔worker relay. Wraps any Duplex so tests run over in-memory streams. */
|
|
78
114
|
export class IdentityRelay {
|
|
79
115
|
stream;
|
|
80
116
|
buffer = "";
|
|
81
117
|
closed = false;
|
|
118
|
+
failure = null;
|
|
82
119
|
wake = null;
|
|
120
|
+
onData;
|
|
83
121
|
constructor(stream) {
|
|
84
122
|
this.stream = stream;
|
|
85
123
|
// One persistent listener for the relay's lifetime: a flowing stream DROPS chunks
|
|
86
124
|
// emitted while nobody listens, so attach-per-read would lose lines between reads.
|
|
87
|
-
|
|
125
|
+
this.onData = (chunk) => {
|
|
88
126
|
this.buffer += typeof chunk === "string" ? chunk : chunk.toString("utf8");
|
|
127
|
+
if (this.buffer.length > MAX_RELAY_LINE_BYTES && !this.buffer.includes("\n")) {
|
|
128
|
+
// The unterminated tail can never become a legal line — fail now instead of
|
|
129
|
+
// buffering without bound (the wire cap, mirrored; init is trusted, so this is a
|
|
130
|
+
// bug or corruption, not an attack to survive).
|
|
131
|
+
this.fail(new Error(`identity relay line exceeds ${MAX_RELAY_LINE_BYTES} bytes`));
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
89
134
|
this.wake?.();
|
|
90
|
-
}
|
|
91
|
-
|
|
135
|
+
};
|
|
136
|
+
stream.on("data", this.onData);
|
|
137
|
+
stream.on("end", () => {
|
|
92
138
|
this.closed = true;
|
|
93
139
|
this.wake?.();
|
|
94
|
-
};
|
|
95
|
-
stream.on("
|
|
96
|
-
|
|
140
|
+
});
|
|
141
|
+
stream.on("error", (err) => {
|
|
142
|
+
// Keep the error's detail — "closed" alone hides why the socket died.
|
|
143
|
+
this.fail(err);
|
|
144
|
+
});
|
|
97
145
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
this.
|
|
146
|
+
fail(err) {
|
|
147
|
+
this.failure ??= err;
|
|
148
|
+
this.closed = true;
|
|
149
|
+
this.stream.destroy();
|
|
150
|
+
this.wake?.();
|
|
151
|
+
}
|
|
152
|
+
/** Announce the pre-identity park (with the diagnostics payload when provided).
|
|
153
|
+
* Init forwards this as the base-snapshot gate. */
|
|
154
|
+
announceReady(diagnostics) {
|
|
155
|
+
this.writeLine(diagnostics === undefined
|
|
156
|
+
? { type: "worker_ready" }
|
|
157
|
+
: { type: "worker_ready", payload: diagnostics });
|
|
101
158
|
}
|
|
102
159
|
/**
|
|
103
160
|
* Block until init relays the run identity. THIS is the point the base snapshot freezes:
|
|
@@ -137,6 +194,64 @@ export class IdentityRelay {
|
|
|
137
194
|
acceptIdentity() {
|
|
138
195
|
this.writeLine({ type: "identity_accepted" });
|
|
139
196
|
}
|
|
197
|
+
/**
|
|
198
|
+
* Attach the post-identity consumer: the suspend/wake channel (the same fd, the same JSON
|
|
199
|
+
* lines). From here the relay dispatches init→worker messages to the handlers —
|
|
200
|
+
* `wake` (resolve a frozen seam) and `suspend_abort` (the seam falls back to holding) —
|
|
201
|
+
* and the returned channel sends the worker→init halves (`suspend_request`,
|
|
202
|
+
* `wake_accepted`). Unknown types are ignored (forward-compatible); malformed lines are
|
|
203
|
+
* logged and skipped, exactly as pre-identity. Call once, after {@link acceptIdentity}.
|
|
204
|
+
*/
|
|
205
|
+
openChannel(handlers) {
|
|
206
|
+
void this.pumpChannel(handlers);
|
|
207
|
+
return {
|
|
208
|
+
sendSuspendRequest: (payload) => {
|
|
209
|
+
this.writeLine({ type: "suspend_request", payload });
|
|
210
|
+
},
|
|
211
|
+
sendWakeAccepted: () => {
|
|
212
|
+
this.writeLine({ type: "wake_accepted" });
|
|
213
|
+
},
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
async pumpChannel(handlers) {
|
|
217
|
+
for (;;) {
|
|
218
|
+
let line;
|
|
219
|
+
try {
|
|
220
|
+
line = await this.readLine();
|
|
221
|
+
}
|
|
222
|
+
catch (err) {
|
|
223
|
+
// The relay dying mid-run means init is gone — the guest is being torn down; the
|
|
224
|
+
// worker's own exit follows. Log, don't throw into nowhere.
|
|
225
|
+
log.warn("relay_channel_closed", {
|
|
226
|
+
error: err instanceof Error ? err.message : String(err),
|
|
227
|
+
});
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
let parsed;
|
|
231
|
+
try {
|
|
232
|
+
parsed = JSON.parse(line);
|
|
233
|
+
}
|
|
234
|
+
catch {
|
|
235
|
+
log.warn("relay_malformed_line_skipped", { length: line.length });
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
const message = relayMessageSchema.safeParse(parsed);
|
|
239
|
+
if (!message.success) {
|
|
240
|
+
log.warn("relay_malformed_message_skipped", {});
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
switch (message.data.type) {
|
|
244
|
+
case "wake":
|
|
245
|
+
handlers.onWake(message.data.payload);
|
|
246
|
+
break;
|
|
247
|
+
case "suspend_abort":
|
|
248
|
+
handlers.onSuspendAbort(message.data.payload);
|
|
249
|
+
break;
|
|
250
|
+
default:
|
|
251
|
+
log.warn("relay_unexpected_type_ignored", { type: message.data.type });
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
140
255
|
writeLine(message) {
|
|
141
256
|
this.stream.write(JSON.stringify(message) + "\n");
|
|
142
257
|
}
|
|
@@ -146,7 +261,15 @@ export class IdentityRelay {
|
|
|
146
261
|
if (newline >= 0) {
|
|
147
262
|
const line = this.buffer.slice(0, newline);
|
|
148
263
|
this.buffer = this.buffer.slice(newline + 1);
|
|
149
|
-
|
|
264
|
+
if (line.length > MAX_RELAY_LINE_BYTES) {
|
|
265
|
+
this.fail(new Error(`identity relay line exceeds ${MAX_RELAY_LINE_BYTES} bytes`));
|
|
266
|
+
}
|
|
267
|
+
else {
|
|
268
|
+
return line;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
if (this.failure !== null) {
|
|
272
|
+
throw this.failure;
|
|
150
273
|
}
|
|
151
274
|
if (this.closed) {
|
|
152
275
|
throw new Error("identity relay closed before the run identity arrived");
|
package/dist/runtime/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AuthContext } from "./support/index.js";
|
|
2
2
|
import type { Run } from "./wire/run.js";
|
|
3
|
+
import { type IdentityRelay } from "./identity_relay.js";
|
|
3
4
|
import type { ByoInferenceProvider } from "../contract.js";
|
|
4
5
|
import { type ProgramWorkerDeps } from "./program_worker.js";
|
|
5
6
|
/** Constructed primitives the pure assembly consumes (real ones in main(), fakes in tests). The
|
|
@@ -27,6 +28,10 @@ export interface WorkerRuntime {
|
|
|
27
28
|
/** vCPUs provisioned for this task (the dispatcher's resolved machine size ÷ 1024 cpu units). Runtime
|
|
28
29
|
* is billed per vCPU-SECOND, so the RuntimeFlusher scales wall-clock by this. Defaults to 1. */
|
|
29
30
|
vcpus: number;
|
|
31
|
+
/** The in-guest identity relay, present ONLY on the snapshot-based microVM substrate (relay-mode
|
|
32
|
+
* boot). When set, the worker suspends by FREEZING — the FreezeCoordinator parks seams over this
|
|
33
|
+
* relay's suspend/wake channel — instead of the exit-and-replay path. */
|
|
34
|
+
freezeRelay?: IdentityRelay;
|
|
30
35
|
}
|
|
31
36
|
/** Durable events are deferred (run_step_events table) — live fan-out via the broker only. */
|
|
32
37
|
/** Synthesize the run's AuthContext. The run was already authorized at trigger time; the
|
package/dist/runtime/index.js
CHANGED
|
@@ -35,7 +35,8 @@ import { SecretRedactor } from "./agent/secret_redactor.js";
|
|
|
35
35
|
import { RecordingSecretResolver } from "./recording_secret_resolver.js";
|
|
36
36
|
import { EngineLeafExecutor } from "./leaf_executor.js";
|
|
37
37
|
import { parseByoProviders } from "./direct_inference.js";
|
|
38
|
-
import { applyIdentityToEnv, connectIdentityRelayFd, relayFdFromEnv } from "./identity_relay.js";
|
|
38
|
+
import { applyIdentityToEnv, connectIdentityRelayFd, relayFdFromEnv, workerDiagnostics, } from "./identity_relay.js";
|
|
39
|
+
import { FreezeCoordinator } from "./freeze_coordinator.js";
|
|
39
40
|
import { WorkerWorkflowHost } from "./workflow_host.js";
|
|
40
41
|
import { runProgramWorker, } from "./program_worker.js";
|
|
41
42
|
import { RunnerControlClient } from "./runner_control_client.js";
|
|
@@ -95,6 +96,24 @@ export function assembleWorkerDeps(runtime) {
|
|
|
95
96
|
// a resumed run suppresses observability for seams it already ran. Captured at claim, read by
|
|
96
97
|
// buildHost (which runs right after claim) when constructing the per-run host.
|
|
97
98
|
let replayFrontier = 0;
|
|
99
|
+
// Snapshot-substrate suspension (relay-mode boot only): one coordinator for the process, wired to
|
|
100
|
+
// the relay's suspend/wake channel now; its per-run hooks (token swap, meter rebase, workspace
|
|
101
|
+
// persist) late-bind in buildHost/startRuntimeFlush once those objects exist. The circular
|
|
102
|
+
// channel↔handler reference resolves through the thunks.
|
|
103
|
+
let freeze;
|
|
104
|
+
let activeFlusher = null;
|
|
105
|
+
if (runtime.freezeRelay !== undefined) {
|
|
106
|
+
const channel = runtime.freezeRelay.openChannel({
|
|
107
|
+
onWake: (payload) => {
|
|
108
|
+
freeze?.onWake(payload);
|
|
109
|
+
},
|
|
110
|
+
onSuspendAbort: (payload) => {
|
|
111
|
+
freeze?.onSuspendAbort(payload);
|
|
112
|
+
},
|
|
113
|
+
});
|
|
114
|
+
freeze = new FreezeCoordinator({ channel });
|
|
115
|
+
log.info("freeze_mode_enabled", { runId: runtime.runId });
|
|
116
|
+
}
|
|
98
117
|
// Live agent-event stream → /telemetry (broker publishes to Redis server-side). Inference,
|
|
99
118
|
// web_search, artifacts, and the model call are all brokered — no Redis / model creds / Tavily / S3.
|
|
100
119
|
const eventPublisher = new BrokerEventPublisher({
|
|
@@ -111,7 +130,8 @@ export function assembleWorkerDeps(runtime) {
|
|
|
111
130
|
// can't read it. The program reaches it ONLY via `runtime.apiToken()` (built below); we still
|
|
112
131
|
// record it in each run's redactor so a value the program threads into a tool can't echo back out
|
|
113
132
|
// of a tool result. Absent in the dev no-signing-key path. (the run env/credential rules)
|
|
114
|
-
|
|
133
|
+
// MUTABLE: on the snapshot substrate a wake carries a fresh token (the frozen one expired).
|
|
134
|
+
let runApiKey = runtime.controlPlane.apiToken;
|
|
115
135
|
// The broker (Runner Control API) is hosted on the api-server, so the public API shares its
|
|
116
136
|
// origin; `runtime.apiUrl` exposes it and the program appends `/v1` or `/mcp/v1`.
|
|
117
137
|
const apiUrl = publicApiOrigin(runtime.controlPlane.baseUrl);
|
|
@@ -288,8 +308,39 @@ export function assembleWorkerDeps(runtime) {
|
|
|
288
308
|
},
|
|
289
309
|
}
|
|
290
310
|
: {}),
|
|
311
|
+
// Snapshot-substrate suspension: seams freeze in place under the quiescence gate instead of
|
|
312
|
+
// raising onSuspend (which stays wired as the transitional/local path's fallback).
|
|
313
|
+
...(freeze !== undefined ? { freeze } : {}),
|
|
291
314
|
phases: phaseTracker,
|
|
292
315
|
});
|
|
316
|
+
// Late-bind the coordinator's per-run hooks now that the run-scoped objects exist.
|
|
317
|
+
if (freeze !== undefined) {
|
|
318
|
+
const coordinator = freeze;
|
|
319
|
+
let freezeWallMs = 0;
|
|
320
|
+
coordinator.setHooks({
|
|
321
|
+
// At quiescence, immediately before the freeze: book the runtime tail (suspended time must
|
|
322
|
+
// never appear billed — SUSPEND_POLICY) and persist the workspace (crash-during-suspension
|
|
323
|
+
// recovery parity with the sleep-hold path). The wall stamp anchors the idle rebase below.
|
|
324
|
+
onBeforeFreeze: async () => {
|
|
325
|
+
freezeWallMs = Date.now();
|
|
326
|
+
await activeFlusher?.flushNow();
|
|
327
|
+
await workspaceStore?.persist();
|
|
328
|
+
},
|
|
329
|
+
// On wake: swap the fresh tokens onto the broker client + the program-facing apiToken()
|
|
330
|
+
// (recording the new values in the run's redactor, same discipline as boot), and exclude
|
|
331
|
+
// the frozen window from billed runtime using the wake's authoritative wall clock (the
|
|
332
|
+
// guest's own clock was stopped).
|
|
333
|
+
onAfterWake: (wake) => {
|
|
334
|
+
broker.swapRunToken(wake.run_token);
|
|
335
|
+
redactor.record(wake.run_token);
|
|
336
|
+
if (wake.api_token !== undefined && wake.api_token !== "") {
|
|
337
|
+
runApiKey = wake.api_token;
|
|
338
|
+
redactor.record(wake.api_token);
|
|
339
|
+
}
|
|
340
|
+
activeFlusher?.excludeIdle(wake.wall_clock_ms - freezeWallMs);
|
|
341
|
+
},
|
|
342
|
+
});
|
|
343
|
+
}
|
|
293
344
|
// Hand the redactor back too: the worker scrubs a terminal error's message with it;
|
|
294
345
|
// workspace (when opted in) hydrates at start + persists at terminal.
|
|
295
346
|
return Promise.resolve({
|
|
@@ -376,6 +427,9 @@ export function assembleWorkerDeps(runtime) {
|
|
|
376
427
|
now: Date.now,
|
|
377
428
|
report: (deltaSeconds, identifier) => broker.reportUsage(deltaSeconds, identifier),
|
|
378
429
|
});
|
|
430
|
+
// The freeze coordinator's hooks flush this before a snapshot + rebase it past the frozen
|
|
431
|
+
// window on wake (suspended time must never appear as billed runtime).
|
|
432
|
+
activeFlusher = flusher;
|
|
379
433
|
flusher.start();
|
|
380
434
|
return { stop: () => flusher.stop(), flushFinal: () => flusher.flushFinal() };
|
|
381
435
|
},
|
|
@@ -492,12 +546,16 @@ export async function main() {
|
|
|
492
546
|
// daemon) the fd is absent and the worker env-boots exactly as before. The post-restore
|
|
493
547
|
// uniqueness reseed will hook in at this boundary, before any run code executes.
|
|
494
548
|
const relayFd = relayFdFromEnv(process.env);
|
|
549
|
+
let freezeRelay;
|
|
495
550
|
if (relayFd !== null) {
|
|
496
551
|
const relay = connectIdentityRelayFd(relayFd);
|
|
497
|
-
relay.announceReady();
|
|
552
|
+
relay.announceReady(workerDiagnostics());
|
|
498
553
|
const identity = await relay.awaitIdentity();
|
|
499
554
|
applyIdentityToEnv(identity, process.env);
|
|
500
555
|
relay.acceptIdentity();
|
|
556
|
+
// The relay now becomes the suspend/wake channel: assembleWorkerDeps opens it into the
|
|
557
|
+
// FreezeCoordinator, and the host's suspending seams freeze in place instead of exiting.
|
|
558
|
+
freezeRelay = relay;
|
|
501
559
|
}
|
|
502
560
|
// Capture the platform context into private state and remove it from process.env BEFORE anything
|
|
503
561
|
// else — so no user program / agent tool / subprocess we later spawn can read the run token or
|
|
@@ -513,6 +571,7 @@ export async function main() {
|
|
|
513
571
|
controlPlane: platform.controlPlane,
|
|
514
572
|
vcpus: platform.vcpus,
|
|
515
573
|
...(byoProviders.length > 0 ? { byoProviders } : {}),
|
|
574
|
+
...(freezeRelay !== undefined ? { freezeRelay } : {}),
|
|
516
575
|
});
|
|
517
576
|
// The only thing to drain is the batched telemetry buffer — the runner opens no DB/Redis/SQS.
|
|
518
577
|
const cleanup = async () => {
|
|
@@ -44,7 +44,12 @@ export declare class RunnerControlClient {
|
|
|
44
44
|
private readonly cfg;
|
|
45
45
|
private readonly base;
|
|
46
46
|
private readonly fetchImpl;
|
|
47
|
+
/** The live bearer. Mutable: on the snapshot substrate a wake carries a FRESH run token (the
|
|
48
|
+
* frozen one expired while suspended) and the worker swaps it at runtime. */
|
|
49
|
+
private runToken;
|
|
47
50
|
constructor(cfg: RunnerControlClientConfig);
|
|
51
|
+
/** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
|
|
52
|
+
swapRunToken(token: string): void;
|
|
48
53
|
/** Claim the run's lease. Returns the run on success, or null when it isn't claimable (409 —
|
|
49
54
|
* another worker has it, or it isn't pending), which the worker treats as "claim lost". */
|
|
50
55
|
claim(workerId: string, leaseSeconds: number): Promise<{
|
|
@@ -16,10 +16,18 @@ export class RunnerControlClient {
|
|
|
16
16
|
cfg;
|
|
17
17
|
base;
|
|
18
18
|
fetchImpl;
|
|
19
|
+
/** The live bearer. Mutable: on the snapshot substrate a wake carries a FRESH run token (the
|
|
20
|
+
* frozen one expired while suspended) and the worker swaps it at runtime. */
|
|
21
|
+
runToken;
|
|
19
22
|
constructor(cfg) {
|
|
20
23
|
this.cfg = cfg;
|
|
21
24
|
this.base = cfg.baseUrl.replace(/\/+$/, "");
|
|
22
25
|
this.fetchImpl = cfg.fetchImpl ?? fetch;
|
|
26
|
+
this.runToken = cfg.runToken;
|
|
27
|
+
}
|
|
28
|
+
/** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
|
|
29
|
+
swapRunToken(token) {
|
|
30
|
+
this.runToken = token;
|
|
23
31
|
}
|
|
24
32
|
/** Claim the run's lease. Returns the run on success, or null when it isn't claimable (409 —
|
|
25
33
|
* another worker has it, or it isn't pending), which the worker treats as "claim lost". */
|
|
@@ -409,7 +417,7 @@ export class RunnerControlClient {
|
|
|
409
417
|
}
|
|
410
418
|
headers(json) {
|
|
411
419
|
const h = {
|
|
412
|
-
authorization: `Bearer ${this.
|
|
420
|
+
authorization: `Bearer ${this.runToken}`,
|
|
413
421
|
accept: "application/json",
|
|
414
422
|
};
|
|
415
423
|
if (json)
|
|
@@ -26,7 +26,16 @@ export declare class RuntimeFlusher {
|
|
|
26
26
|
private flushedSeconds;
|
|
27
27
|
/** Per-flush sequence — advances only on a successful flush, so a retry reuses the same id. */
|
|
28
28
|
private seq;
|
|
29
|
+
/** Wall-clock ms excluded from billing — frozen (suspended) time on the snapshot substrate. */
|
|
30
|
+
private excludedMs;
|
|
29
31
|
constructor(deps: RuntimeFlusherDeps);
|
|
32
|
+
/** Book everything unbilled right now (the pre-freeze flush: suspended time must never appear as
|
|
33
|
+
* billed runtime, so the tail is booked BEFORE the snapshot — SUSPEND_POLICY billing rule). */
|
|
34
|
+
flushNow(): Promise<void>;
|
|
35
|
+
/** Exclude `ms` of wall-clock from billing — the frozen window on a wake (computed from the wake's
|
|
36
|
+
* authoritative wall clock, since the guest's own clock was stopped). Never lets elapsed go
|
|
37
|
+
* negative. */
|
|
38
|
+
excludeIdle(ms: number): void;
|
|
30
39
|
/** Begin periodic delta flushing. */
|
|
31
40
|
start(): void;
|
|
32
41
|
/** Stop the periodic timer and drain any in-flight flush. Does NOT book the final tail — call
|
|
@@ -30,9 +30,23 @@ export class RuntimeFlusher {
|
|
|
30
30
|
flushedSeconds = 0;
|
|
31
31
|
/** Per-flush sequence — advances only on a successful flush, so a retry reuses the same id. */
|
|
32
32
|
seq = 0;
|
|
33
|
+
/** Wall-clock ms excluded from billing — frozen (suspended) time on the snapshot substrate. */
|
|
34
|
+
excludedMs = 0;
|
|
33
35
|
constructor(deps) {
|
|
34
36
|
this.deps = deps;
|
|
35
37
|
}
|
|
38
|
+
/** Book everything unbilled right now (the pre-freeze flush: suspended time must never appear as
|
|
39
|
+
* billed runtime, so the tail is booked BEFORE the snapshot — SUSPEND_POLICY billing rule). */
|
|
40
|
+
async flushNow() {
|
|
41
|
+
await this.flush(false);
|
|
42
|
+
}
|
|
43
|
+
/** Exclude `ms` of wall-clock from billing — the frozen window on a wake (computed from the wake's
|
|
44
|
+
* authoritative wall clock, since the guest's own clock was stopped). Never lets elapsed go
|
|
45
|
+
* negative. */
|
|
46
|
+
excludeIdle(ms) {
|
|
47
|
+
if (ms > 0)
|
|
48
|
+
this.excludedMs += ms;
|
|
49
|
+
}
|
|
36
50
|
/** Begin periodic delta flushing. */
|
|
37
51
|
start() {
|
|
38
52
|
if (this.timer !== null)
|
|
@@ -67,7 +81,7 @@ export class RuntimeFlusher {
|
|
|
67
81
|
// vCPU-seconds = wall-clock seconds × vCPUs (billed per vCPU-second). Rounding the cumulative
|
|
68
82
|
// product (not per-delta) keeps the booked total aligned with wall-clock×vcpus over many flushes.
|
|
69
83
|
const vcpus = this.deps.vcpus ?? 1;
|
|
70
|
-
const total = Math.max(0, Math.round(((this.deps.now() - this.deps.startedAtMs) / 1000) * vcpus));
|
|
84
|
+
const total = Math.max(0, Math.round(((this.deps.now() - this.deps.startedAtMs - this.excludedMs) / 1000) * vcpus));
|
|
71
85
|
const delta = total - this.flushedSeconds;
|
|
72
86
|
if (delta < 1)
|
|
73
87
|
return;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { WorkflowHost, AgentOptions, ArtifactBody, ArtifactRef, CallOptions, HumanInputOptions, HumanInputResult, PhaseOptions, SleepArg } from "@boardwalk-labs/workflow/runtime";
|
|
2
2
|
import { type LeafResume } from "@boardwalk-labs/engine/core";
|
|
3
3
|
import { type JournalSeam, type SuspendSignal } from "./suspension.js";
|
|
4
|
+
import type { FreezeCoordinator } from "./freeze_coordinator.js";
|
|
4
5
|
/** Parse a `humanInput({ timeout })` string (`"48h"`, `"30m"`, `"90s"`, `"7d"`) to milliseconds, or
|
|
5
6
|
* null when absent/unparseable (the gate then waits indefinitely). */
|
|
6
7
|
export declare function parseTimeoutMs(timeout: string | undefined): number | null;
|
|
@@ -126,6 +127,14 @@ export interface WorkerWorkflowHostDeps {
|
|
|
126
127
|
* rejects with {@link SuspendError} instead (the local/test path).
|
|
127
128
|
*/
|
|
128
129
|
onSuspend?: (signal: SuspendSignal) => void;
|
|
130
|
+
/**
|
|
131
|
+
* Snapshot-substrate suspension (the microVM freeze model): when present, a suspending seam
|
|
132
|
+
* BLOCKS on the coordinator instead of raising `onSuspend` — the platform freezes the whole VM
|
|
133
|
+
* and a wake resolves the seam in place, heap intact (no exit, no journal replay). Every hook
|
|
134
|
+
* also runs under the coordinator's quiescence gate: a freeze never captures a live platform
|
|
135
|
+
* stream, and work arriving while a freeze is pending queues until the wake.
|
|
136
|
+
*/
|
|
137
|
+
freeze?: FreezeCoordinator;
|
|
129
138
|
/**
|
|
130
139
|
* The highest journaled seq at claim (0 on a fresh run). While re-running seams up to this
|
|
131
140
|
* frontier on a resume, the host is REPLAYING and observability (phase markers, program logs) is
|
|
@@ -152,8 +161,16 @@ export declare class WorkerWorkflowHost implements WorkflowHost {
|
|
|
152
161
|
private suspend;
|
|
153
162
|
/** Run `fn` only if the run isn't aborted; otherwise REJECT (never throw synchronously) with the
|
|
154
163
|
* signal's RunAbortedError — every Promise-returning hook funnels through this so callers always
|
|
155
|
-
* get a rejected promise on abort, not a sync throw.
|
|
164
|
+
* get a rejected promise on abort, not a sync throw. On the snapshot substrate the body also runs
|
|
165
|
+
* under the freeze coordinator's quiescence gate (see {@link WorkerWorkflowHostDeps.freeze}). */
|
|
156
166
|
private guarded;
|
|
167
|
+
/** A suspending seam's freeze wait: step out of the "work" count around the park (the wait itself
|
|
168
|
+
* is what the gate waits FOR, not work that blocks it), then rejoin on resume. */
|
|
169
|
+
private freezeWait;
|
|
170
|
+
/** The wake's answer for one gate key. A wake whose value is missing the parked gate means the
|
|
171
|
+
* control plane and the snapshot disagree about what this run was waiting for — a platform bug,
|
|
172
|
+
* failed loudly, never a retry. */
|
|
173
|
+
private gateAnswer;
|
|
157
174
|
setPhase(name: string, opts: PhaseOptions | undefined): void;
|
|
158
175
|
agent(prompt: string, opts: AgentOptions | undefined): Promise<unknown>;
|
|
159
176
|
/** The `agent()` durable seam: a journal hit returns the memoized result (never re-runs the LLM); a
|
|
@@ -97,7 +97,8 @@ export class WorkerWorkflowHost {
|
|
|
97
97
|
}
|
|
98
98
|
/** Run `fn` only if the run isn't aborted; otherwise REJECT (never throw synchronously) with the
|
|
99
99
|
* signal's RunAbortedError — every Promise-returning hook funnels through this so callers always
|
|
100
|
-
* get a rejected promise on abort, not a sync throw.
|
|
100
|
+
* get a rejected promise on abort, not a sync throw. On the snapshot substrate the body also runs
|
|
101
|
+
* under the freeze coordinator's quiescence gate (see {@link WorkerWorkflowHostDeps.freeze}). */
|
|
101
102
|
guarded(fn) {
|
|
102
103
|
try {
|
|
103
104
|
throwIfAborted(this.deps.signal);
|
|
@@ -106,10 +107,35 @@ export class WorkerWorkflowHost {
|
|
|
106
107
|
return Promise.reject(err instanceof Error ? err : new Error(String(err)));
|
|
107
108
|
}
|
|
108
109
|
const phases = this.deps.phases;
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
const phaseId = phases
|
|
112
|
-
|
|
110
|
+
// Capture the phase at CALL time — BEFORE any gate queueing — so a hook that queued across a
|
|
111
|
+
// freeze still lands in the phase its call site was in.
|
|
112
|
+
const phaseId = phases?.capture() ?? null;
|
|
113
|
+
const body = phases === undefined ? fn : () => phases.runInPhase(phaseId, fn);
|
|
114
|
+
const freeze = this.deps.freeze;
|
|
115
|
+
if (freeze === undefined)
|
|
116
|
+
return body();
|
|
117
|
+
return freeze.trackWork(body);
|
|
118
|
+
}
|
|
119
|
+
/** A suspending seam's freeze wait: step out of the "work" count around the park (the wait itself
|
|
120
|
+
* is what the gate waits FOR, not work that blocks it), then rejoin on resume. */
|
|
121
|
+
async freezeWait(freeze, signal) {
|
|
122
|
+
freeze.endWork();
|
|
123
|
+
try {
|
|
124
|
+
return await freeze.suspendingWait(signal);
|
|
125
|
+
}
|
|
126
|
+
finally {
|
|
127
|
+
freeze.beginWork();
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
/** The wake's answer for one gate key. A wake whose value is missing the parked gate means the
|
|
131
|
+
* control plane and the snapshot disagree about what this run was waiting for — a platform bug,
|
|
132
|
+
* failed loudly, never a retry. */
|
|
133
|
+
gateAnswer(wake, key) {
|
|
134
|
+
const answer = wake.answers?.[key];
|
|
135
|
+
if (wake.kind !== "human_input" || answer === undefined) {
|
|
136
|
+
throw new AppError(ErrorCode.INTERNAL_ERROR, `Wake value does not answer the parked gate "${key}" (got kind "${wake.kind}")`, { kind: "wake_mismatch", key });
|
|
137
|
+
}
|
|
138
|
+
return answer;
|
|
113
139
|
}
|
|
114
140
|
setPhase(name, opts) {
|
|
115
141
|
throwIfAborted(this.deps.signal);
|
|
@@ -146,21 +172,23 @@ export class WorkerWorkflowHost {
|
|
|
146
172
|
resume = leafResumeSchema.parse(existing.result);
|
|
147
173
|
}
|
|
148
174
|
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
175
|
+
for (;;) {
|
|
176
|
+
try {
|
|
177
|
+
const result = await this.deps.leaf.run(prompt, opts, this.deps.signal, resume);
|
|
178
|
+
await this.deps.journal?.put({
|
|
179
|
+
seq,
|
|
180
|
+
kind: "agent",
|
|
181
|
+
fingerprint,
|
|
182
|
+
label: prompt.slice(0, 120),
|
|
183
|
+
result,
|
|
184
|
+
});
|
|
185
|
+
return result;
|
|
186
|
+
}
|
|
187
|
+
catch (err) {
|
|
188
|
+
if (!(err instanceof LeafParked))
|
|
189
|
+
throw err;
|
|
190
|
+
// The model paused for a person (the in-leaf `human_input` tool).
|
|
191
|
+
const signal = {
|
|
164
192
|
reason: "human_input",
|
|
165
193
|
seq,
|
|
166
194
|
fingerprint,
|
|
@@ -170,9 +198,25 @@ export class WorkerWorkflowHost {
|
|
|
170
198
|
prompt: err.request.prompt,
|
|
171
199
|
inputSpec: err.request.inputSpec,
|
|
172
200
|
},
|
|
173
|
-
}
|
|
201
|
+
};
|
|
202
|
+
const freeze = this.deps.freeze;
|
|
203
|
+
if (freeze === undefined)
|
|
204
|
+
return this.suspend(signal);
|
|
205
|
+
// Snapshot substrate: freeze in place; the wake carries the answers and the leaf
|
|
206
|
+
// re-enters from its checkpoint — heap intact, no journal round-trip. A leaf may park
|
|
207
|
+
// again on a later turn, so this loops.
|
|
208
|
+
if (err.checkpoint === undefined) {
|
|
209
|
+
throw new AppError(ErrorCode.INTERNAL_ERROR, "Parked agent leaf has no checkpoint to resume from", { kind: "leaf_parked_no_checkpoint", seq });
|
|
210
|
+
}
|
|
211
|
+
const outcome = await this.freezeWait(freeze, signal);
|
|
212
|
+
if (outcome.kind !== "wake") {
|
|
213
|
+
throw new AppError(ErrorCode.INTERNAL_ERROR, "Suspend aborted surfaced to a human-input seam (the coordinator retries these)", { kind: "unexpected_abort", seq });
|
|
214
|
+
}
|
|
215
|
+
// Validate OUR gate is answered, but hand the leaf the whole batch (spec: a wake
|
|
216
|
+
// carries every answer the suspension raised, keyed by tool-call id).
|
|
217
|
+
this.gateAnswer(outcome.wake, err.request.toolCallId);
|
|
218
|
+
resume = { checkpoint: err.checkpoint, answers: { ...(outcome.wake.answers ?? {}) } };
|
|
174
219
|
}
|
|
175
|
-
throw err;
|
|
176
220
|
}
|
|
177
221
|
}
|
|
178
222
|
/** Program-level `humanInput()`: suspend the run on a gate, resume with the validated answer. The
|
|
@@ -197,7 +241,7 @@ export class WorkerWorkflowHost {
|
|
|
197
241
|
}
|
|
198
242
|
}
|
|
199
243
|
const expiresAt = this.timeoutExpiry(opts.timeout);
|
|
200
|
-
|
|
244
|
+
const signal = {
|
|
201
245
|
reason: "human_input",
|
|
202
246
|
seq,
|
|
203
247
|
fingerprint,
|
|
@@ -209,7 +253,17 @@ export class WorkerWorkflowHost {
|
|
|
209
253
|
// A timeout only matters with a wake to fire at; carry onTimeout alongside expiresAt.
|
|
210
254
|
...(expiresAt !== null ? { expiresAt, onTimeout: opts.onTimeout ?? "fail" } : {}),
|
|
211
255
|
},
|
|
212
|
-
}
|
|
256
|
+
};
|
|
257
|
+
const freeze = this.deps.freeze;
|
|
258
|
+
if (freeze !== undefined) {
|
|
259
|
+
// Snapshot substrate: park in place; the wake carries the validated answer.
|
|
260
|
+
const outcome = await this.freezeWait(freeze, signal);
|
|
261
|
+
if (outcome.kind !== "wake") {
|
|
262
|
+
throw new AppError(ErrorCode.INTERNAL_ERROR, "Suspend aborted surfaced to a human-input seam (the coordinator retries these)", { kind: "unexpected_abort", seq });
|
|
263
|
+
}
|
|
264
|
+
return normalizeHumanInputResult(this.gateAnswer(outcome.wake, key));
|
|
265
|
+
}
|
|
266
|
+
return this.suspend(signal);
|
|
213
267
|
}
|
|
214
268
|
/** Absolute wake time for a `humanInput({ timeout })`, or null when there is none / it's unparseable. */
|
|
215
269
|
timeoutExpiry(timeout) {
|
|
@@ -290,13 +344,30 @@ export class WorkerWorkflowHost {
|
|
|
290
344
|
if (child.status === "failed" || child.status === "cancelled") {
|
|
291
345
|
throw new AppError(ErrorCode.VALIDATION_FAILED, `Called workflow "${slug}" ${child.status} (run ${child.childRunId})`, { childRunId: child.childRunId, status: child.status });
|
|
292
346
|
}
|
|
293
|
-
// Still running → suspend
|
|
294
|
-
|
|
347
|
+
// Still running → suspend; the child's finalize wakes us.
|
|
348
|
+
const signal = {
|
|
295
349
|
reason: "workflow_call",
|
|
296
350
|
seq,
|
|
297
351
|
fingerprint,
|
|
298
352
|
childRunId: child.childRunId,
|
|
299
|
-
}
|
|
353
|
+
};
|
|
354
|
+
const freeze = this.deps.freeze;
|
|
355
|
+
if (freeze !== undefined) {
|
|
356
|
+
// Snapshot substrate: park in place; the wake carries the finalized child directly (no
|
|
357
|
+
// journal write for the wake-derived value — the heap holds it and nothing replays).
|
|
358
|
+
const outcome = await this.freezeWait(freeze, signal);
|
|
359
|
+
if (outcome.kind !== "wake") {
|
|
360
|
+
throw new AppError(ErrorCode.INTERNAL_ERROR, "Suspend aborted surfaced to a child-wait seam (the coordinator retries these)", { kind: "unexpected_abort", seq });
|
|
361
|
+
}
|
|
362
|
+
const woken = outcome.wake.child;
|
|
363
|
+
if (outcome.wake.kind !== "workflow_call" || woken === undefined) {
|
|
364
|
+
throw new AppError(ErrorCode.INTERNAL_ERROR, `Wake value does not carry the awaited child run (got kind "${outcome.wake.kind}")`, { kind: "wake_mismatch", childRunId: child.childRunId });
|
|
365
|
+
}
|
|
366
|
+
if (woken.status === "completed")
|
|
367
|
+
return woken.output;
|
|
368
|
+
throw new AppError(ErrorCode.VALIDATION_FAILED, `Called workflow "${slug}" ${woken.status} (run ${woken.run_id})`, { childRunId: woken.run_id, status: woken.status });
|
|
369
|
+
}
|
|
370
|
+
return this.suspend(signal);
|
|
300
371
|
}
|
|
301
372
|
/** Fire-and-forget trigger of another workflow; resolves to the new run's id (no hold/poll). */
|
|
302
373
|
runWorkflow(slug, input, opts) {
|
|
@@ -355,9 +426,32 @@ export class WorkerWorkflowHost {
|
|
|
355
426
|
const holdMs = Math.max(0, ms);
|
|
356
427
|
if (holdMs === 0)
|
|
357
428
|
return;
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
429
|
+
const freeze = this.deps.freeze;
|
|
430
|
+
if (holdMs >= SUSPEND_THRESHOLD_MS && freeze !== undefined) {
|
|
431
|
+
// Snapshot substrate: freeze in place; the wake (when the sleep is due) resolves this very
|
|
432
|
+
// await, heap intact. An abort falls back to holding the REMAINDER in-process — the
|
|
433
|
+
// two-ways rule: snapshot or hold, never replay.
|
|
434
|
+
const requestedAt = this.now();
|
|
435
|
+
const outcome = await this.freezeWait(freeze, {
|
|
436
|
+
reason: "sleep",
|
|
437
|
+
seq,
|
|
438
|
+
fingerprint,
|
|
439
|
+
durationMs: holdMs,
|
|
440
|
+
});
|
|
441
|
+
if (outcome.kind === "wake")
|
|
442
|
+
return;
|
|
443
|
+
const remaining = holdMs - (this.now() - requestedAt);
|
|
444
|
+
if (remaining <= 0)
|
|
445
|
+
return;
|
|
446
|
+
if (this.deps.onBeforeSleep !== undefined)
|
|
447
|
+
await this.deps.onBeforeSleep();
|
|
448
|
+
await this.sleeper.hold(remaining, this.deps.signal);
|
|
449
|
+
return;
|
|
450
|
+
}
|
|
451
|
+
// Long wait (transitional substrate): SUSPEND (release the task). The broker records the
|
|
452
|
+
// (resolved) sleep journal entry + the wake time transactionally, so on wake this seam replays
|
|
453
|
+
// past it (the journal hit above). Only when a journal is wired — without it there is no
|
|
454
|
+
// resume, so fall back to holding.
|
|
361
455
|
if (holdMs >= SUSPEND_THRESHOLD_MS && this.deps.journal !== undefined) {
|
|
362
456
|
return this.suspend({ reason: "sleep", seq, fingerprint, durationMs: holdMs });
|
|
363
457
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@boardwalk-labs/runner",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.11",
|
|
4
4
|
"description": "Boardwalk self-hosted runner: execute cloud-scheduled runs on your own machines. Currently: the canonical runner contract types.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|