@boardwalk-labs/runner 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +12 -0
- package/dist/runtime/agent/budget.d.ts +5 -5
- package/dist/runtime/agent/budget.js +8 -8
- package/dist/runtime/agent/model_rates.js +4 -5
- package/dist/runtime/agent/secret_redactor.js +1 -1
- package/dist/runtime/cancel_watcher.js +1 -1
- package/dist/runtime/credit_watcher.d.ts +1 -1
- package/dist/runtime/credit_watcher.js +5 -5
- package/dist/runtime/freeze_coordinator.d.ts +124 -0
- package/dist/runtime/freeze_coordinator.js +276 -0
- package/dist/runtime/identity_relay.d.ts +45 -2
- package/dist/runtime/identity_relay.js +137 -14
- package/dist/runtime/index.d.ts +11 -6
- package/dist/runtime/index.js +119 -39
- package/dist/runtime/leaf_executor.d.ts +2 -2
- package/dist/runtime/program_runner.d.ts +1 -1
- package/dist/runtime/program_runner.js +1 -1
- package/dist/runtime/program_worker.d.ts +3 -3
- package/dist/runtime/program_worker.js +7 -7
- package/dist/runtime/recording_secret_resolver.js +1 -1
- package/dist/runtime/run_abort.js +3 -3
- package/dist/runtime/runner_control_client.d.ts +34 -4
- package/dist/runtime/runner_control_client.js +85 -33
- package/dist/runtime/runtime_flusher.d.ts +9 -0
- package/dist/runtime/runtime_flusher.js +17 -3
- package/dist/runtime/support/index.js +5 -6
- package/dist/runtime/tools/artifacts.js +1 -1
- package/dist/runtime/tools/types.d.ts +5 -5
- package/dist/runtime/tools/types.js +7 -7
- package/dist/runtime/tools/web_search.js +5 -6
- package/dist/runtime/uniqueness_reseed.d.ts +7 -0
- package/dist/runtime/uniqueness_reseed.js +65 -0
- package/dist/runtime/wire/artifact_storage.js +2 -2
- package/dist/runtime/wire/inference_proxy.d.ts +1 -1
- package/dist/runtime/wire/inference_proxy.js +2 -2
- package/dist/runtime/workflow_host.d.ts +50 -1
- package/dist/runtime/workflow_host.js +185 -31
- package/native/reseed.c +52 -0
- package/package.json +12 -5
- package/prebuilds/linux-x64/@boardwalk-labs+runner.node +0 -0
|
@@ -7,14 +7,23 @@
|
|
|
7
7
|
// AF_UNIX socketpair; init tells us its fd via `BOARDWALK_IDENTITY_RELAY_FD`. Wire: one JSON
|
|
8
8
|
// object per LF-terminated line —
|
|
9
9
|
//
|
|
10
|
-
// {"type":"worker_ready"}
|
|
10
|
+
// {"type":"worker_ready","payload":{...}} worker → init the pre-identity park is reached
|
|
11
|
+
// (payload: optional version diagnostics,
|
|
12
|
+
// forwarded verbatim by init)
|
|
11
13
|
// {"type":"identity","payload":{...}} init → worker the run identity (see schema below)
|
|
12
14
|
// {"type":"identity_accepted"} worker → init captured; init acks its host
|
|
15
|
+
// {"type":"suspend_request","payload":{...}} worker → init a quiescent seam asks to be frozen
|
|
16
|
+
// {"type":"suspend_abort","payload":{...}} init → worker snapshot failed; the seam holds instead
|
|
17
|
+
// {"type":"wake","payload":{...}} init → worker resolve the frozen seam (fresh tokens
|
|
18
|
+
// + the wake value, verbatim)
|
|
19
|
+
// {"type":"wake_accepted"} worker → init tokens swapped, seam resolved; init acks
|
|
13
20
|
//
|
|
14
|
-
// The payload carries exactly the platform env contract (snake_case) plus the resolved
|
|
15
|
-
// env; `applyIdentityToEnv` maps it onto `process.env` so `capturePlatformContext` runs
|
|
21
|
+
// The identity payload carries exactly the platform env contract (snake_case) plus the resolved
|
|
22
|
+
// user env; `applyIdentityToEnv` maps it onto `process.env` so `capturePlatformContext` runs
|
|
16
23
|
// UNCHANGED afterward — same fields, same capture-and-delete discipline, two transports.
|
|
17
|
-
// The stream stays open after identity:
|
|
24
|
+
// The stream stays open after identity: `openChannel` attaches the suspend/wake consumer
|
|
25
|
+
// (the FreezeCoordinator drives the choreography above it).
|
|
26
|
+
import { createRequire } from "node:module";
|
|
18
27
|
import { Socket } from "node:net";
|
|
19
28
|
import { z } from "zod";
|
|
20
29
|
import { createLogger } from "./support/index.js";
|
|
@@ -22,6 +31,11 @@ const log = createLogger("identity_relay");
|
|
|
22
31
|
/** Env var naming the inherited relay fd. Set by the guest init; absent everywhere else
|
|
23
32
|
* (Fargate, self-hosted daemon), where the worker env-boots as always. */
|
|
24
33
|
export const RELAY_FD_ENV = "BOARDWALK_IDENTITY_RELAY_FD";
|
|
34
|
+
/** Max bytes of one relay line — the wire protocol's 32 MiB frame cap, mirrored in-guest.
|
|
35
|
+
* An oversized line costs the connection (LF framing cannot resynchronize past it), which
|
|
36
|
+
* for this relay means a hard bootstrap failure. Measured in UTF-16 code units, which for
|
|
37
|
+
* this ASCII-JSON wire is the byte count; the cap is a guard, not exact accounting. */
|
|
38
|
+
export const MAX_RELAY_LINE_BYTES = 32 * 1024 * 1024;
|
|
25
39
|
/** The identity payload — the env contract's fields, as JSON instead of env vars. */
|
|
26
40
|
export const relayIdentitySchema = z.object({
|
|
27
41
|
run_id: z.string().min(1),
|
|
@@ -74,30 +88,73 @@ export function applyIdentityToEnv(identity, env) {
|
|
|
74
88
|
env.BOARDWALK_BYO_PROVIDERS = JSON.stringify(identity.byo_providers);
|
|
75
89
|
}
|
|
76
90
|
}
|
|
91
|
+
/** Collect the worker_ready diagnostics. Version lookups are best-effort: the runner's own
|
|
92
|
+
* package.json sits two levels above this module in both the src and published dist
|
|
93
|
+
* layouts; the SDK's is reachable only if its exports expose ./package.json. */
|
|
94
|
+
export function workerDiagnostics() {
|
|
95
|
+
const require = createRequire(import.meta.url);
|
|
96
|
+
const versionOf = (spec) => {
|
|
97
|
+
try {
|
|
98
|
+
const pkg = require(spec);
|
|
99
|
+
return typeof pkg.version === "string" ? pkg.version : undefined;
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
return undefined;
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
const worker = versionOf("../../package.json");
|
|
106
|
+
const sdk = versionOf("@boardwalk-labs/workflow/package.json");
|
|
107
|
+
return {
|
|
108
|
+
...(worker !== undefined ? { worker_version: worker } : {}),
|
|
109
|
+
node_version: process.version,
|
|
110
|
+
...(sdk !== undefined ? { sdk_version: sdk } : {}),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
77
113
|
/** One end of the init↔worker relay. Wraps any Duplex so tests run over in-memory streams. */
|
|
78
114
|
export class IdentityRelay {
|
|
79
115
|
stream;
|
|
80
116
|
buffer = "";
|
|
81
117
|
closed = false;
|
|
118
|
+
failure = null;
|
|
82
119
|
wake = null;
|
|
120
|
+
onData;
|
|
83
121
|
constructor(stream) {
|
|
84
122
|
this.stream = stream;
|
|
85
123
|
// One persistent listener for the relay's lifetime: a flowing stream DROPS chunks
|
|
86
124
|
// emitted while nobody listens, so attach-per-read would lose lines between reads.
|
|
87
|
-
|
|
125
|
+
this.onData = (chunk) => {
|
|
88
126
|
this.buffer += typeof chunk === "string" ? chunk : chunk.toString("utf8");
|
|
127
|
+
if (this.buffer.length > MAX_RELAY_LINE_BYTES && !this.buffer.includes("\n")) {
|
|
128
|
+
// The unterminated tail can never become a legal line — fail now instead of
|
|
129
|
+
// buffering without bound (the wire cap, mirrored; init is trusted, so this is a
|
|
130
|
+
// bug or corruption, not an attack to survive).
|
|
131
|
+
this.fail(new Error(`identity relay line exceeds ${MAX_RELAY_LINE_BYTES} bytes`));
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
89
134
|
this.wake?.();
|
|
90
|
-
}
|
|
91
|
-
|
|
135
|
+
};
|
|
136
|
+
stream.on("data", this.onData);
|
|
137
|
+
stream.on("end", () => {
|
|
92
138
|
this.closed = true;
|
|
93
139
|
this.wake?.();
|
|
94
|
-
};
|
|
95
|
-
stream.on("
|
|
96
|
-
|
|
140
|
+
});
|
|
141
|
+
stream.on("error", (err) => {
|
|
142
|
+
// Keep the error's detail — "closed" alone hides why the socket died.
|
|
143
|
+
this.fail(err);
|
|
144
|
+
});
|
|
97
145
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
this.
|
|
146
|
+
fail(err) {
|
|
147
|
+
this.failure ??= err;
|
|
148
|
+
this.closed = true;
|
|
149
|
+
this.stream.destroy();
|
|
150
|
+
this.wake?.();
|
|
151
|
+
}
|
|
152
|
+
/** Announce the pre-identity park (with the diagnostics payload when provided).
|
|
153
|
+
* Init forwards this as the base-snapshot gate. */
|
|
154
|
+
announceReady(diagnostics) {
|
|
155
|
+
this.writeLine(diagnostics === undefined
|
|
156
|
+
? { type: "worker_ready" }
|
|
157
|
+
: { type: "worker_ready", payload: diagnostics });
|
|
101
158
|
}
|
|
102
159
|
/**
|
|
103
160
|
* Block until init relays the run identity. THIS is the point the base snapshot freezes:
|
|
@@ -137,6 +194,64 @@ export class IdentityRelay {
|
|
|
137
194
|
acceptIdentity() {
|
|
138
195
|
this.writeLine({ type: "identity_accepted" });
|
|
139
196
|
}
|
|
197
|
+
/**
|
|
198
|
+
* Attach the post-identity consumer: the suspend/wake channel (the same fd, the same JSON
|
|
199
|
+
* lines). From here the relay dispatches init→worker messages to the handlers —
|
|
200
|
+
* `wake` (resolve a frozen seam) and `suspend_abort` (the seam falls back to holding) —
|
|
201
|
+
* and the returned channel sends the worker→init halves (`suspend_request`,
|
|
202
|
+
* `wake_accepted`). Unknown types are ignored (forward-compatible); malformed lines are
|
|
203
|
+
* logged and skipped, exactly as pre-identity. Call once, after {@link acceptIdentity}.
|
|
204
|
+
*/
|
|
205
|
+
openChannel(handlers) {
|
|
206
|
+
void this.pumpChannel(handlers);
|
|
207
|
+
return {
|
|
208
|
+
sendSuspendRequest: (payload) => {
|
|
209
|
+
this.writeLine({ type: "suspend_request", payload });
|
|
210
|
+
},
|
|
211
|
+
sendWakeAccepted: () => {
|
|
212
|
+
this.writeLine({ type: "wake_accepted" });
|
|
213
|
+
},
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
async pumpChannel(handlers) {
|
|
217
|
+
for (;;) {
|
|
218
|
+
let line;
|
|
219
|
+
try {
|
|
220
|
+
line = await this.readLine();
|
|
221
|
+
}
|
|
222
|
+
catch (err) {
|
|
223
|
+
// The relay dying mid-run means init is gone — the guest is being torn down; the
|
|
224
|
+
// worker's own exit follows. Log, don't throw into nowhere.
|
|
225
|
+
log.warn("relay_channel_closed", {
|
|
226
|
+
error: err instanceof Error ? err.message : String(err),
|
|
227
|
+
});
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
let parsed;
|
|
231
|
+
try {
|
|
232
|
+
parsed = JSON.parse(line);
|
|
233
|
+
}
|
|
234
|
+
catch {
|
|
235
|
+
log.warn("relay_malformed_line_skipped", { length: line.length });
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
const message = relayMessageSchema.safeParse(parsed);
|
|
239
|
+
if (!message.success) {
|
|
240
|
+
log.warn("relay_malformed_message_skipped", {});
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
switch (message.data.type) {
|
|
244
|
+
case "wake":
|
|
245
|
+
handlers.onWake(message.data.payload);
|
|
246
|
+
break;
|
|
247
|
+
case "suspend_abort":
|
|
248
|
+
handlers.onSuspendAbort(message.data.payload);
|
|
249
|
+
break;
|
|
250
|
+
default:
|
|
251
|
+
log.warn("relay_unexpected_type_ignored", { type: message.data.type });
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
140
255
|
writeLine(message) {
|
|
141
256
|
this.stream.write(JSON.stringify(message) + "\n");
|
|
142
257
|
}
|
|
@@ -146,7 +261,15 @@ export class IdentityRelay {
|
|
|
146
261
|
if (newline >= 0) {
|
|
147
262
|
const line = this.buffer.slice(0, newline);
|
|
148
263
|
this.buffer = this.buffer.slice(newline + 1);
|
|
149
|
-
|
|
264
|
+
if (line.length > MAX_RELAY_LINE_BYTES) {
|
|
265
|
+
this.fail(new Error(`identity relay line exceeds ${MAX_RELAY_LINE_BYTES} bytes`));
|
|
266
|
+
}
|
|
267
|
+
else {
|
|
268
|
+
return line;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
if (this.failure !== null) {
|
|
272
|
+
throw this.failure;
|
|
150
273
|
}
|
|
151
274
|
if (this.closed) {
|
|
152
275
|
throw new Error("identity relay closed before the run identity arrived");
|
package/dist/runtime/index.d.ts
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import type { AuthContext } from "./support/index.js";
|
|
2
2
|
import type { Run } from "./wire/run.js";
|
|
3
|
+
import { type IdentityRelay } from "./identity_relay.js";
|
|
3
4
|
import type { ByoInferenceProvider } from "../contract.js";
|
|
4
5
|
import { type ProgramWorkerDeps } from "./program_worker.js";
|
|
5
6
|
/** Constructed primitives the pure assembly consumes (real ones in main(), fakes in tests). The
|
|
6
7
|
* runner holds NO platform credential — only the per-run control-plane handle (run token + broker
|
|
7
8
|
* URL); everything else is reached through the Runner Control API (the Runner Credential Broker model). */
|
|
8
9
|
export interface WorkerRuntime {
|
|
9
|
-
/** Stable worker identity stamped onto the lease (
|
|
10
|
+
/** Stable worker identity stamped onto the lease (task ARN / hostname / run-derived). */
|
|
10
11
|
workerId: string;
|
|
11
12
|
/** Sandbox workspace root for filesystem/shell/git tools. */
|
|
12
13
|
workspaceRoot: string;
|
|
@@ -27,14 +28,18 @@ export interface WorkerRuntime {
|
|
|
27
28
|
/** vCPUs provisioned for this task (the dispatcher's resolved machine size ÷ 1024 cpu units). Runtime
|
|
28
29
|
* is billed per vCPU-SECOND, so the RuntimeFlusher scales wall-clock by this. Defaults to 1. */
|
|
29
30
|
vcpus: number;
|
|
31
|
+
/** The in-guest identity relay, present ONLY on the snapshot-based microVM substrate (relay-mode
|
|
32
|
+
* boot). When set, the worker suspends by FREEZING — the FreezeCoordinator parks seams over this
|
|
33
|
+
* relay's suspend/wake channel — instead of the exit-and-replay path. */
|
|
34
|
+
freezeRelay?: IdentityRelay;
|
|
30
35
|
}
|
|
31
|
-
/** Durable events are deferred (
|
|
36
|
+
/** Durable events are deferred (durable event storage) — live fan-out via the broker only. */
|
|
32
37
|
/** Synthesize the run's AuthContext. The run was already authorized at trigger time; the
|
|
33
38
|
* worker acts on the org's behalf, so it carries the org + an owner role. Tool-level
|
|
34
39
|
* boundaries (the broker's server-side manifest allowlist) are the real guard.
|
|
35
40
|
*
|
|
36
41
|
* source='workflow' (NOT 'session_jwt'): the program must never perform SESSION_JWT_ONLY
|
|
37
|
-
* credential mutations
|
|
42
|
+
* credential mutations, so a tool that ever exposed such a service is denied by
|
|
38
43
|
* construction regardless of the owner role. */
|
|
39
44
|
export declare function workerAuthContext(run: Run): AuthContext;
|
|
40
45
|
/**
|
|
@@ -52,10 +57,10 @@ export declare function workerAuthContext(run: Run): AuthContext;
|
|
|
52
57
|
*/
|
|
53
58
|
export declare function tokenMeterIdentifiers(runId: string, sessionId: string): (leafIndex: number) => string;
|
|
54
59
|
/** Pure wiring: build the full ProgramWorkerDeps from the control-plane handle. The runner reaches
|
|
55
|
-
* every privileged seam through the broker over its run token — no
|
|
60
|
+
* every privileged seam through the broker over its run token — no database, cache, billing, or model creds. */
|
|
56
61
|
export declare function assembleWorkerDeps(runtime: WorkerRuntime): ProgramWorkerDeps;
|
|
57
62
|
/**
|
|
58
|
-
* The per-run platform values the dispatcher injects as container env
|
|
63
|
+
* The per-run platform values the dispatcher injects as container env. They
|
|
59
64
|
* are captured into private worker state at bootstrap and DELETED from `process.env` before any user
|
|
60
65
|
* program / agent leaf / subprocess can run — the run token + API token are credentials, and nothing
|
|
61
66
|
* untrusted run code touches should inherit them (the run env/credential rules). The user owns the rest
|
|
@@ -73,7 +78,7 @@ export interface PlatformContext {
|
|
|
73
78
|
vcpus: number;
|
|
74
79
|
}
|
|
75
80
|
/** The public-API origin a run uses for raw API / MCP / CLI calls. The broker (Runner Control API)
|
|
76
|
-
*
|
|
81
|
+
* shares an origin with the public API, so the program appends `/v1` or
|
|
77
82
|
* `/mcp/v1`. Falls back to the broker URL unchanged if it can't be parsed. */
|
|
78
83
|
export declare function publicApiOrigin(controlPlaneBaseUrl: string): string;
|
|
79
84
|
/**
|
package/dist/runtime/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// Worker composition root (the workflow runtime design). The
|
|
2
|
-
// worker
|
|
1
|
+
// Worker composition root (the workflow runtime design). The hosted worker container runs the
|
|
2
|
+
// compiled worker entrypoint; the dispatcher launches it per run with
|
|
3
3
|
// RUN_ID + the per-run control-plane handle. This file assembles real implementations behind every
|
|
4
4
|
// seam `runProgramWorker(runId, deps)` injects, then runs the one workflow program to terminal.
|
|
5
5
|
//
|
|
@@ -11,21 +11,21 @@
|
|
|
11
11
|
// - secrets.get() → broker /secrets/resolve (allowlist enforced server-side).
|
|
12
12
|
// - workflows.call()→ broker /children (durable child run, hold + poll).
|
|
13
13
|
// - events.emit() → broker /events (fan-out server-side).
|
|
14
|
-
// - artifacts / web_search → broker /artifacts + /tools/web_search (no
|
|
14
|
+
// - artifacts / web_search → broker /artifacts + /tools/web_search (no storage or search-provider creds here).
|
|
15
15
|
// - lifecycle / usage / telemetry → broker /claim,/version,/finalize,/usage,/telemetry.
|
|
16
|
-
// There is no
|
|
17
|
-
// the metadata-endpoint escape has nothing to steal. The legacy direct (pre-broker) path is removed.
|
|
16
|
+
// There is no database, cache, billing, or model-provider client on the runner — so its task role is
|
|
17
|
+
// near-zero and the metadata-endpoint escape has nothing to steal. The legacy direct (pre-broker) path is removed.
|
|
18
18
|
//
|
|
19
19
|
// Split: `assembleWorkerDeps(runtime)` is pure wiring (unit-tested with a fake fetch); `main()` is the
|
|
20
20
|
// bootstrap shell (read the control-plane env, install signal handlers).
|
|
21
21
|
//
|
|
22
|
-
// Per-session loops wired here (all brokered): a UsageFlusher meters token deltas (
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
22
|
+
// Per-session loops wired here (all brokered): a UsageFlusher meters token deltas (→ /usage/tokens), a
|
|
23
|
+
// CreditWatcher polls funding (→ /credit, aborting the run on exhaustion), and a CancelWatcher polls
|
|
24
|
+
// for a user cancel (→ /cancel, aborting the run when the user cancels — the brokered worker holds no
|
|
25
|
+
// cache client, so it can't receive the platform's cancel publish directly).
|
|
26
26
|
//
|
|
27
|
-
// Documented v0 deferral (a clear seam): durable events — a no-op AgentEventStore (live
|
|
28
|
-
// via the broker only) until
|
|
27
|
+
// Documented v0 deferral (a clear seam): durable events — a no-op AgentEventStore (live fan-out
|
|
28
|
+
// via the broker only) until durable event storage lands.
|
|
29
29
|
import { createLogger, newId, AppError, ErrorCode, DEFAULT_LEASE_MS } from "./support/index.js";
|
|
30
30
|
import { LspService } from "@boardwalk-labs/engine/core";
|
|
31
31
|
import { BudgetMeter } from "./agent/budget.js";
|
|
@@ -35,7 +35,9 @@ import { SecretRedactor } from "./agent/secret_redactor.js";
|
|
|
35
35
|
import { RecordingSecretResolver } from "./recording_secret_resolver.js";
|
|
36
36
|
import { EngineLeafExecutor } from "./leaf_executor.js";
|
|
37
37
|
import { parseByoProviders } from "./direct_inference.js";
|
|
38
|
-
import { applyIdentityToEnv, connectIdentityRelayFd, relayFdFromEnv } from "./identity_relay.js";
|
|
38
|
+
import { applyIdentityToEnv, connectIdentityRelayFd, relayFdFromEnv, workerDiagnostics, } from "./identity_relay.js";
|
|
39
|
+
import { FreezeCoordinator } from "./freeze_coordinator.js";
|
|
40
|
+
import { reseedUserspaceCsprng } from "./uniqueness_reseed.js";
|
|
39
41
|
import { WorkerWorkflowHost } from "./workflow_host.js";
|
|
40
42
|
import { runProgramWorker, } from "./program_worker.js";
|
|
41
43
|
import { RunnerControlClient } from "./runner_control_client.js";
|
|
@@ -53,13 +55,13 @@ import { PhaseTracker } from "./phase_tracker.js";
|
|
|
53
55
|
import { mkdir } from "node:fs/promises";
|
|
54
56
|
import { join } from "node:path";
|
|
55
57
|
const log = createLogger("worker_entrypoint");
|
|
56
|
-
/** Durable events are deferred (
|
|
58
|
+
/** Durable events are deferred (durable event storage) — live fan-out via the broker only. */
|
|
57
59
|
/** Synthesize the run's AuthContext. The run was already authorized at trigger time; the
|
|
58
60
|
* worker acts on the org's behalf, so it carries the org + an owner role. Tool-level
|
|
59
61
|
* boundaries (the broker's server-side manifest allowlist) are the real guard.
|
|
60
62
|
*
|
|
61
63
|
* source='workflow' (NOT 'session_jwt'): the program must never perform SESSION_JWT_ONLY
|
|
62
|
-
* credential mutations
|
|
64
|
+
* credential mutations, so a tool that ever exposed such a service is denied by
|
|
63
65
|
* construction regardless of the owner role. */
|
|
64
66
|
export function workerAuthContext(run) {
|
|
65
67
|
const userId = run.actor.type === "user" ? run.actor.user_id : `workflow:${run.workflowId}`;
|
|
@@ -83,7 +85,7 @@ export function tokenMeterIdentifiers(runId, sessionId) {
|
|
|
83
85
|
return (leafIndex) => `${runId}:${sessionId}:${String(leafIndex)}:${String(seq++)}`;
|
|
84
86
|
}
|
|
85
87
|
/** Pure wiring: build the full ProgramWorkerDeps from the control-plane handle. The runner reaches
|
|
86
|
-
* every privileged seam through the broker over its run token — no
|
|
88
|
+
* every privileged seam through the broker over its run token — no database, cache, billing, or model creds. */
|
|
87
89
|
export function assembleWorkerDeps(runtime) {
|
|
88
90
|
const broker = new RunnerControlClient({
|
|
89
91
|
baseUrl: runtime.controlPlane.baseUrl,
|
|
@@ -95,8 +97,27 @@ export function assembleWorkerDeps(runtime) {
|
|
|
95
97
|
// a resumed run suppresses observability for seams it already ran. Captured at claim, read by
|
|
96
98
|
// buildHost (which runs right after claim) when constructing the per-run host.
|
|
97
99
|
let replayFrontier = 0;
|
|
98
|
-
//
|
|
99
|
-
//
|
|
100
|
+
// Snapshot-substrate suspension (relay-mode boot only): one coordinator for the process, wired to
|
|
101
|
+
// the relay's suspend/wake channel now; its per-run hooks (token swap, meter rebase, workspace
|
|
102
|
+
// persist) late-bind in buildHost/startRuntimeFlush once those objects exist. The circular
|
|
103
|
+
// channel↔handler reference resolves through the thunks.
|
|
104
|
+
let freeze;
|
|
105
|
+
let activeFlusher = null;
|
|
106
|
+
if (runtime.freezeRelay !== undefined) {
|
|
107
|
+
const channel = runtime.freezeRelay.openChannel({
|
|
108
|
+
onWake: (payload) => {
|
|
109
|
+
freeze?.onWake(payload);
|
|
110
|
+
},
|
|
111
|
+
onSuspendAbort: (payload) => {
|
|
112
|
+
freeze?.onSuspendAbort(payload);
|
|
113
|
+
},
|
|
114
|
+
});
|
|
115
|
+
freeze = new FreezeCoordinator({ channel });
|
|
116
|
+
log.info("freeze_mode_enabled", { runId: runtime.runId });
|
|
117
|
+
}
|
|
118
|
+
// Live agent-event stream → /telemetry (broker fans out server-side). Inference,
|
|
119
|
+
// web_search, artifacts, and the model call are all brokered — no cache client, model creds,
|
|
120
|
+
// search-provider creds, or object-storage creds here.
|
|
100
121
|
const eventPublisher = new BrokerEventPublisher({
|
|
101
122
|
send: (frames) => broker.publishTelemetry(frames),
|
|
102
123
|
});
|
|
@@ -110,10 +131,11 @@ export function assembleWorkerDeps(runtime) {
|
|
|
110
131
|
// bootstrap captured + scrubbed it (capturePlatformContext) so the agent's bash / subprocesses
|
|
111
132
|
// can't read it. The program reaches it ONLY via `runtime.apiToken()` (built below); we still
|
|
112
133
|
// record it in each run's redactor so a value the program threads into a tool can't echo back out
|
|
113
|
-
// of a tool result. Absent in the dev no-signing-key path.
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
//
|
|
134
|
+
// of a tool result. Absent in the dev no-signing-key path.
|
|
135
|
+
// MUTABLE: on the snapshot substrate a wake carries a fresh token (the frozen one expired).
|
|
136
|
+
let runApiKey = runtime.controlPlane.apiToken;
|
|
137
|
+
// The broker (Runner Control API) shares an origin with the public API;
|
|
138
|
+
// `runtime.apiUrl` exposes it and the program appends `/v1` or `/mcp/v1`.
|
|
117
139
|
const apiUrl = publicApiOrigin(runtime.controlPlane.baseUrl);
|
|
118
140
|
// Per-run host: agent() leaf + sleep-hold + secrets + children + events, all brokered. `signal`
|
|
119
141
|
// carries cooperative cancellation (credit exhaustion) into every host hook.
|
|
@@ -123,7 +145,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
123
145
|
// Fallback budget-cap rate. A managed turn caps on the broker's EXACT upstream cost (forwarded on
|
|
124
146
|
// the inference result frame → BudgetMeter `realCostUsd`); this representative sonnet-class rate
|
|
125
147
|
// applies only to a turn with no upstream cost (BYO / unavailable). `max_usd` is a guardrail, not
|
|
126
|
-
// the bill —
|
|
148
|
+
// the bill — the platform meters the actual per-leaf usage.
|
|
127
149
|
rate: BUDGET_GUARDRAIL_RATE,
|
|
128
150
|
startedAt: Date.now(),
|
|
129
151
|
// deadline_seconds is WALL-CLOCK from the run's ORIGINAL start (incl. suspended idle), so a run
|
|
@@ -140,7 +162,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
140
162
|
const nextMeterIdentifier = tokenMeterIdentifiers(run.id, meteringSessionId);
|
|
141
163
|
// Per-run secret boundary: every value resolved (program `secrets.get` OR a tool's
|
|
142
164
|
// ctx.secrets.resolve) is recorded into one shared redactor; the leaf seeds a fresh engine
|
|
143
|
-
// Redactor from it so the loop scrubs those values out of all model-bound content.
|
|
165
|
+
// Redactor from it so the loop scrubs those values out of all model-bound content.
|
|
144
166
|
const redactor = new SecretRedactor();
|
|
145
167
|
// Record the run's own API key so a prompt-injected agent can't echo it back to the model.
|
|
146
168
|
if (runApiKey !== undefined && runApiKey !== "")
|
|
@@ -190,9 +212,9 @@ export function assembleWorkerDeps(runtime) {
|
|
|
190
212
|
// until the artifact extracts. The empty `/workspace` and this dir are SEPARATE dirs on hosted,
|
|
191
213
|
// exactly the two-tier case the engine's AGENTS.md loader is built for.
|
|
192
214
|
programDir: () => programDir,
|
|
193
|
-
// Per-leaf, per-model token metering — reported THROUGH the broker (the worker holds no
|
|
194
|
-
// credential); the broker decides `billed_by_boardwalk` per model + meters to
|
|
195
|
-
// no-op there). Fire-and-forget: a metering hiccup must never fail the run.
|
|
215
|
+
// Per-leaf, per-model token metering — reported THROUGH the broker (the worker holds no billing
|
|
216
|
+
// credential); the broker decides `billed_by_boardwalk` per model + meters usage to the platform
|
|
217
|
+
// (BYO models no-op there). Fire-and-forget: a metering hiccup must never fail the run.
|
|
196
218
|
meterUsage: ({ model, inputTokens, outputTokens, cachedReadTokens, cachedWriteTokens, leafIndex, }) => {
|
|
197
219
|
void broker
|
|
198
220
|
.meterTokens({
|
|
@@ -217,7 +239,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
217
239
|
// comes from the org's connection vault via the Runner Control API — never stored on the worker.
|
|
218
240
|
brokerMcpToken: (serverUrl, invalidateToken) => broker.mcpToken(serverUrl, invalidateToken),
|
|
219
241
|
});
|
|
220
|
-
// Per-workflow persistent /workspace
|
|
242
|
+
// Per-workflow persistent /workspace: only when the manifest opts in. The BROKER additionally
|
|
221
243
|
// gates on hosted (self-hosted runs get null URLs), and snapshots are keyed per-workflow + scoped
|
|
222
244
|
// by the run token — so even the untrusted in-process program can't reach another tenant's data.
|
|
223
245
|
const workspaceStore = manifest.workspace?.persist === true
|
|
@@ -288,8 +310,52 @@ export function assembleWorkerDeps(runtime) {
|
|
|
288
310
|
},
|
|
289
311
|
}
|
|
290
312
|
: {}),
|
|
313
|
+
// Snapshot-substrate suspension: seams freeze in place under the quiescence gate instead of
|
|
314
|
+
// raising onSuspend (which stays wired as the transitional/local path's fallback).
|
|
315
|
+
...(freeze !== undefined ? { freeze } : {}),
|
|
316
|
+
// Register-without-release for held HITL gates — only on the
|
|
317
|
+
// freeze substrate, backed by the broker's inputs endpoints.
|
|
318
|
+
...(freeze !== undefined
|
|
319
|
+
? {
|
|
320
|
+
heldInput: {
|
|
321
|
+
register: (seq, gate) => broker.registerInput(seq, gate),
|
|
322
|
+
poll: (seq) => broker.pollInputAnswers(seq),
|
|
323
|
+
},
|
|
324
|
+
}
|
|
325
|
+
: {}),
|
|
291
326
|
phases: phaseTracker,
|
|
292
327
|
});
|
|
328
|
+
// Late-bind the coordinator's per-run hooks now that the run-scoped objects exist.
|
|
329
|
+
if (freeze !== undefined) {
|
|
330
|
+
const coordinator = freeze;
|
|
331
|
+
let freezeWallMs = 0;
|
|
332
|
+
coordinator.setHooks({
|
|
333
|
+
// At quiescence, immediately before the freeze: book the runtime tail (suspended time must
|
|
334
|
+
// never appear billed) and persist the workspace (crash-during-suspension
|
|
335
|
+
// recovery parity with the sleep-hold path). The wall stamp anchors the idle rebase below.
|
|
336
|
+
onBeforeFreeze: async () => {
|
|
337
|
+
freezeWallMs = Date.now();
|
|
338
|
+
await activeFlusher?.flushNow();
|
|
339
|
+
await workspaceStore?.persist();
|
|
340
|
+
},
|
|
341
|
+
// On wake: reseed the userspace CSPRNG (clause 3 — a suspend snapshot restored more than
|
|
342
|
+
// once, e.g. a re-dispatch retry, would otherwise repeat its post-wake `crypto.*` draws),
|
|
343
|
+
// swap the fresh tokens onto the broker client + the program-facing apiToken() (recording
|
|
344
|
+
// the new values in the run's redactor, same discipline as boot), and exclude the frozen
|
|
345
|
+
// window from billed runtime using the wake's authoritative wall clock. The reseed runs
|
|
346
|
+
// BEFORE the seam resolves, so no woken author code draws from the stale (pre-suspend) DRBG.
|
|
347
|
+
onAfterWake: (wake) => {
|
|
348
|
+
reseedUserspaceCsprng();
|
|
349
|
+
broker.swapRunToken(wake.run_token);
|
|
350
|
+
redactor.record(wake.run_token);
|
|
351
|
+
if (wake.api_token !== undefined && wake.api_token !== "") {
|
|
352
|
+
runApiKey = wake.api_token;
|
|
353
|
+
redactor.record(wake.api_token);
|
|
354
|
+
}
|
|
355
|
+
activeFlusher?.excludeIdle(wake.wall_clock_ms - freezeWallMs);
|
|
356
|
+
},
|
|
357
|
+
});
|
|
358
|
+
}
|
|
293
359
|
// Hand the redactor back too: the worker scrubs a terminal error's message with it;
|
|
294
360
|
// workspace (when opted in) hydrates at start + persists at terminal.
|
|
295
361
|
return Promise.resolve({
|
|
@@ -361,7 +427,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
361
427
|
suspender: {
|
|
362
428
|
suspend: (signal, workerId) => broker.suspend(signal, workerId),
|
|
363
429
|
},
|
|
364
|
-
// Runtime metering
|
|
430
|
+
// Runtime metering: flush runtime as periodic deltas (+ a terminal tail) through the broker,
|
|
365
431
|
// idempotent per flush, so a long/perpetual run bills as it burns and the credit watcher sees it —
|
|
366
432
|
// not a single charge at terminal (which a never-terminating run never reached). A fresh per-session
|
|
367
433
|
// id keeps a restarted run's sessions distinct in the idempotency key (distinct ids sum).
|
|
@@ -376,11 +442,14 @@ export function assembleWorkerDeps(runtime) {
|
|
|
376
442
|
now: Date.now,
|
|
377
443
|
report: (deltaSeconds, identifier) => broker.reportUsage(deltaSeconds, identifier),
|
|
378
444
|
});
|
|
445
|
+
// The freeze coordinator's hooks flush this before a snapshot + rebase it past the frozen
|
|
446
|
+
// window on wake (suspended time must never appear as billed runtime).
|
|
447
|
+
activeFlusher = flusher;
|
|
379
448
|
flusher.start();
|
|
380
449
|
return { stop: () => flusher.stop(), flushFinal: () => flusher.flushFinal() };
|
|
381
450
|
},
|
|
382
451
|
buildHost,
|
|
383
|
-
// Mid-run credit watching
|
|
452
|
+
// Mid-run credit watching: a CreditWatcher polls the broker's GET /credit on a timer; when
|
|
384
453
|
// the org runs out, onExhausted aborts the run (the orchestrator wires it to AbortController).
|
|
385
454
|
startCreditWatch: ({ run, onExhausted }) => {
|
|
386
455
|
const watcher = new CreditWatcher({
|
|
@@ -391,10 +460,10 @@ export function assembleWorkerDeps(runtime) {
|
|
|
391
460
|
watcher.start();
|
|
392
461
|
return { stop: () => watcher.stop() };
|
|
393
462
|
},
|
|
394
|
-
// Mid-run user-cancel watching
|
|
463
|
+
// Mid-run user-cancel watching: a CancelWatcher polls the broker's GET /cancel on a timer;
|
|
395
464
|
// when the user cancels (the run is flipped to `cancelling`), onCancelled aborts the run. This is
|
|
396
|
-
// how the brokered (
|
|
397
|
-
// reach it.
|
|
465
|
+
// how the brokered (cache-less) worker learns of a cancel — the platform's cancel publish can't
|
|
466
|
+
// reach it directly.
|
|
398
467
|
startCancelWatch: ({ run, onCancelled }) => {
|
|
399
468
|
const watcher = new CancelWatcher({
|
|
400
469
|
runId: run.id,
|
|
@@ -404,7 +473,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
404
473
|
watcher.start();
|
|
405
474
|
return { stop: () => watcher.stop() };
|
|
406
475
|
},
|
|
407
|
-
// Lease renewal
|
|
476
|
+
// Lease renewal: a LeaseRenewer extends the lease through the broker's POST /renew on a timer
|
|
408
477
|
// (well under the lease), so a run longer than the 5-min lease isn't reclaimed mid-flight by the
|
|
409
478
|
// recovery sweep. `renew` re-extends with the SAME workerId that claimed it; a null result (the
|
|
410
479
|
// run is no longer ours) fires onLost → the run aborts `lease_lost` without finalizing.
|
|
@@ -427,7 +496,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
427
496
|
}
|
|
428
497
|
// ---- Bootstrap (real container only) -------------------------------------------------
|
|
429
498
|
/**
|
|
430
|
-
* The per-run platform values the dispatcher injects as container env
|
|
499
|
+
* The per-run platform values the dispatcher injects as container env. They
|
|
431
500
|
* are captured into private worker state at bootstrap and DELETED from `process.env` before any user
|
|
432
501
|
* program / agent leaf / subprocess can run — the run token + API token are credentials, and nothing
|
|
433
502
|
* untrusted run code touches should inherit them (the run env/credential rules). The user owns the rest
|
|
@@ -441,7 +510,7 @@ export const PLATFORM_ENV_KEYS = [
|
|
|
441
510
|
"BOARDWALK_TASK_CPU_UNITS",
|
|
442
511
|
];
|
|
443
512
|
/** The public-API origin a run uses for raw API / MCP / CLI calls. The broker (Runner Control API)
|
|
444
|
-
*
|
|
513
|
+
* shares an origin with the public API, so the program appends `/v1` or
|
|
445
514
|
* `/mcp/v1`. Falls back to the broker URL unchanged if it can't be parsed. */
|
|
446
515
|
export function publicApiOrigin(controlPlaneBaseUrl) {
|
|
447
516
|
try {
|
|
@@ -465,7 +534,7 @@ export function capturePlatformContext(env) {
|
|
|
465
534
|
return value.trim();
|
|
466
535
|
};
|
|
467
536
|
// The dispatcher always injects the control-plane handle (the Runner Credential Broker model). The runner is
|
|
468
|
-
// brokered-only — there is no
|
|
537
|
+
// brokered-only — there is no database, cache, or billing fallback — so a missing handle is a hard config error.
|
|
469
538
|
const runId = read("RUN_ID");
|
|
470
539
|
const apiToken = env.BOARDWALK_API_KEY?.trim();
|
|
471
540
|
const controlPlane = {
|
|
@@ -492,12 +561,22 @@ export async function main() {
|
|
|
492
561
|
// daemon) the fd is absent and the worker env-boots exactly as before. The post-restore
|
|
493
562
|
// uniqueness reseed will hook in at this boundary, before any run code executes.
|
|
494
563
|
const relayFd = relayFdFromEnv(process.env);
|
|
564
|
+
let freezeRelay;
|
|
495
565
|
if (relayFd !== null) {
|
|
496
566
|
const relay = connectIdentityRelayFd(relayFd);
|
|
497
|
-
relay.announceReady();
|
|
567
|
+
relay.announceReady(workerDiagnostics());
|
|
498
568
|
const identity = await relay.awaitIdentity();
|
|
499
569
|
applyIdentityToEnv(identity, process.env);
|
|
570
|
+
// Clause 3 (SNAPSHOT_UNIQUENESS_CONTRACT): this run was restored from the SHARED base
|
|
571
|
+
// snapshot, so its OpenSSL DRBG is identical to every other run's. Reseed it from the
|
|
572
|
+
// (VMGenID-diverged) OS entropy BEFORE `acceptIdentity` releases the worker into the brokered
|
|
573
|
+
// lifecycle — so no `crypto.*` draw by the SDK, the agent, or author code can collide across
|
|
574
|
+
// clones. Every wake reseeds again (the after-wake hook).
|
|
575
|
+
reseedUserspaceCsprng();
|
|
500
576
|
relay.acceptIdentity();
|
|
577
|
+
// The relay now becomes the suspend/wake channel: assembleWorkerDeps opens it into the
|
|
578
|
+
// FreezeCoordinator, and the host's suspending seams freeze in place instead of exiting.
|
|
579
|
+
freezeRelay = relay;
|
|
501
580
|
}
|
|
502
581
|
// Capture the platform context into private state and remove it from process.env BEFORE anything
|
|
503
582
|
// else — so no user program / agent tool / subprocess we later spawn can read the run token or
|
|
@@ -513,12 +592,13 @@ export async function main() {
|
|
|
513
592
|
controlPlane: platform.controlPlane,
|
|
514
593
|
vcpus: platform.vcpus,
|
|
515
594
|
...(byoProviders.length > 0 ? { byoProviders } : {}),
|
|
595
|
+
...(freezeRelay !== undefined ? { freezeRelay } : {}),
|
|
516
596
|
});
|
|
517
|
-
// The only thing to drain is the batched telemetry buffer — the runner opens no
|
|
597
|
+
// The only thing to drain is the batched telemetry buffer — the runner opens no database, cache, or queue.
|
|
518
598
|
const cleanup = async () => {
|
|
519
599
|
await deps.flushTelemetry?.().catch(() => undefined);
|
|
520
600
|
};
|
|
521
|
-
// SIGTERM:
|
|
601
|
+
// SIGTERM: the orchestrator is stopping the task. Hold-and-pay has no mid-run checkpoint; we exit and let the
|
|
522
602
|
// lease expire → the scheduler-sweep reclaims it and a fresh worker RESTARTS the run from the
|
|
523
603
|
// top (Lambda/GHA semantics; durable children re-attach via idempotency). Crash-safe by design.
|
|
524
604
|
let shuttingDown = false;
|
|
@@ -32,7 +32,7 @@ export interface EngineLeafExecutorDeps {
|
|
|
32
32
|
budget: BudgetMeter;
|
|
33
33
|
/** RUN-level redactor (shared with the secret resolver): every resolved secret value is recorded
|
|
34
34
|
* here. We seed a fresh engine `Redactor` from it per leaf so the loop scrubs known values out of
|
|
35
|
-
* the prompt, tool args/results, and transcript before they reach the model
|
|
35
|
+
* the prompt, tool args/results, and transcript before they reach the model. */
|
|
36
36
|
redactor: SecretRedactor;
|
|
37
37
|
/** Builds this leaf's event sink; `leafIndex` (1-based) is only a metering identifier — the sink
|
|
38
38
|
* owns the run-global cursor. `identity` names the leaf on its turn frames. */
|
|
@@ -51,7 +51,7 @@ export interface EngineLeafExecutorDeps {
|
|
|
51
51
|
* omitted ⇒ no bundled tier (only the workspace AGENTS.md applies). */
|
|
52
52
|
programDir?: () => string | null;
|
|
53
53
|
/** Per-leaf token metering seam (fire-and-forget). Reports THIS leaf's tokens + its model to the
|
|
54
|
-
* broker, which decides `billed_by_boardwalk` per model + meters to
|
|
54
|
+
* broker, which decides `billed_by_boardwalk` per model + meters usage to the platform. Omitted in tests. */
|
|
55
55
|
meterUsage?: (input: MeterUsageInput) => void;
|
|
56
56
|
/** Backend for the engine's host-backed built-in tools (`webfetch` / `web_search` / `artifacts`):
|
|
57
57
|
* set as the leaf's `capabilities.host` so the engine registers them. Broker-backed on hosted runs
|
|
@@ -53,7 +53,7 @@ export interface ProgramRunnerDeps {
|
|
|
53
53
|
* Scrubs known secret values out of a string (the run's `SecretRedactor.redactText`). Applied to a
|
|
54
54
|
* top-level throw's message before it is logged AND before it is returned to the worker — a program
|
|
55
55
|
* that resolves a secret and then throws it in an error message must NOT land that secret raw in the
|
|
56
|
-
* logs or the finalized run output
|
|
56
|
+
* logs or the finalized run output. Defaults to identity (tests/local).
|
|
57
57
|
*/
|
|
58
58
|
redactText?: (text: string) => string;
|
|
59
59
|
/**
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// WorkflowProgramRunner — executes a workflow program (the JS-body model, the workflow runtime design).
|
|
2
2
|
//
|
|
3
|
-
// A run is the execution of a built program ARTIFACT
|
|
3
|
+
// A run is the execution of a built program ARTIFACT: the worker is handed the VERIFIED tarball
|
|
4
4
|
// (its sha256 already checked against the pinned digest by the orchestrator) plus the entry module
|
|
5
5
|
// name. This module is the mechanism: it installs the host adapter + trigger payload onto the
|
|
6
6
|
// `@boardwalk-labs/workflow` singleton, extracts the artifact into a unique temp dir under a
|