@boardwalk-labs/runner 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +12 -0
- package/dist/runtime/agent/budget.d.ts +5 -5
- package/dist/runtime/agent/budget.js +8 -8
- package/dist/runtime/agent/model_rates.js +4 -5
- package/dist/runtime/agent/secret_redactor.js +1 -1
- package/dist/runtime/cancel_watcher.js +1 -1
- package/dist/runtime/credit_watcher.d.ts +1 -1
- package/dist/runtime/credit_watcher.js +5 -5
- package/dist/runtime/freeze_coordinator.d.ts +124 -0
- package/dist/runtime/freeze_coordinator.js +276 -0
- package/dist/runtime/identity_relay.d.ts +45 -2
- package/dist/runtime/identity_relay.js +137 -14
- package/dist/runtime/index.d.ts +11 -6
- package/dist/runtime/index.js +119 -39
- package/dist/runtime/leaf_executor.d.ts +2 -2
- package/dist/runtime/program_runner.d.ts +1 -1
- package/dist/runtime/program_runner.js +1 -1
- package/dist/runtime/program_worker.d.ts +3 -3
- package/dist/runtime/program_worker.js +7 -7
- package/dist/runtime/recording_secret_resolver.js +1 -1
- package/dist/runtime/run_abort.js +3 -3
- package/dist/runtime/runner_control_client.d.ts +34 -4
- package/dist/runtime/runner_control_client.js +85 -33
- package/dist/runtime/runtime_flusher.d.ts +9 -0
- package/dist/runtime/runtime_flusher.js +17 -3
- package/dist/runtime/support/index.js +5 -6
- package/dist/runtime/tools/artifacts.js +1 -1
- package/dist/runtime/tools/types.d.ts +5 -5
- package/dist/runtime/tools/types.js +7 -7
- package/dist/runtime/tools/web_search.js +5 -6
- package/dist/runtime/uniqueness_reseed.d.ts +7 -0
- package/dist/runtime/uniqueness_reseed.js +65 -0
- package/dist/runtime/wire/artifact_storage.js +2 -2
- package/dist/runtime/wire/inference_proxy.d.ts +1 -1
- package/dist/runtime/wire/inference_proxy.js +2 -2
- package/dist/runtime/workflow_host.d.ts +50 -1
- package/dist/runtime/workflow_host.js +185 -31
- package/native/reseed.c +52 -0
- package/package.json +12 -5
- package/prebuilds/linux-x64/@boardwalk-labs+runner.node +0 -0
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
// Boardwalk's internal Tool contract. Concrete tools (`echo`, `http`, `web_search`, `sleep`,
|
|
2
|
-
// `workflows.call`, …) implement this interface; the worker registers a
|
|
3
|
-
// adapter into
|
|
2
|
+
// `workflows.call`, …) implement this interface; the worker registers a schema-typed
|
|
3
|
+
// adapter into the agent leaf at construction time.
|
|
4
4
|
//
|
|
5
|
-
// Why a Boardwalk-side interface instead of
|
|
6
|
-
// * Tests can drive tool logic without instantiating
|
|
5
|
+
// Why a Boardwalk-side interface instead of the leaf's native tool type?
|
|
6
|
+
// * Tests can drive tool logic without instantiating the agent leaf.
|
|
7
7
|
// * Control-signal tools (`sleep`, `workflows.call`) need to return a
|
|
8
8
|
// specially-shaped value the worker interprets — they don't really "run";
|
|
9
|
-
// they bubble a signal up to the engine. Our adapter
|
|
10
|
-
//
|
|
9
|
+
// they bubble a signal up to the engine. Our adapter translates between
|
|
10
|
+
// this interface and the agent leaf's tool surface.
|
|
11
11
|
//
|
|
12
|
-
//
|
|
12
|
+
// Tools NEVER see secret values. Secrets resolve to
|
|
13
13
|
// short-lived bearer tokens, ARNs, etc. via the injected `SecretResolver`.
|
|
14
14
|
export function isControlSignal(value) {
|
|
15
15
|
return (typeof value === "object" &&
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
// web_search — wraps the Tavily API (https://docs.tavily.com/docs/api-reference).
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
// key fetched from
|
|
5
|
-
//
|
|
6
|
-
// conversation.
|
|
3
|
+
// The platform's default web-search provider. API
|
|
4
|
+
// key fetched from the org's secret store and cached at the module level per
|
|
5
|
+
// container, NEVER appearing in the agent's conversation.
|
|
7
6
|
//
|
|
8
|
-
// 429 from
|
|
9
|
-
// backoff. 5xx → `AppError(TOOL_ERROR)`. Network-level fetch failures fall
|
|
7
|
+
// 429 from the provider → `AppError(RATE_LIMIT)` so the agent loop's retry strategy
|
|
8
|
+
// handles backoff. 5xx → `AppError(TOOL_ERROR)`. Network-level fetch failures fall
|
|
10
9
|
// back to `TOOL_ERROR`.
|
|
11
10
|
import { z } from "zod";
|
|
12
11
|
import { AppError, ErrorCode } from "../support/index.js";
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reseed the userspace CSPRNG at a restore boundary (fresh-run identity injection AND every wake —
|
|
3
|
+
* clause 3), before any run code draws randomness. Idempotent and side-effect-free beyond the
|
|
4
|
+
* reseed; safe to call repeatedly. Returns whether the reseed ran (false = addon unavailable, the
|
|
5
|
+
* degraded no-op path). Never throws.
|
|
6
|
+
*/
|
|
7
|
+
export declare function reseedUserspaceCsprng(): boolean;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// Snapshot-uniqueness reseed — clause 3 of the SNAPSHOT_UNIQUENESS_CONTRACT, the one sliver of
|
|
2
|
+
// the determinism contract that survives the journal's deletion.
|
|
3
|
+
//
|
|
4
|
+
// A memory snapshot freezes OpenSSL's DRBG, so two clones of one base snapshot hand every run
|
|
5
|
+
// byte-identical `crypto.*` output (proven, contract). The kernel CSPRNG diverges across
|
|
6
|
+
// clones (VMGenID) but that reseed never reaches OpenSSL, and a pure-JS monkeypatch was proven
|
|
7
|
+
// insufficient (named ESM imports of `node:crypto` bypass it — measured 2026-07-09). So the robust
|
|
8
|
+
// fix is a native step that reseeds OpenSSL's DRBG UNDERNEATH every caller: the `bw_reseed` addon
|
|
9
|
+
// (native/reseed.c) chains EVP_RAND_reseed over the primary/public/private DRBGs from the
|
|
10
|
+
// (VMGenID-diverged) OS entropy.
|
|
11
|
+
//
|
|
12
|
+
// This module is the platform-owned JS entry point. It loads the prebuilt addon opportunistically
|
|
13
|
+
// and NO-OPS with a warning when it is unavailable (a platform with no prebuild — e.g. the ARM64
|
|
14
|
+
// Fargate worker, where there is no snapshot and thus nothing to reseed; or a self-host on an
|
|
15
|
+
// unsupported arch). It never throws: a reseed that cannot run must not fail a run.
|
|
16
|
+
import { createRequire } from "node:module";
|
|
17
|
+
import { fileURLToPath } from "node:url";
|
|
18
|
+
import { dirname, join } from "node:path";
|
|
19
|
+
import { createLogger } from "./support/index.js";
|
|
20
|
+
const log = createLogger("uniqueness_reseed");
|
|
21
|
+
/** undefined = not yet attempted; null = unavailable (no prebuild for this platform). */
|
|
22
|
+
let addon;
|
|
23
|
+
function loadAddon() {
|
|
24
|
+
if (addon !== undefined)
|
|
25
|
+
return addon;
|
|
26
|
+
try {
|
|
27
|
+
const require = createRequire(import.meta.url);
|
|
28
|
+
// node-gyp-build resolves prebuilds/<platform>-<arch>/*.node (production) or build/Release
|
|
29
|
+
// (a local dev build), throwing when neither exists for this platform.
|
|
30
|
+
const nodeGypBuild = require("node-gyp-build");
|
|
31
|
+
// dist/runtime/uniqueness_reseed.js → the package root two levels up (holds prebuilds/ + build/).
|
|
32
|
+
const pkgRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
33
|
+
addon = nodeGypBuild(pkgRoot);
|
|
34
|
+
}
|
|
35
|
+
catch (err) {
|
|
36
|
+
addon = null;
|
|
37
|
+
log.warn("uniqueness_reseed_addon_unavailable", {
|
|
38
|
+
error: err instanceof Error ? err.message : String(err),
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
return addon;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Reseed the userspace CSPRNG at a restore boundary (fresh-run identity injection AND every wake —
|
|
45
|
+
* clause 3), before any run code draws randomness. Idempotent and side-effect-free beyond the
|
|
46
|
+
* reseed; safe to call repeatedly. Returns whether the reseed ran (false = addon unavailable, the
|
|
47
|
+
* degraded no-op path). Never throws.
|
|
48
|
+
*/
|
|
49
|
+
export function reseedUserspaceCsprng() {
|
|
50
|
+
const a = loadAddon();
|
|
51
|
+
if (a === null)
|
|
52
|
+
return false;
|
|
53
|
+
try {
|
|
54
|
+
const ok = a.reseed();
|
|
55
|
+
if (!ok)
|
|
56
|
+
log.warn("uniqueness_reseed_partial", {});
|
|
57
|
+
return ok;
|
|
58
|
+
}
|
|
59
|
+
catch (err) {
|
|
60
|
+
log.error("uniqueness_reseed_failed", {
|
|
61
|
+
error: err instanceof Error ? err.message : String(err),
|
|
62
|
+
});
|
|
63
|
+
return false;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Artifact storage policy — the server-side rules for turning an agent's artifact-write request into
|
|
2
|
-
// a stored S3 object
|
|
2
|
+
// a stored S3 object. These used to live in the `artifacts` TOOL,
|
|
3
3
|
// which runs on the UNTRUSTED runner; under the Runner Credential Broker (the Runner Credential Broker model) the
|
|
4
4
|
// broker owns them so a malicious runner can't bypass content-type neutralization or escape its
|
|
5
5
|
// run's key prefix. Pure functions — unit-tested exhaustively.
|
|
@@ -10,7 +10,7 @@ import * as nodePath from "node:path";
|
|
|
10
10
|
export function workspaceS3Key(orgId, workflowId) {
|
|
11
11
|
return `orgs/${orgId}/workflows/${workflowId}/workspace.tar.gz`;
|
|
12
12
|
}
|
|
13
|
-
// ---- run-artifact retention tag (
|
|
13
|
+
// ---- run-artifact retention tag (S3 lifecycle safety net) ----
|
|
14
14
|
//
|
|
15
15
|
// Run artifacts share the `orgs/{org}/...` prefix with workspace snapshots (`.../workflows/...`) and
|
|
16
16
|
// program artifacts (`.../program-artifacts/...`), and S3 lifecycle prefix filters can't express
|
|
@@ -57,7 +57,7 @@ export declare function parseInferenceRequest(body: string): InferenceProxyReque
|
|
|
57
57
|
/** Serialize one streamed text delta as a single NDJSON line (trailing "\n" included). */
|
|
58
58
|
export declare function serializeDeltaFrame(text: string): string;
|
|
59
59
|
/** Serialize the single terminal turn result as one NDJSON line. `costMicros` is the turn's EXACT
|
|
60
|
-
* upstream cost (
|
|
60
|
+
* upstream cost (the managed provider's per-request cost × 1e6) on the managed lane — 0 for BYO or when unavailable.
|
|
61
61
|
* The worker feeds it to the budget guardrail so `max_usd` tracks real spend, not a token estimate. */
|
|
62
62
|
export declare function serializeResultFrame(turn: ChatTurn, modelRef: string, costMicros?: number): string;
|
|
63
63
|
/** Serialize a terminal error as a single NDJSON line. */
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
//
|
|
14
14
|
// Transport: the response is **newline-delimited JSON** (one frame per line). A model turn streams
|
|
15
15
|
// many text deltas; NDJSON lets the runner consume them incrementally over the raw socket (the
|
|
16
|
-
// buffered REST path can't stream
|
|
16
|
+
// buffered REST path can't stream). Each frame is one of:
|
|
17
17
|
// { "t": "delta", "text": <string> } — a streamed assistant-text chunk
|
|
18
18
|
// { "t": "result", "turn": <ChatTurn>, "modelRef": <string>, "costMicros": <number> } — terminal turn
|
|
19
19
|
// { "t": "error", "error": { code, message } } — a terminal model/broker error
|
|
@@ -77,7 +77,7 @@ export function serializeDeltaFrame(text) {
|
|
|
77
77
|
return `${JSON.stringify({ t: "delta", text })}\n`;
|
|
78
78
|
}
|
|
79
79
|
/** Serialize the single terminal turn result as one NDJSON line. `costMicros` is the turn's EXACT
|
|
80
|
-
* upstream cost (
|
|
80
|
+
* upstream cost (the managed provider's per-request cost × 1e6) on the managed lane — 0 for BYO or when unavailable.
|
|
81
81
|
* The worker feeds it to the budget guardrail so `max_usd` tracks real spend, not a token estimate. */
|
|
82
82
|
export function serializeResultFrame(turn, modelRef, costMicros = 0) {
|
|
83
83
|
return `${JSON.stringify({ t: "result", turn, modelRef, costMicros })}\n`;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { WorkflowHost, AgentOptions, ArtifactBody, ArtifactRef, CallOptions, HumanInputOptions, HumanInputResult, PhaseOptions, SleepArg } from "@boardwalk-labs/workflow/runtime";
|
|
2
2
|
import { type LeafResume } from "@boardwalk-labs/engine/core";
|
|
3
3
|
import { type JournalSeam, type SuspendSignal } from "./suspension.js";
|
|
4
|
+
import type { FreezeCoordinator } from "./freeze_coordinator.js";
|
|
4
5
|
/** Parse a `humanInput({ timeout })` string (`"48h"`, `"30m"`, `"90s"`, `"7d"`) to milliseconds, or
|
|
5
6
|
* null when absent/unparseable (the gate then waits indefinitely). */
|
|
6
7
|
export declare function parseTimeoutMs(timeout: string | undefined): number | null;
|
|
@@ -51,6 +52,13 @@ export interface ChildDispatcher {
|
|
|
51
52
|
export interface SecretAccessor {
|
|
52
53
|
get(name: string): Promise<string>;
|
|
53
54
|
}
|
|
55
|
+
/** Register-without-release: register a HELD HITL gate so it is
|
|
56
|
+
* answerable while the run keeps running, and poll for the answer. Backed by the broker's
|
|
57
|
+
* `inputs` endpoints. Absent ⇒ a held gate is only answerable once it freezes. */
|
|
58
|
+
export interface HeldInputPort {
|
|
59
|
+
register(seq: number, gate: SuspendSignal["humanInput"]): Promise<unknown>;
|
|
60
|
+
poll(seq: number): Promise<Record<string, unknown>>;
|
|
61
|
+
}
|
|
54
62
|
/** Holds the process for `ms` milliseconds. The seam exists so tests don't wait on real time. An
|
|
55
63
|
* abort fires the hold REJECT (with the signal's RunAbortedError) and clears the timer — so a
|
|
56
64
|
* multi-day sleep aborted early doesn't leave a live timer pinning the event loop open. */
|
|
@@ -126,6 +134,19 @@ export interface WorkerWorkflowHostDeps {
|
|
|
126
134
|
* rejects with {@link SuspendError} instead (the local/test path).
|
|
127
135
|
*/
|
|
128
136
|
onSuspend?: (signal: SuspendSignal) => void;
|
|
137
|
+
/**
|
|
138
|
+
* Snapshot-substrate suspension (the microVM freeze model): when present, a suspending seam
|
|
139
|
+
* BLOCKS on the coordinator instead of raising `onSuspend` — the platform freezes the whole VM
|
|
140
|
+
* and a wake resolves the seam in place, heap intact (no exit, no journal replay). Every hook
|
|
141
|
+
* also runs under the coordinator's quiescence gate: a freeze never captures a live platform
|
|
142
|
+
* stream, and work arriving while a freeze is pending queues until the wake.
|
|
143
|
+
*/
|
|
144
|
+
freeze?: FreezeCoordinator;
|
|
145
|
+
/** Register-without-release for HELD human-input gates. Only
|
|
146
|
+
* meaningful alongside `freeze`; absent ⇒ a held gate is answerable only once it freezes. */
|
|
147
|
+
heldInput?: HeldInputPort;
|
|
148
|
+
/** Poll interval for a held gate's answer (default 3s). */
|
|
149
|
+
heldPollIntervalMs?: number;
|
|
129
150
|
/**
|
|
130
151
|
* The highest journaled seq at claim (0 on a fresh run). While re-running seams up to this
|
|
131
152
|
* frontier on a resume, the host is REPLAYING and observability (phase markers, program logs) is
|
|
@@ -138,6 +159,7 @@ export declare class WorkerWorkflowHost implements WorkflowHost {
|
|
|
138
159
|
private readonly sleeper;
|
|
139
160
|
private readonly now;
|
|
140
161
|
private readonly maxSleepMs;
|
|
162
|
+
private readonly heldPollIntervalMs;
|
|
141
163
|
/** Synchronous durable-seam counter + silent-replay live flag (the durable-suspension design). */
|
|
142
164
|
private readonly seq;
|
|
143
165
|
/** Run context + on-demand public-API bearer the SDK `runtime` accessor reads off the host. */
|
|
@@ -152,8 +174,35 @@ export declare class WorkerWorkflowHost implements WorkflowHost {
|
|
|
152
174
|
private suspend;
|
|
153
175
|
/** Run `fn` only if the run isn't aborted; otherwise REJECT (never throw synchronously) with the
|
|
154
176
|
* signal's RunAbortedError — every Promise-returning hook funnels through this so callers always
|
|
155
|
-
* get a rejected promise on abort, not a sync throw.
|
|
177
|
+
* get a rejected promise on abort, not a sync throw. On the snapshot substrate the body also runs
|
|
178
|
+
* under the freeze coordinator's quiescence gate (see {@link WorkerWorkflowHostDeps.freeze}). */
|
|
156
179
|
private guarded;
|
|
180
|
+
/** A suspending seam's freeze wait: step out of the "work" count around the park (the wait itself
|
|
181
|
+
* is what the gate waits FOR, not work that blocks it), then rejoin on resume. */
|
|
182
|
+
private freezeWait;
|
|
183
|
+
/**
|
|
184
|
+
* Snapshot-substrate `humanInput()` with REGISTER-WITHOUT-RELEASE.
|
|
185
|
+
* A gate reached while a sibling seam is still in flight HOLDS (the quiescence gate won't freeze
|
|
186
|
+
* yet), but a human must still be able to answer during that hold. So: register the gate with the
|
|
187
|
+
* broker immediately (it surfaces in the inbox/API at once), then race two outcomes —
|
|
188
|
+
* - the answer arrives (brokered poll) while holding ⇒ WITHDRAW the freeze wait and resolve
|
|
189
|
+
* in-process (the run never froze); or
|
|
190
|
+
* - quiescence is reached with no answer ⇒ the wait freezes, and the wake carries the answer.
|
|
191
|
+
* Once frozen the poll is frozen too (same process), so the race is only live during the hold.
|
|
192
|
+
* A register/poll failure degrades to the plain freeze wait — the gate still works, just without
|
|
193
|
+
* the answerable-while-held property.
|
|
194
|
+
*/
|
|
195
|
+
private freezeHumanInput;
|
|
196
|
+
/** Poll the broker for a held gate's answer until it arrives or the wait withdraws. Resolves with
|
|
197
|
+
* the answer value; never rejects the run (a transient poll error just retries next tick). Runs
|
|
198
|
+
* OUTSIDE the quiescence gate (it is not run work — it must not block a freeze). */
|
|
199
|
+
private pollHeldAnswer;
|
|
200
|
+
/** Map a froze/aborted freeze outcome to the gate's answer (or throw on an unexpected abort). */
|
|
201
|
+
private resolveFreezeAnswer;
|
|
202
|
+
/** The wake's answer for one gate key. A wake whose value is missing the parked gate means the
|
|
203
|
+
* control plane and the snapshot disagree about what this run was waiting for — a platform bug,
|
|
204
|
+
* failed loudly, never a retry. */
|
|
205
|
+
private gateAnswer;
|
|
157
206
|
setPhase(name: string, opts: PhaseOptions | undefined): void;
|
|
158
207
|
agent(prompt: string, opts: AgentOptions | undefined): Promise<unknown>;
|
|
159
208
|
/** The `agent()` durable seam: a journal hit returns the memoized result (never re-runs the LLM); a
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// WorkerWorkflowHost — the real WorkflowHost the worker installs onto @boardwalk-labs/workflow
|
|
2
2
|
// before running a program (the workflow runtime design). The program's hooks delegate here:
|
|
3
3
|
//
|
|
4
|
-
// agent(prompt, opts) → an ephemeral
|
|
4
|
+
// agent(prompt, opts) → an ephemeral agent leaf (the demoted agent loop)
|
|
5
5
|
// sleep(arg) → an IN-PROCESS hold (hold-and-pay; no checkpoint, no exit)
|
|
6
6
|
// workflows.call(slug, in) → a durable child run (parent holds while it runs)
|
|
7
7
|
// secrets.get(name) → the run's fail-closed secret resolver
|
|
@@ -67,6 +67,7 @@ export class WorkerWorkflowHost {
|
|
|
67
67
|
sleeper;
|
|
68
68
|
now;
|
|
69
69
|
maxSleepMs;
|
|
70
|
+
heldPollIntervalMs;
|
|
70
71
|
/** Synchronous durable-seam counter + silent-replay live flag (the durable-suspension design). */
|
|
71
72
|
seq;
|
|
72
73
|
/** Run context + on-demand public-API bearer the SDK `runtime` accessor reads off the host. */
|
|
@@ -76,6 +77,7 @@ export class WorkerWorkflowHost {
|
|
|
76
77
|
this.sleeper = deps.sleeper ?? new TimerSleepController();
|
|
77
78
|
this.now = deps.now ?? Date.now;
|
|
78
79
|
this.maxSleepMs = deps.maxSleepMs ?? MAX_SLEEP_MS;
|
|
80
|
+
this.heldPollIntervalMs = deps.heldPollIntervalMs ?? 3_000;
|
|
79
81
|
this.seq = new SeamSequencer(deps.replayFrontier ?? 0);
|
|
80
82
|
this.runtime = deps.runtime;
|
|
81
83
|
}
|
|
@@ -97,7 +99,8 @@ export class WorkerWorkflowHost {
|
|
|
97
99
|
}
|
|
98
100
|
/** Run `fn` only if the run isn't aborted; otherwise REJECT (never throw synchronously) with the
|
|
99
101
|
* signal's RunAbortedError — every Promise-returning hook funnels through this so callers always
|
|
100
|
-
* get a rejected promise on abort, not a sync throw.
|
|
102
|
+
* get a rejected promise on abort, not a sync throw. On the snapshot substrate the body also runs
|
|
103
|
+
* under the freeze coordinator's quiescence gate (see {@link WorkerWorkflowHostDeps.freeze}). */
|
|
101
104
|
guarded(fn) {
|
|
102
105
|
try {
|
|
103
106
|
throwIfAborted(this.deps.signal);
|
|
@@ -106,10 +109,98 @@ export class WorkerWorkflowHost {
|
|
|
106
109
|
return Promise.reject(err instanceof Error ? err : new Error(String(err)));
|
|
107
110
|
}
|
|
108
111
|
const phases = this.deps.phases;
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
const phaseId = phases
|
|
112
|
-
|
|
112
|
+
// Capture the phase at CALL time — BEFORE any gate queueing — so a hook that queued across a
|
|
113
|
+
// freeze still lands in the phase its call site was in.
|
|
114
|
+
const phaseId = phases?.capture() ?? null;
|
|
115
|
+
const body = phases === undefined ? fn : () => phases.runInPhase(phaseId, fn);
|
|
116
|
+
const freeze = this.deps.freeze;
|
|
117
|
+
if (freeze === undefined)
|
|
118
|
+
return body();
|
|
119
|
+
return freeze.trackWork(body);
|
|
120
|
+
}
|
|
121
|
+
/** A suspending seam's freeze wait: step out of the "work" count around the park (the wait itself
|
|
122
|
+
* is what the gate waits FOR, not work that blocks it), then rejoin on resume. */
|
|
123
|
+
async freezeWait(freeze, signal, abort) {
|
|
124
|
+
freeze.endWork();
|
|
125
|
+
try {
|
|
126
|
+
return await freeze.suspendingWait(signal, abort);
|
|
127
|
+
}
|
|
128
|
+
finally {
|
|
129
|
+
freeze.beginWork();
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Snapshot-substrate `humanInput()` with REGISTER-WITHOUT-RELEASE.
|
|
134
|
+
* A gate reached while a sibling seam is still in flight HOLDS (the quiescence gate won't freeze
|
|
135
|
+
* yet), but a human must still be able to answer during that hold. So: register the gate with the
|
|
136
|
+
* broker immediately (it surfaces in the inbox/API at once), then race two outcomes —
|
|
137
|
+
* - the answer arrives (brokered poll) while holding ⇒ WITHDRAW the freeze wait and resolve
|
|
138
|
+
* in-process (the run never froze); or
|
|
139
|
+
* - quiescence is reached with no answer ⇒ the wait freezes, and the wake carries the answer.
|
|
140
|
+
* Once frozen the poll is frozen too (same process), so the race is only live during the hold.
|
|
141
|
+
* A register/poll failure degrades to the plain freeze wait — the gate still works, just without
|
|
142
|
+
* the answerable-while-held property.
|
|
143
|
+
*/
|
|
144
|
+
async freezeHumanInput(freeze, signal, key) {
|
|
145
|
+
const held = this.deps.heldInput;
|
|
146
|
+
if (held === undefined) {
|
|
147
|
+
// No held-input port wired: the gate can only be answered once it freezes.
|
|
148
|
+
return this.resolveFreezeAnswer(await this.freezeWait(freeze, signal), key);
|
|
149
|
+
}
|
|
150
|
+
await held.register(signal.seq, signal.humanInput);
|
|
151
|
+
const withdraw = new AbortController();
|
|
152
|
+
const poll = this.pollHeldAnswer(held, signal.seq, key, withdraw.signal);
|
|
153
|
+
const outcome = await Promise.race([
|
|
154
|
+
poll.then((answer) => ({ kind: "answered", answer })),
|
|
155
|
+
this.freezeWait(freeze, signal, withdraw.signal).then((o) => ({ kind: "froze", o })),
|
|
156
|
+
]);
|
|
157
|
+
if (outcome.kind === "answered") {
|
|
158
|
+
withdraw.abort(); // withdraw the still-holding freeze wait (no-op if it already froze)
|
|
159
|
+
return normalizeHumanInputResult(outcome.answer);
|
|
160
|
+
}
|
|
161
|
+
withdraw.abort(); // stop the poll loop; the wake path carries the answer
|
|
162
|
+
return this.resolveFreezeAnswer(outcome.o, key);
|
|
163
|
+
}
|
|
164
|
+
/** Poll the broker for a held gate's answer until it arrives or the wait withdraws. Resolves with
|
|
165
|
+
* the answer value; never rejects the run (a transient poll error just retries next tick). Runs
|
|
166
|
+
* OUTSIDE the quiescence gate (it is not run work — it must not block a freeze). */
|
|
167
|
+
async pollHeldAnswer(held, seq, key, abort) {
|
|
168
|
+
for (;;) {
|
|
169
|
+
if (abort.aborted)
|
|
170
|
+
return await new Promise(() => undefined); // frozen path won: never resolve
|
|
171
|
+
try {
|
|
172
|
+
const answers = await held.poll(seq);
|
|
173
|
+
if (key in answers)
|
|
174
|
+
return answers[key];
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
/* transient — retry next tick */
|
|
178
|
+
}
|
|
179
|
+
await new Promise((resolve) => {
|
|
180
|
+
const t = setTimeout(resolve, this.heldPollIntervalMs);
|
|
181
|
+
abort.addEventListener("abort", () => {
|
|
182
|
+
clearTimeout(t);
|
|
183
|
+
resolve();
|
|
184
|
+
}, { once: true });
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
/** Map a froze/aborted freeze outcome to the gate's answer (or throw on an unexpected abort). */
|
|
189
|
+
resolveFreezeAnswer(outcome, key) {
|
|
190
|
+
if (outcome.kind !== "wake") {
|
|
191
|
+
throw new AppError(ErrorCode.INTERNAL_ERROR, `Freeze wait for a human-input gate ended unexpectedly (${outcome.kind})`, { kind: "unexpected_freeze_outcome", key });
|
|
192
|
+
}
|
|
193
|
+
return normalizeHumanInputResult(this.gateAnswer(outcome.wake, key));
|
|
194
|
+
}
|
|
195
|
+
/** The wake's answer for one gate key. A wake whose value is missing the parked gate means the
|
|
196
|
+
* control plane and the snapshot disagree about what this run was waiting for — a platform bug,
|
|
197
|
+
* failed loudly, never a retry. */
|
|
198
|
+
gateAnswer(wake, key) {
|
|
199
|
+
const answer = wake.answers?.[key];
|
|
200
|
+
if (wake.kind !== "human_input" || answer === undefined) {
|
|
201
|
+
throw new AppError(ErrorCode.INTERNAL_ERROR, `Wake value does not answer the parked gate "${key}" (got kind "${wake.kind}")`, { kind: "wake_mismatch", key });
|
|
202
|
+
}
|
|
203
|
+
return answer;
|
|
113
204
|
}
|
|
114
205
|
setPhase(name, opts) {
|
|
115
206
|
throwIfAborted(this.deps.signal);
|
|
@@ -146,21 +237,23 @@ export class WorkerWorkflowHost {
|
|
|
146
237
|
resume = leafResumeSchema.parse(existing.result);
|
|
147
238
|
}
|
|
148
239
|
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
240
|
+
for (;;) {
|
|
241
|
+
try {
|
|
242
|
+
const result = await this.deps.leaf.run(prompt, opts, this.deps.signal, resume);
|
|
243
|
+
await this.deps.journal?.put({
|
|
244
|
+
seq,
|
|
245
|
+
kind: "agent",
|
|
246
|
+
fingerprint,
|
|
247
|
+
label: prompt.slice(0, 120),
|
|
248
|
+
result,
|
|
249
|
+
});
|
|
250
|
+
return result;
|
|
251
|
+
}
|
|
252
|
+
catch (err) {
|
|
253
|
+
if (!(err instanceof LeafParked))
|
|
254
|
+
throw err;
|
|
255
|
+
// The model paused for a person (the in-leaf `human_input` tool).
|
|
256
|
+
const signal = {
|
|
164
257
|
reason: "human_input",
|
|
165
258
|
seq,
|
|
166
259
|
fingerprint,
|
|
@@ -170,9 +263,25 @@ export class WorkerWorkflowHost {
|
|
|
170
263
|
prompt: err.request.prompt,
|
|
171
264
|
inputSpec: err.request.inputSpec,
|
|
172
265
|
},
|
|
173
|
-
}
|
|
266
|
+
};
|
|
267
|
+
const freeze = this.deps.freeze;
|
|
268
|
+
if (freeze === undefined)
|
|
269
|
+
return this.suspend(signal);
|
|
270
|
+
// Snapshot substrate: freeze in place; the wake carries the answers and the leaf
|
|
271
|
+
// re-enters from its checkpoint — heap intact, no journal round-trip. A leaf may park
|
|
272
|
+
// again on a later turn, so this loops.
|
|
273
|
+
if (err.checkpoint === undefined) {
|
|
274
|
+
throw new AppError(ErrorCode.INTERNAL_ERROR, "Parked agent leaf has no checkpoint to resume from", { kind: "leaf_parked_no_checkpoint", seq });
|
|
275
|
+
}
|
|
276
|
+
const outcome = await this.freezeWait(freeze, signal);
|
|
277
|
+
if (outcome.kind !== "wake") {
|
|
278
|
+
throw new AppError(ErrorCode.INTERNAL_ERROR, "Suspend aborted surfaced to a human-input seam (the coordinator retries these)", { kind: "unexpected_abort", seq });
|
|
279
|
+
}
|
|
280
|
+
// Validate OUR gate is answered, but hand the leaf the whole batch (spec: a wake
|
|
281
|
+
// carries every answer the suspension raised, keyed by tool-call id).
|
|
282
|
+
this.gateAnswer(outcome.wake, err.request.toolCallId);
|
|
283
|
+
resume = { checkpoint: err.checkpoint, answers: { ...(outcome.wake.answers ?? {}) } };
|
|
174
284
|
}
|
|
175
|
-
throw err;
|
|
176
285
|
}
|
|
177
286
|
}
|
|
178
287
|
/** Program-level `humanInput()`: suspend the run on a gate, resume with the validated answer. The
|
|
@@ -197,7 +306,7 @@ export class WorkerWorkflowHost {
|
|
|
197
306
|
}
|
|
198
307
|
}
|
|
199
308
|
const expiresAt = this.timeoutExpiry(opts.timeout);
|
|
200
|
-
|
|
309
|
+
const signal = {
|
|
201
310
|
reason: "human_input",
|
|
202
311
|
seq,
|
|
203
312
|
fingerprint,
|
|
@@ -209,7 +318,12 @@ export class WorkerWorkflowHost {
|
|
|
209
318
|
// A timeout only matters with a wake to fire at; carry onTimeout alongside expiresAt.
|
|
210
319
|
...(expiresAt !== null ? { expiresAt, onTimeout: opts.onTimeout ?? "fail" } : {}),
|
|
211
320
|
},
|
|
212
|
-
}
|
|
321
|
+
};
|
|
322
|
+
const freeze = this.deps.freeze;
|
|
323
|
+
if (freeze !== undefined) {
|
|
324
|
+
return await this.freezeHumanInput(freeze, signal, key);
|
|
325
|
+
}
|
|
326
|
+
return this.suspend(signal);
|
|
213
327
|
}
|
|
214
328
|
/** Absolute wake time for a `humanInput({ timeout })`, or null when there is none / it's unparseable. */
|
|
215
329
|
timeoutExpiry(timeout) {
|
|
@@ -290,13 +404,30 @@ export class WorkerWorkflowHost {
|
|
|
290
404
|
if (child.status === "failed" || child.status === "cancelled") {
|
|
291
405
|
throw new AppError(ErrorCode.VALIDATION_FAILED, `Called workflow "${slug}" ${child.status} (run ${child.childRunId})`, { childRunId: child.childRunId, status: child.status });
|
|
292
406
|
}
|
|
293
|
-
// Still running → suspend
|
|
294
|
-
|
|
407
|
+
// Still running → suspend; the child's finalize wakes us.
|
|
408
|
+
const signal = {
|
|
295
409
|
reason: "workflow_call",
|
|
296
410
|
seq,
|
|
297
411
|
fingerprint,
|
|
298
412
|
childRunId: child.childRunId,
|
|
299
|
-
}
|
|
413
|
+
};
|
|
414
|
+
const freeze = this.deps.freeze;
|
|
415
|
+
if (freeze !== undefined) {
|
|
416
|
+
// Snapshot substrate: park in place; the wake carries the finalized child directly (no
|
|
417
|
+
// journal write for the wake-derived value — the heap holds it and nothing replays).
|
|
418
|
+
const outcome = await this.freezeWait(freeze, signal);
|
|
419
|
+
if (outcome.kind !== "wake") {
|
|
420
|
+
throw new AppError(ErrorCode.INTERNAL_ERROR, "Suspend aborted surfaced to a child-wait seam (the coordinator retries these)", { kind: "unexpected_abort", seq });
|
|
421
|
+
}
|
|
422
|
+
const woken = outcome.wake.child;
|
|
423
|
+
if (outcome.wake.kind !== "workflow_call" || woken === undefined) {
|
|
424
|
+
throw new AppError(ErrorCode.INTERNAL_ERROR, `Wake value does not carry the awaited child run (got kind "${outcome.wake.kind}")`, { kind: "wake_mismatch", childRunId: child.childRunId });
|
|
425
|
+
}
|
|
426
|
+
if (woken.status === "completed")
|
|
427
|
+
return woken.output;
|
|
428
|
+
throw new AppError(ErrorCode.VALIDATION_FAILED, `Called workflow "${slug}" ${woken.status} (run ${woken.run_id})`, { childRunId: woken.run_id, status: woken.status });
|
|
429
|
+
}
|
|
430
|
+
return this.suspend(signal);
|
|
300
431
|
}
|
|
301
432
|
/** Fire-and-forget trigger of another workflow; resolves to the new run's id (no hold/poll). */
|
|
302
433
|
runWorkflow(slug, input, opts) {
|
|
@@ -355,9 +486,32 @@ export class WorkerWorkflowHost {
|
|
|
355
486
|
const holdMs = Math.max(0, ms);
|
|
356
487
|
if (holdMs === 0)
|
|
357
488
|
return;
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
489
|
+
const freeze = this.deps.freeze;
|
|
490
|
+
if (holdMs >= SUSPEND_THRESHOLD_MS && freeze !== undefined) {
|
|
491
|
+
// Snapshot substrate: freeze in place; the wake (when the sleep is due) resolves this very
|
|
492
|
+
// await, heap intact. An abort falls back to holding the REMAINDER in-process — the
|
|
493
|
+
// two-ways rule: snapshot or hold, never replay.
|
|
494
|
+
const requestedAt = this.now();
|
|
495
|
+
const outcome = await this.freezeWait(freeze, {
|
|
496
|
+
reason: "sleep",
|
|
497
|
+
seq,
|
|
498
|
+
fingerprint,
|
|
499
|
+
durationMs: holdMs,
|
|
500
|
+
});
|
|
501
|
+
if (outcome.kind === "wake")
|
|
502
|
+
return;
|
|
503
|
+
const remaining = holdMs - (this.now() - requestedAt);
|
|
504
|
+
if (remaining <= 0)
|
|
505
|
+
return;
|
|
506
|
+
if (this.deps.onBeforeSleep !== undefined)
|
|
507
|
+
await this.deps.onBeforeSleep();
|
|
508
|
+
await this.sleeper.hold(remaining, this.deps.signal);
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
// Long wait (transitional substrate): SUSPEND (release the task). The broker records the
|
|
512
|
+
// (resolved) sleep journal entry + the wake time transactionally, so on wake this seam replays
|
|
513
|
+
// past it (the journal hit above). Only when a journal is wired — without it there is no
|
|
514
|
+
// resume, so fall back to holding.
|
|
361
515
|
if (holdMs >= SUSPEND_THRESHOLD_MS && this.deps.journal !== undefined) {
|
|
362
516
|
return this.suspend({ reason: "sleep", seq, fingerprint, durationMs: holdMs });
|
|
363
517
|
}
|
package/native/reseed.c
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* bw_reseed — the clause-3 userspace-CSPRNG reseed (SNAPSHOT_UNIQUENESS_CONTRACT).
|
|
3
|
+
*
|
|
4
|
+
* A memory snapshot freezes OpenSSL's DRBG state, so two clones of one base snapshot hand every
|
|
5
|
+
* run byte-identical `crypto.*` output. The kernel CSPRNG diverges across clones (VMGenID), but
|
|
6
|
+
* that reseed never reaches OpenSSL. A pure-JS monkeypatch was proven insufficient (named ESM
|
|
7
|
+
* imports of node:crypto bypass it), so this native step reseeds OpenSSL's DRBG UNDERNEATH every
|
|
8
|
+
* caller — the only robust fix.
|
|
9
|
+
*
|
|
10
|
+
* OpenSSL 3 DRBG chain: the primary DRBG seeds from the OS entropy source; the public + private
|
|
11
|
+
* DRBGs (what RAND_bytes / Node crypto actually draw from) seed from the primary. We reseed the
|
|
12
|
+
* primary first (pulling the now-diverged OS entropy) then the public + private so the fresh
|
|
13
|
+
* entropy propagates immediately, and call RAND_poll() as a belt. Everything is guarded on NULL so
|
|
14
|
+
* a non-default RAND provider degrades to whatever succeeded rather than crashing.
|
|
15
|
+
*
|
|
16
|
+
* Symbols (RAND_get0_*, EVP_RAND_reseed, RAND_poll) come from the OpenSSL that Node links; on a
|
|
17
|
+
* Node build that does not export them the addon fails to load and the JS layer no-ops with a warn
|
|
18
|
+
* (uniqueness_reseed.ts). Linux-only in practice (the snapshot substrate is Linux microVMs).
|
|
19
|
+
*/
|
|
20
|
+
#include <node_api.h>
|
|
21
|
+
#include <openssl/rand.h>
|
|
22
|
+
#include <openssl/evp.h>
|
|
23
|
+
|
|
24
|
+
static int reseed_ctx(EVP_RAND_CTX *ctx) {
|
|
25
|
+
if (ctx == NULL) return 1; /* absent layer: nothing to do, not a failure */
|
|
26
|
+
return EVP_RAND_reseed(ctx, /*prediction_resistance=*/0, NULL, 0, NULL, 0);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
static napi_value Reseed(napi_env env, napi_callback_info info) {
|
|
30
|
+
int ok = 1;
|
|
31
|
+
/* Reseed the primary FIRST (it pulls fresh OS entropy), then the layers that serve draws. */
|
|
32
|
+
if (!reseed_ctx(RAND_get0_primary(NULL))) ok = 0;
|
|
33
|
+
if (!reseed_ctx(RAND_get0_public(NULL))) ok = 0;
|
|
34
|
+
if (!reseed_ctx(RAND_get0_private(NULL))) ok = 0;
|
|
35
|
+
/* Legacy belt: also nudge the default method's entropy pool. */
|
|
36
|
+
RAND_poll();
|
|
37
|
+
|
|
38
|
+
napi_value result;
|
|
39
|
+
napi_get_boolean(env, ok ? true : false, &result);
|
|
40
|
+
return result;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
static napi_value Init(napi_env env, napi_value exports) {
|
|
44
|
+
napi_value fn;
|
|
45
|
+
if (napi_create_function(env, "reseed", NAPI_AUTO_LENGTH, Reseed, NULL, &fn) != napi_ok) {
|
|
46
|
+
return exports;
|
|
47
|
+
}
|
|
48
|
+
napi_set_named_property(env, exports, "reseed", fn);
|
|
49
|
+
return exports;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
NAPI_MODULE(NODE_GYP_MODULE_NAME, Init)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@boardwalk-labs/runner",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.12",
|
|
4
4
|
"description": "Boardwalk self-hosted runner: execute cloud-scheduled runs on your own machines. Currently: the canonical runner contract types.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -37,7 +37,10 @@
|
|
|
37
37
|
"files": [
|
|
38
38
|
"dist",
|
|
39
39
|
"README.md",
|
|
40
|
-
"LICENSE"
|
|
40
|
+
"LICENSE",
|
|
41
|
+
"native",
|
|
42
|
+
"binding.gyp",
|
|
43
|
+
"prebuilds"
|
|
41
44
|
],
|
|
42
45
|
"scripts": {
|
|
43
46
|
"build": "tsc -p tsconfig.build.json",
|
|
@@ -46,13 +49,15 @@
|
|
|
46
49
|
"format": "prettier --write .",
|
|
47
50
|
"format:check": "prettier --check .",
|
|
48
51
|
"test": "vitest run --coverage",
|
|
49
|
-
"test:watch": "vitest"
|
|
52
|
+
"test:watch": "vitest",
|
|
53
|
+
"prebuild": "prebuildify --napi --strip -t 24.0.0"
|
|
50
54
|
},
|
|
51
55
|
"dependencies": {
|
|
52
56
|
"zod": "^4.0.0",
|
|
53
57
|
"@boardwalk-labs/engine": "^0.1.29",
|
|
54
58
|
"@boardwalk-labs/workflow": "^0.1.17",
|
|
55
|
-
"tar": "^7.5.16"
|
|
59
|
+
"tar": "^7.5.16",
|
|
60
|
+
"node-gyp-build": "^4.8.4"
|
|
56
61
|
},
|
|
57
62
|
"devDependencies": {
|
|
58
63
|
"@eslint/js": "^9.18.0",
|
|
@@ -62,7 +67,9 @@
|
|
|
62
67
|
"prettier": "^3.4.0",
|
|
63
68
|
"typescript": "^5.6.0",
|
|
64
69
|
"typescript-eslint": "^8.20.0",
|
|
65
|
-
"vitest": "^3.0.0"
|
|
70
|
+
"vitest": "^3.0.0",
|
|
71
|
+
"prebuildify": "^6.0.1",
|
|
72
|
+
"node-gyp": "^11.0.0"
|
|
66
73
|
},
|
|
67
74
|
"bin": {
|
|
68
75
|
"boardwalk-runner": "./dist/bin.js"
|
|
Binary file
|