@boardwalk-labs/runner 0.1.11 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +12 -0
- package/dist/runtime/agent/budget.d.ts +5 -5
- package/dist/runtime/agent/budget.js +8 -8
- package/dist/runtime/agent/model_rates.js +4 -5
- package/dist/runtime/agent/secret_redactor.js +1 -1
- package/dist/runtime/cancel_watcher.js +1 -1
- package/dist/runtime/credit_watcher.d.ts +1 -1
- package/dist/runtime/credit_watcher.js +5 -5
- package/dist/runtime/freeze_coordinator.d.ts +6 -1
- package/dist/runtime/freeze_coordinator.js +18 -4
- package/dist/runtime/index.d.ts +6 -6
- package/dist/runtime/index.js +62 -41
- package/dist/runtime/leaf_executor.d.ts +2 -2
- package/dist/runtime/program_runner.d.ts +1 -1
- package/dist/runtime/program_runner.js +1 -1
- package/dist/runtime/program_worker.d.ts +3 -3
- package/dist/runtime/program_worker.js +7 -7
- package/dist/runtime/recording_secret_resolver.js +1 -1
- package/dist/runtime/run_abort.js +3 -3
- package/dist/runtime/runner_control_client.d.ts +29 -4
- package/dist/runtime/runner_control_client.js +76 -32
- package/dist/runtime/runtime_flusher.d.ts +1 -1
- package/dist/runtime/runtime_flusher.js +3 -3
- package/dist/runtime/support/index.js +5 -6
- package/dist/runtime/tools/artifacts.js +1 -1
- package/dist/runtime/tools/types.d.ts +5 -5
- package/dist/runtime/tools/types.js +7 -7
- package/dist/runtime/tools/web_search.js +5 -6
- package/dist/runtime/uniqueness_reseed.d.ts +7 -0
- package/dist/runtime/uniqueness_reseed.js +65 -0
- package/dist/runtime/wire/artifact_storage.js +2 -2
- package/dist/runtime/wire/inference_proxy.d.ts +1 -1
- package/dist/runtime/wire/inference_proxy.js +2 -2
- package/dist/runtime/workflow_host.d.ts +32 -0
- package/dist/runtime/workflow_host.js +69 -9
- package/native/reseed.c +52 -0
- package/package.json +12 -5
- package/prebuilds/linux-x64/@boardwalk-labs+runner.node +0 -0
package/binding.gyp
ADDED
|
@@ -74,7 +74,7 @@ export declare class BudgetMeter {
|
|
|
74
74
|
constructor(opts: BudgetMeterOptions);
|
|
75
75
|
/**
|
|
76
76
|
* Add a usage delta to the accumulator + recompute USD. `realCostUsd`, when provided, is the EXACT
|
|
77
|
-
* upstream cost the broker observed for this turn (
|
|
77
|
+
* upstream cost the broker observed for this turn (the managed provider's per-request cost) — used
|
|
78
78
|
* verbatim so the `max_usd` cap tracks ACTUAL spend (already cache-discounted, model-correct).
|
|
79
79
|
* Omitted for a BYO turn (no upstream price) or a managed turn whose cost the broker couldn't read,
|
|
80
80
|
* which fall back to the representative-rate {@link costFor} estimate. Token counts accumulate on
|
|
@@ -83,7 +83,7 @@ export declare class BudgetMeter {
|
|
|
83
83
|
addUsage(delta: UsageDelta, realCostUsd?: number): void;
|
|
84
84
|
/**
|
|
85
85
|
* THIS SESSION's accumulator snapshot (excludes prior-session usage). Read by per-session token
|
|
86
|
-
* metering — which reports token deltas to
|
|
86
|
+
* metering — which reports token deltas to the platform (deferred; not yet wired into the brokered
|
|
87
87
|
* loop) — and by audit/post-run cost rollups. Cap enforcement uses {@link cumulative} instead.
|
|
88
88
|
*/
|
|
89
89
|
snapshot(): BudgetSnapshot;
|
|
@@ -92,7 +92,7 @@ export declare class BudgetMeter {
|
|
|
92
92
|
* construction. Cap enforcement ({@link assertWithinCaps}) and checkpoint persistence use
|
|
93
93
|
* this so a run that resumes after a sleep/wait/crash is bounded by its declared caps across
|
|
94
94
|
* the whole run — not once per session. `snapshot()` stays session-local because
|
|
95
|
-
* per-session token metering reports token deltas; seeding it would double-report
|
|
95
|
+
* per-session token metering reports token deltas; seeding it would double-report usage.
|
|
96
96
|
*/
|
|
97
97
|
cumulative(): BudgetSnapshot;
|
|
98
98
|
/**
|
|
@@ -116,8 +116,8 @@ export declare class BudgetMeter {
|
|
|
116
116
|
/**
|
|
117
117
|
* USD cost for a single delta — the representative-rate ESTIMATE used only when the broker reports
|
|
118
118
|
* no real upstream cost for the turn (a BYO-provider turn, or a managed turn whose cost tap missed).
|
|
119
|
-
* Managed turns instead carry
|
|
120
|
-
* the loop can stamp the per-step
|
|
119
|
+
* Managed turns instead carry the managed provider's exact per-request cost through to {@link addUsage}. Public so
|
|
120
|
+
* the loop can stamp the per-step cost without re-deriving the rate table.
|
|
121
121
|
*/
|
|
122
122
|
costFor(delta: UsageDelta): number;
|
|
123
123
|
}
|
|
@@ -5,12 +5,12 @@
|
|
|
5
5
|
// `AppError(BUDGET_EXCEEDED)` and the loop tears down with a `turn_ended
|
|
6
6
|
// reason='error'` event.
|
|
7
7
|
//
|
|
8
|
-
// USD here tracks the bill closely: for a MANAGED turn the meter is fed
|
|
9
|
-
//
|
|
8
|
+
// USD here tracks the bill closely: for a MANAGED turn the meter is fed the managed provider's exact
|
|
9
|
+
// per-request cost (already cache-discounted + model-correct), so the `max_usd` cap reflects real spend —
|
|
10
10
|
// not a hand-rolled token estimate. Only a BYO turn (no upstream price) or a managed turn whose cost
|
|
11
11
|
// the broker couldn't read falls back to a single flat representative rate
|
|
12
12
|
// (`model_rates.ts::BUDGET_GUARDRAIL_RATE`), since a per-`agent()`-model run has no one model. Actual
|
|
13
|
-
// billing
|
|
13
|
+
// billing is metered by the platform, not by the runner. The meter takes
|
|
14
14
|
// the fallback rate as a constructor arg so tests inject a fixed rate.
|
|
15
15
|
import { AppError, ErrorCode } from "../support/index.js";
|
|
16
16
|
const ZERO_PRIOR = {
|
|
@@ -44,7 +44,7 @@ export class BudgetMeter {
|
|
|
44
44
|
}
|
|
45
45
|
/**
|
|
46
46
|
* Add a usage delta to the accumulator + recompute USD. `realCostUsd`, when provided, is the EXACT
|
|
47
|
-
* upstream cost the broker observed for this turn (
|
|
47
|
+
* upstream cost the broker observed for this turn (the managed provider's per-request cost) — used
|
|
48
48
|
* verbatim so the `max_usd` cap tracks ACTUAL spend (already cache-discounted, model-correct).
|
|
49
49
|
* Omitted for a BYO turn (no upstream price) or a managed turn whose cost the broker couldn't read,
|
|
50
50
|
* which fall back to the representative-rate {@link costFor} estimate. Token counts accumulate on
|
|
@@ -59,7 +59,7 @@ export class BudgetMeter {
|
|
|
59
59
|
}
|
|
60
60
|
/**
|
|
61
61
|
* THIS SESSION's accumulator snapshot (excludes prior-session usage). Read by per-session token
|
|
62
|
-
* metering — which reports token deltas to
|
|
62
|
+
* metering — which reports token deltas to the platform (deferred; not yet wired into the brokered
|
|
63
63
|
* loop) — and by audit/post-run cost rollups. Cap enforcement uses {@link cumulative} instead.
|
|
64
64
|
*/
|
|
65
65
|
snapshot() {
|
|
@@ -78,7 +78,7 @@ export class BudgetMeter {
|
|
|
78
78
|
* construction. Cap enforcement ({@link assertWithinCaps}) and checkpoint persistence use
|
|
79
79
|
* this so a run that resumes after a sleep/wait/crash is bounded by its declared caps across
|
|
80
80
|
* the whole run — not once per session. `snapshot()` stays session-local because
|
|
81
|
-
* per-session token metering reports token deltas; seeding it would double-report
|
|
81
|
+
* per-session token metering reports token deltas; seeding it would double-report usage.
|
|
82
82
|
*/
|
|
83
83
|
cumulative() {
|
|
84
84
|
const s = this.snapshot();
|
|
@@ -159,8 +159,8 @@ export class BudgetMeter {
|
|
|
159
159
|
/**
|
|
160
160
|
* USD cost for a single delta — the representative-rate ESTIMATE used only when the broker reports
|
|
161
161
|
* no real upstream cost for the turn (a BYO-provider turn, or a managed turn whose cost tap missed).
|
|
162
|
-
* Managed turns instead carry
|
|
163
|
-
* the loop can stamp the per-step
|
|
162
|
+
* Managed turns instead carry the managed provider's exact per-request cost through to {@link addUsage}. Public so
|
|
163
|
+
* the loop can stamp the per-step cost without re-deriving the rate table.
|
|
164
164
|
*/
|
|
165
165
|
costFor(delta) {
|
|
166
166
|
const inputCost = ((delta.inputTokens ?? 0) * this.rate.inputPerMillion) / 1_000_000;
|
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
// The flat budget-guardrail rate — the FALLBACK basis for the in-run `max_usd` cap (BudgetMeter).
|
|
2
2
|
//
|
|
3
|
-
// A MANAGED turn now caps on
|
|
3
|
+
// A MANAGED turn now caps on the managed provider's EXACT per-request cost, forwarded from the broker to
|
|
4
4
|
// the worker (inference_proxy result frame → BudgetMeter.addUsage `realCostUsd`), so the cap tracks
|
|
5
5
|
// real spend — already cache-discounted + model-correct. This flat rate is used ONLY when a turn has
|
|
6
6
|
// no upstream cost: a BYO-provider turn (the org pays its own key), or a managed turn whose cost the
|
|
7
|
-
// broker couldn't read. It bounds runaway loops; it is NOT the bill. Actual billing is
|
|
8
|
-
//
|
|
9
|
-
// run_usage_service). the platform spec
|
|
7
|
+
// broker couldn't read. It bounds runaway loops; it is NOT the bill. Actual billing is metered by the
|
|
8
|
+
// platform, not by the runner.
|
|
10
9
|
//
|
|
11
10
|
// Deliberately NOT a per-model table with a silent fallback: a lookup that returns a Sonnet-class
|
|
12
11
|
// default for any unknown id only created the ILLUSION of precision. The real per-request cost (above)
|
|
13
12
|
// is the precise path; this rate is just the model-agnostic backstop. Un-metered managed models are
|
|
14
|
-
// rejected fail-closed at resolution
|
|
13
|
+
// rejected fail-closed at resolution by the broker, so the cap never has to price
|
|
15
14
|
// a model we don't support.
|
|
16
15
|
/**
|
|
17
16
|
* The flat representative rate (USD / million tokens) the BudgetMeter applies to the `max_usd` cap
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// SecretRedactor — scrubs known secret values from everything bound for an LLM
|
|
1
|
+
// SecretRedactor — scrubs known secret values from everything bound for an LLM.
|
|
2
2
|
//
|
|
3
3
|
// The architectural guarantee is structural: secrets live ONLY in the workflow PROGRAM (the trusted
|
|
4
4
|
// deterministic tool layer), which holds them via `secrets.get` / `ctx.secrets.resolve`. The
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// CancelWatcher — stops a run when the user cancels it mid-flight
|
|
1
|
+
// CancelWatcher — stops a run when the user cancels it mid-flight.
|
|
2
2
|
//
|
|
3
3
|
// The user-cancel counterpart to CreditWatcher. One per run session. On a timer it asks — through the
|
|
4
4
|
// broker (`GET /cancel`, since the runner holds no DB/Redis credential) — whether the run has been
|
|
@@ -4,7 +4,7 @@ export interface CreditWatcherDeps {
|
|
|
4
4
|
/** The run being watched (for correlation/logging). */
|
|
5
5
|
runId: string;
|
|
6
6
|
/** Resolves true while the org can keep spending; false once it's out of credit. Brokered
|
|
7
|
-
* (`RunnerControlClient.checkCredit` → `GET /credit`), so the runner never reads
|
|
7
|
+
* (`RunnerControlClient.checkCredit` → `GET /credit`), so the runner never reads billing state directly. */
|
|
8
8
|
isFunded: () => Promise<boolean>;
|
|
9
9
|
/** Fired exactly once when the org is first seen to be out of credit. */
|
|
10
10
|
onExhausted: () => void;
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
// CreditWatcher — stops a run when its org runs out of prepaid credit mid-flight
|
|
1
|
+
// CreditWatcher — stops a run when its org runs out of prepaid credit mid-flight.
|
|
2
2
|
//
|
|
3
3
|
// One per run session. On a timer it asks — through the broker (`GET /credit`, since the runner holds
|
|
4
|
-
// no
|
|
4
|
+
// no billing credential) — whether the org is still funded; the FIRST time it isn't, it fires
|
|
5
5
|
// `onExhausted` once (the orchestrator wires this to `AbortController.abort`, which the WorkflowHost
|
|
6
6
|
// honors cooperatively) and stops checking. This is the org-level counterpart to the per-run
|
|
7
|
-
// BudgetMeter cap: the run can't see
|
|
7
|
+
// BudgetMeter cap: the run can't see its billing balance, so funding is enforced here, out of band,
|
|
8
8
|
// against the live balance that incremental token metering keeps fresh.
|
|
9
9
|
//
|
|
10
|
-
// "Prompt, not instant":
|
|
11
|
-
// abort at the next hook boundary, so a run stops within ~one check interval (plus
|
|
10
|
+
// "Prompt, not instant": the platform's usage metering is eventually-consistent and the host honors the
|
|
11
|
+
// abort at the next hook boundary, so a run stops within ~one check interval (plus metering lag) of
|
|
12
12
|
// going unfunded — bounding overshoot, not eliminating it. Applies to MANAGED and BYOK runs alike,
|
|
13
13
|
// since runtime always burns credit (the gate requires the runtime floor regardless of token billing).
|
|
14
14
|
import { createLogger } from "./support/index.js";
|
|
@@ -51,6 +51,11 @@ export type FreezeOutcome = {
|
|
|
51
51
|
} | {
|
|
52
52
|
kind: "aborted";
|
|
53
53
|
reason: string;
|
|
54
|
+
}
|
|
55
|
+
/** The caller withdrew the wait via its abort signal BEFORE it froze (register-without-release:
|
|
56
|
+
* a held gate got its answer during the hold, so it resolves in-process instead of freezing). */
|
|
57
|
+
| {
|
|
58
|
+
kind: "withdrawn";
|
|
54
59
|
};
|
|
55
60
|
export interface FreezeCoordinatorHooks {
|
|
56
61
|
/** Runs at quiescence, immediately before the freeze is requested: flush billable runtime
|
|
@@ -104,7 +109,7 @@ export declare class FreezeCoordinator {
|
|
|
104
109
|
* the abort (the seam then holds in-process); other reasons retry the freeze after a
|
|
105
110
|
* backoff. Concurrent suspending waits serialize through an internal chain.
|
|
106
111
|
*/
|
|
107
|
-
suspendingWait(signal: SuspendSignal): Promise<FreezeOutcome>;
|
|
112
|
+
suspendingWait(signal: SuspendSignal, abort?: AbortSignal): Promise<FreezeOutcome>;
|
|
108
113
|
/** Relay handler: a wake injection landed. Validates, runs the after-wake hook (token
|
|
109
114
|
* swap + meter rebase), confirms to init, and resolves the parked seam. A wake with no
|
|
110
115
|
* parked seam is a duplicate delivery — re-confirm (idempotent), never crash. */
|
|
@@ -126,7 +126,7 @@ export class FreezeCoordinator {
|
|
|
126
126
|
* the abort (the seam then holds in-process); other reasons retry the freeze after a
|
|
127
127
|
* backoff. Concurrent suspending waits serialize through an internal chain.
|
|
128
128
|
*/
|
|
129
|
-
suspendingWait(signal) {
|
|
129
|
+
suspendingWait(signal, abort) {
|
|
130
130
|
const turn = this.chain;
|
|
131
131
|
let release = () => undefined;
|
|
132
132
|
this.chain = new Promise((resolve) => {
|
|
@@ -137,7 +137,14 @@ export class FreezeCoordinator {
|
|
|
137
137
|
try {
|
|
138
138
|
let backoff = ABORT_RETRY_INITIAL_MS;
|
|
139
139
|
for (;;) {
|
|
140
|
-
|
|
140
|
+
// Withdraw window: the abort can cancel the wait any time BEFORE it sends
|
|
141
|
+
// suspend_request (once sent, the VM freezes and the abort is moot — the process is
|
|
142
|
+
// paused). This is register-without-release: a held gate answered during the hold.
|
|
143
|
+
// awaitQuiescence short-circuits when already aborted, so one check after it covers
|
|
144
|
+
// both loop entry and a mid-hold abort.
|
|
145
|
+
await this.awaitQuiescence(abort);
|
|
146
|
+
if (abort?.aborted === true)
|
|
147
|
+
return { kind: "withdrawn" };
|
|
141
148
|
try {
|
|
142
149
|
await this.hooks.onBeforeFreeze?.();
|
|
143
150
|
}
|
|
@@ -167,6 +174,10 @@ export class FreezeCoordinator {
|
|
|
167
174
|
this.releaseGate();
|
|
168
175
|
if (outcome.kind === "wake")
|
|
169
176
|
return outcome;
|
|
177
|
+
// The park only ever resolves wake or aborted (never withdrawn — that returns before the
|
|
178
|
+
// park). Narrow explicitly so `.reason` is safe and a stray outcome propagates.
|
|
179
|
+
if (outcome.kind !== "aborted")
|
|
180
|
+
return outcome;
|
|
170
181
|
if (signal.reason === "sleep")
|
|
171
182
|
return outcome;
|
|
172
183
|
log.warn("suspend_aborted_retrying", {
|
|
@@ -228,11 +239,14 @@ export class FreezeCoordinator {
|
|
|
228
239
|
this.parked = null;
|
|
229
240
|
resolve({ kind: "aborted", reason });
|
|
230
241
|
}
|
|
231
|
-
awaitQuiescence() {
|
|
232
|
-
if (this.inFlight === 0)
|
|
242
|
+
awaitQuiescence(abort) {
|
|
243
|
+
if (this.inFlight === 0 || abort?.aborted === true)
|
|
233
244
|
return Promise.resolve();
|
|
234
245
|
return new Promise((resolve) => {
|
|
246
|
+
// Resolve on quiescence OR abort (a held wait wakes immediately to withdraw). Resolving
|
|
247
|
+
// twice is harmless; endWork may still call the queued waiter later.
|
|
235
248
|
this.quiescenceWaiters.push(resolve);
|
|
249
|
+
abort?.addEventListener("abort", () => resolve(), { once: true });
|
|
236
250
|
});
|
|
237
251
|
}
|
|
238
252
|
releaseGate() {
|
package/dist/runtime/index.d.ts
CHANGED
|
@@ -7,7 +7,7 @@ import { type ProgramWorkerDeps } from "./program_worker.js";
|
|
|
7
7
|
* runner holds NO platform credential — only the per-run control-plane handle (run token + broker
|
|
8
8
|
* URL); everything else is reached through the Runner Control API (the Runner Credential Broker model). */
|
|
9
9
|
export interface WorkerRuntime {
|
|
10
|
-
/** Stable worker identity stamped onto the lease (
|
|
10
|
+
/** Stable worker identity stamped onto the lease (task ARN / hostname / run-derived). */
|
|
11
11
|
workerId: string;
|
|
12
12
|
/** Sandbox workspace root for filesystem/shell/git tools. */
|
|
13
13
|
workspaceRoot: string;
|
|
@@ -33,13 +33,13 @@ export interface WorkerRuntime {
|
|
|
33
33
|
* relay's suspend/wake channel — instead of the exit-and-replay path. */
|
|
34
34
|
freezeRelay?: IdentityRelay;
|
|
35
35
|
}
|
|
36
|
-
/** Durable events are deferred (
|
|
36
|
+
/** Durable events are deferred (durable event storage) — live fan-out via the broker only. */
|
|
37
37
|
/** Synthesize the run's AuthContext. The run was already authorized at trigger time; the
|
|
38
38
|
* worker acts on the org's behalf, so it carries the org + an owner role. Tool-level
|
|
39
39
|
* boundaries (the broker's server-side manifest allowlist) are the real guard.
|
|
40
40
|
*
|
|
41
41
|
* source='workflow' (NOT 'session_jwt'): the program must never perform SESSION_JWT_ONLY
|
|
42
|
-
* credential mutations
|
|
42
|
+
* credential mutations, so a tool that ever exposed such a service is denied by
|
|
43
43
|
* construction regardless of the owner role. */
|
|
44
44
|
export declare function workerAuthContext(run: Run): AuthContext;
|
|
45
45
|
/**
|
|
@@ -57,10 +57,10 @@ export declare function workerAuthContext(run: Run): AuthContext;
|
|
|
57
57
|
*/
|
|
58
58
|
export declare function tokenMeterIdentifiers(runId: string, sessionId: string): (leafIndex: number) => string;
|
|
59
59
|
/** Pure wiring: build the full ProgramWorkerDeps from the control-plane handle. The runner reaches
|
|
60
|
-
* every privileged seam through the broker over its run token — no
|
|
60
|
+
* every privileged seam through the broker over its run token — no database, cache, billing, or model creds. */
|
|
61
61
|
export declare function assembleWorkerDeps(runtime: WorkerRuntime): ProgramWorkerDeps;
|
|
62
62
|
/**
|
|
63
|
-
* The per-run platform values the dispatcher injects as container env
|
|
63
|
+
* The per-run platform values the dispatcher injects as container env. They
|
|
64
64
|
* are captured into private worker state at bootstrap and DELETED from `process.env` before any user
|
|
65
65
|
* program / agent leaf / subprocess can run — the run token + API token are credentials, and nothing
|
|
66
66
|
* untrusted run code touches should inherit them (the run env/credential rules). The user owns the rest
|
|
@@ -78,7 +78,7 @@ export interface PlatformContext {
|
|
|
78
78
|
vcpus: number;
|
|
79
79
|
}
|
|
80
80
|
/** The public-API origin a run uses for raw API / MCP / CLI calls. The broker (Runner Control API)
|
|
81
|
-
*
|
|
81
|
+
* shares an origin with the public API, so the program appends `/v1` or
|
|
82
82
|
* `/mcp/v1`. Falls back to the broker URL unchanged if it can't be parsed. */
|
|
83
83
|
export declare function publicApiOrigin(controlPlaneBaseUrl: string): string;
|
|
84
84
|
/**
|
package/dist/runtime/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// Worker composition root (the workflow runtime design). The
|
|
2
|
-
// worker
|
|
1
|
+
// Worker composition root (the workflow runtime design). The hosted worker container runs the
|
|
2
|
+
// compiled worker entrypoint; the dispatcher launches it per run with
|
|
3
3
|
// RUN_ID + the per-run control-plane handle. This file assembles real implementations behind every
|
|
4
4
|
// seam `runProgramWorker(runId, deps)` injects, then runs the one workflow program to terminal.
|
|
5
5
|
//
|
|
@@ -11,21 +11,21 @@
|
|
|
11
11
|
// - secrets.get() → broker /secrets/resolve (allowlist enforced server-side).
|
|
12
12
|
// - workflows.call()→ broker /children (durable child run, hold + poll).
|
|
13
13
|
// - events.emit() → broker /events (fan-out server-side).
|
|
14
|
-
// - artifacts / web_search → broker /artifacts + /tools/web_search (no
|
|
14
|
+
// - artifacts / web_search → broker /artifacts + /tools/web_search (no storage or search-provider creds here).
|
|
15
15
|
// - lifecycle / usage / telemetry → broker /claim,/version,/finalize,/usage,/telemetry.
|
|
16
|
-
// There is no
|
|
17
|
-
// the metadata-endpoint escape has nothing to steal. The legacy direct (pre-broker) path is removed.
|
|
16
|
+
// There is no database, cache, billing, or model-provider client on the runner — so its task role is
|
|
17
|
+
// near-zero and the metadata-endpoint escape has nothing to steal. The legacy direct (pre-broker) path is removed.
|
|
18
18
|
//
|
|
19
19
|
// Split: `assembleWorkerDeps(runtime)` is pure wiring (unit-tested with a fake fetch); `main()` is the
|
|
20
20
|
// bootstrap shell (read the control-plane env, install signal handlers).
|
|
21
21
|
//
|
|
22
|
-
// Per-session loops wired here (all brokered): a UsageFlusher meters token deltas (
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
22
|
+
// Per-session loops wired here (all brokered): a UsageFlusher meters token deltas (→ /usage/tokens), a
|
|
23
|
+
// CreditWatcher polls funding (→ /credit, aborting the run on exhaustion), and a CancelWatcher polls
|
|
24
|
+
// for a user cancel (→ /cancel, aborting the run when the user cancels — the brokered worker holds no
|
|
25
|
+
// cache client, so it can't receive the platform's cancel publish directly).
|
|
26
26
|
//
|
|
27
|
-
// Documented v0 deferral (a clear seam): durable events — a no-op AgentEventStore (live
|
|
28
|
-
// via the broker only) until
|
|
27
|
+
// Documented v0 deferral (a clear seam): durable events — a no-op AgentEventStore (live fan-out
|
|
28
|
+
// via the broker only) until durable event storage lands.
|
|
29
29
|
import { createLogger, newId, AppError, ErrorCode, DEFAULT_LEASE_MS } from "./support/index.js";
|
|
30
30
|
import { LspService } from "@boardwalk-labs/engine/core";
|
|
31
31
|
import { BudgetMeter } from "./agent/budget.js";
|
|
@@ -37,6 +37,7 @@ import { EngineLeafExecutor } from "./leaf_executor.js";
|
|
|
37
37
|
import { parseByoProviders } from "./direct_inference.js";
|
|
38
38
|
import { applyIdentityToEnv, connectIdentityRelayFd, relayFdFromEnv, workerDiagnostics, } from "./identity_relay.js";
|
|
39
39
|
import { FreezeCoordinator } from "./freeze_coordinator.js";
|
|
40
|
+
import { reseedUserspaceCsprng } from "./uniqueness_reseed.js";
|
|
40
41
|
import { WorkerWorkflowHost } from "./workflow_host.js";
|
|
41
42
|
import { runProgramWorker, } from "./program_worker.js";
|
|
42
43
|
import { RunnerControlClient } from "./runner_control_client.js";
|
|
@@ -54,13 +55,13 @@ import { PhaseTracker } from "./phase_tracker.js";
|
|
|
54
55
|
import { mkdir } from "node:fs/promises";
|
|
55
56
|
import { join } from "node:path";
|
|
56
57
|
const log = createLogger("worker_entrypoint");
|
|
57
|
-
/** Durable events are deferred (
|
|
58
|
+
/** Durable events are deferred (durable event storage) — live fan-out via the broker only. */
|
|
58
59
|
/** Synthesize the run's AuthContext. The run was already authorized at trigger time; the
|
|
59
60
|
* worker acts on the org's behalf, so it carries the org + an owner role. Tool-level
|
|
60
61
|
* boundaries (the broker's server-side manifest allowlist) are the real guard.
|
|
61
62
|
*
|
|
62
63
|
* source='workflow' (NOT 'session_jwt'): the program must never perform SESSION_JWT_ONLY
|
|
63
|
-
* credential mutations
|
|
64
|
+
* credential mutations, so a tool that ever exposed such a service is denied by
|
|
64
65
|
* construction regardless of the owner role. */
|
|
65
66
|
export function workerAuthContext(run) {
|
|
66
67
|
const userId = run.actor.type === "user" ? run.actor.user_id : `workflow:${run.workflowId}`;
|
|
@@ -84,7 +85,7 @@ export function tokenMeterIdentifiers(runId, sessionId) {
|
|
|
84
85
|
return (leafIndex) => `${runId}:${sessionId}:${String(leafIndex)}:${String(seq++)}`;
|
|
85
86
|
}
|
|
86
87
|
/** Pure wiring: build the full ProgramWorkerDeps from the control-plane handle. The runner reaches
|
|
87
|
-
* every privileged seam through the broker over its run token — no
|
|
88
|
+
* every privileged seam through the broker over its run token — no database, cache, billing, or model creds. */
|
|
88
89
|
export function assembleWorkerDeps(runtime) {
|
|
89
90
|
const broker = new RunnerControlClient({
|
|
90
91
|
baseUrl: runtime.controlPlane.baseUrl,
|
|
@@ -114,8 +115,9 @@ export function assembleWorkerDeps(runtime) {
|
|
|
114
115
|
freeze = new FreezeCoordinator({ channel });
|
|
115
116
|
log.info("freeze_mode_enabled", { runId: runtime.runId });
|
|
116
117
|
}
|
|
117
|
-
// Live agent-event stream → /telemetry (broker
|
|
118
|
-
// web_search, artifacts, and the model call are all brokered — no
|
|
118
|
+
// Live agent-event stream → /telemetry (broker fans out server-side). Inference,
|
|
119
|
+
// web_search, artifacts, and the model call are all brokered — no cache client, model creds,
|
|
120
|
+
// search-provider creds, or object-storage creds here.
|
|
119
121
|
const eventPublisher = new BrokerEventPublisher({
|
|
120
122
|
send: (frames) => broker.publishTelemetry(frames),
|
|
121
123
|
});
|
|
@@ -129,11 +131,11 @@ export function assembleWorkerDeps(runtime) {
|
|
|
129
131
|
// bootstrap captured + scrubbed it (capturePlatformContext) so the agent's bash / subprocesses
|
|
130
132
|
// can't read it. The program reaches it ONLY via `runtime.apiToken()` (built below); we still
|
|
131
133
|
// record it in each run's redactor so a value the program threads into a tool can't echo back out
|
|
132
|
-
// of a tool result. Absent in the dev no-signing-key path.
|
|
134
|
+
// of a tool result. Absent in the dev no-signing-key path.
|
|
133
135
|
// MUTABLE: on the snapshot substrate a wake carries a fresh token (the frozen one expired).
|
|
134
136
|
let runApiKey = runtime.controlPlane.apiToken;
|
|
135
|
-
// The broker (Runner Control API)
|
|
136
|
-
//
|
|
137
|
+
// The broker (Runner Control API) shares an origin with the public API;
|
|
138
|
+
// `runtime.apiUrl` exposes it and the program appends `/v1` or `/mcp/v1`.
|
|
137
139
|
const apiUrl = publicApiOrigin(runtime.controlPlane.baseUrl);
|
|
138
140
|
// Per-run host: agent() leaf + sleep-hold + secrets + children + events, all brokered. `signal`
|
|
139
141
|
// carries cooperative cancellation (credit exhaustion) into every host hook.
|
|
@@ -143,7 +145,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
143
145
|
// Fallback budget-cap rate. A managed turn caps on the broker's EXACT upstream cost (forwarded on
|
|
144
146
|
// the inference result frame → BudgetMeter `realCostUsd`); this representative sonnet-class rate
|
|
145
147
|
// applies only to a turn with no upstream cost (BYO / unavailable). `max_usd` is a guardrail, not
|
|
146
|
-
// the bill —
|
|
148
|
+
// the bill — the platform meters the actual per-leaf usage.
|
|
147
149
|
rate: BUDGET_GUARDRAIL_RATE,
|
|
148
150
|
startedAt: Date.now(),
|
|
149
151
|
// deadline_seconds is WALL-CLOCK from the run's ORIGINAL start (incl. suspended idle), so a run
|
|
@@ -160,7 +162,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
160
162
|
const nextMeterIdentifier = tokenMeterIdentifiers(run.id, meteringSessionId);
|
|
161
163
|
// Per-run secret boundary: every value resolved (program `secrets.get` OR a tool's
|
|
162
164
|
// ctx.secrets.resolve) is recorded into one shared redactor; the leaf seeds a fresh engine
|
|
163
|
-
// Redactor from it so the loop scrubs those values out of all model-bound content.
|
|
165
|
+
// Redactor from it so the loop scrubs those values out of all model-bound content.
|
|
164
166
|
const redactor = new SecretRedactor();
|
|
165
167
|
// Record the run's own API key so a prompt-injected agent can't echo it back to the model.
|
|
166
168
|
if (runApiKey !== undefined && runApiKey !== "")
|
|
@@ -210,9 +212,9 @@ export function assembleWorkerDeps(runtime) {
|
|
|
210
212
|
// until the artifact extracts. The empty `/workspace` and this dir are SEPARATE dirs on hosted,
|
|
211
213
|
// exactly the two-tier case the engine's AGENTS.md loader is built for.
|
|
212
214
|
programDir: () => programDir,
|
|
213
|
-
// Per-leaf, per-model token metering — reported THROUGH the broker (the worker holds no
|
|
214
|
-
// credential); the broker decides `billed_by_boardwalk` per model + meters to
|
|
215
|
-
// no-op there). Fire-and-forget: a metering hiccup must never fail the run.
|
|
215
|
+
// Per-leaf, per-model token metering — reported THROUGH the broker (the worker holds no billing
|
|
216
|
+
// credential); the broker decides `billed_by_boardwalk` per model + meters usage to the platform
|
|
217
|
+
// (BYO models no-op there). Fire-and-forget: a metering hiccup must never fail the run.
|
|
216
218
|
meterUsage: ({ model, inputTokens, outputTokens, cachedReadTokens, cachedWriteTokens, leafIndex, }) => {
|
|
217
219
|
void broker
|
|
218
220
|
.meterTokens({
|
|
@@ -237,7 +239,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
237
239
|
// comes from the org's connection vault via the Runner Control API — never stored on the worker.
|
|
238
240
|
brokerMcpToken: (serverUrl, invalidateToken) => broker.mcpToken(serverUrl, invalidateToken),
|
|
239
241
|
});
|
|
240
|
-
// Per-workflow persistent /workspace
|
|
242
|
+
// Per-workflow persistent /workspace: only when the manifest opts in. The BROKER additionally
|
|
241
243
|
// gates on hosted (self-hosted runs get null URLs), and snapshots are keyed per-workflow + scoped
|
|
242
244
|
// by the run token — so even the untrusted in-process program can't reach another tenant's data.
|
|
243
245
|
const workspaceStore = manifest.workspace?.persist === true
|
|
@@ -311,6 +313,16 @@ export function assembleWorkerDeps(runtime) {
|
|
|
311
313
|
// Snapshot-substrate suspension: seams freeze in place under the quiescence gate instead of
|
|
312
314
|
// raising onSuspend (which stays wired as the transitional/local path's fallback).
|
|
313
315
|
...(freeze !== undefined ? { freeze } : {}),
|
|
316
|
+
// Register-without-release for held HITL gates — only on the
|
|
317
|
+
// freeze substrate, backed by the broker's inputs endpoints.
|
|
318
|
+
...(freeze !== undefined
|
|
319
|
+
? {
|
|
320
|
+
heldInput: {
|
|
321
|
+
register: (seq, gate) => broker.registerInput(seq, gate),
|
|
322
|
+
poll: (seq) => broker.pollInputAnswers(seq),
|
|
323
|
+
},
|
|
324
|
+
}
|
|
325
|
+
: {}),
|
|
314
326
|
phases: phaseTracker,
|
|
315
327
|
});
|
|
316
328
|
// Late-bind the coordinator's per-run hooks now that the run-scoped objects exist.
|
|
@@ -319,18 +331,21 @@ export function assembleWorkerDeps(runtime) {
|
|
|
319
331
|
let freezeWallMs = 0;
|
|
320
332
|
coordinator.setHooks({
|
|
321
333
|
// At quiescence, immediately before the freeze: book the runtime tail (suspended time must
|
|
322
|
-
// never appear billed
|
|
334
|
+
// never appear billed) and persist the workspace (crash-during-suspension
|
|
323
335
|
// recovery parity with the sleep-hold path). The wall stamp anchors the idle rebase below.
|
|
324
336
|
onBeforeFreeze: async () => {
|
|
325
337
|
freezeWallMs = Date.now();
|
|
326
338
|
await activeFlusher?.flushNow();
|
|
327
339
|
await workspaceStore?.persist();
|
|
328
340
|
},
|
|
329
|
-
// On wake:
|
|
330
|
-
//
|
|
331
|
-
// the
|
|
332
|
-
//
|
|
341
|
+
// On wake: reseed the userspace CSPRNG (clause 3 — a suspend snapshot restored more than
|
|
342
|
+
// once, e.g. a re-dispatch retry, would otherwise repeat its post-wake `crypto.*` draws),
|
|
343
|
+
// swap the fresh tokens onto the broker client + the program-facing apiToken() (recording
|
|
344
|
+
// the new values in the run's redactor, same discipline as boot), and exclude the frozen
|
|
345
|
+
// window from billed runtime using the wake's authoritative wall clock. The reseed runs
|
|
346
|
+
// BEFORE the seam resolves, so no woken author code draws from the stale (pre-suspend) DRBG.
|
|
333
347
|
onAfterWake: (wake) => {
|
|
348
|
+
reseedUserspaceCsprng();
|
|
334
349
|
broker.swapRunToken(wake.run_token);
|
|
335
350
|
redactor.record(wake.run_token);
|
|
336
351
|
if (wake.api_token !== undefined && wake.api_token !== "") {
|
|
@@ -412,7 +427,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
412
427
|
suspender: {
|
|
413
428
|
suspend: (signal, workerId) => broker.suspend(signal, workerId),
|
|
414
429
|
},
|
|
415
|
-
// Runtime metering
|
|
430
|
+
// Runtime metering: flush runtime as periodic deltas (+ a terminal tail) through the broker,
|
|
416
431
|
// idempotent per flush, so a long/perpetual run bills as it burns and the credit watcher sees it —
|
|
417
432
|
// not a single charge at terminal (which a never-terminating run never reached). A fresh per-session
|
|
418
433
|
// id keeps a restarted run's sessions distinct in the idempotency key (distinct ids sum).
|
|
@@ -434,7 +449,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
434
449
|
return { stop: () => flusher.stop(), flushFinal: () => flusher.flushFinal() };
|
|
435
450
|
},
|
|
436
451
|
buildHost,
|
|
437
|
-
// Mid-run credit watching
|
|
452
|
+
// Mid-run credit watching: a CreditWatcher polls the broker's GET /credit on a timer; when
|
|
438
453
|
// the org runs out, onExhausted aborts the run (the orchestrator wires it to AbortController).
|
|
439
454
|
startCreditWatch: ({ run, onExhausted }) => {
|
|
440
455
|
const watcher = new CreditWatcher({
|
|
@@ -445,10 +460,10 @@ export function assembleWorkerDeps(runtime) {
|
|
|
445
460
|
watcher.start();
|
|
446
461
|
return { stop: () => watcher.stop() };
|
|
447
462
|
},
|
|
448
|
-
// Mid-run user-cancel watching
|
|
463
|
+
// Mid-run user-cancel watching: a CancelWatcher polls the broker's GET /cancel on a timer;
|
|
449
464
|
// when the user cancels (the run is flipped to `cancelling`), onCancelled aborts the run. This is
|
|
450
|
-
// how the brokered (
|
|
451
|
-
// reach it.
|
|
465
|
+
// how the brokered (cache-less) worker learns of a cancel — the platform's cancel publish can't
|
|
466
|
+
// reach it directly.
|
|
452
467
|
startCancelWatch: ({ run, onCancelled }) => {
|
|
453
468
|
const watcher = new CancelWatcher({
|
|
454
469
|
runId: run.id,
|
|
@@ -458,7 +473,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
458
473
|
watcher.start();
|
|
459
474
|
return { stop: () => watcher.stop() };
|
|
460
475
|
},
|
|
461
|
-
// Lease renewal
|
|
476
|
+
// Lease renewal: a LeaseRenewer extends the lease through the broker's POST /renew on a timer
|
|
462
477
|
// (well under the lease), so a run longer than the 5-min lease isn't reclaimed mid-flight by the
|
|
463
478
|
// recovery sweep. `renew` re-extends with the SAME workerId that claimed it; a null result (the
|
|
464
479
|
// run is no longer ours) fires onLost → the run aborts `lease_lost` without finalizing.
|
|
@@ -481,7 +496,7 @@ export function assembleWorkerDeps(runtime) {
|
|
|
481
496
|
}
|
|
482
497
|
// ---- Bootstrap (real container only) -------------------------------------------------
|
|
483
498
|
/**
|
|
484
|
-
* The per-run platform values the dispatcher injects as container env
|
|
499
|
+
* The per-run platform values the dispatcher injects as container env. They
|
|
485
500
|
* are captured into private worker state at bootstrap and DELETED from `process.env` before any user
|
|
486
501
|
* program / agent leaf / subprocess can run — the run token + API token are credentials, and nothing
|
|
487
502
|
* untrusted run code touches should inherit them (the run env/credential rules). The user owns the rest
|
|
@@ -495,7 +510,7 @@ export const PLATFORM_ENV_KEYS = [
|
|
|
495
510
|
"BOARDWALK_TASK_CPU_UNITS",
|
|
496
511
|
];
|
|
497
512
|
/** The public-API origin a run uses for raw API / MCP / CLI calls. The broker (Runner Control API)
|
|
498
|
-
*
|
|
513
|
+
* shares an origin with the public API, so the program appends `/v1` or
|
|
499
514
|
* `/mcp/v1`. Falls back to the broker URL unchanged if it can't be parsed. */
|
|
500
515
|
export function publicApiOrigin(controlPlaneBaseUrl) {
|
|
501
516
|
try {
|
|
@@ -519,7 +534,7 @@ export function capturePlatformContext(env) {
|
|
|
519
534
|
return value.trim();
|
|
520
535
|
};
|
|
521
536
|
// The dispatcher always injects the control-plane handle (the Runner Credential Broker model). The runner is
|
|
522
|
-
// brokered-only — there is no
|
|
537
|
+
// brokered-only — there is no database, cache, or billing fallback — so a missing handle is a hard config error.
|
|
523
538
|
const runId = read("RUN_ID");
|
|
524
539
|
const apiToken = env.BOARDWALK_API_KEY?.trim();
|
|
525
540
|
const controlPlane = {
|
|
@@ -552,6 +567,12 @@ export async function main() {
|
|
|
552
567
|
relay.announceReady(workerDiagnostics());
|
|
553
568
|
const identity = await relay.awaitIdentity();
|
|
554
569
|
applyIdentityToEnv(identity, process.env);
|
|
570
|
+
// Clause 3 (SNAPSHOT_UNIQUENESS_CONTRACT): this run was restored from the SHARED base
|
|
571
|
+
// snapshot, so its OpenSSL DRBG is identical to every other run's. Reseed it from the
|
|
572
|
+
// (VMGenID-diverged) OS entropy BEFORE `acceptIdentity` releases the worker into the brokered
|
|
573
|
+
// lifecycle — so no `crypto.*` draw by the SDK, the agent, or author code can collide across
|
|
574
|
+
// clones. Every wake reseeds again (the after-wake hook).
|
|
575
|
+
reseedUserspaceCsprng();
|
|
555
576
|
relay.acceptIdentity();
|
|
556
577
|
// The relay now becomes the suspend/wake channel: assembleWorkerDeps opens it into the
|
|
557
578
|
// FreezeCoordinator, and the host's suspending seams freeze in place instead of exiting.
|
|
@@ -573,11 +594,11 @@ export async function main() {
|
|
|
573
594
|
...(byoProviders.length > 0 ? { byoProviders } : {}),
|
|
574
595
|
...(freezeRelay !== undefined ? { freezeRelay } : {}),
|
|
575
596
|
});
|
|
576
|
-
// The only thing to drain is the batched telemetry buffer — the runner opens no
|
|
597
|
+
// The only thing to drain is the batched telemetry buffer — the runner opens no database, cache, or queue.
|
|
577
598
|
const cleanup = async () => {
|
|
578
599
|
await deps.flushTelemetry?.().catch(() => undefined);
|
|
579
600
|
};
|
|
580
|
-
// SIGTERM:
|
|
601
|
+
// SIGTERM: the orchestrator is stopping the task. Hold-and-pay has no mid-run checkpoint; we exit and let the
|
|
581
602
|
// lease expire → the scheduler-sweep reclaims it and a fresh worker RESTARTS the run from the
|
|
582
603
|
// top (Lambda/GHA semantics; durable children re-attach via idempotency). Crash-safe by design.
|
|
583
604
|
let shuttingDown = false;
|
|
@@ -32,7 +32,7 @@ export interface EngineLeafExecutorDeps {
|
|
|
32
32
|
budget: BudgetMeter;
|
|
33
33
|
/** RUN-level redactor (shared with the secret resolver): every resolved secret value is recorded
|
|
34
34
|
* here. We seed a fresh engine `Redactor` from it per leaf so the loop scrubs known values out of
|
|
35
|
-
* the prompt, tool args/results, and transcript before they reach the model
|
|
35
|
+
* the prompt, tool args/results, and transcript before they reach the model. */
|
|
36
36
|
redactor: SecretRedactor;
|
|
37
37
|
/** Builds this leaf's event sink; `leafIndex` (1-based) is only a metering identifier — the sink
|
|
38
38
|
* owns the run-global cursor. `identity` names the leaf on its turn frames. */
|
|
@@ -51,7 +51,7 @@ export interface EngineLeafExecutorDeps {
|
|
|
51
51
|
* omitted ⇒ no bundled tier (only the workspace AGENTS.md applies). */
|
|
52
52
|
programDir?: () => string | null;
|
|
53
53
|
/** Per-leaf token metering seam (fire-and-forget). Reports THIS leaf's tokens + its model to the
|
|
54
|
-
* broker, which decides `billed_by_boardwalk` per model + meters to
|
|
54
|
+
* broker, which decides `billed_by_boardwalk` per model + meters usage to the platform. Omitted in tests. */
|
|
55
55
|
meterUsage?: (input: MeterUsageInput) => void;
|
|
56
56
|
/** Backend for the engine's host-backed built-in tools (`webfetch` / `web_search` / `artifacts`):
|
|
57
57
|
* set as the leaf's `capabilities.host` so the engine registers them. Broker-backed on hosted runs
|
|
@@ -53,7 +53,7 @@ export interface ProgramRunnerDeps {
|
|
|
53
53
|
* Scrubs known secret values out of a string (the run's `SecretRedactor.redactText`). Applied to a
|
|
54
54
|
* top-level throw's message before it is logged AND before it is returned to the worker — a program
|
|
55
55
|
* that resolves a secret and then throws it in an error message must NOT land that secret raw in the
|
|
56
|
-
* logs or the finalized run output
|
|
56
|
+
* logs or the finalized run output. Defaults to identity (tests/local).
|
|
57
57
|
*/
|
|
58
58
|
redactText?: (text: string) => string;
|
|
59
59
|
/**
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// WorkflowProgramRunner — executes a workflow program (the JS-body model, the workflow runtime design).
|
|
2
2
|
//
|
|
3
|
-
// A run is the execution of a built program ARTIFACT
|
|
3
|
+
// A run is the execution of a built program ARTIFACT: the worker is handed the VERIFIED tarball
|
|
4
4
|
// (its sha256 already checked against the pinned digest by the orchestrator) plus the entry module
|
|
5
5
|
// name. This module is the mechanism: it installs the host adapter + trigger payload onto the
|
|
6
6
|
// `@boardwalk-labs/workflow` singleton, extracts the artifact into a unique temp dir under a
|