@boardwalk-labs/runner 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +68 -0
- package/dist/bin.d.ts +2 -0
- package/dist/bin.js +214 -0
- package/dist/contract.d.ts +154 -0
- package/dist/contract.js +196 -0
- package/dist/daemon/daemon.d.ts +45 -0
- package/dist/daemon/daemon.js +129 -0
- package/dist/daemon/identity.d.ts +15 -0
- package/dist/daemon/identity.js +44 -0
- package/dist/daemon/index.d.ts +3 -0
- package/dist/daemon/index.js +4 -0
- package/dist/daemon/pool_client.d.ts +31 -0
- package/dist/daemon/pool_client.js +101 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +8 -0
- package/dist/runtime/agent/budget.d.ts +123 -0
- package/dist/runtime/agent/budget.js +174 -0
- package/dist/runtime/agent/events.d.ts +46 -0
- package/dist/runtime/agent/events.js +8 -0
- package/dist/runtime/agent/model_rates.d.ts +7 -0
- package/dist/runtime/agent/model_rates.js +21 -0
- package/dist/runtime/agent/secret_redactor.d.ts +42 -0
- package/dist/runtime/agent/secret_redactor.js +98 -0
- package/dist/runtime/broker_artifact_store.d.ts +17 -0
- package/dist/runtime/broker_artifact_store.js +46 -0
- package/dist/runtime/broker_child_dispatcher.d.ts +31 -0
- package/dist/runtime/broker_child_dispatcher.js +88 -0
- package/dist/runtime/broker_event_publisher.d.ts +25 -0
- package/dist/runtime/broker_event_publisher.js +69 -0
- package/dist/runtime/broker_tool_host.d.ts +49 -0
- package/dist/runtime/broker_tool_host.js +240 -0
- package/dist/runtime/cancel_watcher.d.ts +31 -0
- package/dist/runtime/cancel_watcher.js +77 -0
- package/dist/runtime/credit_watcher.d.ts +30 -0
- package/dist/runtime/credit_watcher.js +78 -0
- package/dist/runtime/direct_inference.d.ts +29 -0
- package/dist/runtime/direct_inference.js +78 -0
- package/dist/runtime/index.d.ts +85 -0
- package/dist/runtime/index.js +525 -0
- package/dist/runtime/inference_transport.d.ts +5 -0
- package/dist/runtime/inference_transport.js +9 -0
- package/dist/runtime/leaf_executor.d.ts +104 -0
- package/dist/runtime/leaf_executor.js +333 -0
- package/dist/runtime/lease_renewer.d.ts +31 -0
- package/dist/runtime/lease_renewer.js +80 -0
- package/dist/runtime/main.d.ts +1 -0
- package/dist/runtime/main.js +7 -0
- package/dist/runtime/phase_tracker.d.ts +26 -0
- package/dist/runtime/phase_tracker.js +45 -0
- package/dist/runtime/program_log_capture.d.ts +20 -0
- package/dist/runtime/program_log_capture.js +79 -0
- package/dist/runtime/program_runner.d.ts +82 -0
- package/dist/runtime/program_runner.js +116 -0
- package/dist/runtime/program_worker.d.ts +177 -0
- package/dist/runtime/program_worker.js +272 -0
- package/dist/runtime/recording_secret_resolver.d.ts +9 -0
- package/dist/runtime/recording_secret_resolver.js +22 -0
- package/dist/runtime/run_abort.d.ts +16 -0
- package/dist/runtime/run_abort.js +38 -0
- package/dist/runtime/run_event_emitter.d.ts +24 -0
- package/dist/runtime/run_event_emitter.js +63 -0
- package/dist/runtime/runner_control_client.d.ts +184 -0
- package/dist/runtime/runner_control_client.js +501 -0
- package/dist/runtime/runtime_flusher.d.ts +41 -0
- package/dist/runtime/runtime_flusher.js +90 -0
- package/dist/runtime/sandbox_config.d.ts +21 -0
- package/dist/runtime/sandbox_config.js +34 -0
- package/dist/runtime/support/index.d.ts +53 -0
- package/dist/runtime/support/index.js +114 -0
- package/dist/runtime/suspension.d.ts +141 -0
- package/dist/runtime/suspension.js +120 -0
- package/dist/runtime/testing_artifact_build.d.ts +27 -0
- package/dist/runtime/testing_artifact_build.js +96 -0
- package/dist/runtime/tools/artifacts.d.ts +146 -0
- package/dist/runtime/tools/artifacts.js +125 -0
- package/dist/runtime/tools/types.d.ts +114 -0
- package/dist/runtime/tools/types.js +61 -0
- package/dist/runtime/tools/web_search.d.ts +66 -0
- package/dist/runtime/tools/web_search.js +165 -0
- package/dist/runtime/wire/artifact_storage.d.ts +37 -0
- package/dist/runtime/wire/artifact_storage.js +93 -0
- package/dist/runtime/wire/artifact_verify.d.ts +1 -0
- package/dist/runtime/wire/artifact_verify.js +8 -0
- package/dist/runtime/wire/human_input.d.ts +62 -0
- package/dist/runtime/wire/human_input.js +156 -0
- package/dist/runtime/wire/inference_proxy.d.ts +70 -0
- package/dist/runtime/wire/inference_proxy.js +157 -0
- package/dist/runtime/wire/manifest.d.ts +16 -0
- package/dist/runtime/wire/manifest.js +18 -0
- package/dist/runtime/wire/run.d.ts +54 -0
- package/dist/runtime/wire/run.js +2 -0
- package/dist/runtime/workflow_host.d.ts +194 -0
- package/dist/runtime/workflow_host.js +382 -0
- package/dist/runtime/workspace_store.d.ts +73 -0
- package/dist/runtime/workspace_store.js +118 -0
- package/package.json +70 -0
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
// EngineLeafExecutor — the `agent()` leaf, run by the OPEN-SOURCE engine's agent loop.
|
|
2
|
+
//
|
|
3
|
+
// In the JS-body model the run is the workflow PROGRAM; the agent loop is no longer "the run" — it
|
|
4
|
+
// is an ephemeral leaf the program invokes via `agent(prompt, opts)`. This is the LeafExecutor the
|
|
5
|
+
// worker's WorkflowHost delegates `agent()` to. It runs ONE leaf to completion via the engine's
|
|
6
|
+
// `runAgentLeaf` (`@boardwalk-labs/engine/core`) — the SAME loop `boardwalk dev` and the self-hosted
|
|
7
|
+
// server run — supplying a broker-backed `LeafIo`: the model call routes through the Runner Control
|
|
8
|
+
// API (the worker holds no model creds), events flow onto the run's v1 event stream, usage meters
|
|
9
|
+
// per-leaf + per-model through the broker, and secrets are redacted out of all model-bound content.
|
|
10
|
+
//
|
|
11
|
+
// What it deliberately does NOT do:
|
|
12
|
+
// - no checkpoint / resume (hold-and-pay; a crash restarts the run from the top).
|
|
13
|
+
// - no control-flow tools — `sleep` / `workflows.call` are PROGRAM hooks (they pause the run),
|
|
14
|
+
// not LLM tools; the engine's tool set is per-call inline ToolDefs / skills / memory only.
|
|
15
|
+
// - no run-level billing/credit/outcome — those live one level up, in the worker.
|
|
16
|
+
//
|
|
17
|
+
// MCP servers ARE supported on hosted runs, but constrained: only the `http` transport, and only to
|
|
18
|
+
// hosts in the egress allowlist (the worker can reach nothing else through the forward proxy). A leaf
|
|
19
|
+
// naming a `stdio` server or a non-allowlisted host fails up-front with a clear error. Static bearer
|
|
20
|
+
// auth (a token in the server's `headers`, typically from `secrets.get`) needs no further wiring — the
|
|
21
|
+
// engine tries those first. OAuth servers broker a short-lived token via `brokerMcpToken` (the
|
|
22
|
+
// control-plane vault); the token state never lives on the worker.
|
|
23
|
+
import { join } from "node:path";
|
|
24
|
+
import { runAgentLeaf, Redactor, EngineError, } from "@boardwalk-labs/engine/core";
|
|
25
|
+
import { AppError, ErrorCode } from "./support/index.js";
|
|
26
|
+
import { directProviderFor, streamDirectTurn, } from "./direct_inference.js";
|
|
27
|
+
import { throwIfAborted } from "./run_abort.js";
|
|
28
|
+
/**
|
|
29
|
+
* Per-run leaf executor. The worker constructs one bound to the run + run-level budget, and wires it
|
|
30
|
+
* as the `LeafExecutor` on the run's WorkflowHost, so every `agent()` the program calls runs here
|
|
31
|
+
* through the engine's loop.
|
|
32
|
+
*/
|
|
33
|
+
export class EngineLeafExecutor {
|
|
34
|
+
deps;
|
|
35
|
+
leafCount = 0;
|
|
36
|
+
constructor(deps) {
|
|
37
|
+
this.deps = deps;
|
|
38
|
+
}
|
|
39
|
+
async run(prompt, opts, signal, resume) {
|
|
40
|
+
// Cooperative cancellation: don't even start the leaf if the run is already aborted.
|
|
41
|
+
throwIfAborted(signal);
|
|
42
|
+
// Validate any named MCP servers up-front (before the engine tries to connect) so a leaf naming
|
|
43
|
+
// an unsupported transport or a non-allowlisted host fails clearly and deterministically at the
|
|
44
|
+
// leaf boundary, rather than deep in the engine loop. The broker re-checks the host authoritatively
|
|
45
|
+
// when it vends a token (the worker can't widen egress).
|
|
46
|
+
assertHostedMcpAllowed(opts?.mcp);
|
|
47
|
+
const leafIndex = ++this.leafCount;
|
|
48
|
+
const agentName = opts?.name;
|
|
49
|
+
const identity = {
|
|
50
|
+
agentId: `agent-${String(leafIndex)}`,
|
|
51
|
+
...(agentName !== undefined ? { agentName } : {}),
|
|
52
|
+
};
|
|
53
|
+
const sink = this.deps.makeEventSink(leafIndex, identity);
|
|
54
|
+
// Seed a fresh engine Redactor from the run's recorded secret values — the loop scrubs them out
|
|
55
|
+
// of every model-bound string (prompt, tool args/results) before the model sees them. Labels are
|
|
56
|
+
// immaterial (the placeholder never reveals which secret matched); index them for distinctness.
|
|
57
|
+
const redactor = new Redactor();
|
|
58
|
+
this.deps.redactor.values.forEach((value, i) => {
|
|
59
|
+
redactor.add(`secret-${String(i)}`, value);
|
|
60
|
+
});
|
|
61
|
+
const io = this.buildLeafIo({ identity, sink, redactor, leafIndex, signal });
|
|
62
|
+
try {
|
|
63
|
+
// `resume` (a tool-level human-input resume) re-enters a parked leaf from its checkpoint + the
|
|
64
|
+
// answers. A leaf that PARKS throws LeafParked, which propagates to the host (the executor never
|
|
65
|
+
// catches it — the host turns it into a suspend).
|
|
66
|
+
return await runAgentLeaf(prompt, opts, io, resume);
|
|
67
|
+
}
|
|
68
|
+
finally {
|
|
69
|
+
// The run's AbortSignal is authoritative: if it fired during the leaf (credit exhaustion / a
|
|
70
|
+
// user cancel), unwind the program even if the loop returned (e.g. an aborted stream that
|
|
71
|
+
// resolved). The streamModel seam below rejects in-flight calls when the signal is aborted.
|
|
72
|
+
throwIfAborted(signal);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/** Assemble the broker-backed `LeafIo` the engine loop drives for one leaf call. */
|
|
76
|
+
buildLeafIo(ctx) {
|
|
77
|
+
const { identity, sink, redactor, leafIndex, signal } = ctx;
|
|
78
|
+
const skillsDir = this.deps.skillsDir?.() ?? null;
|
|
79
|
+
const programDir = this.deps.programDir?.() ?? null;
|
|
80
|
+
// The broker stamps each turn's exact upstream cost on the result frame, but the engine hands
|
|
81
|
+
// `reportUsage` only `ChatTurn.usage` (input/output tokens, no cost). So `streamModel` stashes the
|
|
82
|
+
// result frame's `costMicros` here for the matching `reportUsage` to consume — one slot per leaf
|
|
83
|
+
// io (children get their own via `forChild`), and the loop is strictly streamModel→reportUsage per
|
|
84
|
+
// turn, so a single slot never races. Null ⇒ no upstream cost for the next turn (BYO / unavailable).
|
|
85
|
+
let pendingCostMicros = null;
|
|
86
|
+
return {
|
|
87
|
+
identity,
|
|
88
|
+
redactor,
|
|
89
|
+
// `host` lights up the host-backed built-ins (webfetch/web_search/artifacts); `lspService` lights
|
|
90
|
+
// up the engine-native `diagnostics` tool + diagnostics-after-edit. Each is omitted when its
|
|
91
|
+
// backend isn't wired (then that surface stays absent — the correct degradation). `workspaceDir`
|
|
92
|
+
// drives the WORKSPACE `AGENTS.md` tier (a codebase the run cloned, root + nested) and `programDir`
|
|
93
|
+
// the BUNDLED tier (the package-root AGENTS.md, the author's standing instructions) — both
|
|
94
|
+
// default-on, read by every agent(). `exactOptionalPropertyTypes` ⇒ conditional-spread, never an
|
|
95
|
+
// explicit `undefined`.
|
|
96
|
+
capabilities: {
|
|
97
|
+
workspaceDir: this.deps.workspaceRoot,
|
|
98
|
+
skillsDir,
|
|
99
|
+
...(programDir !== null ? { programDir } : {}),
|
|
100
|
+
...(this.deps.toolHost !== undefined ? { host: this.deps.toolHost } : {}),
|
|
101
|
+
...(this.deps.lspService !== undefined ? { lspService: this.deps.lspService } : {}),
|
|
102
|
+
},
|
|
103
|
+
// One model turn → the broker (the runner holds no model creds). The broker resolves the real
|
|
104
|
+
// model server-side, invokes the matching adapter, and streams text back; we surface each delta
|
|
105
|
+
// through providerIo.onDelta and return the terminal turn. An aborted run rejects in-flight.
|
|
106
|
+
streamModel: async (req, providerIo) => {
|
|
107
|
+
throwIfAborted(signal);
|
|
108
|
+
return this.streamModel(req, providerIo, signal, (costMicros) => {
|
|
109
|
+
pendingCostMicros = (pendingCostMicros ?? 0) + costMicros;
|
|
110
|
+
});
|
|
111
|
+
},
|
|
112
|
+
// turn_started rides a NEW stride block (the supervisor opens it); subsequent leaf events ride
|
|
113
|
+
// the same block. The shared emitter owns the run-global cursor.
|
|
114
|
+
startTurn: (turnId) => {
|
|
115
|
+
sink.beginTurn(turnId, { kind: "turn_started", ...identity });
|
|
116
|
+
},
|
|
117
|
+
// Engine LeafEventBody → the platform's v1 RunEventBody. The platform already adopted the SDK
|
|
118
|
+
// v1 wire format, and the engine emits the same kinds, so this is a near-identity mapping.
|
|
119
|
+
emit: (turnId, body) => {
|
|
120
|
+
sink.emit(toRunEventBody(body, identity), turnId);
|
|
121
|
+
},
|
|
122
|
+
// Usage flows to the budget authority after EVERY model call. Feed the run-level meter and, if
|
|
123
|
+
// a cap is now breached, THROW — the engine loop propagates it and the run fails BUDGET_EXCEEDED
|
|
124
|
+
// before another model call is dispatched. Also fire per-leaf, per-model metering to the broker.
|
|
125
|
+
reportUsage: (modelRefForUsage, usage) => {
|
|
126
|
+
const delta = toUsageDelta(usage);
|
|
127
|
+
// Cap on the turn's REAL upstream cost when the broker reported one (the result frame's
|
|
128
|
+
// costMicros, stashed by streamModel just above) so max_usd tracks actual spend rather than a
|
|
129
|
+
// representative-rate token estimate that's blind to prompt-cache discounts. Null ⇒ BYO / no
|
|
130
|
+
// upstream price ⇒ addUsage falls back to the estimate. Consume-once so the next turn is clean.
|
|
131
|
+
const realCostUsd = pendingCostMicros === null ? undefined : pendingCostMicros / 1_000_000;
|
|
132
|
+
pendingCostMicros = null;
|
|
133
|
+
this.deps.budget.addUsage(delta, realCostUsd);
|
|
134
|
+
const breach = this.deps.budget.capBreachReason();
|
|
135
|
+
if (breach !== null) {
|
|
136
|
+
throw new EngineError("BUDGET_EXCEEDED", `agent() leaf exceeded the run budget cap (${breach})`);
|
|
137
|
+
}
|
|
138
|
+
this.deps.meterUsage?.({
|
|
139
|
+
model: modelRefForUsage,
|
|
140
|
+
inputTokens: delta.inputTokens ?? 0,
|
|
141
|
+
outputTokens: delta.outputTokens ?? 0,
|
|
142
|
+
...(delta.cacheReadTokens === undefined
|
|
143
|
+
? {}
|
|
144
|
+
: { cachedReadTokens: delta.cacheReadTokens }),
|
|
145
|
+
...(delta.cacheWriteTokens === undefined
|
|
146
|
+
? {}
|
|
147
|
+
: { cachedWriteTokens: delta.cacheWriteTokens }),
|
|
148
|
+
leafIndex,
|
|
149
|
+
});
|
|
150
|
+
},
|
|
151
|
+
// A memory dir (`agent({ memory })`) is workspace-relative; the run's whole `/workspace` is
|
|
152
|
+
// already persisted by the worker when the manifest opts in (workspace.persist), so this only
|
|
153
|
+
// needs to validate the dir is inside the workspace (defense-in-depth) — there is no separate
|
|
154
|
+
// per-dir snapshot to register. The engine's buildToolSet already shape-validates the path.
|
|
155
|
+
memoryUsed: (dir) => {
|
|
156
|
+
const abs = join(this.deps.workspaceRoot, dir);
|
|
157
|
+
if (abs !== this.deps.workspaceRoot && !abs.startsWith(this.deps.workspaceRoot + "/")) {
|
|
158
|
+
throw new EngineError("VALIDATION", `agent() memory dir "${dir}" escapes the workspace.`);
|
|
159
|
+
}
|
|
160
|
+
},
|
|
161
|
+
// OAuth bearer brokering for a hosted MCP server. Called REACTIVELY by the engine — only after a
|
|
162
|
+
// server 401s the static `headers` — so static-bearer / no-auth servers never reach here. When
|
|
163
|
+
// the broker hook is wired, the token comes from the control-plane vault (never stored on the
|
|
164
|
+
// worker); `invalidateToken` names a just-rejected token so the broker forces a refresh. Absent
|
|
165
|
+
// hook ⇒ no token, and the engine surfaces a clean failure with the hint (correct degradation).
|
|
166
|
+
mcpToken: (serverUrl, invalidateToken) => this.deps.brokerMcpToken?.(serverUrl, invalidateToken) ??
|
|
167
|
+
Promise.resolve({
|
|
168
|
+
accessToken: null,
|
|
169
|
+
hint: `No OAuth connection is configured for MCP server "${serverUrl}". Connect it in the ` +
|
|
170
|
+
`Boardwalk console, or provide a bearer token via the server's headers.`,
|
|
171
|
+
}),
|
|
172
|
+
// Derive a child leaf io for a `subagent` tool call (engine ≥0.1.11): a fresh run-unique
|
|
173
|
+
// identity over the SAME sinks. `makeEventSink` returns the one run-global emitter, so every
|
|
174
|
+
// leaf's events ride a single monotonic cursor; the child's usage feeds the SAME run budget
|
|
175
|
+
// and meters under its own `leafIndex`. Reuses the parent leaf's redactor (same run secrets)
|
|
176
|
+
// and AbortSignal. A child gets no `subagent` tool, so its own forkLeaf is never invoked.
|
|
177
|
+
forkLeaf: (childOpts) => {
|
|
178
|
+
const childIndex = ++this.leafCount;
|
|
179
|
+
const childIdentity = {
|
|
180
|
+
agentId: `agent-${String(childIndex)}`,
|
|
181
|
+
...(childOpts.name !== undefined ? { agentName: childOpts.name } : {}),
|
|
182
|
+
};
|
|
183
|
+
return this.buildLeafIo({
|
|
184
|
+
identity: childIdentity,
|
|
185
|
+
sink: this.deps.makeEventSink(childIndex, childIdentity),
|
|
186
|
+
redactor,
|
|
187
|
+
leafIndex: childIndex,
|
|
188
|
+
signal,
|
|
189
|
+
});
|
|
190
|
+
},
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
/** POST one model turn to the broker's `/inference` and adapt its NDJSON stream into the engine's
|
|
194
|
+
* ModelTurnResult: each `delta` frame drives providerIo.onDelta; the terminal `result` frame is
|
|
195
|
+
* the turn. An `error` frame throws (the broker already classified it). An abort mid-stream throws. */
|
|
196
|
+
async streamModel(req, providerIo, signal, onCost) {
|
|
197
|
+
// Runner-direct BYO (D7): the org's own endpoint + key, called with the same engine adapters
|
|
198
|
+
// the broker uses. No platform cost (BYO is never metered) — onCost stays untouched.
|
|
199
|
+
// A direct turn needs an explicit model (an omitted model means the managed auto lane,
|
|
200
|
+
// which is always brokered).
|
|
201
|
+
const directModel = req.model;
|
|
202
|
+
const direct = this.deps.byo === undefined || directModel === undefined
|
|
203
|
+
? null
|
|
204
|
+
: directProviderFor(this.deps.byo.registry, req.provider);
|
|
205
|
+
if (direct !== null && this.deps.byo !== undefined && directModel !== undefined) {
|
|
206
|
+
throwIfAborted(signal);
|
|
207
|
+
const out = await streamDirectTurn(this.deps.byo, direct, {
|
|
208
|
+
model: directModel,
|
|
209
|
+
messages: req.messages,
|
|
210
|
+
tools: req.tools,
|
|
211
|
+
...(req.reasoning !== undefined ? { reasoning: req.reasoning } : {}),
|
|
212
|
+
}, providerIo.onDelta);
|
|
213
|
+
throwIfAborted(signal);
|
|
214
|
+
return { turn: out.turn, modelRef: out.modelRef };
|
|
215
|
+
}
|
|
216
|
+
let result = null;
|
|
217
|
+
for await (const frame of this.deps.inference.streamInference({
|
|
218
|
+
model: req.model,
|
|
219
|
+
provider: req.provider,
|
|
220
|
+
messages: req.messages,
|
|
221
|
+
tools: req.tools,
|
|
222
|
+
...(req.reasoning !== undefined ? { reasoning: req.reasoning } : {}),
|
|
223
|
+
})) {
|
|
224
|
+
throwIfAborted(signal);
|
|
225
|
+
if (frame.kind === "delta") {
|
|
226
|
+
providerIo.onDelta?.(frame.text);
|
|
227
|
+
}
|
|
228
|
+
else if (frame.kind === "result") {
|
|
229
|
+
result = { turn: frame.turn, modelRef: frame.modelRef };
|
|
230
|
+
// The broker stamps the turn's exact upstream cost on the result frame (0 ⇒ none / BYO); hand
|
|
231
|
+
// it to the budget guardrail via the caller's stash so max_usd caps on real spend.
|
|
232
|
+
if (frame.costMicros > 0)
|
|
233
|
+
onCost?.(frame.costMicros);
|
|
234
|
+
}
|
|
235
|
+
else if (frame.kind === "error") {
|
|
236
|
+
// The broker already classified this into a clean, customer-facing message — surface it
|
|
237
|
+
// verbatim (the loop records it as the run's error).
|
|
238
|
+
throw new Error(frame.error.message);
|
|
239
|
+
}
|
|
240
|
+
// else: a `ping` heartbeat keeping the connection alive during a long turn — nothing to do.
|
|
241
|
+
}
|
|
242
|
+
if (result === null) {
|
|
243
|
+
throw new Error("Inference stream ended without a result frame");
|
|
244
|
+
}
|
|
245
|
+
return result;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Gate the MCP servers a hosted leaf may use: only the `http` transport (no arbitrary `stdio`
|
|
250
|
+
* processes on the worker) and a parseable URL. Throws a clear VALIDATION_FAILED for the first
|
|
251
|
+
* offending ref so the leaf fails at its boundary, before the engine connects out.
|
|
252
|
+
*
|
|
253
|
+
* Reachability of the host is NOT gated here: a hosted run's egress is OPEN by default (the forward
|
|
254
|
+
* proxy allows all public destinations; a workflow restricts it via manifest.egress), and the proxy
|
|
255
|
+
* is the single enforcement point — a server blocked by a restrictive egress fails at the proxy when
|
|
256
|
+
* the engine connects, not via an MCP-specific allowlist that would be stricter than the platform.
|
|
257
|
+
*/
|
|
258
|
+
export function assertHostedMcpAllowed(refs) {
|
|
259
|
+
for (const ref of refs ?? []) {
|
|
260
|
+
if (ref.transport !== "http") {
|
|
261
|
+
throw new AppError(ErrorCode.VALIDATION_FAILED, `MCP server "${ref.name}" uses the "${ref.transport}" transport, which hosted runs do not ` +
|
|
262
|
+
`support — use transport: "http".`);
|
|
263
|
+
}
|
|
264
|
+
if (!isParsableUrl(ref.url)) {
|
|
265
|
+
throw new AppError(ErrorCode.VALIDATION_FAILED, `MCP server "${ref.name}" has an invalid URL: ${ref.url}`);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/** True when `url` parses (a malformed server URL should fail the leaf early). */
|
|
270
|
+
function isParsableUrl(url) {
|
|
271
|
+
try {
|
|
272
|
+
new URL(url);
|
|
273
|
+
return true;
|
|
274
|
+
}
|
|
275
|
+
catch {
|
|
276
|
+
return false;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
/** Map the engine loop's `TokenUsage` to the worker's `UsageDelta` (only numeric fields kept). */
|
|
280
|
+
function toUsageDelta(usage) {
|
|
281
|
+
const delta = {};
|
|
282
|
+
if (typeof usage.inputTokens === "number")
|
|
283
|
+
delta.inputTokens = usage.inputTokens;
|
|
284
|
+
if (typeof usage.outputTokens === "number")
|
|
285
|
+
delta.outputTokens = usage.outputTokens;
|
|
286
|
+
if (typeof usage.cacheReadTokens === "number")
|
|
287
|
+
delta.cacheReadTokens = usage.cacheReadTokens;
|
|
288
|
+
if (typeof usage.cacheCreationTokens === "number")
|
|
289
|
+
delta.cacheWriteTokens = usage.cacheCreationTokens;
|
|
290
|
+
return delta;
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* Project an engine `LeafEventBody` onto the platform's v1 `RunEventBody`. Both are the SDK's v1
|
|
294
|
+
* event kinds (the platform consumes `@boardwalk-labs/workflow`; the engine emits the same shapes),
|
|
295
|
+
* so this is a near-identity copy — `turn_ended` re-stamps the leaf identity the platform tracks,
|
|
296
|
+
* and the body is otherwise passed through verbatim. A discriminated switch keeps it exhaustive
|
|
297
|
+
* (any new engine kind surfaces as a compile error here, not a silent drop).
|
|
298
|
+
*/
|
|
299
|
+
function toRunEventBody(body, identity) {
|
|
300
|
+
switch (body.kind) {
|
|
301
|
+
case "turn_ended":
|
|
302
|
+
return {
|
|
303
|
+
kind: "turn_ended",
|
|
304
|
+
...identity,
|
|
305
|
+
reason: body.reason,
|
|
306
|
+
...(body.usage === undefined ? {} : { usage: body.usage }),
|
|
307
|
+
...(body.error === undefined ? {} : { error: body.error }),
|
|
308
|
+
};
|
|
309
|
+
case "text_start":
|
|
310
|
+
return { kind: "text_start", blockId: body.blockId };
|
|
311
|
+
case "text_delta":
|
|
312
|
+
return { kind: "text_delta", blockId: body.blockId, text: body.text };
|
|
313
|
+
case "text_end":
|
|
314
|
+
return { kind: "text_end", blockId: body.blockId };
|
|
315
|
+
case "tool_call_start":
|
|
316
|
+
return { kind: "tool_call_start", toolCallId: body.toolCallId, toolName: body.toolName };
|
|
317
|
+
case "tool_call_input_complete":
|
|
318
|
+
return { kind: "tool_call_input_complete", toolCallId: body.toolCallId, input: body.input };
|
|
319
|
+
case "tool_call_executing":
|
|
320
|
+
return { kind: "tool_call_executing", toolCallId: body.toolCallId };
|
|
321
|
+
case "tool_output_delta":
|
|
322
|
+
return {
|
|
323
|
+
kind: "tool_output_delta",
|
|
324
|
+
toolCallId: body.toolCallId,
|
|
325
|
+
stream: body.stream,
|
|
326
|
+
text: body.text,
|
|
327
|
+
};
|
|
328
|
+
case "tool_call_result":
|
|
329
|
+
return { kind: "tool_call_result", toolCallId: body.toolCallId, result: body.result };
|
|
330
|
+
case "tool_call_error":
|
|
331
|
+
return { kind: "tool_call_error", toolCallId: body.toolCallId, error: body.error };
|
|
332
|
+
}
|
|
333
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/** Renew cadence — well under the 5-min lease so a renewed lease always has minutes of headroom (a
|
|
2
|
+
* missed tick still has time to recover before the lease expires). */
|
|
3
|
+
export declare const DEFAULT_LEASE_RENEW_INTERVAL_MS = 120000;
|
|
4
|
+
export interface LeaseRenewerDeps {
|
|
5
|
+
/** The run being kept alive (for correlation/logging). */
|
|
6
|
+
runId: string;
|
|
7
|
+
/** Extend the lease; resolves true while we still hold it, false once it's definitively lost
|
|
8
|
+
* (brokered: `RunnerControlClient.renewLease` → `POST /renew`). */
|
|
9
|
+
renew: () => Promise<boolean>;
|
|
10
|
+
/** Fired exactly once the first time the lease is seen to be lost. */
|
|
11
|
+
onLost: () => void;
|
|
12
|
+
intervalMs?: number;
|
|
13
|
+
}
|
|
14
|
+
export declare class LeaseRenewer {
|
|
15
|
+
private readonly deps;
|
|
16
|
+
private timer;
|
|
17
|
+
private firing;
|
|
18
|
+
private lost;
|
|
19
|
+
private stopped;
|
|
20
|
+
constructor(deps: LeaseRenewerDeps);
|
|
21
|
+
/** Begin periodic lease renewal. */
|
|
22
|
+
start(): void;
|
|
23
|
+
/** Stop renewing and drain any in-flight renew. Idempotent. */
|
|
24
|
+
stop(): Promise<void>;
|
|
25
|
+
/** Whether the lease was definitively lost (the orchestrator may read this to shape the outcome). */
|
|
26
|
+
isLost(): boolean;
|
|
27
|
+
/** True once the renewer should no longer act. A method (re-evaluated after an `await`) so `stop()`
|
|
28
|
+
* can flip `stopped` mid-tick. */
|
|
29
|
+
private done;
|
|
30
|
+
private tick;
|
|
31
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// LeaseRenewer — keeps a long run's lease fresh so the recovery sweep doesn't reclaim a STILL-ALIVE
|
|
2
|
+
// worker (docs/RUNNER_BROKER.md; MASTER_SPEC §6).
|
|
3
|
+
//
|
|
4
|
+
// A run is claimed with a fixed lease (DEFAULT_LEASE_MS, 5 min). The worker never used to renew it,
|
|
5
|
+
// so any run longer than the lease (an Opus agentic loop is routinely 7+ min) had its lease expire
|
|
6
|
+
// mid-flight; the 1-min recovery sweep then reclaimed the live run and re-dispatched it — a redundant
|
|
7
|
+
// concurrent attempt that doubled the spend. This watcher (the heartbeat counterpart to the credit /
|
|
8
|
+
// cancel watchers) re-extends the lease on a timer through the broker (`POST /renew`, since the runner
|
|
9
|
+
// holds no DB credential).
|
|
10
|
+
//
|
|
11
|
+
// On a transient renew failure it just retries next tick (the renew cadence leaves several minutes of
|
|
12
|
+
// lease headroom, so a blip is harmless). On a DEFINITIVE loss (the broker says the run is no longer
|
|
13
|
+
// ours — another worker reclaimed it), it fires `onLost` once: the orchestrator aborts with
|
|
14
|
+
// `lease_lost`, and the worker stops WITHOUT finalizing so it can't clobber the new owner's run.
|
|
15
|
+
import { createLogger } from "./support/index.js";
|
|
16
|
+
const log = createLogger("LeaseRenewer");
|
|
17
|
+
/** Renew cadence — well under the 5-min lease so a renewed lease always has minutes of headroom (a
|
|
18
|
+
* missed tick still has time to recover before the lease expires). */
|
|
19
|
+
export const DEFAULT_LEASE_RENEW_INTERVAL_MS = 120_000;
|
|
20
|
+
export class LeaseRenewer {
|
|
21
|
+
deps;
|
|
22
|
+
timer = null;
|
|
23
|
+
firing = Promise.resolve();
|
|
24
|
+
lost = false;
|
|
25
|
+
stopped = false;
|
|
26
|
+
constructor(deps) {
|
|
27
|
+
this.deps = deps;
|
|
28
|
+
}
|
|
29
|
+
/** Begin periodic lease renewal. */
|
|
30
|
+
start() {
|
|
31
|
+
if (this.timer !== null)
|
|
32
|
+
return;
|
|
33
|
+
const interval = this.deps.intervalMs ?? DEFAULT_LEASE_RENEW_INTERVAL_MS;
|
|
34
|
+
this.timer = setInterval(() => {
|
|
35
|
+
this.firing = this.firing.then(() => this.tick());
|
|
36
|
+
}, interval);
|
|
37
|
+
// Don't keep the worker process alive solely for the renew timer.
|
|
38
|
+
this.timer.unref();
|
|
39
|
+
}
|
|
40
|
+
/** Stop renewing and drain any in-flight renew. Idempotent. */
|
|
41
|
+
async stop() {
|
|
42
|
+
this.stopped = true;
|
|
43
|
+
if (this.timer !== null) {
|
|
44
|
+
clearInterval(this.timer);
|
|
45
|
+
this.timer = null;
|
|
46
|
+
}
|
|
47
|
+
await this.firing.catch(() => undefined);
|
|
48
|
+
}
|
|
49
|
+
/** Whether the lease was definitively lost (the orchestrator may read this to shape the outcome). */
|
|
50
|
+
isLost() {
|
|
51
|
+
return this.lost;
|
|
52
|
+
}
|
|
53
|
+
/** True once the renewer should no longer act. A method (re-evaluated after an `await`) so `stop()`
|
|
54
|
+
* can flip `stopped` mid-tick. */
|
|
55
|
+
done() {
|
|
56
|
+
return this.stopped || this.lost;
|
|
57
|
+
}
|
|
58
|
+
async tick() {
|
|
59
|
+
if (this.done())
|
|
60
|
+
return;
|
|
61
|
+
let held;
|
|
62
|
+
try {
|
|
63
|
+
held = await this.deps.renew();
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
// A transient renew failure must NOT kill a live run — log and let the next tick retry while
|
|
67
|
+
// the current lease still has headroom.
|
|
68
|
+
log.warn("lease_renew_failed", {
|
|
69
|
+
runId: this.deps.runId,
|
|
70
|
+
error: err instanceof Error ? err.message : String(err),
|
|
71
|
+
});
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
if (held || this.done())
|
|
75
|
+
return;
|
|
76
|
+
this.lost = true;
|
|
77
|
+
log.info("run_lease_lost", { runId: this.deps.runId });
|
|
78
|
+
this.deps.onLost();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Process entrypoint for one run: `node .../runtime/main.js` with the platform env contract
|
|
3
|
+
// (RUN_ID + BOARDWALK_CONTROL_PLANE_URL + BOARDWALK_RUN_TOKEN [+ BOARDWALK_API_KEY]). Used by
|
|
4
|
+
// the Boardwalk-hosted Fargate container AND spawned per-run by the self-hosted daemon —
|
|
5
|
+
// one worker, two homes (docs/RUNNER_BROKER.md §6). Import `./index.js` for the library.
|
|
6
|
+
import { main } from "./index.js";
|
|
7
|
+
void main();
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { PhaseOptions } from "@boardwalk-labs/workflow/runtime";
|
|
2
|
+
import type { TurnEventSink } from "./agent/events.js";
|
|
3
|
+
export type PhaseCloseStatus = "completed" | "failed" | "cancelled";
|
|
4
|
+
export interface PhaseTrackerOptions {
|
|
5
|
+
/** The run's shared event emitter — phases ride the one ordered stream. */
|
|
6
|
+
sink: TurnEventSink;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Tracks the author-visible `phase("...")` marker for live run details. v1 wire semantics: a
|
|
10
|
+
* `phase` event is a MARKER — everything after it belongs to that phase until the next marker or
|
|
11
|
+
* run end (no phase_completed/failed events on the wire; consumers derive spans from positions).
|
|
12
|
+
* This is observability only: it never checkpoints JS execution or replays/skips user code.
|
|
13
|
+
*/
|
|
14
|
+
export declare class PhaseTracker {
|
|
15
|
+
private readonly sink;
|
|
16
|
+
private readonly storage;
|
|
17
|
+
private readonly seenIds;
|
|
18
|
+
private current;
|
|
19
|
+
private seq;
|
|
20
|
+
constructor(opts: PhaseTrackerOptions);
|
|
21
|
+
set(name: string, opts: PhaseOptions | undefined): void;
|
|
22
|
+
/** v1 phases are markers — nothing to emit at close; clear the current span. */
|
|
23
|
+
close(_status: PhaseCloseStatus): void;
|
|
24
|
+
capture(): string | null;
|
|
25
|
+
runInPhase<T>(phaseId: string | null, fn: () => Promise<T>): Promise<T>;
|
|
26
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { AppError, ErrorCode } from "./support/index.js";
|
|
3
|
+
/**
|
|
4
|
+
* Tracks the author-visible `phase("...")` marker for live run details. v1 wire semantics: a
|
|
5
|
+
* `phase` event is a MARKER — everything after it belongs to that phase until the next marker or
|
|
6
|
+
* run end (no phase_completed/failed events on the wire; consumers derive spans from positions).
|
|
7
|
+
* This is observability only: it never checkpoints JS execution or replays/skips user code.
|
|
8
|
+
*/
|
|
9
|
+
export class PhaseTracker {
|
|
10
|
+
sink;
|
|
11
|
+
storage = new AsyncLocalStorage();
|
|
12
|
+
seenIds = new Set();
|
|
13
|
+
current = null;
|
|
14
|
+
seq = 0;
|
|
15
|
+
constructor(opts) {
|
|
16
|
+
this.sink = opts.sink;
|
|
17
|
+
}
|
|
18
|
+
set(name, opts) {
|
|
19
|
+
const trimmed = name.trim();
|
|
20
|
+
if (trimmed.length === 0) {
|
|
21
|
+
throw new AppError(ErrorCode.VALIDATION_FAILED, "Phase name must not be empty");
|
|
22
|
+
}
|
|
23
|
+
// Use the caller's id only when it's a non-empty string after trimming; an empty trim falls
|
|
24
|
+
// back to a generated id (so `??` would be wrong here — it wouldn't catch `""`).
|
|
25
|
+
const trimmedId = opts?.id?.trim();
|
|
26
|
+
const id = trimmedId !== undefined && trimmedId !== "" ? trimmedId : `phase-${String(this.seq + 1)}`;
|
|
27
|
+
if (this.seenIds.has(id)) {
|
|
28
|
+
throw new AppError(ErrorCode.VALIDATION_FAILED, `Phase id already used: ${id}`);
|
|
29
|
+
}
|
|
30
|
+
this.seq += 1;
|
|
31
|
+
this.seenIds.add(id);
|
|
32
|
+
this.current = { id, name: trimmed };
|
|
33
|
+
this.sink.emit({ kind: "phase", name: trimmed, id });
|
|
34
|
+
}
|
|
35
|
+
/** v1 phases are markers — nothing to emit at close; clear the current span. */
|
|
36
|
+
close(_status) {
|
|
37
|
+
this.current = null;
|
|
38
|
+
}
|
|
39
|
+
capture() {
|
|
40
|
+
return this.current?.id ?? null;
|
|
41
|
+
}
|
|
42
|
+
runInPhase(phaseId, fn) {
|
|
43
|
+
return this.storage.run(phaseId, fn);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { TurnEventSink } from "./agent/events.js";
|
|
2
|
+
export type LogStream = "stdout" | "stderr";
|
|
3
|
+
/**
|
|
4
|
+
* Patch the global console so each call is forwarded to the original AND handed (formatted) to
|
|
5
|
+
* `sink`. Returns a restore function — ALWAYS call it (the worker runs `restore()` in a finally).
|
|
6
|
+
*/
|
|
7
|
+
export declare function captureConsole(sink: (stream: LogStream, text: string) => void): () => void;
|
|
8
|
+
export interface ProgramLogSinkOptions {
|
|
9
|
+
/** The run's shared event emitter — program logs ride the one ordered stream (`log` channel). */
|
|
10
|
+
sink: TurnEventSink;
|
|
11
|
+
/** Stop emitting after this many frames (the rest still print to CloudWatch). Default 10_000. */
|
|
12
|
+
maxFrames?: number;
|
|
13
|
+
/** Truncate a single line longer than this many chars. Default 8 KiB. */
|
|
14
|
+
maxLineLength?: number;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Build the sink `captureConsole` feeds: turn each formatted console line into a v1
|
|
18
|
+
* `program_output` event (best-effort). Caps + truncates to stay efficient.
|
|
19
|
+
*/
|
|
20
|
+
export declare function createProgramLogSink(opts: ProgramLogSinkOptions): (stream: LogStream, text: string) => void;
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// Capture a workflow program's console output as run-events, so a plain `console.log` program shows
|
|
2
|
+
// up in the run's activity tail (not just the worker's CloudWatch logs).
|
|
3
|
+
//
|
|
4
|
+
// Boundary: we patch the GLOBAL `console` only for the duration of the program body. The worker's
|
|
5
|
+
// own structured logger (Powertools) captured its console reference at construction (module load,
|
|
6
|
+
// before the program runs), so its logs are NOT captured here — only the program's `console.*`,
|
|
7
|
+
// which resolves the global at call time. Each call is still forwarded to the original console (so
|
|
8
|
+
// it also lands in CloudWatch), then formatted and handed to the sink.
|
|
9
|
+
//
|
|
10
|
+
// Performance: lines ride the run's shared event emitter (→ the batched BrokerEventPublisher the
|
|
11
|
+
// run-event path uses), so logging is off the per-line HTTP hot path. We cap total frames + truncate
|
|
12
|
+
// huge lines so a runaway `console.log` loop can't flood Redis/S3/the broker. Frames are v1
|
|
13
|
+
// `program_output` events (`log` channel) interleaved on the run's single ordered cursor stream.
|
|
14
|
+
import { format } from "node:util";
|
|
15
|
+
/** `console` methods mapped to a stream. log/info/debug → stdout; warn/error → stderr. */
|
|
16
|
+
const STREAM_BY_METHOD = {
|
|
17
|
+
log: "stdout",
|
|
18
|
+
info: "stdout",
|
|
19
|
+
debug: "stdout",
|
|
20
|
+
warn: "stderr",
|
|
21
|
+
error: "stderr",
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Patch the global console so each call is forwarded to the original AND handed (formatted) to
|
|
25
|
+
* `sink`. Returns a restore function — ALWAYS call it (the worker runs `restore()` in a finally).
|
|
26
|
+
*/
|
|
27
|
+
export function captureConsole(sink) {
|
|
28
|
+
const console_ = globalThis.console;
|
|
29
|
+
const originals = {};
|
|
30
|
+
for (const [method, stream] of Object.entries(STREAM_BY_METHOD)) {
|
|
31
|
+
const original = console_[method];
|
|
32
|
+
if (typeof original !== "function")
|
|
33
|
+
continue;
|
|
34
|
+
originals[method] = original;
|
|
35
|
+
console_[method] = (...args) => {
|
|
36
|
+
original.apply(console_, args); // still print to the container stdout (CloudWatch)
|
|
37
|
+
try {
|
|
38
|
+
sink(stream, format(...args));
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
// best-effort — a telemetry hiccup must never break the program's own logging
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
return () => {
|
|
46
|
+
for (const [method, original] of Object.entries(originals)) {
|
|
47
|
+
console_[method] = original;
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
const DEFAULT_MAX_FRAMES = 10_000;
|
|
52
|
+
const DEFAULT_MAX_LINE = 8 * 1024;
|
|
53
|
+
/**
|
|
54
|
+
* Build the sink `captureConsole` feeds: turn each formatted console line into a v1
|
|
55
|
+
* `program_output` event (best-effort). Caps + truncates to stay efficient.
|
|
56
|
+
*/
|
|
57
|
+
export function createProgramLogSink(opts) {
|
|
58
|
+
const maxFrames = opts.maxFrames ?? DEFAULT_MAX_FRAMES;
|
|
59
|
+
const maxLine = opts.maxLineLength ?? DEFAULT_MAX_LINE;
|
|
60
|
+
let frames = 0;
|
|
61
|
+
let truncationNoticeSent = false;
|
|
62
|
+
return (stream, text) => {
|
|
63
|
+
if (frames >= maxFrames) {
|
|
64
|
+
if (!truncationNoticeSent) {
|
|
65
|
+
// Emit one truncation notice, then go quiet (still prints to CloudWatch above).
|
|
66
|
+
truncationNoticeSent = true;
|
|
67
|
+
opts.sink.emit({
|
|
68
|
+
kind: "program_output",
|
|
69
|
+
stream: "stderr",
|
|
70
|
+
text: "… console output truncated (too many lines)",
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
const line = text.length > maxLine ? `${text.slice(0, maxLine)}… (truncated)` : text;
|
|
76
|
+
frames += 1;
|
|
77
|
+
opts.sink.emit({ kind: "program_output", stream, text: line });
|
|
78
|
+
};
|
|
79
|
+
}
|