@boardwalk-labs/runner 0.1.11 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +12 -0
- package/dist/runtime/agent/budget.d.ts +5 -5
- package/dist/runtime/agent/budget.js +8 -8
- package/dist/runtime/agent/model_rates.js +4 -5
- package/dist/runtime/agent/secret_redactor.js +1 -1
- package/dist/runtime/cancel_watcher.js +1 -1
- package/dist/runtime/credit_watcher.d.ts +1 -1
- package/dist/runtime/credit_watcher.js +5 -5
- package/dist/runtime/freeze_coordinator.d.ts +6 -1
- package/dist/runtime/freeze_coordinator.js +18 -4
- package/dist/runtime/index.d.ts +6 -6
- package/dist/runtime/index.js +62 -41
- package/dist/runtime/leaf_executor.d.ts +2 -2
- package/dist/runtime/program_runner.d.ts +1 -1
- package/dist/runtime/program_runner.js +1 -1
- package/dist/runtime/program_worker.d.ts +3 -3
- package/dist/runtime/program_worker.js +7 -7
- package/dist/runtime/recording_secret_resolver.js +1 -1
- package/dist/runtime/run_abort.js +3 -3
- package/dist/runtime/runner_control_client.d.ts +29 -4
- package/dist/runtime/runner_control_client.js +76 -32
- package/dist/runtime/runtime_flusher.d.ts +1 -1
- package/dist/runtime/runtime_flusher.js +3 -3
- package/dist/runtime/support/index.js +5 -6
- package/dist/runtime/tools/artifacts.js +1 -1
- package/dist/runtime/tools/types.d.ts +5 -5
- package/dist/runtime/tools/types.js +7 -7
- package/dist/runtime/tools/web_search.js +5 -6
- package/dist/runtime/uniqueness_reseed.d.ts +7 -0
- package/dist/runtime/uniqueness_reseed.js +65 -0
- package/dist/runtime/wire/artifact_storage.js +2 -2
- package/dist/runtime/wire/inference_proxy.d.ts +1 -1
- package/dist/runtime/wire/inference_proxy.js +2 -2
- package/dist/runtime/workflow_host.d.ts +32 -0
- package/dist/runtime/workflow_host.js +69 -9
- package/native/reseed.c +52 -0
- package/package.json +12 -5
- package/prebuilds/linux-x64/@boardwalk-labs+runner.node +0 -0
|
@@ -24,7 +24,7 @@ export interface ProgramVersionReader {
|
|
|
24
24
|
program: ProgramRef;
|
|
25
25
|
} | null>;
|
|
26
26
|
}
|
|
27
|
-
/** Books the run's RUNTIME usage as periodic deltas (the worker's RuntimeFlusher
|
|
27
|
+
/** Books the run's RUNTIME usage as periodic deltas (the worker's RuntimeFlusher). The
|
|
28
28
|
* orchestrator drives the lifecycle: the timer flushes mid-run, `stop()` halts it at the body's end,
|
|
29
29
|
* and `flushFinal()` books the tail on a clean terminal (skipped on a `lease_lost` handoff — the new
|
|
30
30
|
* owner books its own runtime). Replaces the old single terminal runtime charge. */
|
|
@@ -51,7 +51,7 @@ export interface RunFinalizer {
|
|
|
51
51
|
export interface RunSuspender {
|
|
52
52
|
suspend(signal: SuspendSignal, workerId: string): Promise<void>;
|
|
53
53
|
}
|
|
54
|
-
/** Restores/snapshots the workflow's persistent `/workspace
|
|
54
|
+
/** Restores/snapshots the workflow's persistent `/workspace`. Best-effort — both no-op when the
|
|
55
55
|
* run isn't eligible (not opted-in / self-hosted), and neither throws. */
|
|
56
56
|
export interface WorkspaceHandle {
|
|
57
57
|
hydrate(): Promise<void>;
|
|
@@ -155,7 +155,7 @@ export interface ProgramWorkerDeps {
|
|
|
155
155
|
/** Emit the program's `console.*` output as `log` run-events while the body runs (optional —
|
|
156
156
|
* absent disables capture). Wired by the entrypoint to the batched telemetry publisher. */
|
|
157
157
|
onProgramLog?: (stream: LogStream, text: string) => void;
|
|
158
|
-
/**
|
|
158
|
+
/** Task ARN (or any stable worker identity). */
|
|
159
159
|
workerId: string;
|
|
160
160
|
/** Drain any buffered telemetry before the worker exits (brokered path's BrokerEventPublisher).
|
|
161
161
|
* Called by the worker entrypoint's cleanup; the orchestrator itself never invokes it. */
|
|
@@ -11,12 +11,12 @@
|
|
|
11
11
|
// aborted mid-flight — credit exhaustion — finalize failed with the abort reason).
|
|
12
12
|
//
|
|
13
13
|
// While the program runs, two per-session loops watch it (both brokered): a UsageFlusher meters token
|
|
14
|
-
// deltas
|
|
14
|
+
// deltas and a CreditWatcher polls the org's funding. When credit hits zero the watcher
|
|
15
15
|
// aborts the run's AbortSignal, which the WorkflowHost honors at every hook (cooperative
|
|
16
16
|
// cancellation, run_abort.ts) — `signal.aborted` is authoritative for the terminal status.
|
|
17
17
|
//
|
|
18
18
|
// What's gone vs. the old worker: no checkpoint load/pause, no resume, no sleep/wait_for_child
|
|
19
|
-
// pause outcomes (the program holds), no per-turn transcript. The
|
|
19
|
+
// pause outcomes (the program holds), no per-turn transcript. The agent loop now lives one
|
|
20
20
|
// level down, behind the host's agent() leaf.
|
|
21
21
|
//
|
|
22
22
|
// v0 deferral (a clear seam): run-level outcome validation (needs program output capture).
|
|
@@ -130,16 +130,16 @@ export async function runProgramWorker(runId, deps) {
|
|
|
130
130
|
if (workspace !== undefined)
|
|
131
131
|
await workspace.hydrate();
|
|
132
132
|
// Token metering is PER-LEAF: each agent() leaf reports its own tokens + model to the broker, which
|
|
133
|
-
// decides `billed_by_boardwalk` per model + meters to
|
|
133
|
+
// decides `billed_by_boardwalk` per model + meters usage to the platform (see leaf_executor `meterUsage`). A
|
|
134
134
|
// workflow has no run-level model, so there is no run-level token metering here.
|
|
135
|
-
// Mid-run credit watching
|
|
135
|
+
// Mid-run credit watching: when the org runs out of credit, abort the run cooperatively.
|
|
136
136
|
const credit = deps.startCreditWatch?.({
|
|
137
137
|
run: claimed,
|
|
138
138
|
onExhausted: () => {
|
|
139
139
|
controller.abort(new RunAbortedError("credit_exhausted"));
|
|
140
140
|
},
|
|
141
141
|
});
|
|
142
|
-
// Mid-run user-cancel watching
|
|
142
|
+
// Mid-run user-cancel watching: when the user cancels, abort the run cooperatively. The host
|
|
143
143
|
// honors the abort at the next hook boundary (a `sleep` hold wakes immediately); the broker then
|
|
144
144
|
// upgrades the terminal write to `cancelled` because the run was flipped to `cancelling`.
|
|
145
145
|
const cancel = deps.startCancelWatch?.({
|
|
@@ -148,7 +148,7 @@ export async function runProgramWorker(runId, deps) {
|
|
|
148
148
|
controller.abort(new RunAbortedError("cancelled"));
|
|
149
149
|
},
|
|
150
150
|
});
|
|
151
|
-
// Lease renewal
|
|
151
|
+
// Lease renewal: heartbeat the lease so a long run isn't reclaimed mid-flight. If the lease is
|
|
152
152
|
// definitively lost (another worker reclaimed it), abort `lease_lost` — the run stops without
|
|
153
153
|
// finalizing (the new owner owns the terminal write; see the lease_lost guard after the body).
|
|
154
154
|
const lease = deps.startLeaseRenew?.({
|
|
@@ -157,7 +157,7 @@ export async function runProgramWorker(runId, deps) {
|
|
|
157
157
|
controller.abort(new RunAbortedError("lease_lost"));
|
|
158
158
|
},
|
|
159
159
|
});
|
|
160
|
-
// Runtime metering
|
|
160
|
+
// Runtime metering: flush runtime as periodic deltas from the claim, so a long/perpetual run
|
|
161
161
|
// bills as it burns (and the credit watcher sees it) instead of only at terminal. The tail is booked
|
|
162
162
|
// by `flushFinal()` after the body (on every path except a lease_lost handoff).
|
|
163
163
|
const runtimeFlush = deps.startRuntimeFlush?.({ run: claimed, startedAtMs: sessionStartMs });
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// RecordingSecretResolver — the redaction feeder
|
|
1
|
+
// RecordingSecretResolver — the redaction feeder.
|
|
2
2
|
//
|
|
3
3
|
// Decorates any SecretResolver so EVERY value a run resolves — whether the workflow program asked
|
|
4
4
|
// via `secrets.get(name)` or a tool asked via `ctx.secrets.resolve(ref)` — is recorded into the
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// run_abort — the provider-agnostic cooperative-cancellation substrate for a run (the Runner Credential Broker model).
|
|
2
2
|
//
|
|
3
|
-
// The cancellation primitive is a Web-standard `AbortSignal` — NOT a
|
|
3
|
+
// The cancellation primitive is a Web-standard `AbortSignal` — NOT a framework/model concept. The worker
|
|
4
4
|
// owns one `AbortController` per run session; a watcher (credit, later user-cancel) calls
|
|
5
5
|
// `controller.abort(new RunAbortedError(reason))`. The WorkflowHost honors that signal at every hook
|
|
6
6
|
// boundary (`agent`/`sleep`/`workflows.call`/…), unwinding the program. The ONLY place that translates
|
|
7
|
-
// the signal into a model-specific stop is the
|
|
8
|
-
//
|
|
7
|
+
// the signal into a model-specific stop is the agent leaf (signal → the leaf's cancel path); a different
|
|
8
|
+
// leaf implementation honors the SAME `AbortSignal` its own way. So nothing here, in the host, or in the
|
|
9
9
|
// program-facing SDK depends on the model backend.
|
|
10
10
|
//
|
|
11
11
|
// `signal.aborted` is AUTHORITATIVE at the orchestrator: a program that catches RunAbortedError and
|
|
@@ -13,6 +13,10 @@ export interface RunnerControlClientConfig {
|
|
|
13
13
|
runId: string;
|
|
14
14
|
/** Injected fetch (defaults to global fetch). */
|
|
15
15
|
fetchImpl?: typeof fetch;
|
|
16
|
+
/** Per-call ceiling for short control calls (default 30s) — bounds a poll frozen mid-flight. */
|
|
17
|
+
controlTimeoutMs?: number;
|
|
18
|
+
/** Per-call ceiling for bulk artifact/workspace transfers (default 5 min). */
|
|
19
|
+
bulkTimeoutMs?: number;
|
|
16
20
|
}
|
|
17
21
|
/** The pinned program's download reference (the worker fetches + verifies + extracts it). */
|
|
18
22
|
export interface BrokerProgram {
|
|
@@ -47,9 +51,24 @@ export declare class RunnerControlClient {
|
|
|
47
51
|
/** The live bearer. Mutable: on the snapshot substrate a wake carries a FRESH run token (the
|
|
48
52
|
* frozen one expired while suspended) and the worker swaps it at runtime. */
|
|
49
53
|
private runToken;
|
|
54
|
+
private readonly controlTimeoutMs;
|
|
55
|
+
private readonly bulkTimeoutMs;
|
|
50
56
|
constructor(cfg: RunnerControlClientConfig);
|
|
51
57
|
/** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
|
|
52
58
|
swapRunToken(token: string): void;
|
|
59
|
+
/**
|
|
60
|
+
* Every SHORT control call (claim / renew / cancel / credit / journal / …) goes through here so
|
|
61
|
+
* it carries a hard timeout. Without one, a poll frozen mid-flight on the snapshot substrate
|
|
62
|
+
* hangs FOREVER on restore (the socket is dead but never reset), and since a watcher serializes
|
|
63
|
+
* its ticks, one hung tick wedges that watcher — the dead-connections gotcha for the
|
|
64
|
+
* background pollers (lease/cancel/credit), which run on untracked timers the quiescence gate
|
|
65
|
+
* doesn't cover. Also plain robustness: no broker call should hang on a network blip. The
|
|
66
|
+
* streaming inference call is the ONE exception (long-lived NDJSON) and bypasses this.
|
|
67
|
+
*/
|
|
68
|
+
private controlFetch;
|
|
69
|
+
/** Bulk transfers (artifact + workspace up/download over presigned S3) — a much larger ceiling
|
|
70
|
+
* than a control call, but still bounded so a dead socket can't hang the run. */
|
|
71
|
+
private bulkFetch;
|
|
53
72
|
/** Claim the run's lease. Returns the run on success, or null when it isn't claimable (409 —
|
|
54
73
|
* another worker has it, or it isn't pending), which the worker treats as "claim lost". */
|
|
55
74
|
claim(workerId: string, leaseSeconds: number): Promise<{
|
|
@@ -89,8 +108,8 @@ export declare class RunnerControlClient {
|
|
|
89
108
|
* retried/duplicate flush idempotent; distinct per-flush ids sum into the run's runtime total. */
|
|
90
109
|
reportUsage(runtimeSeconds: number, identifier: string): Promise<void>;
|
|
91
110
|
/** Report a token-usage delta for incremental in-run metering (the usage flusher → broker). The
|
|
92
|
-
* broker gates on the run's per-connection `billed_by_boardwalk` server-side + meters to
|
|
93
|
-
* `identifier` makes a retried/duplicate flush idempotent. Satisfies {@link TokenUsageReporter}. */
|
|
111
|
+
* broker gates on the run's per-connection `billed_by_boardwalk` server-side + meters usage to the
|
|
112
|
+
* platform; `identifier` makes a retried/duplicate flush idempotent. Satisfies {@link TokenUsageReporter}. */
|
|
94
113
|
meterTokens(input: {
|
|
95
114
|
inputTokens: number;
|
|
96
115
|
outputTokens: number;
|
|
@@ -101,14 +120,20 @@ export declare class RunnerControlClient {
|
|
|
101
120
|
cachedWriteTokens?: number;
|
|
102
121
|
}): Promise<void>;
|
|
103
122
|
/** Check whether the run's org is still funded (the CreditWatcher → broker). The broker reads the
|
|
104
|
-
* live
|
|
123
|
+
* live billing balance server-side; `false` means out of credit (the watcher then aborts the run). */
|
|
105
124
|
checkCredit(): Promise<boolean>;
|
|
106
125
|
/** Check whether the run has been asked to cancel (the CancelWatcher → broker). `true` once the user
|
|
107
126
|
* cancelled the run (the broker flipped it to `cancelling`/`cancelled`); the watcher then aborts the
|
|
108
127
|
* run. Brokered because the runner holds no DB/Redis — this replaces the unreachable Redis channel. */
|
|
109
128
|
checkCancelled(): Promise<boolean>;
|
|
129
|
+
/** Register-without-release: register a HELD HITL gate's request row
|
|
130
|
+
* so it is answerable while the run keeps running — no suspend. Idempotent. Returns whether a new
|
|
131
|
+
* gate was registered. */
|
|
132
|
+
registerInput(seq: number, gate: unknown): Promise<boolean>;
|
|
133
|
+
/** Poll the resolved answers for a held gate at `seq` (empty until a human responds). */
|
|
134
|
+
pollInputAnswers(seq: number): Promise<Record<string, unknown>>;
|
|
110
135
|
/** Mint a presigned GET URL to restore this workflow's last `/workspace` snapshot (workspace
|
|
111
|
-
* persistence
|
|
136
|
+
* persistence). `null` when the run isn't eligible (not opted-in, or self-hosted). */
|
|
112
137
|
workspaceHydrateUrl(): Promise<string | null>;
|
|
113
138
|
/** Mint a presigned PUT URL to snapshot this workflow's `/workspace` (the worker uploads the tarball
|
|
114
139
|
* straight to S3). `null` when the run isn't eligible. `sizeBytes` is the archive's on-disk size
|
|
@@ -19,20 +19,41 @@ export class RunnerControlClient {
|
|
|
19
19
|
/** The live bearer. Mutable: on the snapshot substrate a wake carries a FRESH run token (the
|
|
20
20
|
* frozen one expired while suspended) and the worker swaps it at runtime. */
|
|
21
21
|
runToken;
|
|
22
|
+
controlTimeoutMs;
|
|
23
|
+
bulkTimeoutMs;
|
|
22
24
|
constructor(cfg) {
|
|
23
25
|
this.cfg = cfg;
|
|
24
26
|
this.base = cfg.baseUrl.replace(/\/+$/, "");
|
|
25
27
|
this.fetchImpl = cfg.fetchImpl ?? fetch;
|
|
26
28
|
this.runToken = cfg.runToken;
|
|
29
|
+
this.controlTimeoutMs = cfg.controlTimeoutMs ?? 30_000;
|
|
30
|
+
this.bulkTimeoutMs = cfg.bulkTimeoutMs ?? 300_000;
|
|
27
31
|
}
|
|
28
32
|
/** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
|
|
29
33
|
swapRunToken(token) {
|
|
30
34
|
this.runToken = token;
|
|
31
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* Every SHORT control call (claim / renew / cancel / credit / journal / …) goes through here so
|
|
38
|
+
* it carries a hard timeout. Without one, a poll frozen mid-flight on the snapshot substrate
|
|
39
|
+
* hangs FOREVER on restore (the socket is dead but never reset), and since a watcher serializes
|
|
40
|
+
* its ticks, one hung tick wedges that watcher — the dead-connections gotcha for the
|
|
41
|
+
* background pollers (lease/cancel/credit), which run on untracked timers the quiescence gate
|
|
42
|
+
* doesn't cover. Also plain robustness: no broker call should hang on a network blip. The
|
|
43
|
+
* streaming inference call is the ONE exception (long-lived NDJSON) and bypasses this.
|
|
44
|
+
*/
|
|
45
|
+
controlFetch(url, init) {
|
|
46
|
+
return this.fetchImpl(url, { ...init, signal: AbortSignal.timeout(this.controlTimeoutMs) });
|
|
47
|
+
}
|
|
48
|
+
/** Bulk transfers (artifact + workspace up/download over presigned S3) — a much larger ceiling
|
|
49
|
+
* than a control call, but still bounded so a dead socket can't hang the run. */
|
|
50
|
+
bulkFetch(url, init) {
|
|
51
|
+
return this.fetchImpl(url, { ...init, signal: AbortSignal.timeout(this.bulkTimeoutMs) });
|
|
52
|
+
}
|
|
32
53
|
/** Claim the run's lease. Returns the run on success, or null when it isn't claimable (409 —
|
|
33
54
|
* another worker has it, or it isn't pending), which the worker treats as "claim lost". */
|
|
34
55
|
async claim(workerId, leaseSeconds) {
|
|
35
|
-
const res = await this.
|
|
56
|
+
const res = await this.controlFetch(this.url("claim"), {
|
|
36
57
|
method: "POST",
|
|
37
58
|
headers: this.headers(true),
|
|
38
59
|
body: JSON.stringify({ workerId, leaseSeconds }),
|
|
@@ -54,7 +75,7 @@ export class RunnerControlClient {
|
|
|
54
75
|
* `leaseUntil`, or null when the lease was lost (409 — another worker reclaimed the run), which
|
|
55
76
|
* the worker treats as "stop". */
|
|
56
77
|
async renewLease(workerId, leaseSeconds) {
|
|
57
|
-
const res = await this.
|
|
78
|
+
const res = await this.controlFetch(this.url("renew"), {
|
|
58
79
|
method: "POST",
|
|
59
80
|
headers: this.headers(true),
|
|
60
81
|
body: JSON.stringify({ workerId, leaseSeconds }),
|
|
@@ -70,7 +91,7 @@ export class RunnerControlClient {
|
|
|
70
91
|
* whose lease expired and whose run was reclaimed + re-dispatched to a new owner), so a
|
|
71
92
|
* hung/partitioned worker that later recovers can't clobber the live run or revive a terminal one. */
|
|
72
93
|
async finalize(status, output, workerId) {
|
|
73
|
-
const res = await this.
|
|
94
|
+
const res = await this.controlFetch(this.url("finalize"), {
|
|
74
95
|
method: "POST",
|
|
75
96
|
headers: this.headers(true),
|
|
76
97
|
body: JSON.stringify({ status, output, workerId }),
|
|
@@ -81,7 +102,7 @@ export class RunnerControlClient {
|
|
|
81
102
|
/** Look up a durable-seam journal entry by its seq (the durable-suspension design), or null on a replay miss
|
|
82
103
|
* (404). The broker joins a parked agent leaf's answers into the result server-side. */
|
|
83
104
|
async journalGet(seq) {
|
|
84
|
-
const res = await this.
|
|
105
|
+
const res = await this.controlFetch(this.url(`journal/${encodeURIComponent(String(seq))}`), {
|
|
85
106
|
method: "GET",
|
|
86
107
|
headers: this.headers(false),
|
|
87
108
|
});
|
|
@@ -94,7 +115,7 @@ export class RunnerControlClient {
|
|
|
94
115
|
/** Record a RESOLVED seam result (idempotent on the run + seq server-side; a resolved entry is
|
|
95
116
|
* immutable). The broker writes the memoized value the next replay returns. */
|
|
96
117
|
async journalPut(entry) {
|
|
97
|
-
const res = await this.
|
|
118
|
+
const res = await this.controlFetch(this.url("journal"), {
|
|
98
119
|
method: "POST",
|
|
99
120
|
headers: this.headers(true),
|
|
100
121
|
body: JSON.stringify(entry),
|
|
@@ -106,7 +127,7 @@ export class RunnerControlClient {
|
|
|
106
127
|
* entry + a human-input request row for HITL, or the wake time for a long sleep), flips the run to
|
|
107
128
|
* its suspended status, and releases the lease — transactionally. No finalize; a wake re-dispatches. */
|
|
108
129
|
async suspend(signal, workerId) {
|
|
109
|
-
const res = await this.
|
|
130
|
+
const res = await this.controlFetch(this.url("suspend"), {
|
|
110
131
|
method: "POST",
|
|
111
132
|
headers: this.headers(true),
|
|
112
133
|
body: JSON.stringify({ ...signal, workerId }),
|
|
@@ -123,7 +144,7 @@ export class RunnerControlClient {
|
|
|
123
144
|
}
|
|
124
145
|
/** Fetch the run's pinned manifest + program source, or null when the version is missing (404). */
|
|
125
146
|
async getVersion() {
|
|
126
|
-
const res = await this.
|
|
147
|
+
const res = await this.controlFetch(this.url("version"), {
|
|
127
148
|
method: "GET",
|
|
128
149
|
headers: this.headers(false),
|
|
129
150
|
});
|
|
@@ -136,7 +157,7 @@ export class RunnerControlClient {
|
|
|
136
157
|
/** Book a runtime-seconds DELTA (the worker's RuntimeFlusher → broker). `identifier` makes a
|
|
137
158
|
* retried/duplicate flush idempotent; distinct per-flush ids sum into the run's runtime total. */
|
|
138
159
|
async reportUsage(runtimeSeconds, identifier) {
|
|
139
|
-
const res = await this.
|
|
160
|
+
const res = await this.controlFetch(this.url("usage"), {
|
|
140
161
|
method: "POST",
|
|
141
162
|
headers: this.headers(true),
|
|
142
163
|
body: JSON.stringify({ runtimeSeconds, identifier }),
|
|
@@ -145,10 +166,10 @@ export class RunnerControlClient {
|
|
|
145
166
|
throw await brokerError(res, "usage");
|
|
146
167
|
}
|
|
147
168
|
/** Report a token-usage delta for incremental in-run metering (the usage flusher → broker). The
|
|
148
|
-
* broker gates on the run's per-connection `billed_by_boardwalk` server-side + meters to
|
|
149
|
-
* `identifier` makes a retried/duplicate flush idempotent. Satisfies {@link TokenUsageReporter}. */
|
|
169
|
+
* broker gates on the run's per-connection `billed_by_boardwalk` server-side + meters usage to the
|
|
170
|
+
* platform; `identifier` makes a retried/duplicate flush idempotent. Satisfies {@link TokenUsageReporter}. */
|
|
150
171
|
async meterTokens(input) {
|
|
151
|
-
const res = await this.
|
|
172
|
+
const res = await this.controlFetch(this.url("usage/tokens"), {
|
|
152
173
|
method: "POST",
|
|
153
174
|
headers: this.headers(true),
|
|
154
175
|
body: JSON.stringify(input),
|
|
@@ -157,9 +178,9 @@ export class RunnerControlClient {
|
|
|
157
178
|
throw await brokerError(res, "usage/tokens");
|
|
158
179
|
}
|
|
159
180
|
/** Check whether the run's org is still funded (the CreditWatcher → broker). The broker reads the
|
|
160
|
-
* live
|
|
181
|
+
* live billing balance server-side; `false` means out of credit (the watcher then aborts the run). */
|
|
161
182
|
async checkCredit() {
|
|
162
|
-
const res = await this.
|
|
183
|
+
const res = await this.controlFetch(this.url("credit"), {
|
|
163
184
|
method: "GET",
|
|
164
185
|
headers: this.headers(false),
|
|
165
186
|
});
|
|
@@ -171,7 +192,7 @@ export class RunnerControlClient {
|
|
|
171
192
|
* cancelled the run (the broker flipped it to `cancelling`/`cancelled`); the watcher then aborts the
|
|
172
193
|
* run. Brokered because the runner holds no DB/Redis — this replaces the unreachable Redis channel. */
|
|
173
194
|
async checkCancelled() {
|
|
174
|
-
const res = await this.
|
|
195
|
+
const res = await this.controlFetch(this.url("cancel"), {
|
|
175
196
|
method: "GET",
|
|
176
197
|
headers: this.headers(false),
|
|
177
198
|
});
|
|
@@ -179,10 +200,33 @@ export class RunnerControlClient {
|
|
|
179
200
|
throw await brokerError(res, "cancel");
|
|
180
201
|
return (await res.json()).cancelRequested;
|
|
181
202
|
}
|
|
203
|
+
/** Register-without-release: register a HELD HITL gate's request row
|
|
204
|
+
* so it is answerable while the run keeps running — no suspend. Idempotent. Returns whether a new
|
|
205
|
+
* gate was registered. */
|
|
206
|
+
async registerInput(seq, gate) {
|
|
207
|
+
const res = await this.controlFetch(this.url("inputs"), {
|
|
208
|
+
method: "POST",
|
|
209
|
+
headers: this.headers(true),
|
|
210
|
+
body: JSON.stringify({ seq, humanInput: gate }),
|
|
211
|
+
});
|
|
212
|
+
if (res.status !== 200)
|
|
213
|
+
throw await brokerError(res, "register-input");
|
|
214
|
+
return (await res.json()).registered;
|
|
215
|
+
}
|
|
216
|
+
/** Poll the resolved answers for a held gate at `seq` (empty until a human responds). */
|
|
217
|
+
async pollInputAnswers(seq) {
|
|
218
|
+
const res = await this.controlFetch(this.url(`inputs/${encodeURIComponent(String(seq))}`), {
|
|
219
|
+
method: "GET",
|
|
220
|
+
headers: this.headers(false),
|
|
221
|
+
});
|
|
222
|
+
if (res.status !== 200)
|
|
223
|
+
throw await brokerError(res, "poll-inputs");
|
|
224
|
+
return (await res.json()).answers;
|
|
225
|
+
}
|
|
182
226
|
/** Mint a presigned GET URL to restore this workflow's last `/workspace` snapshot (workspace
|
|
183
|
-
* persistence
|
|
227
|
+
* persistence). `null` when the run isn't eligible (not opted-in, or self-hosted). */
|
|
184
228
|
async workspaceHydrateUrl() {
|
|
185
|
-
const res = await this.
|
|
229
|
+
const res = await this.controlFetch(this.url("workspace/hydrate-url"), {
|
|
186
230
|
method: "POST",
|
|
187
231
|
headers: this.headers(false),
|
|
188
232
|
});
|
|
@@ -195,7 +239,7 @@ export class RunnerControlClient {
|
|
|
195
239
|
* (the worker tars BEFORE requesting the URL) — the broker records it for the org storage counter +
|
|
196
240
|
* daily meter; the snapshot overwrites one per-workflow key, so it's the workflow's full footprint. */
|
|
197
241
|
async workspacePersistUrl(sizeBytes) {
|
|
198
|
-
const res = await this.
|
|
242
|
+
const res = await this.controlFetch(this.url("workspace/persist-url"), {
|
|
199
243
|
method: "POST",
|
|
200
244
|
headers: this.headers(true),
|
|
201
245
|
body: JSON.stringify({ sizeBytes }),
|
|
@@ -208,7 +252,7 @@ export class RunnerControlClient {
|
|
|
208
252
|
/** Download bytes from a presigned S3 URL (workspace hydrate). `null` on 404 (no snapshot yet —
|
|
209
253
|
* e.g. the workflow's first run); throws on any other non-2xx. Goes straight to S3, not the broker. */
|
|
210
254
|
async downloadBytes(url) {
|
|
211
|
-
const res = await this.
|
|
255
|
+
const res = await this.bulkFetch(url, { method: "GET" });
|
|
212
256
|
if (res.status === 404)
|
|
213
257
|
return null;
|
|
214
258
|
if (!res.ok)
|
|
@@ -219,7 +263,7 @@ export class RunnerControlClient {
|
|
|
219
263
|
* third-party-verifiable token (gated server-side on `permissions.id_token: "write"`) — used to
|
|
220
264
|
* federate into the org's OWN cloud (AWS/GCP). DIFFERENT from this client's run token. */
|
|
221
265
|
async requestOidcToken(audience) {
|
|
222
|
-
const res = await this.
|
|
266
|
+
const res = await this.controlFetch(this.url("oidc/token"), {
|
|
223
267
|
method: "POST",
|
|
224
268
|
headers: this.headers(true),
|
|
225
269
|
body: JSON.stringify({ audience }),
|
|
@@ -231,7 +275,7 @@ export class RunnerControlClient {
|
|
|
231
275
|
/** Publish a batch of live agent-event frames (the SSE live-tail source) — the broker publishes
|
|
232
276
|
* them to the run's Redis channel server-side, so the runner holds no Redis credential. */
|
|
233
277
|
async publishTelemetry(frames) {
|
|
234
|
-
const res = await this.
|
|
278
|
+
const res = await this.controlFetch(this.url("telemetry"), {
|
|
235
279
|
method: "POST",
|
|
236
280
|
headers: this.headers(true),
|
|
237
281
|
body: JSON.stringify({ frames }),
|
|
@@ -242,7 +286,7 @@ export class RunnerControlClient {
|
|
|
242
286
|
/** Store a run artifact through the broker (which holds the S3 credential + neutralizes the served
|
|
243
287
|
* content type server-side). Returns the catalog id + a signed download URL. */
|
|
244
288
|
async writeArtifact(input) {
|
|
245
|
-
const res = await this.
|
|
289
|
+
const res = await this.controlFetch(this.url("artifacts"), {
|
|
246
290
|
method: "POST",
|
|
247
291
|
headers: this.headers(true),
|
|
248
292
|
body: JSON.stringify(input),
|
|
@@ -255,7 +299,7 @@ export class RunnerControlClient {
|
|
|
255
299
|
* broker derives the S3 key + neutralizes/pins the served content type; it returns the upload URL +
|
|
256
300
|
* required headers + the `s3Key` to echo back at commit. No catalog row exists yet. */
|
|
257
301
|
async presignArtifact(input) {
|
|
258
|
-
const res = await this.
|
|
302
|
+
const res = await this.controlFetch(this.url("artifacts/presign"), {
|
|
259
303
|
method: "POST",
|
|
260
304
|
headers: this.headers(true),
|
|
261
305
|
body: JSON.stringify(input),
|
|
@@ -268,7 +312,7 @@ export class RunnerControlClient {
|
|
|
268
312
|
* presign response and MUST be sent verbatim — the content type is pinned into the signature, so
|
|
269
313
|
* S3 rejects a mismatch. This call goes straight to S3, not the broker. */
|
|
270
314
|
async uploadBytes(url, headers, body) {
|
|
271
|
-
const res = await this.
|
|
315
|
+
const res = await this.bulkFetch(url, { method: "PUT", headers, body });
|
|
272
316
|
if (!res.ok)
|
|
273
317
|
throw await brokerError(res, "artifacts-upload");
|
|
274
318
|
}
|
|
@@ -277,7 +321,7 @@ export class RunnerControlClient {
|
|
|
277
321
|
* broker re-validates the run prefix + re-neutralizes the content type, then returns the catalog id
|
|
278
322
|
* + a signed download URL. */
|
|
279
323
|
async commitArtifact(input) {
|
|
280
|
-
const res = await this.
|
|
324
|
+
const res = await this.controlFetch(this.url("artifacts/commit"), {
|
|
281
325
|
method: "POST",
|
|
282
326
|
headers: this.headers(true),
|
|
283
327
|
body: JSON.stringify(input),
|
|
@@ -288,7 +332,7 @@ export class RunnerControlClient {
|
|
|
288
332
|
}
|
|
289
333
|
/** List the artifacts this run has produced. */
|
|
290
334
|
async listArtifacts() {
|
|
291
|
-
const res = await this.
|
|
335
|
+
const res = await this.controlFetch(this.url("artifacts"), {
|
|
292
336
|
method: "GET",
|
|
293
337
|
headers: this.headers(false),
|
|
294
338
|
});
|
|
@@ -298,7 +342,7 @@ export class RunnerControlClient {
|
|
|
298
342
|
}
|
|
299
343
|
/** Mint a fresh signed download URL for one of this run's artifacts. */
|
|
300
344
|
async signArtifactUrl(artifactId, ttlSeconds) {
|
|
301
|
-
const res = await this.
|
|
345
|
+
const res = await this.controlFetch(this.url(`artifacts/${encodeURIComponent(artifactId)}/signed-url`), {
|
|
302
346
|
method: "POST",
|
|
303
347
|
headers: this.headers(true),
|
|
304
348
|
body: JSON.stringify({ ttlSeconds }),
|
|
@@ -310,7 +354,7 @@ export class RunnerControlClient {
|
|
|
310
354
|
/** Resolve an org secret the run's manifest allows (the program's `secrets.get`). The broker
|
|
311
355
|
* enforces the allowlist + returns the value; a forbidden/missing secret surfaces as a throw. */
|
|
312
356
|
async resolveSecret(name) {
|
|
313
|
-
const res = await this.
|
|
357
|
+
const res = await this.controlFetch(this.url("secrets/resolve"), {
|
|
314
358
|
method: "POST",
|
|
315
359
|
headers: this.headers(true),
|
|
316
360
|
body: JSON.stringify({ name }),
|
|
@@ -325,7 +369,7 @@ export class RunnerControlClient {
|
|
|
325
369
|
* A 403 (no active connection / non-allowlisted host) degrades to `{ accessToken: null, hint }` so
|
|
326
370
|
* the engine surfaces a clean failure instead of a thrown 500 mid-run; the token is never logged. */
|
|
327
371
|
async mcpToken(serverUrl, invalidateToken) {
|
|
328
|
-
const res = await this.
|
|
372
|
+
const res = await this.controlFetch(this.url("mcp/token"), {
|
|
329
373
|
method: "POST",
|
|
330
374
|
headers: this.headers(true),
|
|
331
375
|
body: JSON.stringify({
|
|
@@ -344,7 +388,7 @@ export class RunnerControlClient {
|
|
|
344
388
|
/** Proxy a web_search through the broker (which holds the Tavily key) — the runner sends the
|
|
345
389
|
* query, the broker calls Tavily and returns the results. */
|
|
346
390
|
async webSearch(input) {
|
|
347
|
-
const res = await this.
|
|
391
|
+
const res = await this.controlFetch(this.url("tools/web_search"), {
|
|
348
392
|
method: "POST",
|
|
349
393
|
headers: this.headers(true),
|
|
350
394
|
body: JSON.stringify(input),
|
|
@@ -355,7 +399,7 @@ export class RunnerControlClient {
|
|
|
355
399
|
}
|
|
356
400
|
/** Create (or idempotently re-attach to) a child run for `workflows.call`. */
|
|
357
401
|
async startChild(slug, input) {
|
|
358
|
-
const res = await this.
|
|
402
|
+
const res = await this.controlFetch(this.url("children"), {
|
|
359
403
|
method: "POST",
|
|
360
404
|
headers: this.headers(true),
|
|
361
405
|
body: JSON.stringify({ slug, input }),
|
|
@@ -367,7 +411,7 @@ export class RunnerControlClient {
|
|
|
367
411
|
}
|
|
368
412
|
/** Provision a durable schedule for `workflows.schedule`; returns the new schedule's id. */
|
|
369
413
|
async scheduleWorkflow(slug, input, spec) {
|
|
370
|
-
const res = await this.
|
|
414
|
+
const res = await this.controlFetch(this.url("schedules"), {
|
|
371
415
|
method: "POST",
|
|
372
416
|
headers: this.headers(true),
|
|
373
417
|
body: JSON.stringify({ slug, input, ...spec }),
|
|
@@ -378,7 +422,7 @@ export class RunnerControlClient {
|
|
|
378
422
|
}
|
|
379
423
|
/** Poll a child run's status/output, or null when it isn't this run's child (404). */
|
|
380
424
|
async getChild(childRunId) {
|
|
381
|
-
const res = await this.
|
|
425
|
+
const res = await this.controlFetch(this.url(`children/${encodeURIComponent(childRunId)}`), {
|
|
382
426
|
method: "GET",
|
|
383
427
|
headers: this.headers(false),
|
|
384
428
|
});
|
|
@@ -30,7 +30,7 @@ export declare class RuntimeFlusher {
|
|
|
30
30
|
private excludedMs;
|
|
31
31
|
constructor(deps: RuntimeFlusherDeps);
|
|
32
32
|
/** Book everything unbilled right now (the pre-freeze flush: suspended time must never appear as
|
|
33
|
-
* billed runtime, so the tail is booked BEFORE the snapshot —
|
|
33
|
+
* billed runtime, so the tail is booked BEFORE the snapshot — the suspension billing rule). */
|
|
34
34
|
flushNow(): Promise<void>;
|
|
35
35
|
/** Exclude `ms` of wall-clock from billing — the frozen window on a wake (computed from the wake's
|
|
36
36
|
* authoritative wall clock, since the guest's own clock was stopped). Never lets elapsed go
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
// RuntimeFlusher — meters a run's held-task runtime as periodic DELTAS, instead of one charge at
|
|
2
|
-
// terminal
|
|
2
|
+
// terminal. The heartbeat counterpart to the credit / cancel
|
|
3
3
|
// / lease watchers.
|
|
4
4
|
//
|
|
5
5
|
// Why: runtime used to be booked once, at terminal (`now - sessionStart`). That left two holes — a run
|
|
6
6
|
// that never terminates (a perpetual loop) NEVER billed its runtime (and the credit watcher, reading
|
|
7
|
-
//
|
|
7
|
+
// the billing balance, never saw the burn so it couldn't stop it); and a session that crashed before finalizing had
|
|
8
8
|
// its whole runtime lost. Metering on a timer closes both: runtime is billed as it accrues, the credit
|
|
9
9
|
// watcher sees it, and a crashed session keeps the deltas it already flushed (the fresh session bills
|
|
10
10
|
// its own — distinct ids sum, so the restart isn't double-charged or under-charged for the overlap-free
|
|
@@ -36,7 +36,7 @@ export class RuntimeFlusher {
|
|
|
36
36
|
this.deps = deps;
|
|
37
37
|
}
|
|
38
38
|
/** Book everything unbilled right now (the pre-freeze flush: suspended time must never appear as
|
|
39
|
-
* billed runtime, so the tail is booked BEFORE the snapshot —
|
|
39
|
+
* billed runtime, so the tail is booked BEFORE the snapshot — the suspension billing rule). */
|
|
40
40
|
async flushNow() {
|
|
41
41
|
await this.flush(false);
|
|
42
42
|
}
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
-
// Runtime support shims —
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// stdout, same call surface as the platform's Powertools child logger).
|
|
2
|
+
// Runtime support shims — a small, self-contained slice of the error taxonomy and logging the
|
|
3
|
+
// worker runtime depends on. The AppError taxonomy mirrors the platform's error codes so brokered
|
|
4
|
+
// responses map cleanly; the logger emits structured JSON to stdout.
|
|
6
5
|
import { randomUUID } from "node:crypto";
|
|
7
|
-
// ---- errors (
|
|
8
|
-
//
|
|
6
|
+
// ---- errors (the shared AppError taxonomy — mirrors the platform's error codes so brokered
|
|
7
|
+
// responses deserialize to the same shape) ----
|
|
9
8
|
export var ErrorCode;
|
|
10
9
|
(function (ErrorCode) {
|
|
11
10
|
ErrorCode["VALIDATION_FAILED"] = "VALIDATION_FAILED";
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// artifacts — persist + reference run outputs
|
|
1
|
+
// artifacts — persist + reference run outputs. Available to any agent
|
|
2
2
|
// (no sandbox required). Three operations:
|
|
3
3
|
// * write(name, content_type, body) → store a file under the run's prefix; returns id + a signed URL.
|
|
4
4
|
// * list() → artifacts produced in THIS run.
|
|
@@ -3,7 +3,7 @@ import type { AuthContext } from "../support/index.js";
|
|
|
3
3
|
import type { SecretRefManifest } from "../wire/manifest.js";
|
|
4
4
|
/**
|
|
5
5
|
* Per-invocation context the worker threads through to a tool. Equivalent to
|
|
6
|
-
*
|
|
6
|
+
* a tool framework's `ToolContext` but typed in Boardwalk's idiom (AuthContext + run id +
|
|
7
7
|
* secret resolver scoped to this run's permissions).
|
|
8
8
|
*/
|
|
9
9
|
export interface ToolContext {
|
|
@@ -27,7 +27,7 @@ export interface SecretResolver {
|
|
|
27
27
|
* * Return a `ToolReturn` body — synchronous "normal" tools (echo, http,
|
|
28
28
|
* web_search). The body lands in the conversation as the assistant's
|
|
29
29
|
* tool-result block.
|
|
30
|
-
* * Return a `ToolControlSignal` — the legacy
|
|
30
|
+
* * Return a `ToolControlSignal` — the legacy tool-level sleep/workflows.call path. The
|
|
31
31
|
* current JS-body worker exposes these as program SDK hooks instead; agent() leaves strip
|
|
32
32
|
* control-flow tools before registering model-callable tools.
|
|
33
33
|
*/
|
|
@@ -41,8 +41,8 @@ export interface BoardwalkTool<TInput = unknown, TOutput = unknown> {
|
|
|
41
41
|
/**
|
|
42
42
|
* Zod schema for the tool's SUCCESS output (the `TOutput` shape — never the
|
|
43
43
|
* control-signal branch). The adapter validates the tool's return value
|
|
44
|
-
* against this before it lands in the LLM conversation
|
|
45
|
-
*
|
|
44
|
+
* against this before it lands in the LLM conversation
|
|
45
|
+
* (LLM-facing output is treated like untrusted external input).
|
|
46
46
|
*/
|
|
47
47
|
readonly outputSchema: z.ZodType<TOutput>;
|
|
48
48
|
/**
|
|
@@ -64,7 +64,7 @@ export interface BoardwalkTool<TInput = unknown, TOutput = unknown> {
|
|
|
64
64
|
normalizeInput?(raw: unknown): unknown;
|
|
65
65
|
/**
|
|
66
66
|
* Invoke the tool. `input` (LLM-supplied) comes first, `ctx` (worker-supplied)
|
|
67
|
-
* second — see the `Tool` naming note in SPEC
|
|
67
|
+
* second — see the `Tool` naming note in SPEC.md. Returns either the typed
|
|
68
68
|
* `TOutput` body or a `ToolControlSignal` (sleep / workflows.call) the worker
|
|
69
69
|
* intercepts before serialization.
|
|
70
70
|
*/
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
// Boardwalk's internal Tool contract. Concrete tools (`echo`, `http`, `web_search`, `sleep`,
|
|
2
|
-
// `workflows.call`, …) implement this interface; the worker registers a
|
|
3
|
-
// adapter into
|
|
2
|
+
// `workflows.call`, …) implement this interface; the worker registers a schema-typed
|
|
3
|
+
// adapter into the agent leaf at construction time.
|
|
4
4
|
//
|
|
5
|
-
// Why a Boardwalk-side interface instead of
|
|
6
|
-
// * Tests can drive tool logic without instantiating
|
|
5
|
+
// Why a Boardwalk-side interface instead of the leaf's native tool type?
|
|
6
|
+
// * Tests can drive tool logic without instantiating the agent leaf.
|
|
7
7
|
// * Control-signal tools (`sleep`, `workflows.call`) need to return a
|
|
8
8
|
// specially-shaped value the worker interprets — they don't really "run";
|
|
9
|
-
// they bubble a signal up to the engine. Our adapter
|
|
10
|
-
//
|
|
9
|
+
// they bubble a signal up to the engine. Our adapter translates between
|
|
10
|
+
// this interface and the agent leaf's tool surface.
|
|
11
11
|
//
|
|
12
|
-
//
|
|
12
|
+
// Tools NEVER see secret values. Secrets resolve to
|
|
13
13
|
// short-lived bearer tokens, ARNs, etc. via the injected `SecretResolver`.
|
|
14
14
|
export function isControlSignal(value) {
|
|
15
15
|
return (typeof value === "object" &&
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
// web_search — wraps the Tavily API (https://docs.tavily.com/docs/api-reference).
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
// key fetched from
|
|
5
|
-
//
|
|
6
|
-
// conversation.
|
|
3
|
+
// The platform's default web-search provider. API
|
|
4
|
+
// key fetched from the org's secret store and cached at the module level per
|
|
5
|
+
// container, NEVER appearing in the agent's conversation.
|
|
7
6
|
//
|
|
8
|
-
// 429 from
|
|
9
|
-
// backoff. 5xx → `AppError(TOOL_ERROR)`. Network-level fetch failures fall
|
|
7
|
+
// 429 from the provider → `AppError(RATE_LIMIT)` so the agent loop's retry strategy
|
|
8
|
+
// handles backoff. 5xx → `AppError(TOOL_ERROR)`. Network-level fetch failures fall
|
|
10
9
|
// back to `TOOL_ERROR`.
|
|
11
10
|
import { z } from "zod";
|
|
12
11
|
import { AppError, ErrorCode } from "../support/index.js";
|