@boardwalk-labs/runner 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +12 -0
- package/dist/runtime/agent/budget.d.ts +5 -5
- package/dist/runtime/agent/budget.js +8 -8
- package/dist/runtime/agent/model_rates.js +4 -5
- package/dist/runtime/agent/secret_redactor.js +1 -1
- package/dist/runtime/cancel_watcher.js +1 -1
- package/dist/runtime/credit_watcher.d.ts +1 -1
- package/dist/runtime/credit_watcher.js +5 -5
- package/dist/runtime/freeze_coordinator.d.ts +124 -0
- package/dist/runtime/freeze_coordinator.js +276 -0
- package/dist/runtime/identity_relay.d.ts +45 -2
- package/dist/runtime/identity_relay.js +137 -14
- package/dist/runtime/index.d.ts +11 -6
- package/dist/runtime/index.js +119 -39
- package/dist/runtime/leaf_executor.d.ts +2 -2
- package/dist/runtime/program_runner.d.ts +1 -1
- package/dist/runtime/program_runner.js +1 -1
- package/dist/runtime/program_worker.d.ts +3 -3
- package/dist/runtime/program_worker.js +7 -7
- package/dist/runtime/recording_secret_resolver.js +1 -1
- package/dist/runtime/run_abort.js +3 -3
- package/dist/runtime/runner_control_client.d.ts +34 -4
- package/dist/runtime/runner_control_client.js +85 -33
- package/dist/runtime/runtime_flusher.d.ts +9 -0
- package/dist/runtime/runtime_flusher.js +17 -3
- package/dist/runtime/support/index.js +5 -6
- package/dist/runtime/tools/artifacts.js +1 -1
- package/dist/runtime/tools/types.d.ts +5 -5
- package/dist/runtime/tools/types.js +7 -7
- package/dist/runtime/tools/web_search.js +5 -6
- package/dist/runtime/uniqueness_reseed.d.ts +7 -0
- package/dist/runtime/uniqueness_reseed.js +65 -0
- package/dist/runtime/wire/artifact_storage.js +2 -2
- package/dist/runtime/wire/inference_proxy.d.ts +1 -1
- package/dist/runtime/wire/inference_proxy.js +2 -2
- package/dist/runtime/workflow_host.d.ts +50 -1
- package/dist/runtime/workflow_host.js +185 -31
- package/native/reseed.c +52 -0
- package/package.json +12 -5
- package/prebuilds/linux-x64/@boardwalk-labs+runner.node +0 -0
|
@@ -24,7 +24,7 @@ export interface ProgramVersionReader {
|
|
|
24
24
|
program: ProgramRef;
|
|
25
25
|
} | null>;
|
|
26
26
|
}
|
|
27
|
-
/** Books the run's RUNTIME usage as periodic deltas (the worker's RuntimeFlusher
|
|
27
|
+
/** Books the run's RUNTIME usage as periodic deltas (the worker's RuntimeFlusher). The
|
|
28
28
|
* orchestrator drives the lifecycle: the timer flushes mid-run, `stop()` halts it at the body's end,
|
|
29
29
|
* and `flushFinal()` books the tail on a clean terminal (skipped on a `lease_lost` handoff — the new
|
|
30
30
|
* owner books its own runtime). Replaces the old single terminal runtime charge. */
|
|
@@ -51,7 +51,7 @@ export interface RunFinalizer {
|
|
|
51
51
|
export interface RunSuspender {
|
|
52
52
|
suspend(signal: SuspendSignal, workerId: string): Promise<void>;
|
|
53
53
|
}
|
|
54
|
-
/** Restores/snapshots the workflow's persistent `/workspace
|
|
54
|
+
/** Restores/snapshots the workflow's persistent `/workspace`. Best-effort — both no-op when the
|
|
55
55
|
* run isn't eligible (not opted-in / self-hosted), and neither throws. */
|
|
56
56
|
export interface WorkspaceHandle {
|
|
57
57
|
hydrate(): Promise<void>;
|
|
@@ -155,7 +155,7 @@ export interface ProgramWorkerDeps {
|
|
|
155
155
|
/** Emit the program's `console.*` output as `log` run-events while the body runs (optional —
|
|
156
156
|
* absent disables capture). Wired by the entrypoint to the batched telemetry publisher. */
|
|
157
157
|
onProgramLog?: (stream: LogStream, text: string) => void;
|
|
158
|
-
/**
|
|
158
|
+
/** Task ARN (or any stable worker identity). */
|
|
159
159
|
workerId: string;
|
|
160
160
|
/** Drain any buffered telemetry before the worker exits (brokered path's BrokerEventPublisher).
|
|
161
161
|
* Called by the worker entrypoint's cleanup; the orchestrator itself never invokes it. */
|
|
@@ -11,12 +11,12 @@
|
|
|
11
11
|
// aborted mid-flight — credit exhaustion — finalize failed with the abort reason).
|
|
12
12
|
//
|
|
13
13
|
// While the program runs, two per-session loops watch it (both brokered): a UsageFlusher meters token
|
|
14
|
-
// deltas
|
|
14
|
+
// deltas and a CreditWatcher polls the org's funding. When credit hits zero the watcher
|
|
15
15
|
// aborts the run's AbortSignal, which the WorkflowHost honors at every hook (cooperative
|
|
16
16
|
// cancellation, run_abort.ts) — `signal.aborted` is authoritative for the terminal status.
|
|
17
17
|
//
|
|
18
18
|
// What's gone vs. the old worker: no checkpoint load/pause, no resume, no sleep/wait_for_child
|
|
19
|
-
// pause outcomes (the program holds), no per-turn transcript. The
|
|
19
|
+
// pause outcomes (the program holds), no per-turn transcript. The agent loop now lives one
|
|
20
20
|
// level down, behind the host's agent() leaf.
|
|
21
21
|
//
|
|
22
22
|
// v0 deferral (a clear seam): run-level outcome validation (needs program output capture).
|
|
@@ -130,16 +130,16 @@ export async function runProgramWorker(runId, deps) {
|
|
|
130
130
|
if (workspace !== undefined)
|
|
131
131
|
await workspace.hydrate();
|
|
132
132
|
// Token metering is PER-LEAF: each agent() leaf reports its own tokens + model to the broker, which
|
|
133
|
-
// decides `billed_by_boardwalk` per model + meters to
|
|
133
|
+
// decides `billed_by_boardwalk` per model + meters usage to the platform (see leaf_executor `meterUsage`). A
|
|
134
134
|
// workflow has no run-level model, so there is no run-level token metering here.
|
|
135
|
-
// Mid-run credit watching
|
|
135
|
+
// Mid-run credit watching: when the org runs out of credit, abort the run cooperatively.
|
|
136
136
|
const credit = deps.startCreditWatch?.({
|
|
137
137
|
run: claimed,
|
|
138
138
|
onExhausted: () => {
|
|
139
139
|
controller.abort(new RunAbortedError("credit_exhausted"));
|
|
140
140
|
},
|
|
141
141
|
});
|
|
142
|
-
// Mid-run user-cancel watching
|
|
142
|
+
// Mid-run user-cancel watching: when the user cancels, abort the run cooperatively. The host
|
|
143
143
|
// honors the abort at the next hook boundary (a `sleep` hold wakes immediately); the broker then
|
|
144
144
|
// upgrades the terminal write to `cancelled` because the run was flipped to `cancelling`.
|
|
145
145
|
const cancel = deps.startCancelWatch?.({
|
|
@@ -148,7 +148,7 @@ export async function runProgramWorker(runId, deps) {
|
|
|
148
148
|
controller.abort(new RunAbortedError("cancelled"));
|
|
149
149
|
},
|
|
150
150
|
});
|
|
151
|
-
// Lease renewal
|
|
151
|
+
// Lease renewal: heartbeat the lease so a long run isn't reclaimed mid-flight. If the lease is
|
|
152
152
|
// definitively lost (another worker reclaimed it), abort `lease_lost` — the run stops without
|
|
153
153
|
// finalizing (the new owner owns the terminal write; see the lease_lost guard after the body).
|
|
154
154
|
const lease = deps.startLeaseRenew?.({
|
|
@@ -157,7 +157,7 @@ export async function runProgramWorker(runId, deps) {
|
|
|
157
157
|
controller.abort(new RunAbortedError("lease_lost"));
|
|
158
158
|
},
|
|
159
159
|
});
|
|
160
|
-
// Runtime metering
|
|
160
|
+
// Runtime metering: flush runtime as periodic deltas from the claim, so a long/perpetual run
|
|
161
161
|
// bills as it burns (and the credit watcher sees it) instead of only at terminal. The tail is booked
|
|
162
162
|
// by `flushFinal()` after the body (on every path except a lease_lost handoff).
|
|
163
163
|
const runtimeFlush = deps.startRuntimeFlush?.({ run: claimed, startedAtMs: sessionStartMs });
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// RecordingSecretResolver — the redaction feeder
|
|
1
|
+
// RecordingSecretResolver — the redaction feeder.
|
|
2
2
|
//
|
|
3
3
|
// Decorates any SecretResolver so EVERY value a run resolves — whether the workflow program asked
|
|
4
4
|
// via `secrets.get(name)` or a tool asked via `ctx.secrets.resolve(ref)` — is recorded into the
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// run_abort — the provider-agnostic cooperative-cancellation substrate for a run (the Runner Credential Broker model).
|
|
2
2
|
//
|
|
3
|
-
// The cancellation primitive is a Web-standard `AbortSignal` — NOT a
|
|
3
|
+
// The cancellation primitive is a Web-standard `AbortSignal` — NOT a framework/model concept. The worker
|
|
4
4
|
// owns one `AbortController` per run session; a watcher (credit, later user-cancel) calls
|
|
5
5
|
// `controller.abort(new RunAbortedError(reason))`. The WorkflowHost honors that signal at every hook
|
|
6
6
|
// boundary (`agent`/`sleep`/`workflows.call`/…), unwinding the program. The ONLY place that translates
|
|
7
|
-
// the signal into a model-specific stop is the
|
|
8
|
-
//
|
|
7
|
+
// the signal into a model-specific stop is the agent leaf (signal → the leaf's cancel path); a different
|
|
8
|
+
// leaf implementation honors the SAME `AbortSignal` its own way. So nothing here, in the host, or in the
|
|
9
9
|
// program-facing SDK depends on the model backend.
|
|
10
10
|
//
|
|
11
11
|
// `signal.aborted` is AUTHORITATIVE at the orchestrator: a program that catches RunAbortedError and
|
|
@@ -13,6 +13,10 @@ export interface RunnerControlClientConfig {
|
|
|
13
13
|
runId: string;
|
|
14
14
|
/** Injected fetch (defaults to global fetch). */
|
|
15
15
|
fetchImpl?: typeof fetch;
|
|
16
|
+
/** Per-call ceiling for short control calls (default 30s) — bounds a poll frozen mid-flight. */
|
|
17
|
+
controlTimeoutMs?: number;
|
|
18
|
+
/** Per-call ceiling for bulk artifact/workspace transfers (default 5 min). */
|
|
19
|
+
bulkTimeoutMs?: number;
|
|
16
20
|
}
|
|
17
21
|
/** The pinned program's download reference (the worker fetches + verifies + extracts it). */
|
|
18
22
|
export interface BrokerProgram {
|
|
@@ -44,7 +48,27 @@ export declare class RunnerControlClient {
|
|
|
44
48
|
private readonly cfg;
|
|
45
49
|
private readonly base;
|
|
46
50
|
private readonly fetchImpl;
|
|
51
|
+
/** The live bearer. Mutable: on the snapshot substrate a wake carries a FRESH run token (the
|
|
52
|
+
* frozen one expired while suspended) and the worker swaps it at runtime. */
|
|
53
|
+
private runToken;
|
|
54
|
+
private readonly controlTimeoutMs;
|
|
55
|
+
private readonly bulkTimeoutMs;
|
|
47
56
|
constructor(cfg: RunnerControlClientConfig);
|
|
57
|
+
/** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
|
|
58
|
+
swapRunToken(token: string): void;
|
|
59
|
+
/**
|
|
60
|
+
* Every SHORT control call (claim / renew / cancel / credit / journal / …) goes through here so
|
|
61
|
+
* it carries a hard timeout. Without one, a poll frozen mid-flight on the snapshot substrate
|
|
62
|
+
* hangs FOREVER on restore (the socket is dead but never reset), and since a watcher serializes
|
|
63
|
+
* its ticks, one hung tick wedges that watcher — the dead-connections gotcha for the
|
|
64
|
+
* background pollers (lease/cancel/credit), which run on untracked timers the quiescence gate
|
|
65
|
+
* doesn't cover. Also plain robustness: no broker call should hang on a network blip. The
|
|
66
|
+
* streaming inference call is the ONE exception (long-lived NDJSON) and bypasses this.
|
|
67
|
+
*/
|
|
68
|
+
private controlFetch;
|
|
69
|
+
/** Bulk transfers (artifact + workspace up/download over presigned S3) — a much larger ceiling
|
|
70
|
+
* than a control call, but still bounded so a dead socket can't hang the run. */
|
|
71
|
+
private bulkFetch;
|
|
48
72
|
/** Claim the run's lease. Returns the run on success, or null when it isn't claimable (409 —
|
|
49
73
|
* another worker has it, or it isn't pending), which the worker treats as "claim lost". */
|
|
50
74
|
claim(workerId: string, leaseSeconds: number): Promise<{
|
|
@@ -84,8 +108,8 @@ export declare class RunnerControlClient {
|
|
|
84
108
|
* retried/duplicate flush idempotent; distinct per-flush ids sum into the run's runtime total. */
|
|
85
109
|
reportUsage(runtimeSeconds: number, identifier: string): Promise<void>;
|
|
86
110
|
/** Report a token-usage delta for incremental in-run metering (the usage flusher → broker). The
|
|
87
|
-
* broker gates on the run's per-connection `billed_by_boardwalk` server-side + meters to
|
|
88
|
-
* `identifier` makes a retried/duplicate flush idempotent. Satisfies {@link TokenUsageReporter}. */
|
|
111
|
+
* broker gates on the run's per-connection `billed_by_boardwalk` server-side + meters usage to the
|
|
112
|
+
* platform; `identifier` makes a retried/duplicate flush idempotent. Satisfies {@link TokenUsageReporter}. */
|
|
89
113
|
meterTokens(input: {
|
|
90
114
|
inputTokens: number;
|
|
91
115
|
outputTokens: number;
|
|
@@ -96,14 +120,20 @@ export declare class RunnerControlClient {
|
|
|
96
120
|
cachedWriteTokens?: number;
|
|
97
121
|
}): Promise<void>;
|
|
98
122
|
/** Check whether the run's org is still funded (the CreditWatcher → broker). The broker reads the
|
|
99
|
-
* live
|
|
123
|
+
* live billing balance server-side; `false` means out of credit (the watcher then aborts the run). */
|
|
100
124
|
checkCredit(): Promise<boolean>;
|
|
101
125
|
/** Check whether the run has been asked to cancel (the CancelWatcher → broker). `true` once the user
|
|
102
126
|
* cancelled the run (the broker flipped it to `cancelling`/`cancelled`); the watcher then aborts the
|
|
103
127
|
* run. Brokered because the runner holds no DB/Redis — this replaces the unreachable Redis channel. */
|
|
104
128
|
checkCancelled(): Promise<boolean>;
|
|
129
|
+
/** Register-without-release: register a HELD HITL gate's request row
|
|
130
|
+
* so it is answerable while the run keeps running — no suspend. Idempotent. Returns whether a new
|
|
131
|
+
* gate was registered. */
|
|
132
|
+
registerInput(seq: number, gate: unknown): Promise<boolean>;
|
|
133
|
+
/** Poll the resolved answers for a held gate at `seq` (empty until a human responds). */
|
|
134
|
+
pollInputAnswers(seq: number): Promise<Record<string, unknown>>;
|
|
105
135
|
/** Mint a presigned GET URL to restore this workflow's last `/workspace` snapshot (workspace
|
|
106
|
-
* persistence
|
|
136
|
+
* persistence). `null` when the run isn't eligible (not opted-in, or self-hosted). */
|
|
107
137
|
workspaceHydrateUrl(): Promise<string | null>;
|
|
108
138
|
/** Mint a presigned PUT URL to snapshot this workflow's `/workspace` (the worker uploads the tarball
|
|
109
139
|
* straight to S3). `null` when the run isn't eligible. `sizeBytes` is the archive's on-disk size
|
|
@@ -16,15 +16,44 @@ export class RunnerControlClient {
|
|
|
16
16
|
cfg;
|
|
17
17
|
base;
|
|
18
18
|
fetchImpl;
|
|
19
|
+
/** The live bearer. Mutable: on the snapshot substrate a wake carries a FRESH run token (the
|
|
20
|
+
* frozen one expired while suspended) and the worker swaps it at runtime. */
|
|
21
|
+
runToken;
|
|
22
|
+
controlTimeoutMs;
|
|
23
|
+
bulkTimeoutMs;
|
|
19
24
|
constructor(cfg) {
|
|
20
25
|
this.cfg = cfg;
|
|
21
26
|
this.base = cfg.baseUrl.replace(/\/+$/, "");
|
|
22
27
|
this.fetchImpl = cfg.fetchImpl ?? fetch;
|
|
28
|
+
this.runToken = cfg.runToken;
|
|
29
|
+
this.controlTimeoutMs = cfg.controlTimeoutMs ?? 30_000;
|
|
30
|
+
this.bulkTimeoutMs = cfg.bulkTimeoutMs ?? 300_000;
|
|
31
|
+
}
|
|
32
|
+
/** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
|
|
33
|
+
swapRunToken(token) {
|
|
34
|
+
this.runToken = token;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Every SHORT control call (claim / renew / cancel / credit / journal / …) goes through here so
|
|
38
|
+
* it carries a hard timeout. Without one, a poll frozen mid-flight on the snapshot substrate
|
|
39
|
+
* hangs FOREVER on restore (the socket is dead but never reset), and since a watcher serializes
|
|
40
|
+
* its ticks, one hung tick wedges that watcher — the dead-connections gotcha for the
|
|
41
|
+
* background pollers (lease/cancel/credit), which run on untracked timers the quiescence gate
|
|
42
|
+
* doesn't cover. Also plain robustness: no broker call should hang on a network blip. The
|
|
43
|
+
* streaming inference call is the ONE exception (long-lived NDJSON) and bypasses this.
|
|
44
|
+
*/
|
|
45
|
+
controlFetch(url, init) {
|
|
46
|
+
return this.fetchImpl(url, { ...init, signal: AbortSignal.timeout(this.controlTimeoutMs) });
|
|
47
|
+
}
|
|
48
|
+
/** Bulk transfers (artifact + workspace up/download over presigned S3) — a much larger ceiling
|
|
49
|
+
* than a control call, but still bounded so a dead socket can't hang the run. */
|
|
50
|
+
bulkFetch(url, init) {
|
|
51
|
+
return this.fetchImpl(url, { ...init, signal: AbortSignal.timeout(this.bulkTimeoutMs) });
|
|
23
52
|
}
|
|
24
53
|
/** Claim the run's lease. Returns the run on success, or null when it isn't claimable (409 —
|
|
25
54
|
* another worker has it, or it isn't pending), which the worker treats as "claim lost". */
|
|
26
55
|
async claim(workerId, leaseSeconds) {
|
|
27
|
-
const res = await this.
|
|
56
|
+
const res = await this.controlFetch(this.url("claim"), {
|
|
28
57
|
method: "POST",
|
|
29
58
|
headers: this.headers(true),
|
|
30
59
|
body: JSON.stringify({ workerId, leaseSeconds }),
|
|
@@ -46,7 +75,7 @@ export class RunnerControlClient {
|
|
|
46
75
|
* `leaseUntil`, or null when the lease was lost (409 — another worker reclaimed the run), which
|
|
47
76
|
* the worker treats as "stop". */
|
|
48
77
|
async renewLease(workerId, leaseSeconds) {
|
|
49
|
-
const res = await this.
|
|
78
|
+
const res = await this.controlFetch(this.url("renew"), {
|
|
50
79
|
method: "POST",
|
|
51
80
|
headers: this.headers(true),
|
|
52
81
|
body: JSON.stringify({ workerId, leaseSeconds }),
|
|
@@ -62,7 +91,7 @@ export class RunnerControlClient {
|
|
|
62
91
|
* whose lease expired and whose run was reclaimed + re-dispatched to a new owner), so a
|
|
63
92
|
* hung/partitioned worker that later recovers can't clobber the live run or revive a terminal one. */
|
|
64
93
|
async finalize(status, output, workerId) {
|
|
65
|
-
const res = await this.
|
|
94
|
+
const res = await this.controlFetch(this.url("finalize"), {
|
|
66
95
|
method: "POST",
|
|
67
96
|
headers: this.headers(true),
|
|
68
97
|
body: JSON.stringify({ status, output, workerId }),
|
|
@@ -73,7 +102,7 @@ export class RunnerControlClient {
|
|
|
73
102
|
/** Look up a durable-seam journal entry by its seq (the durable-suspension design), or null on a replay miss
|
|
74
103
|
* (404). The broker joins a parked agent leaf's answers into the result server-side. */
|
|
75
104
|
async journalGet(seq) {
|
|
76
|
-
const res = await this.
|
|
105
|
+
const res = await this.controlFetch(this.url(`journal/${encodeURIComponent(String(seq))}`), {
|
|
77
106
|
method: "GET",
|
|
78
107
|
headers: this.headers(false),
|
|
79
108
|
});
|
|
@@ -86,7 +115,7 @@ export class RunnerControlClient {
|
|
|
86
115
|
/** Record a RESOLVED seam result (idempotent on the run + seq server-side; a resolved entry is
|
|
87
116
|
* immutable). The broker writes the memoized value the next replay returns. */
|
|
88
117
|
async journalPut(entry) {
|
|
89
|
-
const res = await this.
|
|
118
|
+
const res = await this.controlFetch(this.url("journal"), {
|
|
90
119
|
method: "POST",
|
|
91
120
|
headers: this.headers(true),
|
|
92
121
|
body: JSON.stringify(entry),
|
|
@@ -98,7 +127,7 @@ export class RunnerControlClient {
|
|
|
98
127
|
* entry + a human-input request row for HITL, or the wake time for a long sleep), flips the run to
|
|
99
128
|
* its suspended status, and releases the lease — transactionally. No finalize; a wake re-dispatches. */
|
|
100
129
|
async suspend(signal, workerId) {
|
|
101
|
-
const res = await this.
|
|
130
|
+
const res = await this.controlFetch(this.url("suspend"), {
|
|
102
131
|
method: "POST",
|
|
103
132
|
headers: this.headers(true),
|
|
104
133
|
body: JSON.stringify({ ...signal, workerId }),
|
|
@@ -115,7 +144,7 @@ export class RunnerControlClient {
|
|
|
115
144
|
}
|
|
116
145
|
/** Fetch the run's pinned manifest + program source, or null when the version is missing (404). */
|
|
117
146
|
async getVersion() {
|
|
118
|
-
const res = await this.
|
|
147
|
+
const res = await this.controlFetch(this.url("version"), {
|
|
119
148
|
method: "GET",
|
|
120
149
|
headers: this.headers(false),
|
|
121
150
|
});
|
|
@@ -128,7 +157,7 @@ export class RunnerControlClient {
|
|
|
128
157
|
/** Book a runtime-seconds DELTA (the worker's RuntimeFlusher → broker). `identifier` makes a
|
|
129
158
|
* retried/duplicate flush idempotent; distinct per-flush ids sum into the run's runtime total. */
|
|
130
159
|
async reportUsage(runtimeSeconds, identifier) {
|
|
131
|
-
const res = await this.
|
|
160
|
+
const res = await this.controlFetch(this.url("usage"), {
|
|
132
161
|
method: "POST",
|
|
133
162
|
headers: this.headers(true),
|
|
134
163
|
body: JSON.stringify({ runtimeSeconds, identifier }),
|
|
@@ -137,10 +166,10 @@ export class RunnerControlClient {
|
|
|
137
166
|
throw await brokerError(res, "usage");
|
|
138
167
|
}
|
|
139
168
|
/** Report a token-usage delta for incremental in-run metering (the usage flusher → broker). The
|
|
140
|
-
* broker gates on the run's per-connection `billed_by_boardwalk` server-side + meters to
|
|
141
|
-
* `identifier` makes a retried/duplicate flush idempotent. Satisfies {@link TokenUsageReporter}. */
|
|
169
|
+
* broker gates on the run's per-connection `billed_by_boardwalk` server-side + meters usage to the
|
|
170
|
+
* platform; `identifier` makes a retried/duplicate flush idempotent. Satisfies {@link TokenUsageReporter}. */
|
|
142
171
|
async meterTokens(input) {
|
|
143
|
-
const res = await this.
|
|
172
|
+
const res = await this.controlFetch(this.url("usage/tokens"), {
|
|
144
173
|
method: "POST",
|
|
145
174
|
headers: this.headers(true),
|
|
146
175
|
body: JSON.stringify(input),
|
|
@@ -149,9 +178,9 @@ export class RunnerControlClient {
|
|
|
149
178
|
throw await brokerError(res, "usage/tokens");
|
|
150
179
|
}
|
|
151
180
|
/** Check whether the run's org is still funded (the CreditWatcher → broker). The broker reads the
|
|
152
|
-
* live
|
|
181
|
+
* live billing balance server-side; `false` means out of credit (the watcher then aborts the run). */
|
|
153
182
|
async checkCredit() {
|
|
154
|
-
const res = await this.
|
|
183
|
+
const res = await this.controlFetch(this.url("credit"), {
|
|
155
184
|
method: "GET",
|
|
156
185
|
headers: this.headers(false),
|
|
157
186
|
});
|
|
@@ -163,7 +192,7 @@ export class RunnerControlClient {
|
|
|
163
192
|
* cancelled the run (the broker flipped it to `cancelling`/`cancelled`); the watcher then aborts the
|
|
164
193
|
* run. Brokered because the runner holds no DB/Redis — this replaces the unreachable Redis channel. */
|
|
165
194
|
async checkCancelled() {
|
|
166
|
-
const res = await this.
|
|
195
|
+
const res = await this.controlFetch(this.url("cancel"), {
|
|
167
196
|
method: "GET",
|
|
168
197
|
headers: this.headers(false),
|
|
169
198
|
});
|
|
@@ -171,10 +200,33 @@ export class RunnerControlClient {
|
|
|
171
200
|
throw await brokerError(res, "cancel");
|
|
172
201
|
return (await res.json()).cancelRequested;
|
|
173
202
|
}
|
|
203
|
+
/** Register-without-release: register a HELD HITL gate's request row
|
|
204
|
+
* so it is answerable while the run keeps running — no suspend. Idempotent. Returns whether a new
|
|
205
|
+
* gate was registered. */
|
|
206
|
+
async registerInput(seq, gate) {
|
|
207
|
+
const res = await this.controlFetch(this.url("inputs"), {
|
|
208
|
+
method: "POST",
|
|
209
|
+
headers: this.headers(true),
|
|
210
|
+
body: JSON.stringify({ seq, humanInput: gate }),
|
|
211
|
+
});
|
|
212
|
+
if (res.status !== 200)
|
|
213
|
+
throw await brokerError(res, "register-input");
|
|
214
|
+
return (await res.json()).registered;
|
|
215
|
+
}
|
|
216
|
+
/** Poll the resolved answers for a held gate at `seq` (empty until a human responds). */
|
|
217
|
+
async pollInputAnswers(seq) {
|
|
218
|
+
const res = await this.controlFetch(this.url(`inputs/${encodeURIComponent(String(seq))}`), {
|
|
219
|
+
method: "GET",
|
|
220
|
+
headers: this.headers(false),
|
|
221
|
+
});
|
|
222
|
+
if (res.status !== 200)
|
|
223
|
+
throw await brokerError(res, "poll-inputs");
|
|
224
|
+
return (await res.json()).answers;
|
|
225
|
+
}
|
|
174
226
|
/** Mint a presigned GET URL to restore this workflow's last `/workspace` snapshot (workspace
|
|
175
|
-
* persistence
|
|
227
|
+
* persistence). `null` when the run isn't eligible (not opted-in, or self-hosted). */
|
|
176
228
|
async workspaceHydrateUrl() {
|
|
177
|
-
const res = await this.
|
|
229
|
+
const res = await this.controlFetch(this.url("workspace/hydrate-url"), {
|
|
178
230
|
method: "POST",
|
|
179
231
|
headers: this.headers(false),
|
|
180
232
|
});
|
|
@@ -187,7 +239,7 @@ export class RunnerControlClient {
|
|
|
187
239
|
* (the worker tars BEFORE requesting the URL) — the broker records it for the org storage counter +
|
|
188
240
|
* daily meter; the snapshot overwrites one per-workflow key, so it's the workflow's full footprint. */
|
|
189
241
|
async workspacePersistUrl(sizeBytes) {
|
|
190
|
-
const res = await this.
|
|
242
|
+
const res = await this.controlFetch(this.url("workspace/persist-url"), {
|
|
191
243
|
method: "POST",
|
|
192
244
|
headers: this.headers(true),
|
|
193
245
|
body: JSON.stringify({ sizeBytes }),
|
|
@@ -200,7 +252,7 @@ export class RunnerControlClient {
|
|
|
200
252
|
/** Download bytes from a presigned S3 URL (workspace hydrate). `null` on 404 (no snapshot yet —
|
|
201
253
|
* e.g. the workflow's first run); throws on any other non-2xx. Goes straight to S3, not the broker. */
|
|
202
254
|
async downloadBytes(url) {
|
|
203
|
-
const res = await this.
|
|
255
|
+
const res = await this.bulkFetch(url, { method: "GET" });
|
|
204
256
|
if (res.status === 404)
|
|
205
257
|
return null;
|
|
206
258
|
if (!res.ok)
|
|
@@ -211,7 +263,7 @@ export class RunnerControlClient {
|
|
|
211
263
|
* third-party-verifiable token (gated server-side on `permissions.id_token: "write"`) — used to
|
|
212
264
|
* federate into the org's OWN cloud (AWS/GCP). DIFFERENT from this client's run token. */
|
|
213
265
|
async requestOidcToken(audience) {
|
|
214
|
-
const res = await this.
|
|
266
|
+
const res = await this.controlFetch(this.url("oidc/token"), {
|
|
215
267
|
method: "POST",
|
|
216
268
|
headers: this.headers(true),
|
|
217
269
|
body: JSON.stringify({ audience }),
|
|
@@ -223,7 +275,7 @@ export class RunnerControlClient {
|
|
|
223
275
|
/** Publish a batch of live agent-event frames (the SSE live-tail source) — the broker publishes
|
|
224
276
|
* them to the run's Redis channel server-side, so the runner holds no Redis credential. */
|
|
225
277
|
async publishTelemetry(frames) {
|
|
226
|
-
const res = await this.
|
|
278
|
+
const res = await this.controlFetch(this.url("telemetry"), {
|
|
227
279
|
method: "POST",
|
|
228
280
|
headers: this.headers(true),
|
|
229
281
|
body: JSON.stringify({ frames }),
|
|
@@ -234,7 +286,7 @@ export class RunnerControlClient {
|
|
|
234
286
|
/** Store a run artifact through the broker (which holds the S3 credential + neutralizes the served
|
|
235
287
|
* content type server-side). Returns the catalog id + a signed download URL. */
|
|
236
288
|
async writeArtifact(input) {
|
|
237
|
-
const res = await this.
|
|
289
|
+
const res = await this.controlFetch(this.url("artifacts"), {
|
|
238
290
|
method: "POST",
|
|
239
291
|
headers: this.headers(true),
|
|
240
292
|
body: JSON.stringify(input),
|
|
@@ -247,7 +299,7 @@ export class RunnerControlClient {
|
|
|
247
299
|
* broker derives the S3 key + neutralizes/pins the served content type; it returns the upload URL +
|
|
248
300
|
* required headers + the `s3Key` to echo back at commit. No catalog row exists yet. */
|
|
249
301
|
async presignArtifact(input) {
|
|
250
|
-
const res = await this.
|
|
302
|
+
const res = await this.controlFetch(this.url("artifacts/presign"), {
|
|
251
303
|
method: "POST",
|
|
252
304
|
headers: this.headers(true),
|
|
253
305
|
body: JSON.stringify(input),
|
|
@@ -260,7 +312,7 @@ export class RunnerControlClient {
|
|
|
260
312
|
* presign response and MUST be sent verbatim — the content type is pinned into the signature, so
|
|
261
313
|
* S3 rejects a mismatch. This call goes straight to S3, not the broker. */
|
|
262
314
|
async uploadBytes(url, headers, body) {
|
|
263
|
-
const res = await this.
|
|
315
|
+
const res = await this.bulkFetch(url, { method: "PUT", headers, body });
|
|
264
316
|
if (!res.ok)
|
|
265
317
|
throw await brokerError(res, "artifacts-upload");
|
|
266
318
|
}
|
|
@@ -269,7 +321,7 @@ export class RunnerControlClient {
|
|
|
269
321
|
* broker re-validates the run prefix + re-neutralizes the content type, then returns the catalog id
|
|
270
322
|
* + a signed download URL. */
|
|
271
323
|
async commitArtifact(input) {
|
|
272
|
-
const res = await this.
|
|
324
|
+
const res = await this.controlFetch(this.url("artifacts/commit"), {
|
|
273
325
|
method: "POST",
|
|
274
326
|
headers: this.headers(true),
|
|
275
327
|
body: JSON.stringify(input),
|
|
@@ -280,7 +332,7 @@ export class RunnerControlClient {
|
|
|
280
332
|
}
|
|
281
333
|
/** List the artifacts this run has produced. */
|
|
282
334
|
async listArtifacts() {
|
|
283
|
-
const res = await this.
|
|
335
|
+
const res = await this.controlFetch(this.url("artifacts"), {
|
|
284
336
|
method: "GET",
|
|
285
337
|
headers: this.headers(false),
|
|
286
338
|
});
|
|
@@ -290,7 +342,7 @@ export class RunnerControlClient {
|
|
|
290
342
|
}
|
|
291
343
|
/** Mint a fresh signed download URL for one of this run's artifacts. */
|
|
292
344
|
async signArtifactUrl(artifactId, ttlSeconds) {
|
|
293
|
-
const res = await this.
|
|
345
|
+
const res = await this.controlFetch(this.url(`artifacts/${encodeURIComponent(artifactId)}/signed-url`), {
|
|
294
346
|
method: "POST",
|
|
295
347
|
headers: this.headers(true),
|
|
296
348
|
body: JSON.stringify({ ttlSeconds }),
|
|
@@ -302,7 +354,7 @@ export class RunnerControlClient {
|
|
|
302
354
|
/** Resolve an org secret the run's manifest allows (the program's `secrets.get`). The broker
|
|
303
355
|
* enforces the allowlist + returns the value; a forbidden/missing secret surfaces as a throw. */
|
|
304
356
|
async resolveSecret(name) {
|
|
305
|
-
const res = await this.
|
|
357
|
+
const res = await this.controlFetch(this.url("secrets/resolve"), {
|
|
306
358
|
method: "POST",
|
|
307
359
|
headers: this.headers(true),
|
|
308
360
|
body: JSON.stringify({ name }),
|
|
@@ -317,7 +369,7 @@ export class RunnerControlClient {
|
|
|
317
369
|
* A 403 (no active connection / non-allowlisted host) degrades to `{ accessToken: null, hint }` so
|
|
318
370
|
* the engine surfaces a clean failure instead of a thrown 500 mid-run; the token is never logged. */
|
|
319
371
|
async mcpToken(serverUrl, invalidateToken) {
|
|
320
|
-
const res = await this.
|
|
372
|
+
const res = await this.controlFetch(this.url("mcp/token"), {
|
|
321
373
|
method: "POST",
|
|
322
374
|
headers: this.headers(true),
|
|
323
375
|
body: JSON.stringify({
|
|
@@ -336,7 +388,7 @@ export class RunnerControlClient {
|
|
|
336
388
|
/** Proxy a web_search through the broker (which holds the Tavily key) — the runner sends the
|
|
337
389
|
* query, the broker calls Tavily and returns the results. */
|
|
338
390
|
async webSearch(input) {
|
|
339
|
-
const res = await this.
|
|
391
|
+
const res = await this.controlFetch(this.url("tools/web_search"), {
|
|
340
392
|
method: "POST",
|
|
341
393
|
headers: this.headers(true),
|
|
342
394
|
body: JSON.stringify(input),
|
|
@@ -347,7 +399,7 @@ export class RunnerControlClient {
|
|
|
347
399
|
}
|
|
348
400
|
/** Create (or idempotently re-attach to) a child run for `workflows.call`. */
|
|
349
401
|
async startChild(slug, input) {
|
|
350
|
-
const res = await this.
|
|
402
|
+
const res = await this.controlFetch(this.url("children"), {
|
|
351
403
|
method: "POST",
|
|
352
404
|
headers: this.headers(true),
|
|
353
405
|
body: JSON.stringify({ slug, input }),
|
|
@@ -359,7 +411,7 @@ export class RunnerControlClient {
|
|
|
359
411
|
}
|
|
360
412
|
/** Provision a durable schedule for `workflows.schedule`; returns the new schedule's id. */
|
|
361
413
|
async scheduleWorkflow(slug, input, spec) {
|
|
362
|
-
const res = await this.
|
|
414
|
+
const res = await this.controlFetch(this.url("schedules"), {
|
|
363
415
|
method: "POST",
|
|
364
416
|
headers: this.headers(true),
|
|
365
417
|
body: JSON.stringify({ slug, input, ...spec }),
|
|
@@ -370,7 +422,7 @@ export class RunnerControlClient {
|
|
|
370
422
|
}
|
|
371
423
|
/** Poll a child run's status/output, or null when it isn't this run's child (404). */
|
|
372
424
|
async getChild(childRunId) {
|
|
373
|
-
const res = await this.
|
|
425
|
+
const res = await this.controlFetch(this.url(`children/${encodeURIComponent(childRunId)}`), {
|
|
374
426
|
method: "GET",
|
|
375
427
|
headers: this.headers(false),
|
|
376
428
|
});
|
|
@@ -409,7 +461,7 @@ export class RunnerControlClient {
|
|
|
409
461
|
}
|
|
410
462
|
headers(json) {
|
|
411
463
|
const h = {
|
|
412
|
-
authorization: `Bearer ${this.
|
|
464
|
+
authorization: `Bearer ${this.runToken}`,
|
|
413
465
|
accept: "application/json",
|
|
414
466
|
};
|
|
415
467
|
if (json)
|
|
@@ -26,7 +26,16 @@ export declare class RuntimeFlusher {
|
|
|
26
26
|
private flushedSeconds;
|
|
27
27
|
/** Per-flush sequence — advances only on a successful flush, so a retry reuses the same id. */
|
|
28
28
|
private seq;
|
|
29
|
+
/** Wall-clock ms excluded from billing — frozen (suspended) time on the snapshot substrate. */
|
|
30
|
+
private excludedMs;
|
|
29
31
|
constructor(deps: RuntimeFlusherDeps);
|
|
32
|
+
/** Book everything unbilled right now (the pre-freeze flush: suspended time must never appear as
|
|
33
|
+
* billed runtime, so the tail is booked BEFORE the snapshot — the suspension billing rule). */
|
|
34
|
+
flushNow(): Promise<void>;
|
|
35
|
+
/** Exclude `ms` of wall-clock from billing — the frozen window on a wake (computed from the wake's
|
|
36
|
+
* authoritative wall clock, since the guest's own clock was stopped). Never lets elapsed go
|
|
37
|
+
* negative. */
|
|
38
|
+
excludeIdle(ms: number): void;
|
|
30
39
|
/** Begin periodic delta flushing. */
|
|
31
40
|
start(): void;
|
|
32
41
|
/** Stop the periodic timer and drain any in-flight flush. Does NOT book the final tail — call
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
// RuntimeFlusher — meters a run's held-task runtime as periodic DELTAS, instead of one charge at
|
|
2
|
-
// terminal
|
|
2
|
+
// terminal. The heartbeat counterpart to the credit / cancel
|
|
3
3
|
// / lease watchers.
|
|
4
4
|
//
|
|
5
5
|
// Why: runtime used to be booked once, at terminal (`now - sessionStart`). That left two holes — a run
|
|
6
6
|
// that never terminates (a perpetual loop) NEVER billed its runtime (and the credit watcher, reading
|
|
7
|
-
//
|
|
7
|
+
// the billing balance, never saw the burn so it couldn't stop it); and a session that crashed before finalizing had
|
|
8
8
|
// its whole runtime lost. Metering on a timer closes both: runtime is billed as it accrues, the credit
|
|
9
9
|
// watcher sees it, and a crashed session keeps the deltas it already flushed (the fresh session bills
|
|
10
10
|
// its own — distinct ids sum, so the restart isn't double-charged or under-charged for the overlap-free
|
|
@@ -30,9 +30,23 @@ export class RuntimeFlusher {
|
|
|
30
30
|
flushedSeconds = 0;
|
|
31
31
|
/** Per-flush sequence — advances only on a successful flush, so a retry reuses the same id. */
|
|
32
32
|
seq = 0;
|
|
33
|
+
/** Wall-clock ms excluded from billing — frozen (suspended) time on the snapshot substrate. */
|
|
34
|
+
excludedMs = 0;
|
|
33
35
|
constructor(deps) {
|
|
34
36
|
this.deps = deps;
|
|
35
37
|
}
|
|
38
|
+
/** Book everything unbilled right now (the pre-freeze flush: suspended time must never appear as
|
|
39
|
+
* billed runtime, so the tail is booked BEFORE the snapshot — the suspension billing rule). */
|
|
40
|
+
async flushNow() {
|
|
41
|
+
await this.flush(false);
|
|
42
|
+
}
|
|
43
|
+
/** Exclude `ms` of wall-clock from billing — the frozen window on a wake (computed from the wake's
|
|
44
|
+
* authoritative wall clock, since the guest's own clock was stopped). Never lets elapsed go
|
|
45
|
+
* negative. */
|
|
46
|
+
excludeIdle(ms) {
|
|
47
|
+
if (ms > 0)
|
|
48
|
+
this.excludedMs += ms;
|
|
49
|
+
}
|
|
36
50
|
/** Begin periodic delta flushing. */
|
|
37
51
|
start() {
|
|
38
52
|
if (this.timer !== null)
|
|
@@ -67,7 +81,7 @@ export class RuntimeFlusher {
|
|
|
67
81
|
// vCPU-seconds = wall-clock seconds × vCPUs (billed per vCPU-second). Rounding the cumulative
|
|
68
82
|
// product (not per-delta) keeps the booked total aligned with wall-clock×vcpus over many flushes.
|
|
69
83
|
const vcpus = this.deps.vcpus ?? 1;
|
|
70
|
-
const total = Math.max(0, Math.round(((this.deps.now() - this.deps.startedAtMs) / 1000) * vcpus));
|
|
84
|
+
const total = Math.max(0, Math.round(((this.deps.now() - this.deps.startedAtMs - this.excludedMs) / 1000) * vcpus));
|
|
71
85
|
const delta = total - this.flushedSeconds;
|
|
72
86
|
if (delta < 1)
|
|
73
87
|
return;
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
-
// Runtime support shims —
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// stdout, same call surface as the platform's Powertools child logger).
|
|
2
|
+
// Runtime support shims — a small, self-contained slice of the error taxonomy and logging the
|
|
3
|
+
// worker runtime depends on. The AppError taxonomy mirrors the platform's error codes so brokered
|
|
4
|
+
// responses map cleanly; the logger emits structured JSON to stdout.
|
|
6
5
|
import { randomUUID } from "node:crypto";
|
|
7
|
-
// ---- errors (
|
|
8
|
-
//
|
|
6
|
+
// ---- errors (the shared AppError taxonomy — mirrors the platform's error codes so brokered
|
|
7
|
+
// responses deserialize to the same shape) ----
|
|
9
8
|
export var ErrorCode;
|
|
10
9
|
(function (ErrorCode) {
|
|
11
10
|
ErrorCode["VALIDATION_FAILED"] = "VALIDATION_FAILED";
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// artifacts — persist + reference run outputs
|
|
1
|
+
// artifacts — persist + reference run outputs. Available to any agent
|
|
2
2
|
// (no sandbox required). Three operations:
|
|
3
3
|
// * write(name, content_type, body) → store a file under the run's prefix; returns id + a signed URL.
|
|
4
4
|
// * list() → artifacts produced in THIS run.
|
|
@@ -3,7 +3,7 @@ import type { AuthContext } from "../support/index.js";
|
|
|
3
3
|
import type { SecretRefManifest } from "../wire/manifest.js";
|
|
4
4
|
/**
|
|
5
5
|
* Per-invocation context the worker threads through to a tool. Equivalent to
|
|
6
|
-
*
|
|
6
|
+
* a tool framework's `ToolContext` but typed in Boardwalk's idiom (AuthContext + run id +
|
|
7
7
|
* secret resolver scoped to this run's permissions).
|
|
8
8
|
*/
|
|
9
9
|
export interface ToolContext {
|
|
@@ -27,7 +27,7 @@ export interface SecretResolver {
|
|
|
27
27
|
* * Return a `ToolReturn` body — synchronous "normal" tools (echo, http,
|
|
28
28
|
* web_search). The body lands in the conversation as the assistant's
|
|
29
29
|
* tool-result block.
|
|
30
|
-
* * Return a `ToolControlSignal` — the legacy
|
|
30
|
+
* * Return a `ToolControlSignal` — the legacy tool-level sleep/workflows.call path. The
|
|
31
31
|
* current JS-body worker exposes these as program SDK hooks instead; agent() leaves strip
|
|
32
32
|
* control-flow tools before registering model-callable tools.
|
|
33
33
|
*/
|
|
@@ -41,8 +41,8 @@ export interface BoardwalkTool<TInput = unknown, TOutput = unknown> {
|
|
|
41
41
|
/**
|
|
42
42
|
* Zod schema for the tool's SUCCESS output (the `TOutput` shape — never the
|
|
43
43
|
* control-signal branch). The adapter validates the tool's return value
|
|
44
|
-
* against this before it lands in the LLM conversation
|
|
45
|
-
*
|
|
44
|
+
* against this before it lands in the LLM conversation
|
|
45
|
+
* (LLM-facing output is treated like untrusted external input).
|
|
46
46
|
*/
|
|
47
47
|
readonly outputSchema: z.ZodType<TOutput>;
|
|
48
48
|
/**
|
|
@@ -64,7 +64,7 @@ export interface BoardwalkTool<TInput = unknown, TOutput = unknown> {
|
|
|
64
64
|
normalizeInput?(raw: unknown): unknown;
|
|
65
65
|
/**
|
|
66
66
|
* Invoke the tool. `input` (LLM-supplied) comes first, `ctx` (worker-supplied)
|
|
67
|
-
* second — see the `Tool` naming note in SPEC
|
|
67
|
+
* second — see the `Tool` naming note in SPEC.md. Returns either the typed
|
|
68
68
|
* `TOutput` body or a `ToolControlSignal` (sleep / workflows.call) the worker
|
|
69
69
|
* intercepts before serialization.
|
|
70
70
|
*/
|