@boardwalk-labs/runner 0.1.21 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/runtime/index.d.ts +1 -1
- package/dist/runtime/index.js +8 -40
- package/dist/runtime/program_runner.d.ts +4 -15
- package/dist/runtime/program_runner.js +6 -21
- package/dist/runtime/program_worker.d.ts +0 -18
- package/dist/runtime/program_worker.js +1 -20
- package/dist/runtime/runner_control_client.d.ts +2 -22
- package/dist/runtime/runner_control_client.js +2 -49
- package/dist/runtime/suspension.d.ts +12 -107
- package/dist/runtime/suspension.js +18 -111
- package/dist/runtime/wire/human_input.d.ts +1 -1
- package/dist/runtime/wire/human_input.js +2 -2
- package/dist/runtime/workflow_host.d.ts +42 -60
- package/dist/runtime/workflow_host.js +89 -199
- package/package.json +3 -3
package/dist/runtime/index.d.ts
CHANGED
|
@@ -32,7 +32,7 @@ export interface WorkerRuntime {
|
|
|
32
32
|
vcpus: number;
|
|
33
33
|
/** The in-guest identity relay, present ONLY on the snapshot-based microVM substrate (relay-mode
|
|
34
34
|
* boot). When set, the worker suspends by FREEZING — the FreezeCoordinator parks seams over this
|
|
35
|
-
* relay's suspend/wake channel —
|
|
35
|
+
* relay's suspend/wake channel — the whole VM freezes and the wake resolves seams in place. */
|
|
36
36
|
freezeRelay?: IdentityRelay;
|
|
37
37
|
/** The browser tier's process backend, present ONLY when the runner IMAGE ships the browser stack
|
|
38
38
|
* (Chromium + a pre-installed Playwright MCP + an X display; gated by BOARDWALK_BROWSER_TIER).
|
package/dist/runtime/index.js
CHANGED
|
@@ -99,10 +99,6 @@ export function assembleWorkerDeps(runtime) {
|
|
|
99
99
|
runId: runtime.runId,
|
|
100
100
|
});
|
|
101
101
|
log.info("runner_control_enabled", { runId: runtime.runId });
|
|
102
|
-
// The replay frontier (the highest journaled seq at claim) drives SILENT REPLAY (the durable-suspension design):
|
|
103
|
-
// a resumed run suppresses observability for seams it already ran. Captured at claim, read by
|
|
104
|
-
// buildHost (which runs right after claim) when constructing the per-run host.
|
|
105
|
-
let replayFrontier = 0;
|
|
106
102
|
// Snapshot-substrate suspension (relay-mode boot only): one coordinator for the process, wired to
|
|
107
103
|
// the relay's suspend/wake channel now; its per-run hooks (token swap, meter rebase, workspace
|
|
108
104
|
// persist) late-bind in buildHost/startRuntimeFlush once those objects exist. The circular
|
|
@@ -304,13 +300,6 @@ export function assembleWorkerDeps(runtime) {
|
|
|
304
300
|
workspaceRoot: runtime.workspaceRoot,
|
|
305
301
|
})
|
|
306
302
|
: undefined;
|
|
307
|
-
// Durable suspension (the durable-suspension design): the host raises a suspend OUT OF BAND (onSuspend +
|
|
308
|
-
// a never-resolving seam), so a program's own try/catch can't swallow it; the program runner
|
|
309
|
-
// races `suspendSignal` against the body. One deferred per run — the first seam to suspend wins.
|
|
310
|
-
let resolveSuspend = () => undefined;
|
|
311
|
-
const suspendSignal = new Promise((resolve) => {
|
|
312
|
-
resolveSuspend = resolve;
|
|
313
|
-
});
|
|
314
303
|
// The run's identity + on-demand public-API bearer, surfaced to the program via `import { runtime }`.
|
|
315
304
|
// ids come from the claimed run; the bearer is the captured (scrubbed-from-env) api token, already
|
|
316
305
|
// recorded in this run's redactor above — so threading it into an MCP header keeps it out of LLM
|
|
@@ -341,13 +330,6 @@ export function assembleWorkerDeps(runtime) {
|
|
|
341
330
|
runtime: runtimeContext,
|
|
342
331
|
// workflows.call → broker /children (resolve + callable_by gate server-side), bound to THIS run.
|
|
343
332
|
children: new BrokerChildDispatcher({ client: broker }),
|
|
344
|
-
// The durable-seam journal (memoization + replay) over the broker; the replay frontier drives
|
|
345
|
-
// silent replay; onSuspend resolves the deferred the program runner races.
|
|
346
|
-
journal: broker.journalSeam(),
|
|
347
|
-
replayFrontier,
|
|
348
|
-
onSuspend: (signal) => {
|
|
349
|
-
resolveSuspend(signal);
|
|
350
|
-
},
|
|
351
333
|
// The program's secrets.get(name) → the run's fail-closed RECORDING resolver.
|
|
352
334
|
secrets: { get: (name) => secretResolver.resolve({ name }) },
|
|
353
335
|
// events.emit → broker /events (fan-out server-side).
|
|
@@ -373,19 +355,15 @@ export function assembleWorkerDeps(runtime) {
|
|
|
373
355
|
},
|
|
374
356
|
}
|
|
375
357
|
: {}),
|
|
376
|
-
// Snapshot-substrate suspension: seams freeze in place under the quiescence gate
|
|
377
|
-
//
|
|
358
|
+
// Snapshot-substrate suspension: seams freeze in place under the quiescence gate. Absent
|
|
359
|
+
// (a self-hosted daemon / the Fargate break-glass), waiting seams HOLD the live process.
|
|
378
360
|
...(freeze !== undefined ? { freeze } : {}),
|
|
379
|
-
//
|
|
380
|
-
// freeze substrate,
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
poll: (seq) => broker.pollInputAnswers(seq),
|
|
386
|
-
},
|
|
387
|
-
}
|
|
388
|
-
: {}),
|
|
361
|
+
// Held HITL gates, backed by the broker's inputs endpoints: register-without-release on the
|
|
362
|
+
// freeze substrate, and the WHOLE gate mechanism (register + poll) on the hold path.
|
|
363
|
+
heldInput: {
|
|
364
|
+
register: (seq, gate) => broker.registerInput(seq, gate),
|
|
365
|
+
poll: (seq) => broker.pollInputAnswers(seq),
|
|
366
|
+
},
|
|
389
367
|
// Browser tier: `computer.openBrowser()` + `agent({ session })` resolve through this manager
|
|
390
368
|
// (absent ⇒ the host throws "not available on this runner image").
|
|
391
369
|
...(browserSessions !== undefined ? { browserSessions } : {}),
|
|
@@ -463,8 +441,6 @@ export function assembleWorkerDeps(runtime) {
|
|
|
463
441
|
setProgramDir: (dir) => {
|
|
464
442
|
programDir = dir;
|
|
465
443
|
},
|
|
466
|
-
// Resolves when a host seam suspends the run; the program runner races it against the body.
|
|
467
|
-
suspendSignal,
|
|
468
444
|
...(workspaceStore !== undefined ? { workspace: workspaceStore } : {}),
|
|
469
445
|
// The orchestrator reaps every still-open browser session on terminal (kill Chromium + its
|
|
470
446
|
// Playwright MCP) so no browser process leaks past the run.
|
|
@@ -483,8 +459,6 @@ export function assembleWorkerDeps(runtime) {
|
|
|
483
459
|
// Order this session's frames after a previous (crashed) session's, then announce the
|
|
484
460
|
// lifecycle transition the claim IS — the wire's `running` frame.
|
|
485
461
|
runEvents.resumeAfter(claimed.lastEventCursor);
|
|
486
|
-
// The replay frontier for a resumed run (0 on a fresh run); buildHost reads it next.
|
|
487
|
-
replayFrontier = claimed.lastJournalSeq;
|
|
488
462
|
runEvents.emit({ kind: "run_status", status: "running" });
|
|
489
463
|
return claimed.run;
|
|
490
464
|
},
|
|
@@ -513,12 +487,6 @@ export function assembleWorkerDeps(runtime) {
|
|
|
513
487
|
finalizer: {
|
|
514
488
|
finalize: (_id, status, output) => broker.finalize(status, output, runtime.workerId),
|
|
515
489
|
},
|
|
516
|
-
// Durable suspension (the durable-suspension design): persist the wake condition through the broker (journal
|
|
517
|
-
// entry + a HITL request row, or the wake time for a long sleep) and release the lease — no
|
|
518
|
-
// finalize. A wake (an answer or a timer) re-dispatches the run.
|
|
519
|
-
suspender: {
|
|
520
|
-
suspend: (signal, workerId) => broker.suspend(signal, workerId),
|
|
521
|
-
},
|
|
522
490
|
// Runtime metering: flush runtime as periodic deltas (+ a terminal tail) through the broker,
|
|
523
491
|
// idempotent per flush, so a long/perpetual run bills as it burns and the credit watcher sees it —
|
|
524
492
|
// not a single charge at terminal (which a never-terminating run never reached). A fresh per-session
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { WorkflowHost } from "@boardwalk-labs/workflow/runtime";
|
|
2
|
-
import { type SuspendSignal } from "./suspension.js";
|
|
3
2
|
/**
|
|
4
3
|
* Link THIS runtime's `@boardwalk-labs/workflow` into the exec dir's `node_modules` so the
|
|
5
4
|
* program's bare import resolves anywhere (a self-hosted daemon workspace has no ancestor
|
|
@@ -63,18 +62,11 @@ export interface ProgramRunnerDeps {
|
|
|
63
62
|
* throw (a telemetry hiccup can't change the run's result). Absent ⇒ no output entry.
|
|
64
63
|
*/
|
|
65
64
|
onOutput?: (value: unknown) => void;
|
|
66
|
-
/**
|
|
67
|
-
* Resolves when a host seam SUSPENDS the run (the durable-suspension design) — the worker wires it to the
|
|
68
|
-
* host's `onSuspend` so a suspend is surfaced OUT OF BAND, racing the program body. The program's
|
|
69
|
-
* own `try/catch` can't swallow a suspend this way (the suspending seam never resolves; this signal
|
|
70
|
-
* short-circuits at the runner). Absent ⇒ no suspension wired (a seam that suspends throws
|
|
71
|
-
* {@link SuspendError} instead, which is caught here just the same).
|
|
72
|
-
*/
|
|
73
|
-
suspendSignal?: Promise<SuspendSignal>;
|
|
74
65
|
}
|
|
75
|
-
/** Terminal
|
|
76
|
-
*
|
|
77
|
-
*
|
|
66
|
+
/** Terminal result of running a workflow program. `output` is what the program declared via
|
|
67
|
+
* `output(value)` (null when it never did); for a failure it's null and `error` is set. A waiting
|
|
68
|
+
* seam (sleep / humanInput / workflows.call) never surfaces here — it freezes with the VM or
|
|
69
|
+
* holds the process, and the body simply continues when the wait is over. */
|
|
78
70
|
export type ProgramResult = {
|
|
79
71
|
kind: "completed";
|
|
80
72
|
output: unknown;
|
|
@@ -85,9 +77,6 @@ export type ProgramResult = {
|
|
|
85
77
|
code: string;
|
|
86
78
|
message: string;
|
|
87
79
|
};
|
|
88
|
-
} | {
|
|
89
|
-
kind: "suspended";
|
|
90
|
-
signal: SuspendSignal;
|
|
91
80
|
};
|
|
92
81
|
/**
|
|
93
82
|
* Run a workflow program to completion. Installs the host + input, extracts the VERIFIED artifact +
|
|
@@ -27,7 +27,6 @@ import { pathToFileURL } from "node:url";
|
|
|
27
27
|
import { randomUUID } from "node:crypto";
|
|
28
28
|
import { installHost, installInput, installConfig, takeDeclaredOutput, resetRuntime, } from "@boardwalk-labs/workflow/runtime";
|
|
29
29
|
import { AppError, ErrorCode, createLogger } from "./support/index.js";
|
|
30
|
-
import { SuspendError } from "./suspension.js";
|
|
31
30
|
const log = createLogger("ProgramRunner");
|
|
32
31
|
/** Subdirectory (under the work root) that holds transient extracted program trees. */
|
|
33
32
|
const RUN_DIR = ".bw-runs";
|
|
@@ -112,25 +111,11 @@ export async function runWorkflowProgram(args, deps) {
|
|
|
112
111
|
// runner may point at an arbitrary control plane, so refuse an entry that could escape the
|
|
113
112
|
// extraction dir (absolute path or `..` segment) before importing it.
|
|
114
113
|
const entryPath = resolveEntryPath(dir, args.entry);
|
|
115
|
-
// Run the program body
|
|
116
|
-
//
|
|
117
|
-
|
|
118
|
-
// no-`onSuspend` path. Everything else is the program's natural completion / failure.
|
|
119
|
-
const body = runProgramBody(entryPath, deps);
|
|
120
|
-
const result = deps.suspendSignal === undefined
|
|
121
|
-
? await body
|
|
122
|
-
: await Promise.race([
|
|
123
|
-
body,
|
|
124
|
-
deps.suspendSignal.then((signal) => ({ kind: "suspended", signal })),
|
|
125
|
-
]);
|
|
126
|
-
// If the suspend won the race, the body promise is abandoned (its suspending seam never settles);
|
|
127
|
-
// swallow any late settle so it can't surface as an unhandled rejection before the process exits.
|
|
128
|
-
void body.catch(() => undefined);
|
|
129
|
-
return result;
|
|
114
|
+
// Run the program body to its natural completion / failure. A waiting seam freezes with the
|
|
115
|
+
// VM (snapshot substrate) or holds the process — either way the body's own await continues.
|
|
116
|
+
return await runProgramBody(entryPath, deps);
|
|
130
117
|
}
|
|
131
118
|
catch (err) {
|
|
132
|
-
if (err instanceof SuspendError)
|
|
133
|
-
return { kind: "suspended", signal: err.signal };
|
|
134
119
|
const redactText = deps.redactText ?? ((s) => s);
|
|
135
120
|
// Redact BEFORE both sinks: the message can carry a secret the program resolved then threw.
|
|
136
121
|
const message = redactText(err instanceof Error ? err.message : String(err));
|
|
@@ -157,9 +142,9 @@ export async function runWorkflowProgram(args, deps) {
|
|
|
157
142
|
}
|
|
158
143
|
/**
|
|
159
144
|
* Import (= run) the program entry and capture its declared output. Resolves to a `completed`
|
|
160
|
-
* result; a program failure
|
|
161
|
-
*
|
|
162
|
-
*
|
|
145
|
+
* result; a program failure THROWS and is handled by the caller. A unique dir per run gives a
|
|
146
|
+
* fresh URL so the module cache never returns an already-run program; `@vite-ignore` keeps
|
|
147
|
+
* vitest/vite from statically analyzing the runtime URL.
|
|
163
148
|
*/
|
|
164
149
|
async function runProgramBody(entryPath, deps) {
|
|
165
150
|
await import(/* @vite-ignore */ pathToFileURL(entryPath).href);
|
|
@@ -3,7 +3,6 @@ import { type WorkflowManifest } from "./wire/manifest.js";
|
|
|
3
3
|
import type { WorkflowHost } from "@boardwalk-labs/workflow/runtime";
|
|
4
4
|
import type { SecretRedactor } from "./agent/secret_redactor.js";
|
|
5
5
|
import { type LogStream } from "./program_log_capture.js";
|
|
6
|
-
import type { SuspendSignal } from "./suspension.js";
|
|
7
6
|
/** Default 5-minute lease (matches the engine spec). */
|
|
8
7
|
export declare const DEFAULT_LEASE_MS: number;
|
|
9
8
|
/** Race-safe claim surface — RunRepository satisfies it. */
|
|
@@ -44,13 +43,6 @@ export type RuntimeMeterStarter = (args: {
|
|
|
44
43
|
export interface RunFinalizer {
|
|
45
44
|
finalize(runId: string, status: "completed" | "failed", output: unknown): Promise<void>;
|
|
46
45
|
}
|
|
47
|
-
/** Persists a durable SUSPENSION (the durable-suspension design): the broker records the wake condition (a
|
|
48
|
-
* pending/suspended journal entry + a human-input request row for HITL, or the wake time for a long
|
|
49
|
-
* sleep), flips the run to its suspended status, and releases the lease — all transactionally. The
|
|
50
|
-
* run is NOT finalized; a wake (an answer, a child finalize, or a timer) re-dispatches it. */
|
|
51
|
-
export interface RunSuspender {
|
|
52
|
-
suspend(signal: SuspendSignal, workerId: string): Promise<void>;
|
|
53
|
-
}
|
|
54
46
|
/** Restores/snapshots the workflow's persistent `/workspace`. Best-effort — both no-op when the
|
|
55
47
|
* run isn't eligible (not opted-in / self-hosted), and neither throws. */
|
|
56
48
|
export interface WorkspaceHandle {
|
|
@@ -93,10 +85,6 @@ export type ProgramHostBuilder = (run: Run, manifest: WorkflowManifest, signal:
|
|
|
93
85
|
* leaf can resolve this run's bundled skill files (`<dir>/skills/<name>.md`). The orchestrator wires
|
|
94
86
|
* it to the runner's `onExtracted`. Optional — absent on paths that don't surface bundled files. */
|
|
95
87
|
setProgramDir?: (dir: string) => void;
|
|
96
|
-
/** Resolves when a host seam SUSPENDS the run — wired to the host's `onSuspend`, threaded into the
|
|
97
|
-
* program runner so a suspend short-circuits the body out of band (the durable-suspension design). Absent ⇒
|
|
98
|
-
* no durable suspension on this path (a suspend then surfaces as a thrown SuspendError). */
|
|
99
|
-
suspendSignal?: Promise<SuspendSignal>;
|
|
100
88
|
/** The run's browser-session manager (browser tier). The orchestrator reaps every still-open session
|
|
101
89
|
* on EVERY terminal path so no Chromium / Playwright MCP process leaks past the run. `closeAll` is
|
|
102
90
|
* best-effort + never throws. Absent on images without the browser stack. */
|
|
@@ -154,9 +142,6 @@ export interface ProgramWorkerDeps {
|
|
|
154
142
|
/** Periodic runtime metering (optional — absent disables it, e.g. the local/test path). */
|
|
155
143
|
startRuntimeFlush?: RuntimeMeterStarter;
|
|
156
144
|
finalizer: RunFinalizer;
|
|
157
|
-
/** Persists a durable suspension (the durable-suspension design). Absent ⇒ no suspension support: a run that
|
|
158
|
-
* reaches a suspend fails cleanly rather than stranding (the brokered worker always wires it). */
|
|
159
|
-
suspender?: RunSuspender;
|
|
160
145
|
buildHost: ProgramHostBuilder;
|
|
161
146
|
/** Starts mid-run credit watching for the session (optional — absent disables it). */
|
|
162
147
|
startCreditWatch?: CreditWatchStarter;
|
|
@@ -183,8 +168,5 @@ export type ProgramWorkerOutcome = {
|
|
|
183
168
|
} | {
|
|
184
169
|
kind: "failed";
|
|
185
170
|
reason: string;
|
|
186
|
-
} | {
|
|
187
|
-
kind: "suspended";
|
|
188
|
-
reason: string;
|
|
189
171
|
};
|
|
190
172
|
export declare function runProgramWorker(runId: string, deps: ProgramWorkerDeps): Promise<ProgramWorkerOutcome>;
|
|
@@ -110,7 +110,7 @@ export async function runProgramWorker(runId, deps) {
|
|
|
110
110
|
log.error("worker_host_build_failed", { runId, error: message });
|
|
111
111
|
return { kind: "failed", reason: "host_build_failed" };
|
|
112
112
|
}
|
|
113
|
-
const { host, redactor, workspace, phases, activity, setProgramDir, lsp
|
|
113
|
+
const { host, redactor, workspace, phases, activity, setProgramDir, lsp } = built;
|
|
114
114
|
const browserSessions = built.browserSessions;
|
|
115
115
|
const capture = built.capture;
|
|
116
116
|
// Guarantee the /workspace sandbox dir exists for EVERY run (persist or not) so a program can write
|
|
@@ -193,7 +193,6 @@ export async function runProgramWorker(runId, deps) {
|
|
|
193
193
|
redactText: (text) => redactor.redactText(text),
|
|
194
194
|
extract: deps.extractArchive,
|
|
195
195
|
...(setProgramDir !== undefined ? { onExtracted: setProgramDir } : {}),
|
|
196
|
-
...(suspendSignal !== undefined ? { suspendSignal } : {}),
|
|
197
196
|
...(activity !== undefined
|
|
198
197
|
? {
|
|
199
198
|
onOutput: (value) => {
|
|
@@ -268,24 +267,6 @@ export async function runProgramWorker(runId, deps) {
|
|
|
268
267
|
log.info("worker_run_aborted", { runId, reason });
|
|
269
268
|
return { kind: "failed", reason };
|
|
270
269
|
}
|
|
271
|
-
// Suspended: a host seam released the task (a long `sleep`, a `humanInput()` gate, or the in-leaf
|
|
272
|
-
// `human_input` tool). Persist the wake condition through the broker — NO finalize — and exit
|
|
273
|
-
// cleanly; a wake (an answer or a timer) re-dispatches the run, which restarts from the top and
|
|
274
|
-
// replays the journal past the already-done seams. The runtime tail for THIS session was booked
|
|
275
|
-
// above, so idle time while suspended is not billed. Phases stay open (the run is non-terminal).
|
|
276
|
-
if (result.kind === "suspended") {
|
|
277
|
-
if (deps.suspender === undefined) {
|
|
278
|
-
phases?.close("failed");
|
|
279
|
-
await deps.finalizer.finalize(runId, "failed", {
|
|
280
|
-
error: { code: "SUSPEND_UNSUPPORTED", message: "This runtime cannot suspend a run." },
|
|
281
|
-
});
|
|
282
|
-
log.error("worker_suspend_unsupported", { runId });
|
|
283
|
-
return { kind: "failed", reason: "suspend_unsupported" };
|
|
284
|
-
}
|
|
285
|
-
await deps.suspender.suspend(result.signal, deps.workerId);
|
|
286
|
-
log.info("worker_suspended", { runId, reason: result.signal.reason });
|
|
287
|
-
return { kind: "suspended", reason: result.signal.reason };
|
|
288
|
-
}
|
|
289
270
|
if (result.kind === "completed") {
|
|
290
271
|
phases?.close("completed");
|
|
291
272
|
await deps.finalizer.finalize(runId, "completed", result.output);
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { McpTokenResult } from "@boardwalk-labs/engine/core";
|
|
2
2
|
import type { Run } from "./wire/run.js";
|
|
3
|
-
import { type JournalKind, type JournalLookup, type JournalSeam, type SuspendSignal } from "./suspension.js";
|
|
4
3
|
import type { WebSearchOutput } from "./tools/web_search.js";
|
|
5
4
|
import type { ArtifactCommitInput, ArtifactPresignInput, ArtifactPresignResult, ArtifactSignResult, ArtifactSummary, ArtifactWriteInput, ArtifactWriteResult } from "./tools/artifacts.js";
|
|
6
5
|
import { type InferenceFrame, type InferenceProxyRequest } from "./wire/inference_proxy.js";
|
|
@@ -61,7 +60,7 @@ export declare class RunnerControlClient {
|
|
|
61
60
|
/** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
|
|
62
61
|
swapRunToken(token: string): void;
|
|
63
62
|
/**
|
|
64
|
-
* Every SHORT control call (claim / renew / cancel / credit /
|
|
63
|
+
* Every SHORT control call (claim / renew / cancel / credit / inputs / …) goes through here so
|
|
65
64
|
* it carries a hard timeout. Without one, a poll frozen mid-flight on the snapshot substrate
|
|
66
65
|
* hangs FOREVER on restore (the socket is dead but never reset), and since a watcher serializes
|
|
67
66
|
* its ticks, one hung tick wedges that watcher — the dead-connections gotcha for the
|
|
@@ -78,7 +77,7 @@ export declare class RunnerControlClient {
|
|
|
78
77
|
* reset/refused mid-rollover, our own per-attempt timeout on a dead socket) and the
|
|
79
78
|
* load-balancer's {@link RETRYABLE_STATUSES}. Safe to re-send because every caller's body is a
|
|
80
79
|
* reusable string/byte-array (the streaming inference call bypasses this entirely), and the
|
|
81
|
-
* broker's mutating endpoints are idempotent per worker/identifier (
|
|
80
|
+
* broker's mutating endpoints are idempotent per worker/identifier (gate seq, usage
|
|
82
81
|
* identifier, lease per workerId). Before this, ONE blip during an api-server deploy rollover
|
|
83
82
|
* crashed the worker hard mid-suspend/finalize and only crash-reclaim recovered the run.
|
|
84
83
|
*/
|
|
@@ -88,7 +87,6 @@ export declare class RunnerControlClient {
|
|
|
88
87
|
claim(workerId: string, leaseSeconds: number): Promise<{
|
|
89
88
|
run: Run;
|
|
90
89
|
lastEventCursor: number;
|
|
91
|
-
lastJournalSeq: number;
|
|
92
90
|
} | null>;
|
|
93
91
|
/** Heartbeat: extend our lease so a long run isn't reclaimed mid-flight. Returns the new
|
|
94
92
|
* `leaseUntil`, or null when the lease was lost (409 — another worker reclaimed the run), which
|
|
@@ -98,24 +96,6 @@ export declare class RunnerControlClient {
|
|
|
98
96
|
* whose lease expired and whose run was reclaimed + re-dispatched to a new owner), so a
|
|
99
97
|
* hung/partitioned worker that later recovers can't clobber the live run or revive a terminal one. */
|
|
100
98
|
finalize(status: "completed" | "failed", output: unknown, workerId: string): Promise<void>;
|
|
101
|
-
/** Look up a durable-seam journal entry by its seq (the durable-suspension design), or null on a replay miss
|
|
102
|
-
* (404). The broker joins a parked agent leaf's answers into the result server-side. */
|
|
103
|
-
journalGet(seq: number): Promise<JournalLookup | null>;
|
|
104
|
-
/** Record a RESOLVED seam result (idempotent on the run + seq server-side; a resolved entry is
|
|
105
|
-
* immutable). The broker writes the memoized value the next replay returns. */
|
|
106
|
-
journalPut(entry: {
|
|
107
|
-
seq: number;
|
|
108
|
-
kind: JournalKind;
|
|
109
|
-
fingerprint: string;
|
|
110
|
-
label: string;
|
|
111
|
-
result: unknown;
|
|
112
|
-
}): Promise<void>;
|
|
113
|
-
/** Persist a durable SUSPENSION: the broker records the wake condition (a pending/suspended journal
|
|
114
|
-
* entry + a human-input request row for HITL, or the wake time for a long sleep), flips the run to
|
|
115
|
-
* its suspended status, and releases the lease — transactionally. No finalize; a wake re-dispatches. */
|
|
116
|
-
suspend(signal: SuspendSignal, workerId: string): Promise<void>;
|
|
117
|
-
/** The {@link JournalSeam} the worker host reads/writes — a thin adapter over the broker methods. */
|
|
118
|
-
journalSeam(): JournalSeam;
|
|
119
99
|
/** Fetch the run's pinned manifest + program source, or null when the version is missing (404). */
|
|
120
100
|
getVersion(): Promise<BrokerVersion | null>;
|
|
121
101
|
/** Book a runtime-seconds DELTA (the worker's RuntimeFlusher → broker). `identifier` makes a
|
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
// failures (thrown network errors, LB 502/503/504 — a control-plane deploy rollover) are retried
|
|
12
12
|
// with backoff first, so a blip heals in place instead of crashing the run.
|
|
13
13
|
import { createLogger } from "./support/index.js";
|
|
14
|
-
import { journalLookupSchema, } from "./suspension.js";
|
|
15
14
|
import { INFERENCE_NDJSON_CONTENT_TYPE, parseInferenceFrame, serializeInferenceRequest, } from "./wire/inference_proxy.js";
|
|
16
15
|
const log = createLogger("RunnerControlClient");
|
|
17
16
|
/**
|
|
@@ -49,7 +48,7 @@ export class RunnerControlClient {
|
|
|
49
48
|
this.runToken = token;
|
|
50
49
|
}
|
|
51
50
|
/**
|
|
52
|
-
* Every SHORT control call (claim / renew / cancel / credit /
|
|
51
|
+
* Every SHORT control call (claim / renew / cancel / credit / inputs / …) goes through here so
|
|
53
52
|
* it carries a hard timeout. Without one, a poll frozen mid-flight on the snapshot substrate
|
|
54
53
|
* hangs FOREVER on restore (the socket is dead but never reset), and since a watcher serializes
|
|
55
54
|
* its ticks, one hung tick wedges that watcher — the dead-connections gotcha for the
|
|
@@ -70,7 +69,7 @@ export class RunnerControlClient {
|
|
|
70
69
|
* reset/refused mid-rollover, our own per-attempt timeout on a dead socket) and the
|
|
71
70
|
* load-balancer's {@link RETRYABLE_STATUSES}. Safe to re-send because every caller's body is a
|
|
72
71
|
* reusable string/byte-array (the streaming inference call bypasses this entirely), and the
|
|
73
|
-
* broker's mutating endpoints are idempotent per worker/identifier (
|
|
72
|
+
* broker's mutating endpoints are idempotent per worker/identifier (gate seq, usage
|
|
74
73
|
* identifier, lease per workerId). Before this, ONE blip during an api-server deploy rollover
|
|
75
74
|
* crashed the worker hard mid-suspend/finalize and only crash-reclaim recovered the run.
|
|
76
75
|
*/
|
|
@@ -120,9 +119,6 @@ export class RunnerControlClient {
|
|
|
120
119
|
return {
|
|
121
120
|
run: body.run,
|
|
122
121
|
lastEventCursor: body.lastEventCursor ?? 0,
|
|
123
|
-
// The replay frontier for silent replay (the durable-suspension design): the highest journaled seq, so a
|
|
124
|
-
// resumed run knows which seams already ran (suppress their re-emitted observability).
|
|
125
|
-
lastJournalSeq: body.lastJournalSeq ?? 0,
|
|
126
122
|
};
|
|
127
123
|
}
|
|
128
124
|
/** Heartbeat: extend our lease so a long run isn't reclaimed mid-flight. Returns the new
|
|
@@ -153,49 +149,6 @@ export class RunnerControlClient {
|
|
|
153
149
|
if (res.status !== 204)
|
|
154
150
|
throw await brokerError(res, "finalize");
|
|
155
151
|
}
|
|
156
|
-
/** Look up a durable-seam journal entry by its seq (the durable-suspension design), or null on a replay miss
|
|
157
|
-
* (404). The broker joins a parked agent leaf's answers into the result server-side. */
|
|
158
|
-
async journalGet(seq) {
|
|
159
|
-
const res = await this.controlFetch(this.url(`journal/${encodeURIComponent(String(seq))}`), {
|
|
160
|
-
method: "GET",
|
|
161
|
-
headers: this.headers(false),
|
|
162
|
-
});
|
|
163
|
-
if (res.status === 404)
|
|
164
|
-
return null;
|
|
165
|
-
if (res.status !== 200)
|
|
166
|
-
throw await brokerError(res, "journal-get");
|
|
167
|
-
return journalLookupSchema.parse(await res.json());
|
|
168
|
-
}
|
|
169
|
-
/** Record a RESOLVED seam result (idempotent on the run + seq server-side; a resolved entry is
|
|
170
|
-
* immutable). The broker writes the memoized value the next replay returns. */
|
|
171
|
-
async journalPut(entry) {
|
|
172
|
-
const res = await this.controlFetch(this.url("journal"), {
|
|
173
|
-
method: "POST",
|
|
174
|
-
headers: this.headers(true),
|
|
175
|
-
body: JSON.stringify(entry),
|
|
176
|
-
});
|
|
177
|
-
if (res.status !== 204)
|
|
178
|
-
throw await brokerError(res, "journal-put");
|
|
179
|
-
}
|
|
180
|
-
/** Persist a durable SUSPENSION: the broker records the wake condition (a pending/suspended journal
|
|
181
|
-
* entry + a human-input request row for HITL, or the wake time for a long sleep), flips the run to
|
|
182
|
-
* its suspended status, and releases the lease — transactionally. No finalize; a wake re-dispatches. */
|
|
183
|
-
async suspend(signal, workerId) {
|
|
184
|
-
const res = await this.controlFetch(this.url("suspend"), {
|
|
185
|
-
method: "POST",
|
|
186
|
-
headers: this.headers(true),
|
|
187
|
-
body: JSON.stringify({ ...signal, workerId }),
|
|
188
|
-
});
|
|
189
|
-
if (res.status !== 204)
|
|
190
|
-
throw await brokerError(res, "suspend");
|
|
191
|
-
}
|
|
192
|
-
/** The {@link JournalSeam} the worker host reads/writes — a thin adapter over the broker methods. */
|
|
193
|
-
journalSeam() {
|
|
194
|
-
return {
|
|
195
|
-
get: (seq) => this.journalGet(seq),
|
|
196
|
-
put: (entry) => this.journalPut(entry),
|
|
197
|
-
};
|
|
198
|
-
}
|
|
199
152
|
/** Fetch the run's pinned manifest + program source, or null when the version is missing (404). */
|
|
200
153
|
async getVersion() {
|
|
201
154
|
const res = await this.controlFetch(this.url("version"), {
|
|
@@ -1,20 +1,15 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import type { LeafCheckpoint, LeafResume } from "@boardwalk-labs/engine/core";
|
|
3
|
-
import { AppError } from "./support/index.js";
|
|
1
|
+
import type { LeafCheckpoint } from "@boardwalk-labs/engine/core";
|
|
4
2
|
/**
|
|
5
|
-
* Sleeps at/above this
|
|
6
|
-
*
|
|
7
|
-
* more than it saves.
|
|
8
|
-
* in the same place.
|
|
3
|
+
* Sleeps at/above this boundary SUSPEND on the snapshot substrate (freeze the VM; the wake fires
|
|
4
|
+
* when the sleep is due); shorter ones HOLD the process in-memory, where a snapshot round-trip
|
|
5
|
+
* costs more than it saves. Without a freeze substrate every sleep holds, whatever its length.
|
|
9
6
|
*/
|
|
10
7
|
export declare const SUSPEND_THRESHOLD_MS = 30000;
|
|
11
|
-
/** The durable-seam kinds the journal memoizes (mirrors `run_journal.kind` + the engine's IPC). */
|
|
12
|
-
export type JournalKind = "agent" | "step" | "human_input" | "sleep" | "workflow_call";
|
|
13
8
|
/** Why a seam suspended the run. */
|
|
14
9
|
export type SuspendReason = "human_input" | "sleep" | "workflow_call";
|
|
15
10
|
/** A human-in-the-loop gate carried out of a suspending seam (program-level or the in-leaf tool). */
|
|
16
11
|
export interface HumanInputGate {
|
|
17
|
-
/** The stable key the responder answers by (an author/
|
|
12
|
+
/** The stable key the responder answers by (an author/derived key, or the model's tool-call id). */
|
|
18
13
|
key: string;
|
|
19
14
|
prompt: string;
|
|
20
15
|
/** The response form ({@link import("@boardwalk-labs/workflow").HumanInputSpec}); validated on submit. */
|
|
@@ -29,13 +24,14 @@ export interface HumanInputGate {
|
|
|
29
24
|
/** Everything the broker needs to persist a suspension + the wake condition. */
|
|
30
25
|
export interface SuspendSignal {
|
|
31
26
|
reason: SuspendReason;
|
|
32
|
-
/** The
|
|
27
|
+
/** The suspension's within-run key (a monotonic per-run counter): it keys the HITL gate rows the
|
|
28
|
+
* wake joins answers from. Not a journal seq — there is no journal. */
|
|
33
29
|
seq: number;
|
|
34
|
-
/** The seam's determinism fingerprint (recorded on the pending journal entry). */
|
|
35
|
-
fingerprint: string;
|
|
36
30
|
/** Present for `reason: "human_input"` — the gate to register a request row for. */
|
|
37
31
|
humanInput?: HumanInputGate;
|
|
38
|
-
/** A tool-level gate's leaf transcript checkpoint
|
|
32
|
+
/** A tool-level gate's leaf transcript checkpoint. On the snapshot substrate the transcript
|
|
33
|
+
* rides in the frozen heap — this field is informational for the control plane, not a resume
|
|
34
|
+
* source. */
|
|
39
35
|
leafCheckpoint?: LeafCheckpoint;
|
|
40
36
|
/** Relative wait (ms) for `reason: "sleep"`; the broker computes the absolute wake time. */
|
|
41
37
|
durationMs?: number;
|
|
@@ -43,99 +39,8 @@ export interface SuspendSignal {
|
|
|
43
39
|
* woken when this child finalizes (the sweep wakes a parent whose child is terminal). */
|
|
44
40
|
childRunId?: string;
|
|
45
41
|
}
|
|
46
|
-
/**
|
|
47
|
-
|
|
48
|
-
* supervisor kills out-of-band), the backend worker runs the program IN-PROCESS, so a suspend is
|
|
49
|
-
* surfaced by the host calling `onSuspend(signal)` and returning a NEVER-resolving promise; the
|
|
50
|
-
* worker races that against the program body and tears down. This class exists so a seam that has no
|
|
51
|
-
* `onSuspend` wired (the local/test path) can still raise a typed, catchable signal.
|
|
52
|
-
*/
|
|
53
|
-
export declare class SuspendError extends Error {
|
|
54
|
-
readonly signal: SuspendSignal;
|
|
55
|
-
constructor(signal: SuspendSignal);
|
|
56
|
-
}
|
|
57
|
-
/** Journal-entry states (mirrors `run_journal.state` + the engine's IPC). */
|
|
58
|
-
export declare const JOURNAL_STATES: readonly ["pending", "suspended", "resolved"];
|
|
59
|
-
export type JournalEntryState = (typeof JOURNAL_STATES)[number];
|
|
60
|
-
/** A memoized journal entry, as the host reads it back on replay (mirrors the engine's IPC shape). */
|
|
61
|
-
export interface JournalLookup {
|
|
62
|
-
seq: number;
|
|
63
|
-
kind: JournalKind;
|
|
64
|
-
fingerprint: string;
|
|
65
|
-
/** `resolved` ⇒ `result` is the memoized value; `suspended` ⇒ a parked agent leaf (result is the
|
|
66
|
-
* {@link LeafResume} the host re-enters with); `pending` ⇒ awaiting an external event (re-suspend). */
|
|
67
|
-
state: JournalEntryState;
|
|
68
|
-
result: unknown;
|
|
69
|
-
}
|
|
70
|
-
/** Validate the broker's journal_get response (the worker's run token doesn't exempt the channel from
|
|
71
|
-
* validation). The result is genuinely heterogeneous JSON (an agent return / a LeafResume / a
|
|
72
|
-
* HumanInputResult), parsed per-kind downstream — so it stays `unknown` here, the validation seam. */
|
|
73
|
-
export declare const journalLookupSchema: z.ZodObject<{
|
|
74
|
-
seq: z.ZodNumber;
|
|
75
|
-
kind: z.ZodEnum<{
|
|
76
|
-
sleep: "sleep";
|
|
77
|
-
agent: "agent";
|
|
78
|
-
step: "step";
|
|
79
|
-
human_input: "human_input";
|
|
80
|
-
workflow_call: "workflow_call";
|
|
81
|
-
}>;
|
|
82
|
-
fingerprint: z.ZodString;
|
|
83
|
-
state: z.ZodEnum<{
|
|
84
|
-
pending: "pending";
|
|
85
|
-
suspended: "suspended";
|
|
86
|
-
resolved: "resolved";
|
|
87
|
-
}>;
|
|
88
|
-
result: z.ZodUnknown;
|
|
89
|
-
}, z.core.$strip>;
|
|
90
|
-
/** The journal the host reads (replay lookup) + writes (resolved seam results). Backed by the broker
|
|
91
|
-
* over the run token on hosted runs; absent on the local/test path (no durable suspension). */
|
|
92
|
-
export interface JournalSeam {
|
|
93
|
-
/** The memoized entry for a seam, or null on a replay miss. */
|
|
94
|
-
get(seq: number): Promise<JournalLookup | null>;
|
|
95
|
-
/** Record a RESOLVED seam result (idempotent on the run + seq; a resolved entry is immutable). */
|
|
96
|
-
put(entry: {
|
|
97
|
-
seq: number;
|
|
98
|
-
kind: JournalKind;
|
|
99
|
-
fingerprint: string;
|
|
100
|
-
label: string;
|
|
101
|
-
result: unknown;
|
|
102
|
-
}): Promise<void>;
|
|
103
|
-
}
|
|
104
|
-
/**
|
|
105
|
-
* The synchronous, monotonic durable-seam counter. Incremented at each journaled seam's ENTRY:
|
|
106
|
-
* because a program's synchronous call order is deterministic (even under `Promise.all`, whose
|
|
107
|
-
* `.map(...)` runs left-to-right synchronously), the same logical call gets the same `seq` on every
|
|
108
|
-
* execution — the journal key that lets a resumed run return a memoized result.
|
|
109
|
-
*
|
|
110
|
-
* It also drives SILENT REPLAY: a resume starts suppressed (observability — console output, phase
|
|
111
|
-
* markers — was already emitted last segment) and goes `live` the moment it reaches the suspending
|
|
112
|
-
* seam (the frontier = the highest journaled seq). A fresh run (frontier 0) is live immediately.
|
|
113
|
-
*/
|
|
114
|
-
export declare class SeamSequencer {
|
|
115
|
-
private readonly replayFrontier;
|
|
42
|
+
/** A monotonic per-run counter for suspension/gate keys (the `seq` on {@link SuspendSignal}). */
|
|
43
|
+
export declare class SuspensionCounter {
|
|
116
44
|
private count;
|
|
117
|
-
private liveFlag;
|
|
118
|
-
constructor(replayFrontier?: number);
|
|
119
45
|
next(): number;
|
|
120
|
-
/** True once execution has crossed the replay frontier — output after this point is NEW. */
|
|
121
|
-
get isLive(): boolean;
|
|
122
|
-
/** True while re-running already-journaled seams on a resume (observability suppressed). */
|
|
123
|
-
isReplaying(): boolean;
|
|
124
46
|
}
|
|
125
|
-
/** The child run id a `workflow_call` seam journals while it waits (parsed back on resume). */
|
|
126
|
-
export declare const childRunIdSchema: z.ZodString;
|
|
127
|
-
/** A stable content hash of a seam's salient args — the determinism check on replay. Identical
|
|
128
|
-
* construction to the engine's `seamFingerprint` so a journal written by one runtime validates in
|
|
129
|
-
* the other (the conformance promise). */
|
|
130
|
-
export declare function seamFingerprint(parts: readonly unknown[]): string;
|
|
131
|
-
/** A seam reached on replay didn't match what the journal recorded at that seq — the workflow's code
|
|
132
|
-
* on the path to a suspend changed (a different prompt/model/step name, or a different seam kind).
|
|
133
|
-
* Fails the run loudly rather than returning a stale memoized result for the wrong call. */
|
|
134
|
-
export declare function determinismError(seq: number, got: JournalKind, recorded: JournalKind): AppError;
|
|
135
|
-
/**
|
|
136
|
-
* The shape of a SUSPENDED agent leaf's journal result on resume: the transcript checkpoint plus the
|
|
137
|
-
* answers the broker joined from the resolved request rows, keyed by tool-call id. `messages` is the
|
|
138
|
-
* engine's own serialized transcript round-tripping through JSON — handed straight back to the leaf,
|
|
139
|
-
* not re-validated field-by-field. Structurally a {@link LeafResume}.
|
|
140
|
-
*/
|
|
141
|
-
export declare const leafResumeSchema: z.ZodType<LeafResume>;
|