@boardwalk-labs/runner 0.1.21 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/contract.d.ts +2 -0
- package/dist/contract.js +14 -0
- package/dist/daemon/container.d.ts +3 -2
- package/dist/daemon/container.js +15 -2
- package/dist/daemon/daemon.d.ts +1 -0
- package/dist/daemon/daemon.js +16 -1
- package/dist/runtime/agent/budget.d.ts +14 -1
- package/dist/runtime/agent/budget.js +23 -0
- package/dist/runtime/broker_child_dispatcher.d.ts +0 -2
- package/dist/runtime/broker_child_dispatcher.js +0 -7
- package/dist/runtime/budget_gate.d.ts +51 -0
- package/dist/runtime/budget_gate.js +114 -0
- package/dist/runtime/freeze_coordinator.js +8 -0
- package/dist/runtime/index.d.ts +15 -2
- package/dist/runtime/index.js +80 -57
- package/dist/runtime/leaf_executor.d.ts +20 -0
- package/dist/runtime/leaf_executor.js +13 -4
- package/dist/runtime/local_workspace_store.d.ts +22 -0
- package/dist/runtime/local_workspace_store.js +103 -0
- package/dist/runtime/program_runner.d.ts +32 -28
- package/dist/runtime/program_runner.js +75 -44
- package/dist/runtime/program_worker.d.ts +5 -18
- package/dist/runtime/program_worker.js +3 -20
- package/dist/runtime/runner_control_client.d.ts +2 -22
- package/dist/runtime/runner_control_client.js +2 -49
- package/dist/runtime/suspension.d.ts +20 -109
- package/dist/runtime/suspension.js +18 -111
- package/dist/runtime/wire/human_input.d.ts +1 -1
- package/dist/runtime/wire/human_input.js +2 -2
- package/dist/runtime/workflow_host.d.ts +65 -62
- package/dist/runtime/workflow_host.js +124 -199
- package/dist/runtime/workspace_store.d.ts +31 -3
- package/dist/runtime/workspace_store.js +43 -4
- package/package.json +3 -3
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
// A run is the execution of a built program ARTIFACT: the worker is handed the VERIFIED tarball
|
|
4
4
|
// (its sha256 already checked against the pinned digest by the orchestrator) plus the entry module
|
|
5
5
|
// name. This module is the mechanism: it installs the host adapter + trigger payload onto the
|
|
6
|
-
// `@boardwalk-labs/workflow` singleton, extracts the artifact into a unique temp dir under
|
|
7
|
-
//
|
|
6
|
+
// `@boardwalk-labs/workflow` singleton, extracts the artifact into a unique temp dir under the
|
|
7
|
+
// program root, chdirs to the workspace, and dynamic-imports the entry so the program's body runs. The
|
|
8
8
|
// program's `import { agent, sleep, … } from "@boardwalk-labs/workflow"` resolves to the SAME package
|
|
9
9
|
// instance the host was installed on (one instance per process), so the hooks reach our adapter.
|
|
10
10
|
//
|
|
@@ -13,32 +13,37 @@
|
|
|
13
13
|
// `@boardwalk-labs/workflow` is left external in the bundle, so the imported program resolves it to the SDK
|
|
14
14
|
// package present in the worker image (giving up its own copy would break the dual-adapter).
|
|
15
15
|
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
16
|
+
// Two places, both explicit, neither derived from `process.cwd()` (docs/WORKSPACE_PERSISTENCE.md I1/I2):
|
|
17
|
+
// - `workspaceRoot` — the run's `/workspace`. The working directory + HOME for AUTHOR code, so a
|
|
18
|
+
// relative write is the correct write and lands in what `workspace.persist` archives.
|
|
19
|
+
// - `programRoot` — where the artifact extracts (`<programRoot>/.bw-runs/<runId>-<uuid>`). Must be
|
|
20
|
+
// OUTSIDE the workspace, or the bundle rides into every snapshot and accumulates across runs.
|
|
21
|
+
// The extracted tree needs no node_modules-reachable ancestor: `ensureSdkLink` (below) links the SDK
|
|
22
|
+
// into the exec dir, so the bare `@boardwalk-labs/workflow` import resolves from any root.
|
|
19
23
|
//
|
|
20
24
|
// Durability: the body runs once, in-process. `sleep`/`workflows.call` hold in-process via the host
|
|
21
25
|
// (no checkpoint, no exit). A crash mid-run restarts the run from the top (handled by the
|
|
22
26
|
// worker/scheduler-sweep, not here). Output capture is deferred (v0 returns null).
|
|
23
27
|
import { mkdir, writeFile, rm, symlink, lstat } from "node:fs/promises";
|
|
24
|
-
import { dirname, join, isAbsolute, relative, sep } from "node:path";
|
|
28
|
+
import { dirname, join, isAbsolute, relative, resolve, sep } from "node:path";
|
|
25
29
|
import { createRequire } from "node:module";
|
|
26
30
|
import { pathToFileURL } from "node:url";
|
|
27
31
|
import { randomUUID } from "node:crypto";
|
|
28
32
|
import { installHost, installInput, installConfig, takeDeclaredOutput, resetRuntime, } from "@boardwalk-labs/workflow/runtime";
|
|
29
33
|
import { AppError, ErrorCode, createLogger } from "./support/index.js";
|
|
30
|
-
import { SuspendError } from "./suspension.js";
|
|
31
34
|
const log = createLogger("ProgramRunner");
|
|
32
35
|
/** Subdirectory (under the work root) that holds transient extracted program trees. */
|
|
33
36
|
const RUN_DIR = ".bw-runs";
|
|
34
37
|
/**
|
|
35
|
-
* Link THIS runtime's `@boardwalk-labs/workflow` into the exec dir's `node_modules` so the
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
38
|
+
* Link THIS runtime's `@boardwalk-labs/workflow` into the exec dir's `node_modules` so the program's
|
|
39
|
+
* bare import resolves from ANY program root — this link, not an ancestor `node_modules`, is what
|
|
40
|
+
* makes resolution work. (Worth stating plainly: a stale comment claiming the extraction dir had to
|
|
41
|
+
* sit under `/app` so the import could walk up to `/app/node_modules` outlived this function by
|
|
42
|
+
* several versions and sent a later reader chasing a dependency that no longer exists. The live
|
|
43
|
+
* fleet has no `/app` at all and resolves fine.) A symlink — not a copy — is load-bearing: Node
|
|
44
|
+
* resolves it to the REAL path, so the program gets the same module instance the host adapter was
|
|
45
|
+
* installed on (the singleton contract). `junction` covers Windows without elevation; a failed link
|
|
46
|
+
* is only logged, since a resolvable ancestor may still exist.
|
|
42
47
|
*/
|
|
43
48
|
export async function ensureSdkLink(execDir) {
|
|
44
49
|
// A program tarball that ships its own `node_modules/@boardwalk-labs/workflow` would shadow the
|
|
@@ -83,19 +88,31 @@ export function resolveEntryPath(dir, entry) {
|
|
|
83
88
|
}
|
|
84
89
|
/** Scratch filename for the in-flight artifact tarball inside a run's dir. */
|
|
85
90
|
const ARTIFACT_FILE = "__program.tgz";
|
|
91
|
+
/**
|
|
92
|
+
* Enforce I2: the extracted program must not live inside the run's workspace. Incidental separation
|
|
93
|
+
* is what broke before — the extraction root used to be `process.cwd()`, so it landed inside the
|
|
94
|
+
* workspace on exactly the lanes whose cwd was already correct. Compares RESOLVED paths, and treats
|
|
95
|
+
* "the workspace itself" as inside.
|
|
96
|
+
*/
|
|
97
|
+
export function assertProgramRootOutsideWorkspace(programRoot, workspaceRoot) {
|
|
98
|
+
const rel = relative(resolve(workspaceRoot), resolve(programRoot));
|
|
99
|
+
if (rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel))) {
|
|
100
|
+
throw new AppError(ErrorCode.VALIDATION_FAILED, `The program root "${programRoot}" is inside the workspace "${workspaceRoot}"; the extracted program must live outside it.`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
86
103
|
/**
|
|
87
104
|
* Run a workflow program to completion. Installs the host + input, extracts the VERIFIED artifact +
|
|
88
105
|
* dynamic-imports its entry (which runs the body), and returns the terminal result. Always tears the
|
|
89
106
|
* runtime state down and removes the temp tree afterward.
|
|
90
107
|
*/
|
|
91
108
|
export async function runWorkflowProgram(args, deps) {
|
|
92
|
-
const
|
|
93
|
-
const dir = join(workRoot, RUN_DIR, `${args.runId}-${randomUUID()}`);
|
|
109
|
+
const dir = join(deps.programRoot, RUN_DIR, `${args.runId}-${randomUUID()}`);
|
|
94
110
|
installHost(deps.host);
|
|
95
111
|
installInput(args.input);
|
|
96
112
|
// The run row's config is arbitrary JSON (jsonb); it IS valid JSON, so narrow to the SDK's JsonValue.
|
|
97
113
|
installConfig(args.config);
|
|
98
114
|
try {
|
|
115
|
+
assertProgramRootOutsideWorkspace(deps.programRoot, deps.workspaceRoot);
|
|
99
116
|
await mkdir(dir, { recursive: true });
|
|
100
117
|
const tgzPath = join(dir, ARTIFACT_FILE);
|
|
101
118
|
await writeFile(tgzPath, args.tarball);
|
|
@@ -112,25 +129,11 @@ export async function runWorkflowProgram(args, deps) {
|
|
|
112
129
|
// runner may point at an arbitrary control plane, so refuse an entry that could escape the
|
|
113
130
|
// extraction dir (absolute path or `..` segment) before importing it.
|
|
114
131
|
const entryPath = resolveEntryPath(dir, args.entry);
|
|
115
|
-
// Run the program body
|
|
116
|
-
//
|
|
117
|
-
|
|
118
|
-
// no-`onSuspend` path. Everything else is the program's natural completion / failure.
|
|
119
|
-
const body = runProgramBody(entryPath, deps);
|
|
120
|
-
const result = deps.suspendSignal === undefined
|
|
121
|
-
? await body
|
|
122
|
-
: await Promise.race([
|
|
123
|
-
body,
|
|
124
|
-
deps.suspendSignal.then((signal) => ({ kind: "suspended", signal })),
|
|
125
|
-
]);
|
|
126
|
-
// If the suspend won the race, the body promise is abandoned (its suspending seam never settles);
|
|
127
|
-
// swallow any late settle so it can't surface as an unhandled rejection before the process exits.
|
|
128
|
-
void body.catch(() => undefined);
|
|
129
|
-
return result;
|
|
132
|
+
// Run the program body to its natural completion / failure. A waiting seam freezes with the
|
|
133
|
+
// VM (snapshot substrate) or holds the process — either way the body's own await continues.
|
|
134
|
+
return await runProgramBody(entryPath, deps);
|
|
130
135
|
}
|
|
131
136
|
catch (err) {
|
|
132
|
-
if (err instanceof SuspendError)
|
|
133
|
-
return { kind: "suspended", signal: err.signal };
|
|
134
137
|
const redactText = deps.redactText ?? ((s) => s);
|
|
135
138
|
// Redact BEFORE both sinks: the message can carry a secret the program resolved then threw.
|
|
136
139
|
const message = redactText(err instanceof Error ? err.message : String(err));
|
|
@@ -157,17 +160,45 @@ export async function runWorkflowProgram(args, deps) {
|
|
|
157
160
|
}
|
|
158
161
|
/**
|
|
159
162
|
* Import (= run) the program entry and capture its declared output. Resolves to a `completed`
|
|
160
|
-
* result; a program failure
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
+
* result; a program failure THROWS and is handled by the caller. A unique dir per run gives a
|
|
164
|
+
* fresh URL so the module cache never returns an already-run program; `@vite-ignore` keeps
|
|
165
|
+
* vitest/vite from statically analyzing the runtime URL.
|
|
166
|
+
*
|
|
167
|
+
* The chdir to the workspace (I1) happens HERE, at the boundary where author code starts, and only
|
|
168
|
+
* after the artifact is extracted from {@link ProgramRunnerDeps.programRoot} — so the program's
|
|
169
|
+
* files land in the workspace while the bundle stays out of it. Module resolution is unaffected:
|
|
170
|
+
* Node resolves file-relative (the entry is imported by absolute URL, the SDK via `ensureSdkLink`),
|
|
171
|
+
* never cwd-relative. The engine's local path has run exactly this shape since dev-on-engine
|
|
172
|
+
* (`boardwalk/src/run/child.ts`), and the self-hosted daemon spawns with the same cwd.
|
|
163
173
|
*/
|
|
164
174
|
async function runProgramBody(entryPath, deps) {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
deps.
|
|
172
|
-
|
|
175
|
+
const callerCwd = process.cwd();
|
|
176
|
+
try {
|
|
177
|
+
process.chdir(deps.workspaceRoot);
|
|
178
|
+
}
|
|
179
|
+
catch (err) {
|
|
180
|
+
// Fail loud. Running author code from an arbitrary cwd is what silently threw writes away.
|
|
181
|
+
throw new AppError(ErrorCode.VALIDATION_FAILED, `The run's workspace "${deps.workspaceRoot}" is not usable as the working directory: ${err instanceof Error ? err.message : String(err)}`);
|
|
182
|
+
}
|
|
183
|
+
try {
|
|
184
|
+
await import(/* @vite-ignore */ pathToFileURL(entryPath).href);
|
|
185
|
+
// The program declares its result via `output(value)` (top-level code can't `return`); null when it
|
|
186
|
+
// never called it. This becomes the run's persisted output + a `workflows.call` parent's value, and
|
|
187
|
+
// (when actually declared) an `output` entry in the run's activity log.
|
|
188
|
+
const declared = takeDeclaredOutput();
|
|
189
|
+
if (declared !== null)
|
|
190
|
+
deps.onOutput?.(declared.value);
|
|
191
|
+
return { kind: "completed", output: declared !== null ? declared.value : null };
|
|
192
|
+
}
|
|
193
|
+
finally {
|
|
194
|
+
// One run per process in production, so this matters for tests + the local/dev path — but a
|
|
195
|
+
// function that permanently moves its caller's cwd is a trap either way. Best-effort: the caller's
|
|
196
|
+
// dir may itself be gone, and that must not mask the run's real outcome.
|
|
197
|
+
try {
|
|
198
|
+
process.chdir(callerCwd);
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
/* the caller's cwd vanished mid-run; nothing useful to do */
|
|
202
|
+
}
|
|
203
|
+
}
|
|
173
204
|
}
|
|
@@ -3,7 +3,6 @@ import { type WorkflowManifest } from "./wire/manifest.js";
|
|
|
3
3
|
import type { WorkflowHost } from "@boardwalk-labs/workflow/runtime";
|
|
4
4
|
import type { SecretRedactor } from "./agent/secret_redactor.js";
|
|
5
5
|
import { type LogStream } from "./program_log_capture.js";
|
|
6
|
-
import type { SuspendSignal } from "./suspension.js";
|
|
7
6
|
/** Default 5-minute lease (matches the engine spec). */
|
|
8
7
|
export declare const DEFAULT_LEASE_MS: number;
|
|
9
8
|
/** Race-safe claim surface — RunRepository satisfies it. */
|
|
@@ -44,13 +43,6 @@ export type RuntimeMeterStarter = (args: {
|
|
|
44
43
|
export interface RunFinalizer {
|
|
45
44
|
finalize(runId: string, status: "completed" | "failed", output: unknown): Promise<void>;
|
|
46
45
|
}
|
|
47
|
-
/** Persists a durable SUSPENSION (the durable-suspension design): the broker records the wake condition (a
|
|
48
|
-
* pending/suspended journal entry + a human-input request row for HITL, or the wake time for a long
|
|
49
|
-
* sleep), flips the run to its suspended status, and releases the lease — all transactionally. The
|
|
50
|
-
* run is NOT finalized; a wake (an answer, a child finalize, or a timer) re-dispatches it. */
|
|
51
|
-
export interface RunSuspender {
|
|
52
|
-
suspend(signal: SuspendSignal, workerId: string): Promise<void>;
|
|
53
|
-
}
|
|
54
46
|
/** Restores/snapshots the workflow's persistent `/workspace`. Best-effort — both no-op when the
|
|
55
47
|
* run isn't eligible (not opted-in / self-hosted), and neither throws. */
|
|
56
48
|
export interface WorkspaceHandle {
|
|
@@ -93,10 +85,6 @@ export type ProgramHostBuilder = (run: Run, manifest: WorkflowManifest, signal:
|
|
|
93
85
|
* leaf can resolve this run's bundled skill files (`<dir>/skills/<name>.md`). The orchestrator wires
|
|
94
86
|
* it to the runner's `onExtracted`. Optional — absent on paths that don't surface bundled files. */
|
|
95
87
|
setProgramDir?: (dir: string) => void;
|
|
96
|
-
/** Resolves when a host seam SUSPENDS the run — wired to the host's `onSuspend`, threaded into the
|
|
97
|
-
* program runner so a suspend short-circuits the body out of band (the durable-suspension design). Absent ⇒
|
|
98
|
-
* no durable suspension on this path (a suspend then surfaces as a thrown SuspendError). */
|
|
99
|
-
suspendSignal?: Promise<SuspendSignal>;
|
|
100
88
|
/** The run's browser-session manager (browser tier). The orchestrator reaps every still-open session
|
|
101
89
|
* on EVERY terminal path so no Chromium / Playwright MCP process leaks past the run. `closeAll` is
|
|
102
90
|
* best-effort + never throws. Absent on images without the browser stack. */
|
|
@@ -140,6 +128,11 @@ export type LeaseWatchStarter = (args: {
|
|
|
140
128
|
export interface ProgramWorkerDeps {
|
|
141
129
|
runs: RunClaimer;
|
|
142
130
|
versions: ProgramVersionReader;
|
|
131
|
+
/** The run's `/workspace` — cwd + HOME for author code (docs/WORKSPACE_PERSISTENCE.md I1), and the
|
|
132
|
+
* tree `workspace.persist` archives. Passed through to the program runner. */
|
|
133
|
+
workspaceRoot: string;
|
|
134
|
+
/** Where the program artifact extracts — OUTSIDE the workspace (I2). Passed through to the runner. */
|
|
135
|
+
programRoot: string;
|
|
143
136
|
/** Download the program artifact bytes from the broker's presigned URL (broker.downloadBytes). */
|
|
144
137
|
fetchProgram: (downloadUrl: string) => Promise<Uint8Array>;
|
|
145
138
|
/** Extract a gzipped tar into a dir (system `tar`); passed through to the program runner. */
|
|
@@ -154,9 +147,6 @@ export interface ProgramWorkerDeps {
|
|
|
154
147
|
/** Periodic runtime metering (optional — absent disables it, e.g. the local/test path). */
|
|
155
148
|
startRuntimeFlush?: RuntimeMeterStarter;
|
|
156
149
|
finalizer: RunFinalizer;
|
|
157
|
-
/** Persists a durable suspension (the durable-suspension design). Absent ⇒ no suspension support: a run that
|
|
158
|
-
* reaches a suspend fails cleanly rather than stranding (the brokered worker always wires it). */
|
|
159
|
-
suspender?: RunSuspender;
|
|
160
150
|
buildHost: ProgramHostBuilder;
|
|
161
151
|
/** Starts mid-run credit watching for the session (optional — absent disables it). */
|
|
162
152
|
startCreditWatch?: CreditWatchStarter;
|
|
@@ -183,8 +173,5 @@ export type ProgramWorkerOutcome = {
|
|
|
183
173
|
} | {
|
|
184
174
|
kind: "failed";
|
|
185
175
|
reason: string;
|
|
186
|
-
} | {
|
|
187
|
-
kind: "suspended";
|
|
188
|
-
reason: string;
|
|
189
176
|
};
|
|
190
177
|
export declare function runProgramWorker(runId: string, deps: ProgramWorkerDeps): Promise<ProgramWorkerOutcome>;
|
|
@@ -110,7 +110,7 @@ export async function runProgramWorker(runId, deps) {
|
|
|
110
110
|
log.error("worker_host_build_failed", { runId, error: message });
|
|
111
111
|
return { kind: "failed", reason: "host_build_failed" };
|
|
112
112
|
}
|
|
113
|
-
const { host, redactor, workspace, phases, activity, setProgramDir, lsp
|
|
113
|
+
const { host, redactor, workspace, phases, activity, setProgramDir, lsp } = built;
|
|
114
114
|
const browserSessions = built.browserSessions;
|
|
115
115
|
const capture = built.capture;
|
|
116
116
|
// Guarantee the /workspace sandbox dir exists for EVERY run (persist or not) so a program can write
|
|
@@ -190,10 +190,11 @@ export async function runProgramWorker(runId, deps) {
|
|
|
190
190
|
// onOutput emits the `output` activity entry into the run's log when the program declared one.
|
|
191
191
|
{
|
|
192
192
|
host,
|
|
193
|
+
workspaceRoot: deps.workspaceRoot,
|
|
194
|
+
programRoot: deps.programRoot,
|
|
193
195
|
redactText: (text) => redactor.redactText(text),
|
|
194
196
|
extract: deps.extractArchive,
|
|
195
197
|
...(setProgramDir !== undefined ? { onExtracted: setProgramDir } : {}),
|
|
196
|
-
...(suspendSignal !== undefined ? { suspendSignal } : {}),
|
|
197
198
|
...(activity !== undefined
|
|
198
199
|
? {
|
|
199
200
|
onOutput: (value) => {
|
|
@@ -268,24 +269,6 @@ export async function runProgramWorker(runId, deps) {
|
|
|
268
269
|
log.info("worker_run_aborted", { runId, reason });
|
|
269
270
|
return { kind: "failed", reason };
|
|
270
271
|
}
|
|
271
|
-
// Suspended: a host seam released the task (a long `sleep`, a `humanInput()` gate, or the in-leaf
|
|
272
|
-
// `human_input` tool). Persist the wake condition through the broker — NO finalize — and exit
|
|
273
|
-
// cleanly; a wake (an answer or a timer) re-dispatches the run, which restarts from the top and
|
|
274
|
-
// replays the journal past the already-done seams. The runtime tail for THIS session was booked
|
|
275
|
-
// above, so idle time while suspended is not billed. Phases stay open (the run is non-terminal).
|
|
276
|
-
if (result.kind === "suspended") {
|
|
277
|
-
if (deps.suspender === undefined) {
|
|
278
|
-
phases?.close("failed");
|
|
279
|
-
await deps.finalizer.finalize(runId, "failed", {
|
|
280
|
-
error: { code: "SUSPEND_UNSUPPORTED", message: "This runtime cannot suspend a run." },
|
|
281
|
-
});
|
|
282
|
-
log.error("worker_suspend_unsupported", { runId });
|
|
283
|
-
return { kind: "failed", reason: "suspend_unsupported" };
|
|
284
|
-
}
|
|
285
|
-
await deps.suspender.suspend(result.signal, deps.workerId);
|
|
286
|
-
log.info("worker_suspended", { runId, reason: result.signal.reason });
|
|
287
|
-
return { kind: "suspended", reason: result.signal.reason };
|
|
288
|
-
}
|
|
289
272
|
if (result.kind === "completed") {
|
|
290
273
|
phases?.close("completed");
|
|
291
274
|
await deps.finalizer.finalize(runId, "completed", result.output);
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { McpTokenResult } from "@boardwalk-labs/engine/core";
|
|
2
2
|
import type { Run } from "./wire/run.js";
|
|
3
|
-
import { type JournalKind, type JournalLookup, type JournalSeam, type SuspendSignal } from "./suspension.js";
|
|
4
3
|
import type { WebSearchOutput } from "./tools/web_search.js";
|
|
5
4
|
import type { ArtifactCommitInput, ArtifactPresignInput, ArtifactPresignResult, ArtifactSignResult, ArtifactSummary, ArtifactWriteInput, ArtifactWriteResult } from "./tools/artifacts.js";
|
|
6
5
|
import { type InferenceFrame, type InferenceProxyRequest } from "./wire/inference_proxy.js";
|
|
@@ -61,7 +60,7 @@ export declare class RunnerControlClient {
|
|
|
61
60
|
/** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
|
|
62
61
|
swapRunToken(token: string): void;
|
|
63
62
|
/**
|
|
64
|
-
* Every SHORT control call (claim / renew / cancel / credit /
|
|
63
|
+
* Every SHORT control call (claim / renew / cancel / credit / inputs / …) goes through here so
|
|
65
64
|
* it carries a hard timeout. Without one, a poll frozen mid-flight on the snapshot substrate
|
|
66
65
|
* hangs FOREVER on restore (the socket is dead but never reset), and since a watcher serializes
|
|
67
66
|
* its ticks, one hung tick wedges that watcher — the dead-connections gotcha for the
|
|
@@ -78,7 +77,7 @@ export declare class RunnerControlClient {
|
|
|
78
77
|
* reset/refused mid-rollover, our own per-attempt timeout on a dead socket) and the
|
|
79
78
|
* load-balancer's {@link RETRYABLE_STATUSES}. Safe to re-send because every caller's body is a
|
|
80
79
|
* reusable string/byte-array (the streaming inference call bypasses this entirely), and the
|
|
81
|
-
* broker's mutating endpoints are idempotent per worker/identifier (
|
|
80
|
+
* broker's mutating endpoints are idempotent per worker/identifier (gate seq, usage
|
|
82
81
|
* identifier, lease per workerId). Before this, ONE blip during an api-server deploy rollover
|
|
83
82
|
* crashed the worker hard mid-suspend/finalize and only crash-reclaim recovered the run.
|
|
84
83
|
*/
|
|
@@ -88,7 +87,6 @@ export declare class RunnerControlClient {
|
|
|
88
87
|
claim(workerId: string, leaseSeconds: number): Promise<{
|
|
89
88
|
run: Run;
|
|
90
89
|
lastEventCursor: number;
|
|
91
|
-
lastJournalSeq: number;
|
|
92
90
|
} | null>;
|
|
93
91
|
/** Heartbeat: extend our lease so a long run isn't reclaimed mid-flight. Returns the new
|
|
94
92
|
* `leaseUntil`, or null when the lease was lost (409 — another worker reclaimed the run), which
|
|
@@ -98,24 +96,6 @@ export declare class RunnerControlClient {
|
|
|
98
96
|
* whose lease expired and whose run was reclaimed + re-dispatched to a new owner), so a
|
|
99
97
|
* hung/partitioned worker that later recovers can't clobber the live run or revive a terminal one. */
|
|
100
98
|
finalize(status: "completed" | "failed", output: unknown, workerId: string): Promise<void>;
|
|
101
|
-
/** Look up a durable-seam journal entry by its seq (the durable-suspension design), or null on a replay miss
|
|
102
|
-
* (404). The broker joins a parked agent leaf's answers into the result server-side. */
|
|
103
|
-
journalGet(seq: number): Promise<JournalLookup | null>;
|
|
104
|
-
/** Record a RESOLVED seam result (idempotent on the run + seq server-side; a resolved entry is
|
|
105
|
-
* immutable). The broker writes the memoized value the next replay returns. */
|
|
106
|
-
journalPut(entry: {
|
|
107
|
-
seq: number;
|
|
108
|
-
kind: JournalKind;
|
|
109
|
-
fingerprint: string;
|
|
110
|
-
label: string;
|
|
111
|
-
result: unknown;
|
|
112
|
-
}): Promise<void>;
|
|
113
|
-
/** Persist a durable SUSPENSION: the broker records the wake condition (a pending/suspended journal
|
|
114
|
-
* entry + a human-input request row for HITL, or the wake time for a long sleep), flips the run to
|
|
115
|
-
* its suspended status, and releases the lease — transactionally. No finalize; a wake re-dispatches. */
|
|
116
|
-
suspend(signal: SuspendSignal, workerId: string): Promise<void>;
|
|
117
|
-
/** The {@link JournalSeam} the worker host reads/writes — a thin adapter over the broker methods. */
|
|
118
|
-
journalSeam(): JournalSeam;
|
|
119
99
|
/** Fetch the run's pinned manifest + program source, or null when the version is missing (404). */
|
|
120
100
|
getVersion(): Promise<BrokerVersion | null>;
|
|
121
101
|
/** Book a runtime-seconds DELTA (the worker's RuntimeFlusher → broker). `identifier` makes a
|
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
// failures (thrown network errors, LB 502/503/504 — a control-plane deploy rollover) are retried
|
|
12
12
|
// with backoff first, so a blip heals in place instead of crashing the run.
|
|
13
13
|
import { createLogger } from "./support/index.js";
|
|
14
|
-
import { journalLookupSchema, } from "./suspension.js";
|
|
15
14
|
import { INFERENCE_NDJSON_CONTENT_TYPE, parseInferenceFrame, serializeInferenceRequest, } from "./wire/inference_proxy.js";
|
|
16
15
|
const log = createLogger("RunnerControlClient");
|
|
17
16
|
/**
|
|
@@ -49,7 +48,7 @@ export class RunnerControlClient {
|
|
|
49
48
|
this.runToken = token;
|
|
50
49
|
}
|
|
51
50
|
/**
|
|
52
|
-
* Every SHORT control call (claim / renew / cancel / credit /
|
|
51
|
+
* Every SHORT control call (claim / renew / cancel / credit / inputs / …) goes through here so
|
|
53
52
|
* it carries a hard timeout. Without one, a poll frozen mid-flight on the snapshot substrate
|
|
54
53
|
* hangs FOREVER on restore (the socket is dead but never reset), and since a watcher serializes
|
|
55
54
|
* its ticks, one hung tick wedges that watcher — the dead-connections gotcha for the
|
|
@@ -70,7 +69,7 @@ export class RunnerControlClient {
|
|
|
70
69
|
* reset/refused mid-rollover, our own per-attempt timeout on a dead socket) and the
|
|
71
70
|
* load-balancer's {@link RETRYABLE_STATUSES}. Safe to re-send because every caller's body is a
|
|
72
71
|
* reusable string/byte-array (the streaming inference call bypasses this entirely), and the
|
|
73
|
-
* broker's mutating endpoints are idempotent per worker/identifier (
|
|
72
|
+
* broker's mutating endpoints are idempotent per worker/identifier (gate seq, usage
|
|
74
73
|
* identifier, lease per workerId). Before this, ONE blip during an api-server deploy rollover
|
|
75
74
|
* crashed the worker hard mid-suspend/finalize and only crash-reclaim recovered the run.
|
|
76
75
|
*/
|
|
@@ -120,9 +119,6 @@ export class RunnerControlClient {
|
|
|
120
119
|
return {
|
|
121
120
|
run: body.run,
|
|
122
121
|
lastEventCursor: body.lastEventCursor ?? 0,
|
|
123
|
-
// The replay frontier for silent replay (the durable-suspension design): the highest journaled seq, so a
|
|
124
|
-
// resumed run knows which seams already ran (suppress their re-emitted observability).
|
|
125
|
-
lastJournalSeq: body.lastJournalSeq ?? 0,
|
|
126
122
|
};
|
|
127
123
|
}
|
|
128
124
|
/** Heartbeat: extend our lease so a long run isn't reclaimed mid-flight. Returns the new
|
|
@@ -153,49 +149,6 @@ export class RunnerControlClient {
|
|
|
153
149
|
if (res.status !== 204)
|
|
154
150
|
throw await brokerError(res, "finalize");
|
|
155
151
|
}
|
|
156
|
-
/** Look up a durable-seam journal entry by its seq (the durable-suspension design), or null on a replay miss
|
|
157
|
-
* (404). The broker joins a parked agent leaf's answers into the result server-side. */
|
|
158
|
-
async journalGet(seq) {
|
|
159
|
-
const res = await this.controlFetch(this.url(`journal/${encodeURIComponent(String(seq))}`), {
|
|
160
|
-
method: "GET",
|
|
161
|
-
headers: this.headers(false),
|
|
162
|
-
});
|
|
163
|
-
if (res.status === 404)
|
|
164
|
-
return null;
|
|
165
|
-
if (res.status !== 200)
|
|
166
|
-
throw await brokerError(res, "journal-get");
|
|
167
|
-
return journalLookupSchema.parse(await res.json());
|
|
168
|
-
}
|
|
169
|
-
/** Record a RESOLVED seam result (idempotent on the run + seq server-side; a resolved entry is
|
|
170
|
-
* immutable). The broker writes the memoized value the next replay returns. */
|
|
171
|
-
async journalPut(entry) {
|
|
172
|
-
const res = await this.controlFetch(this.url("journal"), {
|
|
173
|
-
method: "POST",
|
|
174
|
-
headers: this.headers(true),
|
|
175
|
-
body: JSON.stringify(entry),
|
|
176
|
-
});
|
|
177
|
-
if (res.status !== 204)
|
|
178
|
-
throw await brokerError(res, "journal-put");
|
|
179
|
-
}
|
|
180
|
-
/** Persist a durable SUSPENSION: the broker records the wake condition (a pending/suspended journal
|
|
181
|
-
* entry + a human-input request row for HITL, or the wake time for a long sleep), flips the run to
|
|
182
|
-
* its suspended status, and releases the lease — transactionally. No finalize; a wake re-dispatches. */
|
|
183
|
-
async suspend(signal, workerId) {
|
|
184
|
-
const res = await this.controlFetch(this.url("suspend"), {
|
|
185
|
-
method: "POST",
|
|
186
|
-
headers: this.headers(true),
|
|
187
|
-
body: JSON.stringify({ ...signal, workerId }),
|
|
188
|
-
});
|
|
189
|
-
if (res.status !== 204)
|
|
190
|
-
throw await brokerError(res, "suspend");
|
|
191
|
-
}
|
|
192
|
-
/** The {@link JournalSeam} the worker host reads/writes — a thin adapter over the broker methods. */
|
|
193
|
-
journalSeam() {
|
|
194
|
-
return {
|
|
195
|
-
get: (seq) => this.journalGet(seq),
|
|
196
|
-
put: (entry) => this.journalPut(entry),
|
|
197
|
-
};
|
|
198
|
-
}
|
|
199
152
|
/** Fetch the run's pinned manifest + program source, or null when the version is missing (404). */
|
|
200
153
|
async getVersion() {
|
|
201
154
|
const res = await this.controlFetch(this.url("version"), {
|
|
@@ -1,20 +1,21 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import type { LeafCheckpoint, LeafResume } from "@boardwalk-labs/engine/core";
|
|
3
|
-
import { AppError } from "./support/index.js";
|
|
1
|
+
import type { LeafCheckpoint } from "@boardwalk-labs/engine/core";
|
|
4
2
|
/**
|
|
5
|
-
* Sleeps at/above this
|
|
6
|
-
*
|
|
7
|
-
* more than it saves.
|
|
8
|
-
* in the same place.
|
|
3
|
+
* Sleeps at/above this boundary SUSPEND on the snapshot substrate (freeze the VM; the wake fires
|
|
4
|
+
* when the sleep is due); shorter ones HOLD the process in-memory, where a snapshot round-trip
|
|
5
|
+
* costs more than it saves. Without a freeze substrate every sleep holds, whatever its length.
|
|
9
6
|
*/
|
|
10
7
|
export declare const SUSPEND_THRESHOLD_MS = 30000;
|
|
11
|
-
/**
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Why a seam suspended the run. `budget` is the odd one out: it is NOT a seam the program called but
|
|
10
|
+
* an involuntary park — the run hit its `max_usd` cap and is waiting for a person to approve more
|
|
11
|
+
* spend (docs/SUSPEND_POLICY.md Decision 3). It still carries a `humanInput` gate (key `budget`), so
|
|
12
|
+
* the control plane persists, surfaces, and answers it exactly like any other gate; only the reason
|
|
13
|
+
* differs, which is what lets the UI say "budget" instead of "waiting on a human".
|
|
14
|
+
*/
|
|
15
|
+
export type SuspendReason = "human_input" | "sleep" | "workflow_call" | "budget";
|
|
15
16
|
/** A human-in-the-loop gate carried out of a suspending seam (program-level or the in-leaf tool). */
|
|
16
17
|
export interface HumanInputGate {
|
|
17
|
-
/** The stable key the responder answers by (an author/
|
|
18
|
+
/** The stable key the responder answers by (an author/derived key, or the model's tool-call id). */
|
|
18
19
|
key: string;
|
|
19
20
|
prompt: string;
|
|
20
21
|
/** The response form ({@link import("@boardwalk-labs/workflow").HumanInputSpec}); validated on submit. */
|
|
@@ -29,13 +30,14 @@ export interface HumanInputGate {
|
|
|
29
30
|
/** Everything the broker needs to persist a suspension + the wake condition. */
|
|
30
31
|
export interface SuspendSignal {
|
|
31
32
|
reason: SuspendReason;
|
|
32
|
-
/** The
|
|
33
|
+
/** The suspension's within-run key (a monotonic per-run counter): it keys the HITL gate rows the
|
|
34
|
+
* wake joins answers from. Not a journal seq — there is no journal. */
|
|
33
35
|
seq: number;
|
|
34
|
-
/** The seam's determinism fingerprint (recorded on the pending journal entry). */
|
|
35
|
-
fingerprint: string;
|
|
36
36
|
/** Present for `reason: "human_input"` — the gate to register a request row for. */
|
|
37
37
|
humanInput?: HumanInputGate;
|
|
38
|
-
/** A tool-level gate's leaf transcript checkpoint
|
|
38
|
+
/** A tool-level gate's leaf transcript checkpoint. On the snapshot substrate the transcript
|
|
39
|
+
* rides in the frozen heap — this field is informational for the control plane, not a resume
|
|
40
|
+
* source. */
|
|
39
41
|
leafCheckpoint?: LeafCheckpoint;
|
|
40
42
|
/** Relative wait (ms) for `reason: "sleep"`; the broker computes the absolute wake time. */
|
|
41
43
|
durationMs?: number;
|
|
@@ -43,99 +45,8 @@ export interface SuspendSignal {
|
|
|
43
45
|
* woken when this child finalizes (the sweep wakes a parent whose child is terminal). */
|
|
44
46
|
childRunId?: string;
|
|
45
47
|
}
|
|
46
|
-
/**
|
|
47
|
-
|
|
48
|
-
* supervisor kills out-of-band), the backend worker runs the program IN-PROCESS, so a suspend is
|
|
49
|
-
* surfaced by the host calling `onSuspend(signal)` and returning a NEVER-resolving promise; the
|
|
50
|
-
* worker races that against the program body and tears down. This class exists so a seam that has no
|
|
51
|
-
* `onSuspend` wired (the local/test path) can still raise a typed, catchable signal.
|
|
52
|
-
*/
|
|
53
|
-
export declare class SuspendError extends Error {
|
|
54
|
-
readonly signal: SuspendSignal;
|
|
55
|
-
constructor(signal: SuspendSignal);
|
|
56
|
-
}
|
|
57
|
-
/** Journal-entry states (mirrors `run_journal.state` + the engine's IPC). */
|
|
58
|
-
export declare const JOURNAL_STATES: readonly ["pending", "suspended", "resolved"];
|
|
59
|
-
export type JournalEntryState = (typeof JOURNAL_STATES)[number];
|
|
60
|
-
/** A memoized journal entry, as the host reads it back on replay (mirrors the engine's IPC shape). */
|
|
61
|
-
export interface JournalLookup {
|
|
62
|
-
seq: number;
|
|
63
|
-
kind: JournalKind;
|
|
64
|
-
fingerprint: string;
|
|
65
|
-
/** `resolved` ⇒ `result` is the memoized value; `suspended` ⇒ a parked agent leaf (result is the
|
|
66
|
-
* {@link LeafResume} the host re-enters with); `pending` ⇒ awaiting an external event (re-suspend). */
|
|
67
|
-
state: JournalEntryState;
|
|
68
|
-
result: unknown;
|
|
69
|
-
}
|
|
70
|
-
/** Validate the broker's journal_get response (the worker's run token doesn't exempt the channel from
|
|
71
|
-
* validation). The result is genuinely heterogeneous JSON (an agent return / a LeafResume / a
|
|
72
|
-
* HumanInputResult), parsed per-kind downstream — so it stays `unknown` here, the validation seam. */
|
|
73
|
-
export declare const journalLookupSchema: z.ZodObject<{
|
|
74
|
-
seq: z.ZodNumber;
|
|
75
|
-
kind: z.ZodEnum<{
|
|
76
|
-
sleep: "sleep";
|
|
77
|
-
agent: "agent";
|
|
78
|
-
step: "step";
|
|
79
|
-
human_input: "human_input";
|
|
80
|
-
workflow_call: "workflow_call";
|
|
81
|
-
}>;
|
|
82
|
-
fingerprint: z.ZodString;
|
|
83
|
-
state: z.ZodEnum<{
|
|
84
|
-
pending: "pending";
|
|
85
|
-
suspended: "suspended";
|
|
86
|
-
resolved: "resolved";
|
|
87
|
-
}>;
|
|
88
|
-
result: z.ZodUnknown;
|
|
89
|
-
}, z.core.$strip>;
|
|
90
|
-
/** The journal the host reads (replay lookup) + writes (resolved seam results). Backed by the broker
|
|
91
|
-
* over the run token on hosted runs; absent on the local/test path (no durable suspension). */
|
|
92
|
-
export interface JournalSeam {
|
|
93
|
-
/** The memoized entry for a seam, or null on a replay miss. */
|
|
94
|
-
get(seq: number): Promise<JournalLookup | null>;
|
|
95
|
-
/** Record a RESOLVED seam result (idempotent on the run + seq; a resolved entry is immutable). */
|
|
96
|
-
put(entry: {
|
|
97
|
-
seq: number;
|
|
98
|
-
kind: JournalKind;
|
|
99
|
-
fingerprint: string;
|
|
100
|
-
label: string;
|
|
101
|
-
result: unknown;
|
|
102
|
-
}): Promise<void>;
|
|
103
|
-
}
|
|
104
|
-
/**
|
|
105
|
-
* The synchronous, monotonic durable-seam counter. Incremented at each journaled seam's ENTRY:
|
|
106
|
-
* because a program's synchronous call order is deterministic (even under `Promise.all`, whose
|
|
107
|
-
* `.map(...)` runs left-to-right synchronously), the same logical call gets the same `seq` on every
|
|
108
|
-
* execution — the journal key that lets a resumed run return a memoized result.
|
|
109
|
-
*
|
|
110
|
-
* It also drives SILENT REPLAY: a resume starts suppressed (observability — console output, phase
|
|
111
|
-
* markers — was already emitted last segment) and goes `live` the moment it reaches the suspending
|
|
112
|
-
* seam (the frontier = the highest journaled seq). A fresh run (frontier 0) is live immediately.
|
|
113
|
-
*/
|
|
114
|
-
export declare class SeamSequencer {
|
|
115
|
-
private readonly replayFrontier;
|
|
48
|
+
/** A monotonic per-run counter for suspension/gate keys (the `seq` on {@link SuspendSignal}). */
|
|
49
|
+
export declare class SuspensionCounter {
|
|
116
50
|
private count;
|
|
117
|
-
private liveFlag;
|
|
118
|
-
constructor(replayFrontier?: number);
|
|
119
51
|
next(): number;
|
|
120
|
-
/** True once execution has crossed the replay frontier — output after this point is NEW. */
|
|
121
|
-
get isLive(): boolean;
|
|
122
|
-
/** True while re-running already-journaled seams on a resume (observability suppressed). */
|
|
123
|
-
isReplaying(): boolean;
|
|
124
52
|
}
|
|
125
|
-
/** The child run id a `workflow_call` seam journals while it waits (parsed back on resume). */
|
|
126
|
-
export declare const childRunIdSchema: z.ZodString;
|
|
127
|
-
/** A stable content hash of a seam's salient args — the determinism check on replay. Identical
|
|
128
|
-
* construction to the engine's `seamFingerprint` so a journal written by one runtime validates in
|
|
129
|
-
* the other (the conformance promise). */
|
|
130
|
-
export declare function seamFingerprint(parts: readonly unknown[]): string;
|
|
131
|
-
/** A seam reached on replay didn't match what the journal recorded at that seq — the workflow's code
|
|
132
|
-
* on the path to a suspend changed (a different prompt/model/step name, or a different seam kind).
|
|
133
|
-
* Fails the run loudly rather than returning a stale memoized result for the wrong call. */
|
|
134
|
-
export declare function determinismError(seq: number, got: JournalKind, recorded: JournalKind): AppError;
|
|
135
|
-
/**
|
|
136
|
-
* The shape of a SUSPENDED agent leaf's journal result on resume: the transcript checkpoint plus the
|
|
137
|
-
* answers the broker joined from the resolved request rows, keyed by tool-call id. `messages` is the
|
|
138
|
-
* engine's own serialized transcript round-tripping through JSON — handed straight back to the leaf,
|
|
139
|
-
* not re-validated field-by-field. Structurally a {@link LeafResume}.
|
|
140
|
-
*/
|
|
141
|
-
export declare const leafResumeSchema: z.ZodType<LeafResume>;
|