@boardwalk-labs/runner 0.1.21 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/README.md +11 -0
  2. package/dist/contract.d.ts +2 -0
  3. package/dist/contract.js +14 -0
  4. package/dist/daemon/container.d.ts +3 -2
  5. package/dist/daemon/container.js +15 -2
  6. package/dist/daemon/daemon.d.ts +1 -0
  7. package/dist/daemon/daemon.js +16 -1
  8. package/dist/runtime/agent/budget.d.ts +14 -1
  9. package/dist/runtime/agent/budget.js +23 -0
  10. package/dist/runtime/broker_child_dispatcher.d.ts +0 -2
  11. package/dist/runtime/broker_child_dispatcher.js +0 -7
  12. package/dist/runtime/budget_gate.d.ts +51 -0
  13. package/dist/runtime/budget_gate.js +114 -0
  14. package/dist/runtime/freeze_coordinator.js +8 -0
  15. package/dist/runtime/index.d.ts +15 -2
  16. package/dist/runtime/index.js +80 -57
  17. package/dist/runtime/leaf_executor.d.ts +20 -0
  18. package/dist/runtime/leaf_executor.js +13 -4
  19. package/dist/runtime/local_workspace_store.d.ts +22 -0
  20. package/dist/runtime/local_workspace_store.js +103 -0
  21. package/dist/runtime/program_runner.d.ts +32 -28
  22. package/dist/runtime/program_runner.js +75 -44
  23. package/dist/runtime/program_worker.d.ts +5 -18
  24. package/dist/runtime/program_worker.js +3 -20
  25. package/dist/runtime/runner_control_client.d.ts +2 -22
  26. package/dist/runtime/runner_control_client.js +2 -49
  27. package/dist/runtime/suspension.d.ts +20 -109
  28. package/dist/runtime/suspension.js +18 -111
  29. package/dist/runtime/wire/human_input.d.ts +1 -1
  30. package/dist/runtime/wire/human_input.js +2 -2
  31. package/dist/runtime/workflow_host.d.ts +65 -62
  32. package/dist/runtime/workflow_host.js +124 -199
  33. package/dist/runtime/workspace_store.d.ts +31 -3
  34. package/dist/runtime/workspace_store.js +43 -4
  35. package/package.json +3 -3
@@ -3,8 +3,8 @@
3
3
  // A run is the execution of a built program ARTIFACT: the worker is handed the VERIFIED tarball
4
4
  // (its sha256 already checked against the pinned digest by the orchestrator) plus the entry module
5
5
  // name. This module is the mechanism: it installs the host adapter + trigger payload onto the
6
- // `@boardwalk-labs/workflow` singleton, extracts the artifact into a unique temp dir under a
7
- // node_modules-reachable root, and dynamic-imports the entry so the program's body runs. The
6
+ // `@boardwalk-labs/workflow` singleton, extracts the artifact into a unique temp dir under the
7
+ // program root, chdirs to the workspace, and dynamic-imports the entry so the program's body runs. The
8
8
  // program's `import { agent, sleep, … } from "@boardwalk-labs/workflow"` resolves to the SAME package
9
9
  // instance the host was installed on (one instance per process), so the hooks reach our adapter.
10
10
  //
@@ -13,32 +13,37 @@
13
13
  // `@boardwalk-labs/workflow` is left external in the bundle, so the imported program resolves it to the SDK
14
14
  // package present in the worker image (giving up its own copy would break the dual-adapter).
15
15
  //
16
- // Why a temp dir under a node_modules-reachable root (not os.tmpdir): the program imports the bare
17
- // specifier `@boardwalk-labs/workflow`, which Node resolves by walking up from the module's location to
18
- // `node_modules`. The extracted tree must therefore live inside the app tree (/app/.bw-runs/*).
16
+ // Two places, both explicit, neither derived from `process.cwd()` (docs/WORKSPACE_PERSISTENCE.md I1/I2):
17
+ // - `workspaceRoot` the run's `/workspace`. The working directory + HOME for AUTHOR code, so a
18
+ // relative write is the correct write and lands in what `workspace.persist` archives.
19
+ // - `programRoot` — where the artifact extracts (`<programRoot>/.bw-runs/<runId>-<uuid>`). Must be
20
+ // OUTSIDE the workspace, or the bundle rides into every snapshot and accumulates across runs.
21
+ // The extracted tree needs no node_modules-reachable ancestor: `ensureSdkLink` (below) links the SDK
22
+ // into the exec dir, so the bare `@boardwalk-labs/workflow` import resolves from any root.
19
23
  //
20
24
  // Durability: the body runs once, in-process. `sleep`/`workflows.call` hold in-process via the host
21
25
  // (no checkpoint, no exit). A crash mid-run restarts the run from the top (handled by the
22
26
  // worker/scheduler-sweep, not here). Output capture is deferred (v0 returns null).
23
27
  import { mkdir, writeFile, rm, symlink, lstat } from "node:fs/promises";
24
- import { dirname, join, isAbsolute, relative, sep } from "node:path";
28
+ import { dirname, join, isAbsolute, relative, resolve, sep } from "node:path";
25
29
  import { createRequire } from "node:module";
26
30
  import { pathToFileURL } from "node:url";
27
31
  import { randomUUID } from "node:crypto";
28
32
  import { installHost, installInput, installConfig, takeDeclaredOutput, resetRuntime, } from "@boardwalk-labs/workflow/runtime";
29
33
  import { AppError, ErrorCode, createLogger } from "./support/index.js";
30
- import { SuspendError } from "./suspension.js";
31
34
  const log = createLogger("ProgramRunner");
32
35
  /** Subdirectory (under the work root) that holds transient extracted program trees. */
33
36
  const RUN_DIR = ".bw-runs";
34
37
  /**
35
- * Link THIS runtime's `@boardwalk-labs/workflow` into the exec dir's `node_modules` so the
36
- * program's bare import resolves anywhere (a self-hosted daemon workspace has no ancestor
37
- * `node_modules`; hosted images do, making the link a harmless no-op there). A symlink not a
38
- * copy is load-bearing: Node resolves it to the REAL path, so the program gets the same
39
- * module instance the host adapter was installed on (the singleton contract). `junction` covers
40
- * Windows without elevation; a failed link is only logged, since a resolvable ancestor may
41
- * still exist.
38
+ * Link THIS runtime's `@boardwalk-labs/workflow` into the exec dir's `node_modules` so the program's
39
+ * bare import resolves from ANY program root this link, not an ancestor `node_modules`, is what
40
+ * makes resolution work. (Worth stating plainly: a stale comment claiming the extraction dir had to
41
+ * sit under `/app` so the import could walk up to `/app/node_modules` outlived this function by
42
+ * several versions and sent a later reader chasing a dependency that no longer exists. The live
43
+ * fleet has no `/app` at all and resolves fine.) A symlink — not a copy is load-bearing: Node
44
+ * resolves it to the REAL path, so the program gets the same module instance the host adapter was
45
+ * installed on (the singleton contract). `junction` covers Windows without elevation; a failed link
46
+ * is only logged, since a resolvable ancestor may still exist.
42
47
  */
43
48
  export async function ensureSdkLink(execDir) {
44
49
  // A program tarball that ships its own `node_modules/@boardwalk-labs/workflow` would shadow the
@@ -83,19 +88,31 @@ export function resolveEntryPath(dir, entry) {
83
88
  }
84
89
  /** Scratch filename for the in-flight artifact tarball inside a run's dir. */
85
90
  const ARTIFACT_FILE = "__program.tgz";
91
+ /**
92
+ * Enforce I2: the extracted program must not live inside the run's workspace. Incidental separation
93
+ * is what broke before — the extraction root used to be `process.cwd()`, so it landed inside the
94
+ * workspace on exactly the lanes whose cwd was already correct. Compares RESOLVED paths, and treats
95
+ * "the workspace itself" as inside.
96
+ */
97
+ export function assertProgramRootOutsideWorkspace(programRoot, workspaceRoot) {
98
+ const rel = relative(resolve(workspaceRoot), resolve(programRoot));
99
+ if (rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel))) {
100
+ throw new AppError(ErrorCode.VALIDATION_FAILED, `The program root "${programRoot}" is inside the workspace "${workspaceRoot}"; the extracted program must live outside it.`);
101
+ }
102
+ }
86
103
  /**
87
104
  * Run a workflow program to completion. Installs the host + input, extracts the VERIFIED artifact +
88
105
  * dynamic-imports its entry (which runs the body), and returns the terminal result. Always tears the
89
106
  * runtime state down and removes the temp tree afterward.
90
107
  */
91
108
  export async function runWorkflowProgram(args, deps) {
92
- const workRoot = deps.workRoot ?? process.cwd();
93
- const dir = join(workRoot, RUN_DIR, `${args.runId}-${randomUUID()}`);
109
+ const dir = join(deps.programRoot, RUN_DIR, `${args.runId}-${randomUUID()}`);
94
110
  installHost(deps.host);
95
111
  installInput(args.input);
96
112
  // The run row's config is arbitrary JSON (jsonb); it IS valid JSON, so narrow to the SDK's JsonValue.
97
113
  installConfig(args.config);
98
114
  try {
115
+ assertProgramRootOutsideWorkspace(deps.programRoot, deps.workspaceRoot);
99
116
  await mkdir(dir, { recursive: true });
100
117
  const tgzPath = join(dir, ARTIFACT_FILE);
101
118
  await writeFile(tgzPath, args.tarball);
@@ -112,25 +129,11 @@ export async function runWorkflowProgram(args, deps) {
112
129
  // runner may point at an arbitrary control plane, so refuse an entry that could escape the
113
130
  // extraction dir (absolute path or `..` segment) before importing it.
114
131
  const entryPath = resolveEntryPath(dir, args.entry);
115
- // Run the program body. A SUSPEND is surfaced two ways and both land here as a `suspended`
116
- // result: (a) out of band via `suspendSignal` racing the body, immune to a program's own
117
- // try/catch (the suspending seam never resolves); (b) a thrown {@link SuspendError} on the
118
- // no-`onSuspend` path. Everything else is the program's natural completion / failure.
119
- const body = runProgramBody(entryPath, deps);
120
- const result = deps.suspendSignal === undefined
121
- ? await body
122
- : await Promise.race([
123
- body,
124
- deps.suspendSignal.then((signal) => ({ kind: "suspended", signal })),
125
- ]);
126
- // If the suspend won the race, the body promise is abandoned (its suspending seam never settles);
127
- // swallow any late settle so it can't surface as an unhandled rejection before the process exits.
128
- void body.catch(() => undefined);
129
- return result;
132
+ // Run the program body to its natural completion / failure. A waiting seam freezes with the
133
+ // VM (snapshot substrate) or holds the processeither way the body's own await continues.
134
+ return await runProgramBody(entryPath, deps);
130
135
  }
131
136
  catch (err) {
132
- if (err instanceof SuspendError)
133
- return { kind: "suspended", signal: err.signal };
134
137
  const redactText = deps.redactText ?? ((s) => s);
135
138
  // Redact BEFORE both sinks: the message can carry a secret the program resolved then threw.
136
139
  const message = redactText(err instanceof Error ? err.message : String(err));
@@ -157,17 +160,45 @@ export async function runWorkflowProgram(args, deps) {
157
160
  }
158
161
  /**
159
162
  * Import (= run) the program entry and capture its declared output. Resolves to a `completed`
160
- * result; a program failure (or a {@link SuspendError} on the no-`onSuspend` path) THROWS and is
161
- * handled by the caller. A unique dir per run gives a fresh URL so the module cache never returns an
162
- * already-run program; `@vite-ignore` keeps vitest/vite from statically analyzing the runtime URL.
163
+ * result; a program failure THROWS and is handled by the caller. A unique dir per run gives a
164
+ * fresh URL so the module cache never returns an already-run program; `@vite-ignore` keeps
165
+ * vitest/vite from statically analyzing the runtime URL.
166
+ *
167
+ * The chdir to the workspace (I1) happens HERE, at the boundary where author code starts, and only
168
+ * after the artifact is extracted from {@link ProgramRunnerDeps.programRoot} — so the program's
169
+ * files land in the workspace while the bundle stays out of it. Module resolution is unaffected:
170
+ * Node resolves file-relative (the entry is imported by absolute URL, the SDK via `ensureSdkLink`),
171
+ * never cwd-relative. The engine's local path has run exactly this shape since dev-on-engine
172
+ * (`boardwalk/src/run/child.ts`), and the self-hosted daemon spawns with the same cwd.
163
173
  */
164
174
  async function runProgramBody(entryPath, deps) {
165
- await import(/* @vite-ignore */ pathToFileURL(entryPath).href);
166
- // The program declares its result via `output(value)` (top-level code can't `return`); null when it
167
- // never called it. This becomes the run's persisted output + a `workflows.call` parent's value, and
168
- // (when actually declared) an `output` entry in the run's activity log.
169
- const declared = takeDeclaredOutput();
170
- if (declared !== null)
171
- deps.onOutput?.(declared.value);
172
- return { kind: "completed", output: declared !== null ? declared.value : null };
175
+ const callerCwd = process.cwd();
176
+ try {
177
+ process.chdir(deps.workspaceRoot);
178
+ }
179
+ catch (err) {
180
+ // Fail loud. Running author code from an arbitrary cwd is what silently threw writes away.
181
+ throw new AppError(ErrorCode.VALIDATION_FAILED, `The run's workspace "${deps.workspaceRoot}" is not usable as the working directory: ${err instanceof Error ? err.message : String(err)}`);
182
+ }
183
+ try {
184
+ await import(/* @vite-ignore */ pathToFileURL(entryPath).href);
185
+ // The program declares its result via `output(value)` (top-level code can't `return`); null when it
186
+ // never called it. This becomes the run's persisted output + a `workflows.call` parent's value, and
187
+ // (when actually declared) an `output` entry in the run's activity log.
188
+ const declared = takeDeclaredOutput();
189
+ if (declared !== null)
190
+ deps.onOutput?.(declared.value);
191
+ return { kind: "completed", output: declared !== null ? declared.value : null };
192
+ }
193
+ finally {
194
+ // One run per process in production, so this matters for tests + the local/dev path — but a
195
+ // function that permanently moves its caller's cwd is a trap either way. Best-effort: the caller's
196
+ // dir may itself be gone, and that must not mask the run's real outcome.
197
+ try {
198
+ process.chdir(callerCwd);
199
+ }
200
+ catch {
201
+ /* the caller's cwd vanished mid-run; nothing useful to do */
202
+ }
203
+ }
173
204
  }
@@ -3,7 +3,6 @@ import { type WorkflowManifest } from "./wire/manifest.js";
3
3
  import type { WorkflowHost } from "@boardwalk-labs/workflow/runtime";
4
4
  import type { SecretRedactor } from "./agent/secret_redactor.js";
5
5
  import { type LogStream } from "./program_log_capture.js";
6
- import type { SuspendSignal } from "./suspension.js";
7
6
  /** Default 5-minute lease (matches the engine spec). */
8
7
  export declare const DEFAULT_LEASE_MS: number;
9
8
  /** Race-safe claim surface — RunRepository satisfies it. */
@@ -44,13 +43,6 @@ export type RuntimeMeterStarter = (args: {
44
43
  export interface RunFinalizer {
45
44
  finalize(runId: string, status: "completed" | "failed", output: unknown): Promise<void>;
46
45
  }
47
- /** Persists a durable SUSPENSION (the durable-suspension design): the broker records the wake condition (a
48
- * pending/suspended journal entry + a human-input request row for HITL, or the wake time for a long
49
- * sleep), flips the run to its suspended status, and releases the lease — all transactionally. The
50
- * run is NOT finalized; a wake (an answer, a child finalize, or a timer) re-dispatches it. */
51
- export interface RunSuspender {
52
- suspend(signal: SuspendSignal, workerId: string): Promise<void>;
53
- }
54
46
  /** Restores/snapshots the workflow's persistent `/workspace`. Best-effort — both no-op when the
55
47
  * run isn't eligible (not opted-in / self-hosted), and neither throws. */
56
48
  export interface WorkspaceHandle {
@@ -93,10 +85,6 @@ export type ProgramHostBuilder = (run: Run, manifest: WorkflowManifest, signal:
93
85
  * leaf can resolve this run's bundled skill files (`<dir>/skills/<name>.md`). The orchestrator wires
94
86
  * it to the runner's `onExtracted`. Optional — absent on paths that don't surface bundled files. */
95
87
  setProgramDir?: (dir: string) => void;
96
- /** Resolves when a host seam SUSPENDS the run — wired to the host's `onSuspend`, threaded into the
97
- * program runner so a suspend short-circuits the body out of band (the durable-suspension design). Absent ⇒
98
- * no durable suspension on this path (a suspend then surfaces as a thrown SuspendError). */
99
- suspendSignal?: Promise<SuspendSignal>;
100
88
  /** The run's browser-session manager (browser tier). The orchestrator reaps every still-open session
101
89
  * on EVERY terminal path so no Chromium / Playwright MCP process leaks past the run. `closeAll` is
102
90
  * best-effort + never throws. Absent on images without the browser stack. */
@@ -140,6 +128,11 @@ export type LeaseWatchStarter = (args: {
140
128
  export interface ProgramWorkerDeps {
141
129
  runs: RunClaimer;
142
130
  versions: ProgramVersionReader;
131
+ /** The run's `/workspace` — cwd + HOME for author code (docs/WORKSPACE_PERSISTENCE.md I1), and the
132
+ * tree `workspace.persist` archives. Passed through to the program runner. */
133
+ workspaceRoot: string;
134
+ /** Where the program artifact extracts — OUTSIDE the workspace (I2). Passed through to the runner. */
135
+ programRoot: string;
143
136
  /** Download the program artifact bytes from the broker's presigned URL (broker.downloadBytes). */
144
137
  fetchProgram: (downloadUrl: string) => Promise<Uint8Array>;
145
138
  /** Extract a gzipped tar into a dir (system `tar`); passed through to the program runner. */
@@ -154,9 +147,6 @@ export interface ProgramWorkerDeps {
154
147
  /** Periodic runtime metering (optional — absent disables it, e.g. the local/test path). */
155
148
  startRuntimeFlush?: RuntimeMeterStarter;
156
149
  finalizer: RunFinalizer;
157
- /** Persists a durable suspension (the durable-suspension design). Absent ⇒ no suspension support: a run that
158
- * reaches a suspend fails cleanly rather than stranding (the brokered worker always wires it). */
159
- suspender?: RunSuspender;
160
150
  buildHost: ProgramHostBuilder;
161
151
  /** Starts mid-run credit watching for the session (optional — absent disables it). */
162
152
  startCreditWatch?: CreditWatchStarter;
@@ -183,8 +173,5 @@ export type ProgramWorkerOutcome = {
183
173
  } | {
184
174
  kind: "failed";
185
175
  reason: string;
186
- } | {
187
- kind: "suspended";
188
- reason: string;
189
176
  };
190
177
  export declare function runProgramWorker(runId: string, deps: ProgramWorkerDeps): Promise<ProgramWorkerOutcome>;
@@ -110,7 +110,7 @@ export async function runProgramWorker(runId, deps) {
110
110
  log.error("worker_host_build_failed", { runId, error: message });
111
111
  return { kind: "failed", reason: "host_build_failed" };
112
112
  }
113
- const { host, redactor, workspace, phases, activity, setProgramDir, lsp, suspendSignal } = built;
113
+ const { host, redactor, workspace, phases, activity, setProgramDir, lsp } = built;
114
114
  const browserSessions = built.browserSessions;
115
115
  const capture = built.capture;
116
116
  // Guarantee the /workspace sandbox dir exists for EVERY run (persist or not) so a program can write
@@ -190,10 +190,11 @@ export async function runProgramWorker(runId, deps) {
190
190
  // onOutput emits the `output` activity entry into the run's log when the program declared one.
191
191
  {
192
192
  host,
193
+ workspaceRoot: deps.workspaceRoot,
194
+ programRoot: deps.programRoot,
193
195
  redactText: (text) => redactor.redactText(text),
194
196
  extract: deps.extractArchive,
195
197
  ...(setProgramDir !== undefined ? { onExtracted: setProgramDir } : {}),
196
- ...(suspendSignal !== undefined ? { suspendSignal } : {}),
197
198
  ...(activity !== undefined
198
199
  ? {
199
200
  onOutput: (value) => {
@@ -268,24 +269,6 @@ export async function runProgramWorker(runId, deps) {
268
269
  log.info("worker_run_aborted", { runId, reason });
269
270
  return { kind: "failed", reason };
270
271
  }
271
- // Suspended: a host seam released the task (a long `sleep`, a `humanInput()` gate, or the in-leaf
272
- // `human_input` tool). Persist the wake condition through the broker — NO finalize — and exit
273
- // cleanly; a wake (an answer or a timer) re-dispatches the run, which restarts from the top and
274
- // replays the journal past the already-done seams. The runtime tail for THIS session was booked
275
- // above, so idle time while suspended is not billed. Phases stay open (the run is non-terminal).
276
- if (result.kind === "suspended") {
277
- if (deps.suspender === undefined) {
278
- phases?.close("failed");
279
- await deps.finalizer.finalize(runId, "failed", {
280
- error: { code: "SUSPEND_UNSUPPORTED", message: "This runtime cannot suspend a run." },
281
- });
282
- log.error("worker_suspend_unsupported", { runId });
283
- return { kind: "failed", reason: "suspend_unsupported" };
284
- }
285
- await deps.suspender.suspend(result.signal, deps.workerId);
286
- log.info("worker_suspended", { runId, reason: result.signal.reason });
287
- return { kind: "suspended", reason: result.signal.reason };
288
- }
289
272
  if (result.kind === "completed") {
290
273
  phases?.close("completed");
291
274
  await deps.finalizer.finalize(runId, "completed", result.output);
@@ -1,6 +1,5 @@
1
1
  import type { McpTokenResult } from "@boardwalk-labs/engine/core";
2
2
  import type { Run } from "./wire/run.js";
3
- import { type JournalKind, type JournalLookup, type JournalSeam, type SuspendSignal } from "./suspension.js";
4
3
  import type { WebSearchOutput } from "./tools/web_search.js";
5
4
  import type { ArtifactCommitInput, ArtifactPresignInput, ArtifactPresignResult, ArtifactSignResult, ArtifactSummary, ArtifactWriteInput, ArtifactWriteResult } from "./tools/artifacts.js";
6
5
  import { type InferenceFrame, type InferenceProxyRequest } from "./wire/inference_proxy.js";
@@ -61,7 +60,7 @@ export declare class RunnerControlClient {
61
60
  /** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
62
61
  swapRunToken(token: string): void;
63
62
  /**
64
- * Every SHORT control call (claim / renew / cancel / credit / journal / …) goes through here so
63
+ * Every SHORT control call (claim / renew / cancel / credit / inputs / …) goes through here so
65
64
  * it carries a hard timeout. Without one, a poll frozen mid-flight on the snapshot substrate
66
65
  * hangs FOREVER on restore (the socket is dead but never reset), and since a watcher serializes
67
66
  * its ticks, one hung tick wedges that watcher — the dead-connections gotcha for the
@@ -78,7 +77,7 @@ export declare class RunnerControlClient {
78
77
  * reset/refused mid-rollover, our own per-attempt timeout on a dead socket) and the
79
78
  * load-balancer's {@link RETRYABLE_STATUSES}. Safe to re-send because every caller's body is a
80
79
  * reusable string/byte-array (the streaming inference call bypasses this entirely), and the
81
- * broker's mutating endpoints are idempotent per worker/identifier (journal seq, usage
80
+ * broker's mutating endpoints are idempotent per worker/identifier (gate seq, usage
82
81
  * identifier, lease per workerId). Before this, ONE blip during an api-server deploy rollover
83
82
  * crashed the worker hard mid-suspend/finalize and only crash-reclaim recovered the run.
84
83
  */
@@ -88,7 +87,6 @@ export declare class RunnerControlClient {
88
87
  claim(workerId: string, leaseSeconds: number): Promise<{
89
88
  run: Run;
90
89
  lastEventCursor: number;
91
- lastJournalSeq: number;
92
90
  } | null>;
93
91
  /** Heartbeat: extend our lease so a long run isn't reclaimed mid-flight. Returns the new
94
92
  * `leaseUntil`, or null when the lease was lost (409 — another worker reclaimed the run), which
@@ -98,24 +96,6 @@ export declare class RunnerControlClient {
98
96
  * whose lease expired and whose run was reclaimed + re-dispatched to a new owner), so a
99
97
  * hung/partitioned worker that later recovers can't clobber the live run or revive a terminal one. */
100
98
  finalize(status: "completed" | "failed", output: unknown, workerId: string): Promise<void>;
101
- /** Look up a durable-seam journal entry by its seq (the durable-suspension design), or null on a replay miss
102
- * (404). The broker joins a parked agent leaf's answers into the result server-side. */
103
- journalGet(seq: number): Promise<JournalLookup | null>;
104
- /** Record a RESOLVED seam result (idempotent on the run + seq server-side; a resolved entry is
105
- * immutable). The broker writes the memoized value the next replay returns. */
106
- journalPut(entry: {
107
- seq: number;
108
- kind: JournalKind;
109
- fingerprint: string;
110
- label: string;
111
- result: unknown;
112
- }): Promise<void>;
113
- /** Persist a durable SUSPENSION: the broker records the wake condition (a pending/suspended journal
114
- * entry + a human-input request row for HITL, or the wake time for a long sleep), flips the run to
115
- * its suspended status, and releases the lease — transactionally. No finalize; a wake re-dispatches. */
116
- suspend(signal: SuspendSignal, workerId: string): Promise<void>;
117
- /** The {@link JournalSeam} the worker host reads/writes — a thin adapter over the broker methods. */
118
- journalSeam(): JournalSeam;
119
99
  /** Fetch the run's pinned manifest + program source, or null when the version is missing (404). */
120
100
  getVersion(): Promise<BrokerVersion | null>;
121
101
  /** Book a runtime-seconds DELTA (the worker's RuntimeFlusher → broker). `identifier` makes a
@@ -11,7 +11,6 @@
11
11
  // failures (thrown network errors, LB 502/503/504 — a control-plane deploy rollover) are retried
12
12
  // with backoff first, so a blip heals in place instead of crashing the run.
13
13
  import { createLogger } from "./support/index.js";
14
- import { journalLookupSchema, } from "./suspension.js";
15
14
  import { INFERENCE_NDJSON_CONTENT_TYPE, parseInferenceFrame, serializeInferenceRequest, } from "./wire/inference_proxy.js";
16
15
  const log = createLogger("RunnerControlClient");
17
16
  /**
@@ -49,7 +48,7 @@ export class RunnerControlClient {
49
48
  this.runToken = token;
50
49
  }
51
50
  /**
52
- * Every SHORT control call (claim / renew / cancel / credit / journal / …) goes through here so
51
+ * Every SHORT control call (claim / renew / cancel / credit / inputs / …) goes through here so
53
52
  * it carries a hard timeout. Without one, a poll frozen mid-flight on the snapshot substrate
54
53
  * hangs FOREVER on restore (the socket is dead but never reset), and since a watcher serializes
55
54
  * its ticks, one hung tick wedges that watcher — the dead-connections gotcha for the
@@ -70,7 +69,7 @@ export class RunnerControlClient {
70
69
  * reset/refused mid-rollover, our own per-attempt timeout on a dead socket) and the
71
70
  * load-balancer's {@link RETRYABLE_STATUSES}. Safe to re-send because every caller's body is a
72
71
  * reusable string/byte-array (the streaming inference call bypasses this entirely), and the
73
- * broker's mutating endpoints are idempotent per worker/identifier (journal seq, usage
72
+ * broker's mutating endpoints are idempotent per worker/identifier (gate seq, usage
74
73
  * identifier, lease per workerId). Before this, ONE blip during an api-server deploy rollover
75
74
  * crashed the worker hard mid-suspend/finalize and only crash-reclaim recovered the run.
76
75
  */
@@ -120,9 +119,6 @@ export class RunnerControlClient {
120
119
  return {
121
120
  run: body.run,
122
121
  lastEventCursor: body.lastEventCursor ?? 0,
123
- // The replay frontier for silent replay (the durable-suspension design): the highest journaled seq, so a
124
- // resumed run knows which seams already ran (suppress their re-emitted observability).
125
- lastJournalSeq: body.lastJournalSeq ?? 0,
126
122
  };
127
123
  }
128
124
  /** Heartbeat: extend our lease so a long run isn't reclaimed mid-flight. Returns the new
@@ -153,49 +149,6 @@ export class RunnerControlClient {
153
149
  if (res.status !== 204)
154
150
  throw await brokerError(res, "finalize");
155
151
  }
156
- /** Look up a durable-seam journal entry by its seq (the durable-suspension design), or null on a replay miss
157
- * (404). The broker joins a parked agent leaf's answers into the result server-side. */
158
- async journalGet(seq) {
159
- const res = await this.controlFetch(this.url(`journal/${encodeURIComponent(String(seq))}`), {
160
- method: "GET",
161
- headers: this.headers(false),
162
- });
163
- if (res.status === 404)
164
- return null;
165
- if (res.status !== 200)
166
- throw await brokerError(res, "journal-get");
167
- return journalLookupSchema.parse(await res.json());
168
- }
169
- /** Record a RESOLVED seam result (idempotent on the run + seq server-side; a resolved entry is
170
- * immutable). The broker writes the memoized value the next replay returns. */
171
- async journalPut(entry) {
172
- const res = await this.controlFetch(this.url("journal"), {
173
- method: "POST",
174
- headers: this.headers(true),
175
- body: JSON.stringify(entry),
176
- });
177
- if (res.status !== 204)
178
- throw await brokerError(res, "journal-put");
179
- }
180
- /** Persist a durable SUSPENSION: the broker records the wake condition (a pending/suspended journal
181
- * entry + a human-input request row for HITL, or the wake time for a long sleep), flips the run to
182
- * its suspended status, and releases the lease — transactionally. No finalize; a wake re-dispatches. */
183
- async suspend(signal, workerId) {
184
- const res = await this.controlFetch(this.url("suspend"), {
185
- method: "POST",
186
- headers: this.headers(true),
187
- body: JSON.stringify({ ...signal, workerId }),
188
- });
189
- if (res.status !== 204)
190
- throw await brokerError(res, "suspend");
191
- }
192
- /** The {@link JournalSeam} the worker host reads/writes — a thin adapter over the broker methods. */
193
- journalSeam() {
194
- return {
195
- get: (seq) => this.journalGet(seq),
196
- put: (entry) => this.journalPut(entry),
197
- };
198
- }
199
152
  /** Fetch the run's pinned manifest + program source, or null when the version is missing (404). */
200
153
  async getVersion() {
201
154
  const res = await this.controlFetch(this.url("version"), {
@@ -1,20 +1,21 @@
1
- import { z } from "zod";
2
- import type { LeafCheckpoint, LeafResume } from "@boardwalk-labs/engine/core";
3
- import { AppError } from "./support/index.js";
1
+ import type { LeafCheckpoint } from "@boardwalk-labs/engine/core";
4
2
  /**
5
- * Sleeps at/above this hold-vs-release boundary SUSPEND (release the task; a timer re-dispatches the
6
- * run when due); shorter ones HOLD the process in-memory, where a release + replay round-trip costs
7
- * more than it saves. Mirrors the engine's `SUSPEND_THRESHOLD_MS` so the two runtimes draw the line
8
- * in the same place.
3
+ * Sleeps at/above this boundary SUSPEND on the snapshot substrate (freeze the VM; the wake fires
4
+ * when the sleep is due); shorter ones HOLD the process in-memory, where a snapshot round-trip
5
+ * costs more than it saves. Without a freeze substrate every sleep holds, whatever its length.
9
6
  */
10
7
  export declare const SUSPEND_THRESHOLD_MS = 30000;
11
- /** The durable-seam kinds the journal memoizes (mirrors `run_journal.kind` + the engine's IPC). */
12
- export type JournalKind = "agent" | "step" | "human_input" | "sleep" | "workflow_call";
13
- /** Why a seam suspended the run. */
14
- export type SuspendReason = "human_input" | "sleep" | "workflow_call";
8
+ /**
9
+ * Why a seam suspended the run. `budget` is the odd one out: it is NOT a seam the program called but
10
+ * an involuntary park the run hit its `max_usd` cap and is waiting for a person to approve more
11
+ * spend (docs/SUSPEND_POLICY.md Decision 3). It still carries a `humanInput` gate (key `budget`), so
12
+ * the control plane persists, surfaces, and answers it exactly like any other gate; only the reason
13
+ * differs, which is what lets the UI say "budget" instead of "waiting on a human".
14
+ */
15
+ export type SuspendReason = "human_input" | "sleep" | "workflow_call" | "budget";
15
16
  /** A human-in-the-loop gate carried out of a suspending seam (program-level or the in-leaf tool). */
16
17
  export interface HumanInputGate {
17
- /** The stable key the responder answers by (an author/seq key, or the model's tool-call id). */
18
+ /** The stable key the responder answers by (an author/derived key, or the model's tool-call id). */
18
19
  key: string;
19
20
  prompt: string;
20
21
  /** The response form ({@link import("@boardwalk-labs/workflow").HumanInputSpec}); validated on submit. */
@@ -29,13 +30,14 @@ export interface HumanInputGate {
29
30
  /** Everything the broker needs to persist a suspension + the wake condition. */
30
31
  export interface SuspendSignal {
31
32
  reason: SuspendReason;
32
- /** The journal seq of the suspending seam. */
33
+ /** The suspension's within-run key (a monotonic per-run counter): it keys the HITL gate rows the
34
+ * wake joins answers from. Not a journal seq — there is no journal. */
33
35
  seq: number;
34
- /** The seam's determinism fingerprint (recorded on the pending journal entry). */
35
- fingerprint: string;
36
36
  /** Present for `reason: "human_input"` — the gate to register a request row for. */
37
37
  humanInput?: HumanInputGate;
38
- /** A tool-level gate's leaf transcript checkpoint, stored so the leaf resumes where it paused. */
38
+ /** A tool-level gate's leaf transcript checkpoint. On the snapshot substrate the transcript
39
+ * rides in the frozen heap — this field is informational for the control plane, not a resume
40
+ * source. */
39
41
  leafCheckpoint?: LeafCheckpoint;
40
42
  /** Relative wait (ms) for `reason: "sleep"`; the broker computes the absolute wake time. */
41
43
  durationMs?: number;
@@ -43,99 +45,8 @@ export interface SuspendSignal {
43
45
  * woken when this child finalizes (the sweep wakes a parent whose child is terminal). */
44
46
  childRunId?: string;
45
47
  }
46
- /**
47
- * The control signal a host seam raises to suspend the run. Unlike the engine (a child process the
48
- * supervisor kills out-of-band), the backend worker runs the program IN-PROCESS, so a suspend is
49
- * surfaced by the host calling `onSuspend(signal)` and returning a NEVER-resolving promise; the
50
- * worker races that against the program body and tears down. This class exists so a seam that has no
51
- * `onSuspend` wired (the local/test path) can still raise a typed, catchable signal.
52
- */
53
- export declare class SuspendError extends Error {
54
- readonly signal: SuspendSignal;
55
- constructor(signal: SuspendSignal);
56
- }
57
- /** Journal-entry states (mirrors `run_journal.state` + the engine's IPC). */
58
- export declare const JOURNAL_STATES: readonly ["pending", "suspended", "resolved"];
59
- export type JournalEntryState = (typeof JOURNAL_STATES)[number];
60
- /** A memoized journal entry, as the host reads it back on replay (mirrors the engine's IPC shape). */
61
- export interface JournalLookup {
62
- seq: number;
63
- kind: JournalKind;
64
- fingerprint: string;
65
- /** `resolved` ⇒ `result` is the memoized value; `suspended` ⇒ a parked agent leaf (result is the
66
- * {@link LeafResume} the host re-enters with); `pending` ⇒ awaiting an external event (re-suspend). */
67
- state: JournalEntryState;
68
- result: unknown;
69
- }
70
- /** Validate the broker's journal_get response (the worker's run token doesn't exempt the channel from
71
- * validation). The result is genuinely heterogeneous JSON (an agent return / a LeafResume / a
72
- * HumanInputResult), parsed per-kind downstream — so it stays `unknown` here, the validation seam. */
73
- export declare const journalLookupSchema: z.ZodObject<{
74
- seq: z.ZodNumber;
75
- kind: z.ZodEnum<{
76
- sleep: "sleep";
77
- agent: "agent";
78
- step: "step";
79
- human_input: "human_input";
80
- workflow_call: "workflow_call";
81
- }>;
82
- fingerprint: z.ZodString;
83
- state: z.ZodEnum<{
84
- pending: "pending";
85
- suspended: "suspended";
86
- resolved: "resolved";
87
- }>;
88
- result: z.ZodUnknown;
89
- }, z.core.$strip>;
90
- /** The journal the host reads (replay lookup) + writes (resolved seam results). Backed by the broker
91
- * over the run token on hosted runs; absent on the local/test path (no durable suspension). */
92
- export interface JournalSeam {
93
- /** The memoized entry for a seam, or null on a replay miss. */
94
- get(seq: number): Promise<JournalLookup | null>;
95
- /** Record a RESOLVED seam result (idempotent on the run + seq; a resolved entry is immutable). */
96
- put(entry: {
97
- seq: number;
98
- kind: JournalKind;
99
- fingerprint: string;
100
- label: string;
101
- result: unknown;
102
- }): Promise<void>;
103
- }
104
- /**
105
- * The synchronous, monotonic durable-seam counter. Incremented at each journaled seam's ENTRY:
106
- * because a program's synchronous call order is deterministic (even under `Promise.all`, whose
107
- * `.map(...)` runs left-to-right synchronously), the same logical call gets the same `seq` on every
108
- * execution — the journal key that lets a resumed run return a memoized result.
109
- *
110
- * It also drives SILENT REPLAY: a resume starts suppressed (observability — console output, phase
111
- * markers — was already emitted last segment) and goes `live` the moment it reaches the suspending
112
- * seam (the frontier = the highest journaled seq). A fresh run (frontier 0) is live immediately.
113
- */
114
- export declare class SeamSequencer {
115
- private readonly replayFrontier;
48
+ /** A monotonic per-run counter for suspension/gate keys (the `seq` on {@link SuspendSignal}). */
49
+ export declare class SuspensionCounter {
116
50
  private count;
117
- private liveFlag;
118
- constructor(replayFrontier?: number);
119
51
  next(): number;
120
- /** True once execution has crossed the replay frontier — output after this point is NEW. */
121
- get isLive(): boolean;
122
- /** True while re-running already-journaled seams on a resume (observability suppressed). */
123
- isReplaying(): boolean;
124
52
  }
125
- /** The child run id a `workflow_call` seam journals while it waits (parsed back on resume). */
126
- export declare const childRunIdSchema: z.ZodString;
127
- /** A stable content hash of a seam's salient args — the determinism check on replay. Identical
128
- * construction to the engine's `seamFingerprint` so a journal written by one runtime validates in
129
- * the other (the conformance promise). */
130
- export declare function seamFingerprint(parts: readonly unknown[]): string;
131
- /** A seam reached on replay didn't match what the journal recorded at that seq — the workflow's code
132
- * on the path to a suspend changed (a different prompt/model/step name, or a different seam kind).
133
- * Fails the run loudly rather than returning a stale memoized result for the wrong call. */
134
- export declare function determinismError(seq: number, got: JournalKind, recorded: JournalKind): AppError;
135
- /**
136
- * The shape of a SUSPENDED agent leaf's journal result on resume: the transcript checkpoint plus the
137
- * answers the broker joined from the resolved request rows, keyed by tool-call id. `messages` is the
138
- * engine's own serialized transcript round-tripping through JSON — handed straight back to the leaf,
139
- * not re-validated field-by-field. Structurally a {@link LeafResume}.
140
- */
141
- export declare const leafResumeSchema: z.ZodType<LeafResume>;