@boardwalk-labs/runner 0.1.20 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/daemon/daemon.js +4 -2
- package/dist/runtime/freeze_coordinator.d.ts +5 -0
- package/dist/runtime/freeze_coordinator.js +3 -0
- package/dist/runtime/index.d.ts +1 -1
- package/dist/runtime/index.js +20 -40
- package/dist/runtime/program_runner.d.ts +4 -15
- package/dist/runtime/program_runner.js +6 -21
- package/dist/runtime/program_worker.d.ts +0 -18
- package/dist/runtime/program_worker.js +1 -20
- package/dist/runtime/runner_control_client.d.ts +2 -22
- package/dist/runtime/runner_control_client.js +2 -49
- package/dist/runtime/runtime_flusher.d.ts +11 -0
- package/dist/runtime/runtime_flusher.js +20 -0
- package/dist/runtime/suspension.d.ts +12 -107
- package/dist/runtime/suspension.js +18 -111
- package/dist/runtime/wire/human_input.d.ts +1 -1
- package/dist/runtime/wire/human_input.js +2 -2
- package/dist/runtime/workflow_host.d.ts +42 -60
- package/dist/runtime/workflow_host.js +89 -199
- package/package.json +3 -3
package/dist/daemon/daemon.js
CHANGED
|
@@ -7,7 +7,6 @@
|
|
|
7
7
|
// per-run workspace, then tears the workspace down (contract: cleanup "always").
|
|
8
8
|
import { mkdir, rm } from "node:fs/promises";
|
|
9
9
|
import * as path from "node:path";
|
|
10
|
-
import * as os from "node:os";
|
|
11
10
|
import { createLogger } from "../runtime/support/index.js";
|
|
12
11
|
/** Assignment-lease heartbeat cadence — comfortably beats the 300s lease. */
|
|
13
12
|
const HEARTBEAT_MS = 60_000;
|
|
@@ -23,7 +22,10 @@ export function runProcessEnv(claim, opts) {
|
|
|
23
22
|
BOARDWALK_CONTROL_PLANE_URL: claim.control_plane.base_url,
|
|
24
23
|
BOARDWALK_RUN_TOKEN: claim.control_plane.run_token,
|
|
25
24
|
BOARDWALK_API_KEY: claim.control_plane.api_token,
|
|
26
|
-
BOARDWALK_TASK_CPU_UNITS:
|
|
25
|
+
// BOARDWALK_TASK_CPU_UNITS is deliberately ABSENT: self-hosted compute is Boardwalk-unbilled
|
|
26
|
+
// (SELF_HOSTED_RUNNERS.md D5), so the runtime's wall-clock fallback (1 vCPU) is the right
|
|
27
|
+
// display metric. Stamping the host's core count here billed a run at cores × the compute rate
|
|
28
|
+
// for hardware the org owns.
|
|
27
29
|
// The org's BYO inference providers, as data (the runner-direct BYO design) — consumed by the runtime's direct
|
|
28
30
|
// model path. Names + endpoints + secret NAMES only; never a credential value.
|
|
29
31
|
BOARDWALK_BYO_PROVIDERS: JSON.stringify(claim.byo_providers),
|
|
@@ -65,6 +65,11 @@ export interface FreezeCoordinatorHooks {
|
|
|
65
65
|
/** Runs when a wake lands, before the seam resolves: swap the run/api tokens onto the
|
|
66
66
|
* broker client and rebase the runtime meter past the frozen window. */
|
|
67
67
|
onAfterWake?: (wake: WakePayload) => void | Promise<void>;
|
|
68
|
+
/** Runs when a REQUESTED freeze aborts (`suspend_abort` — snapshot/store failure) before the
|
|
69
|
+
* parked seam resolves: undo onBeforeFreeze's reversible effects (resume the paused runtime
|
|
70
|
+
* flusher) since the run now HOLDS in-process instead of freezing. Not called for a failure
|
|
71
|
+
* inside onBeforeFreeze itself — that hook owns its own unwind. Must not throw. */
|
|
72
|
+
onFreezeAborted?: () => void;
|
|
68
73
|
}
|
|
69
74
|
export interface FreezeCoordinatorDeps {
|
|
70
75
|
channel: RelayChannel;
|
|
@@ -237,6 +237,9 @@ export class FreezeCoordinator {
|
|
|
237
237
|
}
|
|
238
238
|
const resolve = this.parked;
|
|
239
239
|
this.parked = null;
|
|
240
|
+
// The freeze is off — the seam will hold in-process, so onBeforeFreeze's reversible prep
|
|
241
|
+
// (the paused runtime flusher) must unwind before the hold starts metering-blind.
|
|
242
|
+
this.hooks.onFreezeAborted?.();
|
|
240
243
|
resolve({ kind: "aborted", reason });
|
|
241
244
|
}
|
|
242
245
|
awaitQuiescence(abort) {
|
package/dist/runtime/index.d.ts
CHANGED
|
@@ -32,7 +32,7 @@ export interface WorkerRuntime {
|
|
|
32
32
|
vcpus: number;
|
|
33
33
|
/** The in-guest identity relay, present ONLY on the snapshot-based microVM substrate (relay-mode
|
|
34
34
|
* boot). When set, the worker suspends by FREEZING — the FreezeCoordinator parks seams over this
|
|
35
|
-
* relay's suspend/wake channel —
|
|
35
|
+
* relay's suspend/wake channel — the whole VM freezes and the wake resolves seams in place. */
|
|
36
36
|
freezeRelay?: IdentityRelay;
|
|
37
37
|
/** The browser tier's process backend, present ONLY when the runner IMAGE ships the browser stack
|
|
38
38
|
* (Chromium + a pre-installed Playwright MCP + an X display; gated by BOARDWALK_BROWSER_TIER).
|
package/dist/runtime/index.js
CHANGED
|
@@ -99,10 +99,6 @@ export function assembleWorkerDeps(runtime) {
|
|
|
99
99
|
runId: runtime.runId,
|
|
100
100
|
});
|
|
101
101
|
log.info("runner_control_enabled", { runId: runtime.runId });
|
|
102
|
-
// The replay frontier (the highest journaled seq at claim) drives SILENT REPLAY (the durable-suspension design):
|
|
103
|
-
// a resumed run suppresses observability for seams it already ran. Captured at claim, read by
|
|
104
|
-
// buildHost (which runs right after claim) when constructing the per-run host.
|
|
105
|
-
let replayFrontier = 0;
|
|
106
102
|
// Snapshot-substrate suspension (relay-mode boot only): one coordinator for the process, wired to
|
|
107
103
|
// the relay's suspend/wake channel now; its per-run hooks (token swap, meter rebase, workspace
|
|
108
104
|
// persist) late-bind in buildHost/startRuntimeFlush once those objects exist. The circular
|
|
@@ -304,13 +300,6 @@ export function assembleWorkerDeps(runtime) {
|
|
|
304
300
|
workspaceRoot: runtime.workspaceRoot,
|
|
305
301
|
})
|
|
306
302
|
: undefined;
|
|
307
|
-
// Durable suspension (the durable-suspension design): the host raises a suspend OUT OF BAND (onSuspend +
|
|
308
|
-
// a never-resolving seam), so a program's own try/catch can't swallow it; the program runner
|
|
309
|
-
// races `suspendSignal` against the body. One deferred per run — the first seam to suspend wins.
|
|
310
|
-
let resolveSuspend = () => undefined;
|
|
311
|
-
const suspendSignal = new Promise((resolve) => {
|
|
312
|
-
resolveSuspend = resolve;
|
|
313
|
-
});
|
|
314
303
|
// The run's identity + on-demand public-API bearer, surfaced to the program via `import { runtime }`.
|
|
315
304
|
// ids come from the claimed run; the bearer is the captured (scrubbed-from-env) api token, already
|
|
316
305
|
// recorded in this run's redactor above — so threading it into an MCP header keeps it out of LLM
|
|
@@ -341,13 +330,6 @@ export function assembleWorkerDeps(runtime) {
|
|
|
341
330
|
runtime: runtimeContext,
|
|
342
331
|
// workflows.call → broker /children (resolve + callable_by gate server-side), bound to THIS run.
|
|
343
332
|
children: new BrokerChildDispatcher({ client: broker }),
|
|
344
|
-
// The durable-seam journal (memoization + replay) over the broker; the replay frontier drives
|
|
345
|
-
// silent replay; onSuspend resolves the deferred the program runner races.
|
|
346
|
-
journal: broker.journalSeam(),
|
|
347
|
-
replayFrontier,
|
|
348
|
-
onSuspend: (signal) => {
|
|
349
|
-
resolveSuspend(signal);
|
|
350
|
-
},
|
|
351
333
|
// The program's secrets.get(name) → the run's fail-closed RECORDING resolver.
|
|
352
334
|
secrets: { get: (name) => secretResolver.resolve({ name }) },
|
|
353
335
|
// events.emit → broker /events (fan-out server-side).
|
|
@@ -373,19 +355,15 @@ export function assembleWorkerDeps(runtime) {
|
|
|
373
355
|
},
|
|
374
356
|
}
|
|
375
357
|
: {}),
|
|
376
|
-
// Snapshot-substrate suspension: seams freeze in place under the quiescence gate
|
|
377
|
-
//
|
|
358
|
+
// Snapshot-substrate suspension: seams freeze in place under the quiescence gate. Absent
|
|
359
|
+
// (a self-hosted daemon / the Fargate break-glass), waiting seams HOLD the live process.
|
|
378
360
|
...(freeze !== undefined ? { freeze } : {}),
|
|
379
|
-
//
|
|
380
|
-
// freeze substrate,
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
poll: (seq) => broker.pollInputAnswers(seq),
|
|
386
|
-
},
|
|
387
|
-
}
|
|
388
|
-
: {}),
|
|
361
|
+
// Held HITL gates, backed by the broker's inputs endpoints: register-without-release on the
|
|
362
|
+
// freeze substrate, and the WHOLE gate mechanism (register + poll) on the hold path.
|
|
363
|
+
heldInput: {
|
|
364
|
+
register: (seq, gate) => broker.registerInput(seq, gate),
|
|
365
|
+
poll: (seq) => broker.pollInputAnswers(seq),
|
|
366
|
+
},
|
|
389
367
|
// Browser tier: `computer.openBrowser()` + `agent({ session })` resolve through this manager
|
|
390
368
|
// (absent ⇒ the host throws "not available on this runner image").
|
|
391
369
|
...(browserSessions !== undefined ? { browserSessions } : {}),
|
|
@@ -406,6 +384,16 @@ export function assembleWorkerDeps(runtime) {
|
|
|
406
384
|
// The recorder never spans a snapshot: finalize + upload the in-flight segment before the
|
|
407
385
|
// freeze (SCREEN_CAPTURE §4.3). Bounded internally, so a slow upload delays, not blocks, it.
|
|
408
386
|
await capture?.stopAndFlush();
|
|
387
|
+
// Pause LAST (nothing after it can throw, so a failed persist never strands a paused
|
|
388
|
+
// flusher): with the timer off, no tick can land in the post-wake sliver between the
|
|
389
|
+
// guest clock resync and excludeIdle below — a tick there would compute its delta over
|
|
390
|
+
// the whole frozen window and bill suspended time. Resumed on wake and on suspend_abort.
|
|
391
|
+
activeFlusher?.pause();
|
|
392
|
+
},
|
|
393
|
+
// The freeze died (snapshot/store failure) — the seam holds in-process, which must keep
|
|
394
|
+
// metering: undo the pre-freeze pause.
|
|
395
|
+
onFreezeAborted: () => {
|
|
396
|
+
activeFlusher?.resume();
|
|
409
397
|
},
|
|
410
398
|
// On wake: reseed the userspace CSPRNG (clause 3 — a suspend snapshot restored more than
|
|
411
399
|
// once, e.g. a re-dispatch retry, would otherwise repeat its post-wake `crypto.*` draws),
|
|
@@ -426,6 +414,8 @@ export function assembleWorkerDeps(runtime) {
|
|
|
426
414
|
redactor.record(wake.api_token);
|
|
427
415
|
}
|
|
428
416
|
activeFlusher?.excludeIdle(wake.wall_clock_ms - freezeWallMs);
|
|
417
|
+
// Only now that the frozen window is excluded may the flush timer run again.
|
|
418
|
+
activeFlusher?.resume();
|
|
429
419
|
// Resume capture in a fresh segment epoch (a suspend/resume boundary is a segment boundary).
|
|
430
420
|
void capture?.startFresh();
|
|
431
421
|
},
|
|
@@ -451,8 +441,6 @@ export function assembleWorkerDeps(runtime) {
|
|
|
451
441
|
setProgramDir: (dir) => {
|
|
452
442
|
programDir = dir;
|
|
453
443
|
},
|
|
454
|
-
// Resolves when a host seam suspends the run; the program runner races it against the body.
|
|
455
|
-
suspendSignal,
|
|
456
444
|
...(workspaceStore !== undefined ? { workspace: workspaceStore } : {}),
|
|
457
445
|
// The orchestrator reaps every still-open browser session on terminal (kill Chromium + its
|
|
458
446
|
// Playwright MCP) so no browser process leaks past the run.
|
|
@@ -471,8 +459,6 @@ export function assembleWorkerDeps(runtime) {
|
|
|
471
459
|
// Order this session's frames after a previous (crashed) session's, then announce the
|
|
472
460
|
// lifecycle transition the claim IS — the wire's `running` frame.
|
|
473
461
|
runEvents.resumeAfter(claimed.lastEventCursor);
|
|
474
|
-
// The replay frontier for a resumed run (0 on a fresh run); buildHost reads it next.
|
|
475
|
-
replayFrontier = claimed.lastJournalSeq;
|
|
476
462
|
runEvents.emit({ kind: "run_status", status: "running" });
|
|
477
463
|
return claimed.run;
|
|
478
464
|
},
|
|
@@ -501,12 +487,6 @@ export function assembleWorkerDeps(runtime) {
|
|
|
501
487
|
finalizer: {
|
|
502
488
|
finalize: (_id, status, output) => broker.finalize(status, output, runtime.workerId),
|
|
503
489
|
},
|
|
504
|
-
// Durable suspension (the durable-suspension design): persist the wake condition through the broker (journal
|
|
505
|
-
// entry + a HITL request row, or the wake time for a long sleep) and release the lease — no
|
|
506
|
-
// finalize. A wake (an answer or a timer) re-dispatches the run.
|
|
507
|
-
suspender: {
|
|
508
|
-
suspend: (signal, workerId) => broker.suspend(signal, workerId),
|
|
509
|
-
},
|
|
510
490
|
// Runtime metering: flush runtime as periodic deltas (+ a terminal tail) through the broker,
|
|
511
491
|
// idempotent per flush, so a long/perpetual run bills as it burns and the credit watcher sees it —
|
|
512
492
|
// not a single charge at terminal (which a never-terminating run never reached). A fresh per-session
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { WorkflowHost } from "@boardwalk-labs/workflow/runtime";
|
|
2
|
-
import { type SuspendSignal } from "./suspension.js";
|
|
3
2
|
/**
|
|
4
3
|
* Link THIS runtime's `@boardwalk-labs/workflow` into the exec dir's `node_modules` so the
|
|
5
4
|
* program's bare import resolves anywhere (a self-hosted daemon workspace has no ancestor
|
|
@@ -63,18 +62,11 @@ export interface ProgramRunnerDeps {
|
|
|
63
62
|
* throw (a telemetry hiccup can't change the run's result). Absent ⇒ no output entry.
|
|
64
63
|
*/
|
|
65
64
|
onOutput?: (value: unknown) => void;
|
|
66
|
-
/**
|
|
67
|
-
* Resolves when a host seam SUSPENDS the run (the durable-suspension design) — the worker wires it to the
|
|
68
|
-
* host's `onSuspend` so a suspend is surfaced OUT OF BAND, racing the program body. The program's
|
|
69
|
-
* own `try/catch` can't swallow a suspend this way (the suspending seam never resolves; this signal
|
|
70
|
-
* short-circuits at the runner). Absent ⇒ no suspension wired (a seam that suspends throws
|
|
71
|
-
* {@link SuspendError} instead, which is caught here just the same).
|
|
72
|
-
*/
|
|
73
|
-
suspendSignal?: Promise<SuspendSignal>;
|
|
74
65
|
}
|
|
75
|
-
/** Terminal
|
|
76
|
-
*
|
|
77
|
-
*
|
|
66
|
+
/** Terminal result of running a workflow program. `output` is what the program declared via
|
|
67
|
+
* `output(value)` (null when it never did); for a failure it's null and `error` is set. A waiting
|
|
68
|
+
* seam (sleep / humanInput / workflows.call) never surfaces here — it freezes with the VM or
|
|
69
|
+
* holds the process, and the body simply continues when the wait is over. */
|
|
78
70
|
export type ProgramResult = {
|
|
79
71
|
kind: "completed";
|
|
80
72
|
output: unknown;
|
|
@@ -85,9 +77,6 @@ export type ProgramResult = {
|
|
|
85
77
|
code: string;
|
|
86
78
|
message: string;
|
|
87
79
|
};
|
|
88
|
-
} | {
|
|
89
|
-
kind: "suspended";
|
|
90
|
-
signal: SuspendSignal;
|
|
91
80
|
};
|
|
92
81
|
/**
|
|
93
82
|
* Run a workflow program to completion. Installs the host + input, extracts the VERIFIED artifact +
|
|
@@ -27,7 +27,6 @@ import { pathToFileURL } from "node:url";
|
|
|
27
27
|
import { randomUUID } from "node:crypto";
|
|
28
28
|
import { installHost, installInput, installConfig, takeDeclaredOutput, resetRuntime, } from "@boardwalk-labs/workflow/runtime";
|
|
29
29
|
import { AppError, ErrorCode, createLogger } from "./support/index.js";
|
|
30
|
-
import { SuspendError } from "./suspension.js";
|
|
31
30
|
const log = createLogger("ProgramRunner");
|
|
32
31
|
/** Subdirectory (under the work root) that holds transient extracted program trees. */
|
|
33
32
|
const RUN_DIR = ".bw-runs";
|
|
@@ -112,25 +111,11 @@ export async function runWorkflowProgram(args, deps) {
|
|
|
112
111
|
// runner may point at an arbitrary control plane, so refuse an entry that could escape the
|
|
113
112
|
// extraction dir (absolute path or `..` segment) before importing it.
|
|
114
113
|
const entryPath = resolveEntryPath(dir, args.entry);
|
|
115
|
-
// Run the program body
|
|
116
|
-
//
|
|
117
|
-
|
|
118
|
-
// no-`onSuspend` path. Everything else is the program's natural completion / failure.
|
|
119
|
-
const body = runProgramBody(entryPath, deps);
|
|
120
|
-
const result = deps.suspendSignal === undefined
|
|
121
|
-
? await body
|
|
122
|
-
: await Promise.race([
|
|
123
|
-
body,
|
|
124
|
-
deps.suspendSignal.then((signal) => ({ kind: "suspended", signal })),
|
|
125
|
-
]);
|
|
126
|
-
// If the suspend won the race, the body promise is abandoned (its suspending seam never settles);
|
|
127
|
-
// swallow any late settle so it can't surface as an unhandled rejection before the process exits.
|
|
128
|
-
void body.catch(() => undefined);
|
|
129
|
-
return result;
|
|
114
|
+
// Run the program body to its natural completion / failure. A waiting seam freezes with the
|
|
115
|
+
// VM (snapshot substrate) or holds the process — either way the body's own await continues.
|
|
116
|
+
return await runProgramBody(entryPath, deps);
|
|
130
117
|
}
|
|
131
118
|
catch (err) {
|
|
132
|
-
if (err instanceof SuspendError)
|
|
133
|
-
return { kind: "suspended", signal: err.signal };
|
|
134
119
|
const redactText = deps.redactText ?? ((s) => s);
|
|
135
120
|
// Redact BEFORE both sinks: the message can carry a secret the program resolved then threw.
|
|
136
121
|
const message = redactText(err instanceof Error ? err.message : String(err));
|
|
@@ -157,9 +142,9 @@ export async function runWorkflowProgram(args, deps) {
|
|
|
157
142
|
}
|
|
158
143
|
/**
|
|
159
144
|
* Import (= run) the program entry and capture its declared output. Resolves to a `completed`
|
|
160
|
-
* result; a program failure
|
|
161
|
-
*
|
|
162
|
-
*
|
|
145
|
+
* result; a program failure THROWS and is handled by the caller. A unique dir per run gives a
|
|
146
|
+
* fresh URL so the module cache never returns an already-run program; `@vite-ignore` keeps
|
|
147
|
+
* vitest/vite from statically analyzing the runtime URL.
|
|
163
148
|
*/
|
|
164
149
|
async function runProgramBody(entryPath, deps) {
|
|
165
150
|
await import(/* @vite-ignore */ pathToFileURL(entryPath).href);
|
|
@@ -3,7 +3,6 @@ import { type WorkflowManifest } from "./wire/manifest.js";
|
|
|
3
3
|
import type { WorkflowHost } from "@boardwalk-labs/workflow/runtime";
|
|
4
4
|
import type { SecretRedactor } from "./agent/secret_redactor.js";
|
|
5
5
|
import { type LogStream } from "./program_log_capture.js";
|
|
6
|
-
import type { SuspendSignal } from "./suspension.js";
|
|
7
6
|
/** Default 5-minute lease (matches the engine spec). */
|
|
8
7
|
export declare const DEFAULT_LEASE_MS: number;
|
|
9
8
|
/** Race-safe claim surface — RunRepository satisfies it. */
|
|
@@ -44,13 +43,6 @@ export type RuntimeMeterStarter = (args: {
|
|
|
44
43
|
export interface RunFinalizer {
|
|
45
44
|
finalize(runId: string, status: "completed" | "failed", output: unknown): Promise<void>;
|
|
46
45
|
}
|
|
47
|
-
/** Persists a durable SUSPENSION (the durable-suspension design): the broker records the wake condition (a
|
|
48
|
-
* pending/suspended journal entry + a human-input request row for HITL, or the wake time for a long
|
|
49
|
-
* sleep), flips the run to its suspended status, and releases the lease — all transactionally. The
|
|
50
|
-
* run is NOT finalized; a wake (an answer, a child finalize, or a timer) re-dispatches it. */
|
|
51
|
-
export interface RunSuspender {
|
|
52
|
-
suspend(signal: SuspendSignal, workerId: string): Promise<void>;
|
|
53
|
-
}
|
|
54
46
|
/** Restores/snapshots the workflow's persistent `/workspace`. Best-effort — both no-op when the
|
|
55
47
|
* run isn't eligible (not opted-in / self-hosted), and neither throws. */
|
|
56
48
|
export interface WorkspaceHandle {
|
|
@@ -93,10 +85,6 @@ export type ProgramHostBuilder = (run: Run, manifest: WorkflowManifest, signal:
|
|
|
93
85
|
* leaf can resolve this run's bundled skill files (`<dir>/skills/<name>.md`). The orchestrator wires
|
|
94
86
|
* it to the runner's `onExtracted`. Optional — absent on paths that don't surface bundled files. */
|
|
95
87
|
setProgramDir?: (dir: string) => void;
|
|
96
|
-
/** Resolves when a host seam SUSPENDS the run — wired to the host's `onSuspend`, threaded into the
|
|
97
|
-
* program runner so a suspend short-circuits the body out of band (the durable-suspension design). Absent ⇒
|
|
98
|
-
* no durable suspension on this path (a suspend then surfaces as a thrown SuspendError). */
|
|
99
|
-
suspendSignal?: Promise<SuspendSignal>;
|
|
100
88
|
/** The run's browser-session manager (browser tier). The orchestrator reaps every still-open session
|
|
101
89
|
* on EVERY terminal path so no Chromium / Playwright MCP process leaks past the run. `closeAll` is
|
|
102
90
|
* best-effort + never throws. Absent on images without the browser stack. */
|
|
@@ -154,9 +142,6 @@ export interface ProgramWorkerDeps {
|
|
|
154
142
|
/** Periodic runtime metering (optional — absent disables it, e.g. the local/test path). */
|
|
155
143
|
startRuntimeFlush?: RuntimeMeterStarter;
|
|
156
144
|
finalizer: RunFinalizer;
|
|
157
|
-
/** Persists a durable suspension (the durable-suspension design). Absent ⇒ no suspension support: a run that
|
|
158
|
-
* reaches a suspend fails cleanly rather than stranding (the brokered worker always wires it). */
|
|
159
|
-
suspender?: RunSuspender;
|
|
160
145
|
buildHost: ProgramHostBuilder;
|
|
161
146
|
/** Starts mid-run credit watching for the session (optional — absent disables it). */
|
|
162
147
|
startCreditWatch?: CreditWatchStarter;
|
|
@@ -183,8 +168,5 @@ export type ProgramWorkerOutcome = {
|
|
|
183
168
|
} | {
|
|
184
169
|
kind: "failed";
|
|
185
170
|
reason: string;
|
|
186
|
-
} | {
|
|
187
|
-
kind: "suspended";
|
|
188
|
-
reason: string;
|
|
189
171
|
};
|
|
190
172
|
export declare function runProgramWorker(runId: string, deps: ProgramWorkerDeps): Promise<ProgramWorkerOutcome>;
|
|
@@ -110,7 +110,7 @@ export async function runProgramWorker(runId, deps) {
|
|
|
110
110
|
log.error("worker_host_build_failed", { runId, error: message });
|
|
111
111
|
return { kind: "failed", reason: "host_build_failed" };
|
|
112
112
|
}
|
|
113
|
-
const { host, redactor, workspace, phases, activity, setProgramDir, lsp
|
|
113
|
+
const { host, redactor, workspace, phases, activity, setProgramDir, lsp } = built;
|
|
114
114
|
const browserSessions = built.browserSessions;
|
|
115
115
|
const capture = built.capture;
|
|
116
116
|
// Guarantee the /workspace sandbox dir exists for EVERY run (persist or not) so a program can write
|
|
@@ -193,7 +193,6 @@ export async function runProgramWorker(runId, deps) {
|
|
|
193
193
|
redactText: (text) => redactor.redactText(text),
|
|
194
194
|
extract: deps.extractArchive,
|
|
195
195
|
...(setProgramDir !== undefined ? { onExtracted: setProgramDir } : {}),
|
|
196
|
-
...(suspendSignal !== undefined ? { suspendSignal } : {}),
|
|
197
196
|
...(activity !== undefined
|
|
198
197
|
? {
|
|
199
198
|
onOutput: (value) => {
|
|
@@ -268,24 +267,6 @@ export async function runProgramWorker(runId, deps) {
|
|
|
268
267
|
log.info("worker_run_aborted", { runId, reason });
|
|
269
268
|
return { kind: "failed", reason };
|
|
270
269
|
}
|
|
271
|
-
// Suspended: a host seam released the task (a long `sleep`, a `humanInput()` gate, or the in-leaf
|
|
272
|
-
// `human_input` tool). Persist the wake condition through the broker — NO finalize — and exit
|
|
273
|
-
// cleanly; a wake (an answer or a timer) re-dispatches the run, which restarts from the top and
|
|
274
|
-
// replays the journal past the already-done seams. The runtime tail for THIS session was booked
|
|
275
|
-
// above, so idle time while suspended is not billed. Phases stay open (the run is non-terminal).
|
|
276
|
-
if (result.kind === "suspended") {
|
|
277
|
-
if (deps.suspender === undefined) {
|
|
278
|
-
phases?.close("failed");
|
|
279
|
-
await deps.finalizer.finalize(runId, "failed", {
|
|
280
|
-
error: { code: "SUSPEND_UNSUPPORTED", message: "This runtime cannot suspend a run." },
|
|
281
|
-
});
|
|
282
|
-
log.error("worker_suspend_unsupported", { runId });
|
|
283
|
-
return { kind: "failed", reason: "suspend_unsupported" };
|
|
284
|
-
}
|
|
285
|
-
await deps.suspender.suspend(result.signal, deps.workerId);
|
|
286
|
-
log.info("worker_suspended", { runId, reason: result.signal.reason });
|
|
287
|
-
return { kind: "suspended", reason: result.signal.reason };
|
|
288
|
-
}
|
|
289
270
|
if (result.kind === "completed") {
|
|
290
271
|
phases?.close("completed");
|
|
291
272
|
await deps.finalizer.finalize(runId, "completed", result.output);
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { McpTokenResult } from "@boardwalk-labs/engine/core";
|
|
2
2
|
import type { Run } from "./wire/run.js";
|
|
3
|
-
import { type JournalKind, type JournalLookup, type JournalSeam, type SuspendSignal } from "./suspension.js";
|
|
4
3
|
import type { WebSearchOutput } from "./tools/web_search.js";
|
|
5
4
|
import type { ArtifactCommitInput, ArtifactPresignInput, ArtifactPresignResult, ArtifactSignResult, ArtifactSummary, ArtifactWriteInput, ArtifactWriteResult } from "./tools/artifacts.js";
|
|
6
5
|
import { type InferenceFrame, type InferenceProxyRequest } from "./wire/inference_proxy.js";
|
|
@@ -61,7 +60,7 @@ export declare class RunnerControlClient {
|
|
|
61
60
|
/** Swap the bearer for a fresh run token (the wake path). Every subsequent call uses it. */
|
|
62
61
|
swapRunToken(token: string): void;
|
|
63
62
|
/**
|
|
64
|
-
* Every SHORT control call (claim / renew / cancel / credit /
|
|
63
|
+
* Every SHORT control call (claim / renew / cancel / credit / inputs / …) goes through here so
|
|
65
64
|
* it carries a hard timeout. Without one, a poll frozen mid-flight on the snapshot substrate
|
|
66
65
|
* hangs FOREVER on restore (the socket is dead but never reset), and since a watcher serializes
|
|
67
66
|
* its ticks, one hung tick wedges that watcher — the dead-connections gotcha for the
|
|
@@ -78,7 +77,7 @@ export declare class RunnerControlClient {
|
|
|
78
77
|
* reset/refused mid-rollover, our own per-attempt timeout on a dead socket) and the
|
|
79
78
|
* load-balancer's {@link RETRYABLE_STATUSES}. Safe to re-send because every caller's body is a
|
|
80
79
|
* reusable string/byte-array (the streaming inference call bypasses this entirely), and the
|
|
81
|
-
* broker's mutating endpoints are idempotent per worker/identifier (
|
|
80
|
+
* broker's mutating endpoints are idempotent per worker/identifier (gate seq, usage
|
|
82
81
|
* identifier, lease per workerId). Before this, ONE blip during an api-server deploy rollover
|
|
83
82
|
* crashed the worker hard mid-suspend/finalize and only crash-reclaim recovered the run.
|
|
84
83
|
*/
|
|
@@ -88,7 +87,6 @@ export declare class RunnerControlClient {
|
|
|
88
87
|
claim(workerId: string, leaseSeconds: number): Promise<{
|
|
89
88
|
run: Run;
|
|
90
89
|
lastEventCursor: number;
|
|
91
|
-
lastJournalSeq: number;
|
|
92
90
|
} | null>;
|
|
93
91
|
/** Heartbeat: extend our lease so a long run isn't reclaimed mid-flight. Returns the new
|
|
94
92
|
* `leaseUntil`, or null when the lease was lost (409 — another worker reclaimed the run), which
|
|
@@ -98,24 +96,6 @@ export declare class RunnerControlClient {
|
|
|
98
96
|
* whose lease expired and whose run was reclaimed + re-dispatched to a new owner), so a
|
|
99
97
|
* hung/partitioned worker that later recovers can't clobber the live run or revive a terminal one. */
|
|
100
98
|
finalize(status: "completed" | "failed", output: unknown, workerId: string): Promise<void>;
|
|
101
|
-
/** Look up a durable-seam journal entry by its seq (the durable-suspension design), or null on a replay miss
|
|
102
|
-
* (404). The broker joins a parked agent leaf's answers into the result server-side. */
|
|
103
|
-
journalGet(seq: number): Promise<JournalLookup | null>;
|
|
104
|
-
/** Record a RESOLVED seam result (idempotent on the run + seq server-side; a resolved entry is
|
|
105
|
-
* immutable). The broker writes the memoized value the next replay returns. */
|
|
106
|
-
journalPut(entry: {
|
|
107
|
-
seq: number;
|
|
108
|
-
kind: JournalKind;
|
|
109
|
-
fingerprint: string;
|
|
110
|
-
label: string;
|
|
111
|
-
result: unknown;
|
|
112
|
-
}): Promise<void>;
|
|
113
|
-
/** Persist a durable SUSPENSION: the broker records the wake condition (a pending/suspended journal
|
|
114
|
-
* entry + a human-input request row for HITL, or the wake time for a long sleep), flips the run to
|
|
115
|
-
* its suspended status, and releases the lease — transactionally. No finalize; a wake re-dispatches. */
|
|
116
|
-
suspend(signal: SuspendSignal, workerId: string): Promise<void>;
|
|
117
|
-
/** The {@link JournalSeam} the worker host reads/writes — a thin adapter over the broker methods. */
|
|
118
|
-
journalSeam(): JournalSeam;
|
|
119
99
|
/** Fetch the run's pinned manifest + program source, or null when the version is missing (404). */
|
|
120
100
|
getVersion(): Promise<BrokerVersion | null>;
|
|
121
101
|
/** Book a runtime-seconds DELTA (the worker's RuntimeFlusher → broker). `identifier` makes a
|
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
// failures (thrown network errors, LB 502/503/504 — a control-plane deploy rollover) are retried
|
|
12
12
|
// with backoff first, so a blip heals in place instead of crashing the run.
|
|
13
13
|
import { createLogger } from "./support/index.js";
|
|
14
|
-
import { journalLookupSchema, } from "./suspension.js";
|
|
15
14
|
import { INFERENCE_NDJSON_CONTENT_TYPE, parseInferenceFrame, serializeInferenceRequest, } from "./wire/inference_proxy.js";
|
|
16
15
|
const log = createLogger("RunnerControlClient");
|
|
17
16
|
/**
|
|
@@ -49,7 +48,7 @@ export class RunnerControlClient {
|
|
|
49
48
|
this.runToken = token;
|
|
50
49
|
}
|
|
51
50
|
/**
|
|
52
|
-
* Every SHORT control call (claim / renew / cancel / credit /
|
|
51
|
+
* Every SHORT control call (claim / renew / cancel / credit / inputs / …) goes through here so
|
|
53
52
|
* it carries a hard timeout. Without one, a poll frozen mid-flight on the snapshot substrate
|
|
54
53
|
* hangs FOREVER on restore (the socket is dead but never reset), and since a watcher serializes
|
|
55
54
|
* its ticks, one hung tick wedges that watcher — the dead-connections gotcha for the
|
|
@@ -70,7 +69,7 @@ export class RunnerControlClient {
|
|
|
70
69
|
* reset/refused mid-rollover, our own per-attempt timeout on a dead socket) and the
|
|
71
70
|
* load-balancer's {@link RETRYABLE_STATUSES}. Safe to re-send because every caller's body is a
|
|
72
71
|
* reusable string/byte-array (the streaming inference call bypasses this entirely), and the
|
|
73
|
-
* broker's mutating endpoints are idempotent per worker/identifier (
|
|
72
|
+
* broker's mutating endpoints are idempotent per worker/identifier (gate seq, usage
|
|
74
73
|
* identifier, lease per workerId). Before this, ONE blip during an api-server deploy rollover
|
|
75
74
|
* crashed the worker hard mid-suspend/finalize and only crash-reclaim recovered the run.
|
|
76
75
|
*/
|
|
@@ -120,9 +119,6 @@ export class RunnerControlClient {
|
|
|
120
119
|
return {
|
|
121
120
|
run: body.run,
|
|
122
121
|
lastEventCursor: body.lastEventCursor ?? 0,
|
|
123
|
-
// The replay frontier for silent replay (the durable-suspension design): the highest journaled seq, so a
|
|
124
|
-
// resumed run knows which seams already ran (suppress their re-emitted observability).
|
|
125
|
-
lastJournalSeq: body.lastJournalSeq ?? 0,
|
|
126
122
|
};
|
|
127
123
|
}
|
|
128
124
|
/** Heartbeat: extend our lease so a long run isn't reclaimed mid-flight. Returns the new
|
|
@@ -153,49 +149,6 @@ export class RunnerControlClient {
|
|
|
153
149
|
if (res.status !== 204)
|
|
154
150
|
throw await brokerError(res, "finalize");
|
|
155
151
|
}
|
|
156
|
-
/** Look up a durable-seam journal entry by its seq (the durable-suspension design), or null on a replay miss
|
|
157
|
-
* (404). The broker joins a parked agent leaf's answers into the result server-side. */
|
|
158
|
-
async journalGet(seq) {
|
|
159
|
-
const res = await this.controlFetch(this.url(`journal/${encodeURIComponent(String(seq))}`), {
|
|
160
|
-
method: "GET",
|
|
161
|
-
headers: this.headers(false),
|
|
162
|
-
});
|
|
163
|
-
if (res.status === 404)
|
|
164
|
-
return null;
|
|
165
|
-
if (res.status !== 200)
|
|
166
|
-
throw await brokerError(res, "journal-get");
|
|
167
|
-
return journalLookupSchema.parse(await res.json());
|
|
168
|
-
}
|
|
169
|
-
/** Record a RESOLVED seam result (idempotent on the run + seq server-side; a resolved entry is
|
|
170
|
-
* immutable). The broker writes the memoized value the next replay returns. */
|
|
171
|
-
async journalPut(entry) {
|
|
172
|
-
const res = await this.controlFetch(this.url("journal"), {
|
|
173
|
-
method: "POST",
|
|
174
|
-
headers: this.headers(true),
|
|
175
|
-
body: JSON.stringify(entry),
|
|
176
|
-
});
|
|
177
|
-
if (res.status !== 204)
|
|
178
|
-
throw await brokerError(res, "journal-put");
|
|
179
|
-
}
|
|
180
|
-
/** Persist a durable SUSPENSION: the broker records the wake condition (a pending/suspended journal
|
|
181
|
-
* entry + a human-input request row for HITL, or the wake time for a long sleep), flips the run to
|
|
182
|
-
* its suspended status, and releases the lease — transactionally. No finalize; a wake re-dispatches. */
|
|
183
|
-
async suspend(signal, workerId) {
|
|
184
|
-
const res = await this.controlFetch(this.url("suspend"), {
|
|
185
|
-
method: "POST",
|
|
186
|
-
headers: this.headers(true),
|
|
187
|
-
body: JSON.stringify({ ...signal, workerId }),
|
|
188
|
-
});
|
|
189
|
-
if (res.status !== 204)
|
|
190
|
-
throw await brokerError(res, "suspend");
|
|
191
|
-
}
|
|
192
|
-
/** The {@link JournalSeam} the worker host reads/writes — a thin adapter over the broker methods. */
|
|
193
|
-
journalSeam() {
|
|
194
|
-
return {
|
|
195
|
-
get: (seq) => this.journalGet(seq),
|
|
196
|
-
put: (entry) => this.journalPut(entry),
|
|
197
|
-
};
|
|
198
|
-
}
|
|
199
152
|
/** Fetch the run's pinned manifest + program source, or null when the version is missing (404). */
|
|
200
153
|
async getVersion() {
|
|
201
154
|
const res = await this.controlFetch(this.url("version"), {
|
|
@@ -38,6 +38,17 @@ export declare class RuntimeFlusher {
|
|
|
38
38
|
excludeIdle(ms: number): void;
|
|
39
39
|
/** Begin periodic delta flushing. */
|
|
40
40
|
start(): void;
|
|
41
|
+
/**
|
|
42
|
+
* Suspend the periodic timer across a VM freeze — the pre-freeze hook pauses AFTER the tail flush
|
|
43
|
+
* (as its last, non-throwing step), and the wake path resumes AFTER {@link excludeIdle}. Without
|
|
44
|
+
* this, a tick landing in the sliver between the guest clock resync and the idle rebase would
|
|
45
|
+
* compute its delta over the whole frozen window and bill suspended time. Reversible, unlike
|
|
46
|
+
* {@link stop}; a freeze that aborts (snapshot failure → the seam holds in-process) must resume so
|
|
47
|
+
* a long hold keeps metering.
|
|
48
|
+
*/
|
|
49
|
+
pause(): void;
|
|
50
|
+
/** Restart periodic flushing after {@link pause} (wake, or a freeze abort). No-op once stopped. */
|
|
51
|
+
resume(): void;
|
|
41
52
|
/** Stop the periodic timer and drain any in-flight flush. Does NOT book the final tail — call
|
|
42
53
|
* {@link flushFinal} for that on a clean terminal. Idempotent. */
|
|
43
54
|
stop(): Promise<void>;
|
|
@@ -58,6 +58,26 @@ export class RuntimeFlusher {
|
|
|
58
58
|
// Don't keep the worker process alive solely for the flush timer.
|
|
59
59
|
this.timer.unref();
|
|
60
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* Suspend the periodic timer across a VM freeze — the pre-freeze hook pauses AFTER the tail flush
|
|
63
|
+
* (as its last, non-throwing step), and the wake path resumes AFTER {@link excludeIdle}. Without
|
|
64
|
+
* this, a tick landing in the sliver between the guest clock resync and the idle rebase would
|
|
65
|
+
* compute its delta over the whole frozen window and bill suspended time. Reversible, unlike
|
|
66
|
+
* {@link stop}; a freeze that aborts (snapshot failure → the seam holds in-process) must resume so
|
|
67
|
+
* a long hold keeps metering.
|
|
68
|
+
*/
|
|
69
|
+
pause() {
|
|
70
|
+
if (this.timer === null)
|
|
71
|
+
return;
|
|
72
|
+
clearInterval(this.timer);
|
|
73
|
+
this.timer = null;
|
|
74
|
+
}
|
|
75
|
+
/** Restart periodic flushing after {@link pause} (wake, or a freeze abort). No-op once stopped. */
|
|
76
|
+
resume() {
|
|
77
|
+
if (this.stopped)
|
|
78
|
+
return;
|
|
79
|
+
this.start();
|
|
80
|
+
}
|
|
61
81
|
/** Stop the periodic timer and drain any in-flight flush. Does NOT book the final tail — call
|
|
62
82
|
* {@link flushFinal} for that on a clean terminal. Idempotent. */
|
|
63
83
|
async stop() {
|