@boardwalk-labs/runner 0.1.16 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/runtime/browser_session_backend.js +6 -1
- package/dist/runtime/http_pool_reset.d.ts +4 -0
- package/dist/runtime/http_pool_reset.js +49 -0
- package/dist/runtime/index.js +18 -1
- package/dist/runtime/run_event_emitter.d.ts +5 -0
- package/dist/runtime/run_event_emitter.js +10 -0
- package/dist/runtime/run_log_file_sink.d.ts +9 -0
- package/dist/runtime/run_log_file_sink.js +106 -0
- package/package.json +2 -1
|
@@ -162,7 +162,12 @@ export function makeGuestBrowserBackend(cfg) {
|
|
|
162
162
|
mcp.once("error", (err) => {
|
|
163
163
|
log.error("browser_mcp_spawn_error", { error: err.message });
|
|
164
164
|
});
|
|
165
|
-
|
|
165
|
+
// Probe the EXACT url the program's client will connect to (the `localhost` form), not a
|
|
166
|
+
// `127.0.0.1` stand-in: if `localhost` doesn't resolve in the guest, connectSessionMcp would
|
|
167
|
+
// otherwise throw a bare `TypeError: fetch failed` while this readiness check passed on a
|
|
168
|
+
// different address (masking the fault). Probing mcpUrl turns that into a diagnosable
|
|
169
|
+
// `browser_backend_timeout`. (The guest ships /etc/hosts mapping localhost → 127.0.0.1.)
|
|
170
|
+
await waitForHttp(mcpUrl, cfg.readyTimeoutMs);
|
|
166
171
|
let killed = false;
|
|
167
172
|
return {
|
|
168
173
|
cdpUrl,
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** Discard the global HTTP connection pool so post-restore requests open fresh sockets. Never
|
|
2
|
+
* throws (a wake must not fail on pool hygiene) and never blocks (the dead pool drains in the
|
|
3
|
+
* background). Returns true if the dispatcher was swapped. */
|
|
4
|
+
export declare function resetHttpConnectionPool(): boolean;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// Reset the global HTTP connection pool after a snapshot restore.
|
|
2
|
+
//
|
|
3
|
+
// The North Star substrate freezes the whole VM (memory + every socket) and restores it later,
|
|
4
|
+
// possibly minutes or days on. Keep-alive TCP sockets do NOT survive that: undici pools them, but
|
|
5
|
+
// on restore the frozen socket state is stale and the peer — the per-host egress proxy (a CONNECT
|
|
6
|
+
// tunnel) and, through it, the broker — long since closed its end. undici would REUSE such a socket
|
|
7
|
+
// for the next request, which then hangs (no bytes flow, no RST arrives on a half-dead tunnel) until
|
|
8
|
+
// the control client's 30s AbortSignal.timeout fires and aborts it. On a woken run's FIRST broker
|
|
9
|
+
// call (finalize) that abort crashes the run — the "socket is dead but never reset" gap the control
|
|
10
|
+
// client's timeout only papered over (see runner_control_client.ts).
|
|
11
|
+
//
|
|
12
|
+
// The fix is to discard the dead pool on resume. Swapping in a fresh EnvHttpProxyAgent — which
|
|
13
|
+
// re-reads the unchanged HTTP(S)_PROXY env, so proxying is preserved — means every post-wake request
|
|
14
|
+
// opens a NEW socket instead of reusing a frozen one. Node's global `fetch` reads the same global
|
|
15
|
+
// dispatcher, so this covers the control client, inference, artifacts, and any author fetch alike.
|
|
16
|
+
import { EnvHttpProxyAgent, getGlobalDispatcher, setGlobalDispatcher } from "undici";
|
|
17
|
+
import { createLogger } from "./support/index.js";
|
|
18
|
+
const log = createLogger("http_pool_reset");
|
|
19
|
+
/** Discard the global HTTP connection pool so post-restore requests open fresh sockets. Never
|
|
20
|
+
* throws (a wake must not fail on pool hygiene) and never blocks (the dead pool drains in the
|
|
21
|
+
* background). Returns true if the dispatcher was swapped. */
|
|
22
|
+
export function resetHttpConnectionPool() {
|
|
23
|
+
try {
|
|
24
|
+
const prev = getGlobalDispatcher();
|
|
25
|
+
setGlobalDispatcher(new EnvHttpProxyAgent());
|
|
26
|
+
// Drain the old pool's dead sockets in the background; never block the wake on it.
|
|
27
|
+
const drain = async () => {
|
|
28
|
+
try {
|
|
29
|
+
await prev?.close?.();
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
try {
|
|
33
|
+
await prev?.destroy?.(new Error("http pool reset on wake"));
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
/* already gone */
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
void drain();
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
catch (err) {
|
|
44
|
+
log.warn("http_pool_reset_failed", {
|
|
45
|
+
error: err instanceof Error ? err.message : String(err),
|
|
46
|
+
});
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
}
|
package/dist/runtime/index.js
CHANGED
|
@@ -38,6 +38,7 @@ import { parseByoProviders } from "./direct_inference.js";
|
|
|
38
38
|
import { applyIdentityToEnv, connectIdentityRelayFd, relayFdFromEnv, workerDiagnostics, } from "./identity_relay.js";
|
|
39
39
|
import { FreezeCoordinator } from "./freeze_coordinator.js";
|
|
40
40
|
import { reseedUserspaceCsprng } from "./uniqueness_reseed.js";
|
|
41
|
+
import { resetHttpConnectionPool } from "./http_pool_reset.js";
|
|
41
42
|
import { WorkerWorkflowHost } from "./workflow_host.js";
|
|
42
43
|
import { BrowserSessionManager } from "./browser_session.js";
|
|
43
44
|
import { loadGuestBrowserConfig, makeGuestBrowserBackend, connectSessionMcp, } from "./browser_session_backend.js";
|
|
@@ -48,6 +49,7 @@ import { RunnerControlClient } from "./runner_control_client.js";
|
|
|
48
49
|
import { BrokerChildDispatcher } from "./broker_child_dispatcher.js";
|
|
49
50
|
import { BrokerEventPublisher } from "./broker_event_publisher.js";
|
|
50
51
|
import { createProgramLogSink } from "./program_log_capture.js";
|
|
52
|
+
import { makeRunLogFileSink } from "./run_log_file_sink.js";
|
|
51
53
|
import { BrokerArtifactStore } from "./broker_artifact_store.js";
|
|
52
54
|
import { buildBrokerToolHost } from "./broker_tool_host.js";
|
|
53
55
|
import { CreditWatcher } from "./credit_watcher.js";
|
|
@@ -129,7 +131,18 @@ export function assembleWorkerDeps(runtime) {
|
|
|
129
131
|
// declared output, and every agent turn stamp through this emitter, so cursors are run-globally
|
|
130
132
|
// monotonic with no separate program band. Frames publish as `{cursor, event}` rows via the
|
|
131
133
|
// broker telemetry path. The claim wrapper below bumps it past a previous session's frames.
|
|
132
|
-
|
|
134
|
+
// Optional on-screen mirror: append the (already-redacted) event stream to a local file an xterm in
|
|
135
|
+
// the ambient desktop tails, so the live-view/recording shows the run working. Opt-in via the env the
|
|
136
|
+
// desktop guest image sets (BOARDWALK_RUN_LOG_FILE); absent everywhere else, so no cost off-desktop.
|
|
137
|
+
const runLogFilePath = process.env.BOARDWALK_RUN_LOG_FILE?.trim();
|
|
138
|
+
const runLogLocalSink = runLogFilePath !== undefined && runLogFilePath !== ""
|
|
139
|
+
? makeRunLogFileSink(runLogFilePath)
|
|
140
|
+
: undefined;
|
|
141
|
+
const runEvents = new WorkerRunEventEmitter({
|
|
142
|
+
runId: runtime.runId,
|
|
143
|
+
publisher: eventPublisher,
|
|
144
|
+
...(runLogLocalSink !== undefined ? { localSink: runLogLocalSink } : {}),
|
|
145
|
+
});
|
|
133
146
|
const phaseTracker = new PhaseTracker({ sink: runEvents });
|
|
134
147
|
// The run's public-API credential (was BOARDWALK_API_KEY). It is NO LONGER in process.env — the
|
|
135
148
|
// bootstrap captured + scrubbed it (capturePlatformContext) so the agent's bash / subprocesses
|
|
@@ -393,6 +406,10 @@ export function assembleWorkerDeps(runtime) {
|
|
|
393
406
|
// BEFORE the seam resolves, so no woken author code draws from the stale (pre-suspend) DRBG.
|
|
394
407
|
onAfterWake: (wake) => {
|
|
395
408
|
reseedUserspaceCsprng();
|
|
409
|
+
// Keep-alive sockets (to the egress proxy + through it to the broker) do NOT survive the
|
|
410
|
+
// freeze; discard the connection pool so the first post-wake broker call (finalize) opens a
|
|
411
|
+
// fresh socket instead of hanging on a stale one until its 30s timeout crashes the run.
|
|
412
|
+
resetHttpConnectionPool();
|
|
396
413
|
broker.swapRunToken(wake.run_token);
|
|
397
414
|
redactor.record(wake.run_token);
|
|
398
415
|
if (wake.api_token !== undefined && wake.api_token !== "") {
|
|
@@ -6,11 +6,16 @@ export interface RunEventEmitterOptions {
|
|
|
6
6
|
resumeAfterCursor?: number;
|
|
7
7
|
/** Injected clock for deterministic tests. Defaults to Date.now. */
|
|
8
8
|
now?: () => number;
|
|
9
|
+
/** Optional local mirror of every (already-redacted) event — used to surface the run's output in an
|
|
10
|
+
* on-screen terminal in the ambient desktop (docs/SCREEN_CAPTURE.md). Best-effort: a sink throw never
|
|
11
|
+
* affects the run or the broker publish. Only fed events that already passed the run's redactor. */
|
|
12
|
+
localSink?: (event: RunEvent) => void;
|
|
9
13
|
}
|
|
10
14
|
export declare class WorkerRunEventEmitter {
|
|
11
15
|
private readonly runId;
|
|
12
16
|
private readonly publisher;
|
|
13
17
|
private readonly now;
|
|
18
|
+
private readonly localSink;
|
|
14
19
|
private turn;
|
|
15
20
|
private seq;
|
|
16
21
|
constructor(opts: RunEventEmitterOptions);
|
|
@@ -20,12 +20,14 @@ export class WorkerRunEventEmitter {
|
|
|
20
20
|
runId;
|
|
21
21
|
publisher;
|
|
22
22
|
now;
|
|
23
|
+
localSink;
|
|
23
24
|
turn;
|
|
24
25
|
seq;
|
|
25
26
|
constructor(opts) {
|
|
26
27
|
this.runId = opts.runId;
|
|
27
28
|
this.publisher = opts.publisher;
|
|
28
29
|
this.now = opts.now ?? Date.now;
|
|
30
|
+
this.localSink = opts.localSink;
|
|
29
31
|
const max = opts.resumeAfterCursor ?? 0;
|
|
30
32
|
this.turn = max > 0 ? Math.floor(max / TURN_CURSOR_STRIDE) + 1 : 0;
|
|
31
33
|
this.seq = 0;
|
|
@@ -44,6 +46,14 @@ export class WorkerRunEventEmitter {
|
|
|
44
46
|
this.publisher
|
|
45
47
|
.publish(`run:${this.runId}`, JSON.stringify({ cursor, event }))
|
|
46
48
|
.catch(() => undefined); // best-effort; the publisher logs its own failures
|
|
49
|
+
if (this.localSink !== undefined) {
|
|
50
|
+
try {
|
|
51
|
+
this.localSink(event); // best-effort on-screen mirror; must never affect the run
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
/* ignore */
|
|
55
|
+
}
|
|
56
|
+
}
|
|
47
57
|
return event;
|
|
48
58
|
}
|
|
49
59
|
/** Restart resume: jump past the previous session's frames (claim carries the store max). */
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { RunEvent } from "./agent/events.js";
|
|
2
|
+
/** One readable, ANSI-colored line for a run event, or null to skip it (streaming text/reasoning
|
|
3
|
+
* deltas are buffered by the sink, not formatted here). Reads only a few fields off the SDK's typed
|
|
4
|
+
* event and degrades gracefully on anything unexpected. */
|
|
5
|
+
export declare function formatRunEventLine(event: RunEvent): string | null;
|
|
6
|
+
/** Build the {@link WorkerRunEventEmitter} local sink: append formatted lines to `path`, buffering
|
|
7
|
+
* streaming agent text/reasoning into whole lines. Best-effort — a broken stream (full disk, closed
|
|
8
|
+
* pipe) is swallowed and disables the sink; it never throws into the run. */
|
|
9
|
+
export declare function makeRunLogFileSink(path: string): (event: RunEvent) => void;
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// On-screen run-output mirror. Formats the run's (already-redacted) event stream into readable,
|
|
2
|
+
// ANSI-colored lines appended to a local file, which an xterm in the ambient desktop tails so the
|
|
3
|
+
// live-view / recording shows the run working (docs/SCREEN_CAPTURE.md). Gated on
|
|
4
|
+
// BOARDWALK_RUN_LOG_FILE — the desktop guest image sets it. This ONLY ever sees events that already
|
|
5
|
+
// passed the run's redactor (the same stream the web run-detail renders), so the file is safe to
|
|
6
|
+
// capture in a durable recording.
|
|
7
|
+
import { createWriteStream } from "node:fs";
|
|
8
|
+
function oneLine(s, n) {
|
|
9
|
+
const t = s.replace(/\s+/g, " ").trim();
|
|
10
|
+
return t.length > n ? t.slice(0, n - 1) + "…" : t;
|
|
11
|
+
}
|
|
12
|
+
function safeJson(v) {
|
|
13
|
+
try {
|
|
14
|
+
return typeof v === "string" ? v : JSON.stringify(v);
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return String(v);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** One readable, ANSI-colored line for a run event, or null to skip it (streaming text/reasoning
|
|
21
|
+
* deltas are buffered by the sink, not formatted here). Reads only a few fields off the SDK's typed
|
|
22
|
+
* event and degrades gracefully on anything unexpected. */
|
|
23
|
+
export function formatRunEventLine(event) {
|
|
24
|
+
const e = event;
|
|
25
|
+
const k = typeof e.kind === "string" ? e.kind : "";
|
|
26
|
+
const s = (v) => (typeof v === "string" ? v : "");
|
|
27
|
+
switch (k) {
|
|
28
|
+
case "run_status":
|
|
29
|
+
return `\x1b[1;32m● ${s(e.status)}\x1b[0m`;
|
|
30
|
+
case "phase":
|
|
31
|
+
return `\x1b[1;36m▸ ${s(e.name)}\x1b[0m`;
|
|
32
|
+
case "program_output": {
|
|
33
|
+
const t = s(e.text).replace(/\s+$/, "");
|
|
34
|
+
return t ? ` ${t}` : null;
|
|
35
|
+
}
|
|
36
|
+
case "turn_started":
|
|
37
|
+
return `\x1b[35m· agent ${s(e.agentName) || s(e.agentId)}\x1b[0m`;
|
|
38
|
+
case "tool_call_start":
|
|
39
|
+
return ` \x1b[33m⚙ ${s(e.toolName)}\x1b[0m`;
|
|
40
|
+
case "tool_output_delta": {
|
|
41
|
+
const t = s(e.text).replace(/\s+$/, "");
|
|
42
|
+
return t ? ` \x1b[90m${oneLine(t, 200)}\x1b[0m` : null;
|
|
43
|
+
}
|
|
44
|
+
case "tool_call_result":
|
|
45
|
+
return ` \x1b[32m✓ done\x1b[0m`;
|
|
46
|
+
case "tool_call_error":
|
|
47
|
+
return ` \x1b[31m✗ tool error\x1b[0m`;
|
|
48
|
+
case "output":
|
|
49
|
+
return `\x1b[1;32m✔ output ${oneLine(safeJson(e.value), 300)}\x1b[0m`;
|
|
50
|
+
case "human_input_requested":
|
|
51
|
+
return `\x1b[1;33m⏸ awaiting human input\x1b[0m`;
|
|
52
|
+
case "human_input_resolved":
|
|
53
|
+
return `\x1b[33m▶ input received\x1b[0m`;
|
|
54
|
+
case "suspended":
|
|
55
|
+
return `\x1b[90m⏸ suspended\x1b[0m`;
|
|
56
|
+
case "resumed":
|
|
57
|
+
return `\x1b[90m▶ resumed\x1b[0m`;
|
|
58
|
+
case "egress_denied":
|
|
59
|
+
return ` \x1b[31m⛔ egress denied\x1b[0m`;
|
|
60
|
+
default:
|
|
61
|
+
return null; // turn_ended, text_*, tool_call_input_*, tool_call_executing — noise for the terminal
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/** Build the {@link WorkerRunEventEmitter} local sink: append formatted lines to `path`, buffering
|
|
65
|
+
* streaming agent text/reasoning into whole lines. Best-effort — a broken stream (full disk, closed
|
|
66
|
+
* pipe) is swallowed and disables the sink; it never throws into the run. */
|
|
67
|
+
export function makeRunLogFileSink(path) {
|
|
68
|
+
let stream = null;
|
|
69
|
+
try {
|
|
70
|
+
stream = createWriteStream(path, { flags: "a" });
|
|
71
|
+
stream.on("error", () => {
|
|
72
|
+
stream = null;
|
|
73
|
+
});
|
|
74
|
+
stream.write("\n\x1b[1;36m── boardwalk run ──\x1b[0m\n");
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
stream = null;
|
|
78
|
+
}
|
|
79
|
+
let textBuf = "";
|
|
80
|
+
const flushText = () => {
|
|
81
|
+
if (textBuf.trim() !== "" && stream !== null) {
|
|
82
|
+
stream.write(` \x1b[37m${oneLine(textBuf, 2000)}\x1b[0m\n`);
|
|
83
|
+
}
|
|
84
|
+
textBuf = "";
|
|
85
|
+
};
|
|
86
|
+
return (event) => {
|
|
87
|
+
if (stream === null)
|
|
88
|
+
return;
|
|
89
|
+
const e = event;
|
|
90
|
+
const k = typeof e.kind === "string" ? e.kind : "";
|
|
91
|
+
if (k === "text_delta" || k === "reasoning_delta") {
|
|
92
|
+
textBuf += typeof e.text === "string" ? e.text : "";
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (k === "text_start")
|
|
96
|
+
return;
|
|
97
|
+
if (k === "text_end") {
|
|
98
|
+
flushText();
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
flushText(); // a non-text event ends any in-flight text line
|
|
102
|
+
const line = formatRunEventLine(event);
|
|
103
|
+
if (line !== null)
|
|
104
|
+
stream.write(line + "\n");
|
|
105
|
+
};
|
|
106
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@boardwalk-labs/runner",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.18",
|
|
4
4
|
"description": "Boardwalk self-hosted runner: execute cloud-scheduled runs on your own machines. Currently: the canonical runner contract types.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -58,6 +58,7 @@
|
|
|
58
58
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
59
59
|
"node-gyp-build": "^4.8.4",
|
|
60
60
|
"tar": "^7.5.16",
|
|
61
|
+
"undici": "^6.27.0",
|
|
61
62
|
"zod": "^4.0.0"
|
|
62
63
|
},
|
|
63
64
|
"devDependencies": {
|