@sagentlab/navarch-runtime 0.1.77 → 0.1.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/dist/browser-evidence.cjs +13 -8
- package/dist/browser-probe.cjs +277 -0
- package/dist/cli.cjs +20 -3
- package/dist/completion-retry.cjs +48 -0
- package/dist/config.cjs +1 -0
- package/dist/heartbeat-loop.cjs +6 -1
- package/dist/session.cjs +6 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -22,6 +22,26 @@ are returned as capture errors; the runtime does not relax host permissions.
|
|
|
22
22
|
Docker sessions continue to use their image's tooling, and an operator-supplied
|
|
23
23
|
`NAVARCH_MCP_CONFIG_PATH` remains authoritative.
|
|
24
24
|
|
|
25
|
+
### Browser capability probe
|
|
26
|
+
|
|
27
|
+
`browser-use` is advertised only after a startup launch check passes. When
|
|
28
|
+
`start` boots, the runtime launches its headless Chromium on the host. Then,
|
|
29
|
+
for every advertised runtime that wraps agent shells in an OS sandbox
|
|
30
|
+
(currently Codex, unless it runs with `--dangerously-bypass-approvals-and-sandbox`
|
|
31
|
+
or `danger-full-access`), it launches Chromium again through `codex sandbox`
|
|
32
|
+
with the same permission policy a session gets. If either launch fails, the
|
|
33
|
+
runtime claims without `browser-use` and reports the reason on every machine
|
|
34
|
+
heartbeat; the Agents page shows it. The result is cached for the process
|
|
35
|
+
lifetime, so restart the agent after fixing the browser to re-probe.
|
|
36
|
+
`navarch-runtime doctor` runs the same check.
|
|
37
|
+
|
|
38
|
+
Codex's macOS seatbelt denies Chromium's Mach port registration
|
|
39
|
+
(`bootstrap_check_in ... MachPortRendezvousServer: Permission denied`), so a
|
|
40
|
+
sandboxed Codex machine on macOS does not take browser tasks. Run those with
|
|
41
|
+
another agent such as Claude Code, or use Docker mode. Docker mode is not
|
|
42
|
+
probed because its image supplies the browser. Set `NAVARCH_BROWSER_PROBE=off`
|
|
43
|
+
to trust the configured capability list without a check.
|
|
44
|
+
|
|
25
45
|
Call the tool with `url`, `source_commit_sha`, and `evidence_role` (`before` or
|
|
26
46
|
`after`). Optional `wait_for_selector` and `settle_ms` let a scene finish loading.
|
|
27
47
|
The URL's deployed SHA must be checked by the agent: the supplied SHA is metadata,
|
|
@@ -403,6 +423,7 @@ unchanged across the deployment.
|
|
|
403
423
|
| `NAVARCH_WORKSPACE_ROOT` | `<config dir>/sandboxes` | Persistent bare repo caches plus isolated per-session worktrees. |
|
|
404
424
|
| `NAVARCH_MAX_SESSIONS` | `5` | Local concurrent-session capacity cap — see `src/capacity.cts`. |
|
|
405
425
|
| `NAVARCH_CAPABILITIES` | `shell,browser-use` (`docker-sandbox,shell` in Docker mode) | Comma list reported at heartbeat/claim time. Docker mode does not advertise browser use until the configured image provides it. |
|
|
426
|
+
| `NAVARCH_BROWSER_PROBE` | on | Advertise `browser-use` only after a startup headless-Chromium launch (host, plus each sandboxed runtime) succeeds. `off` trusts `NAVARCH_CAPABILITIES` without a check. |
|
|
406
427
|
| `NAVARCH_OWNER_ZONE` | `sagentlab` | `sagentlab` or `customer-<slug>-premises` (project-plan.md §3.11). |
|
|
407
428
|
| `NAVARCH_POLL_INTERVAL_MS` | `30000` | Claim-loop poll interval. |
|
|
408
429
|
| `NAVARCH_HEARTBEAT_INTERVAL_MS` | `60000` | Machine-level heartbeat interval. |
|
|
@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.captureSchema = void 0;
|
|
7
7
|
exports.installEvidenceBrowser = installEvidenceBrowser;
|
|
8
|
+
exports.launchCaptureBrowser = launchCaptureBrowser;
|
|
8
9
|
exports.startBrowserEvidence = startBrowserEvidence;
|
|
9
10
|
const node_http_1 = require("node:http");
|
|
10
11
|
const node_crypto_1 = require("node:crypto");
|
|
@@ -21,6 +22,17 @@ async function installEvidenceBrowser() {
|
|
|
21
22
|
timeout: 180_000,
|
|
22
23
|
});
|
|
23
24
|
}
|
|
25
|
+
/** The runtime's own headless Chromium, shared by evidence capture and the startup probe. */
|
|
26
|
+
function launchCaptureBrowser() {
|
|
27
|
+
return playwright_1.chromium.launch({
|
|
28
|
+
headless: true,
|
|
29
|
+
chromiumSandbox: true,
|
|
30
|
+
timeout: 20_000,
|
|
31
|
+
// Do not pass the supervisor's provider or machine credentials to Chrome.
|
|
32
|
+
env: Object.fromEntries(["HOME", "PATH", "TMPDIR", "TEMP", "TMP", "SystemRoot"]
|
|
33
|
+
.flatMap((key) => process.env[key] ? [[key, process.env[key]]] : [])),
|
|
34
|
+
});
|
|
35
|
+
}
|
|
24
36
|
const captureShape = {
|
|
25
37
|
url: zod_1.z.string().url().refine((value) => {
|
|
26
38
|
const url = new URL(value);
|
|
@@ -60,14 +72,7 @@ async function startBrowserEvidence(options) {
|
|
|
60
72
|
const images = [];
|
|
61
73
|
try {
|
|
62
74
|
try {
|
|
63
|
-
browser = await (options.launch?.() ??
|
|
64
|
-
headless: true,
|
|
65
|
-
chromiumSandbox: true,
|
|
66
|
-
timeout: 20_000,
|
|
67
|
-
// Do not pass the supervisor's provider or machine credentials to Chrome.
|
|
68
|
-
env: Object.fromEntries(["HOME", "PATH", "TMPDIR", "TEMP", "TMP", "SystemRoot"]
|
|
69
|
-
.flatMap((key) => process.env[key] ? [[key, process.env[key]]] : [])),
|
|
70
|
-
}));
|
|
75
|
+
browser = await (options.launch?.() ?? launchCaptureBrowser());
|
|
71
76
|
}
|
|
72
77
|
catch {
|
|
73
78
|
throw new Error("Runtime browser could not start. On the agent host run navarch-runtime browser-install. Check host browser permissions; do not change the task repository or CI to install capture tooling.");
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.BROWSER_CAPABILITY = void 0;
|
|
7
|
+
exports.probeBrowserCapability = probeBrowserCapability;
|
|
8
|
+
exports.capabilitiesAfterBrowserProbe = capabilitiesAfterBrowserProbe;
|
|
9
|
+
exports.codexSandboxNeedsProbe = codexSandboxNeedsProbe;
|
|
10
|
+
exports.codexSandboxProbeArgs = codexSandboxProbeArgs;
|
|
11
|
+
exports.summarizeLaunchFailure = summarizeLaunchFailure;
|
|
12
|
+
const node_child_process_1 = require("node:child_process");
|
|
13
|
+
const node_fs_1 = require("node:fs");
|
|
14
|
+
const node_os_1 = __importDefault(require("node:os"));
|
|
15
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
16
|
+
const codex_cjs_1 = require("./adapters/codex.cjs");
|
|
17
|
+
const browser_evidence_cjs_1 = require("./browser-evidence.cjs");
|
|
18
|
+
const logger_cjs_1 = require("./logger.cjs");
|
|
19
|
+
const worktree_guard_cjs_1 = require("./worktree-guard.cjs");
|
|
20
|
+
const log = (0, logger_cjs_1.createLogger)("browser-probe");
|
|
21
|
+
exports.BROWSER_CAPABILITY = "browser-use";
|
|
22
|
+
const HOST_PROBE_TIMEOUT_MS = 30_000;
|
|
23
|
+
const SANDBOX_PROBE_TIMEOUT_MS = 45_000;
|
|
24
|
+
const PROBE_START_MARKER = "NAVARCH_BROWSER_PROBE_START";
|
|
25
|
+
const PROBE_OK_MARKER = "NAVARCH_BROWSER_PROBE_OK";
|
|
26
|
+
const MAX_REASON_LENGTH = 480;
|
|
27
|
+
/**
|
|
28
|
+
* Probes once per runtime process (the result is cached by the caller for the
|
|
29
|
+
* process lifetime; a restart re-probes). A machine on 2026-10-01 advertised
|
|
30
|
+
* browser-use by default while Codex's macOS seatbelt killed every Chromium
|
|
31
|
+
* launch at `bootstrap_check_in ... MachPortRendezvousServer: Permission
|
|
32
|
+
* denied`, so nine milestone QA runs ended BLOCKED for environment reasons.
|
|
33
|
+
* The runtime's own capture browser still worked there because it runs
|
|
34
|
+
* outside the agent sandbox, which is why the probe also launches Chromium
|
|
35
|
+
* through each sandbox an advertised runtime wraps its shell in.
|
|
36
|
+
*/
|
|
37
|
+
async function probeBrowserCapability(config, deps = {}) {
|
|
38
|
+
const now = deps.now ?? (() => new Date());
|
|
39
|
+
const skipped = (reason) => ({
|
|
40
|
+
status: "skipped",
|
|
41
|
+
reason,
|
|
42
|
+
checked_at: now().toISOString(),
|
|
43
|
+
});
|
|
44
|
+
if (!config.capabilities.includes(exports.BROWSER_CAPABILITY)) {
|
|
45
|
+
return skipped("browser-use is not in this machine's configured capabilities.");
|
|
46
|
+
}
|
|
47
|
+
if (!config.browserProbe) {
|
|
48
|
+
return skipped("NAVARCH_BROWSER_PROBE=off: browser-use is advertised without a launch check.");
|
|
49
|
+
}
|
|
50
|
+
if (config.sandboxMode === "docker") {
|
|
51
|
+
return skipped("Docker sessions use the sandbox image's browser; the host was not probed.");
|
|
52
|
+
}
|
|
53
|
+
const launchHost = deps.launchHost ?? (async () => {
|
|
54
|
+
const browser = await (0, browser_evidence_cjs_1.launchCaptureBrowser)();
|
|
55
|
+
try {
|
|
56
|
+
const page = await browser.newPage();
|
|
57
|
+
await page.setContent("<title>navarch-probe</title>");
|
|
58
|
+
}
|
|
59
|
+
finally {
|
|
60
|
+
await browser.close();
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
try {
|
|
64
|
+
await withTimeout(launchHost(), HOST_PROBE_TIMEOUT_MS, "headless Chromium launch timed out");
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
return {
|
|
68
|
+
status: "failed",
|
|
69
|
+
reason: truncate(`Headless Chromium could not start on this machine: ${summarizeLaunchFailure(String(error))} ` +
|
|
70
|
+
"Run `navarch-runtime browser-install`, then restart the agent."),
|
|
71
|
+
checked_at: now().toISOString(),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
const platform = deps.platform ?? process.platform;
|
|
75
|
+
if (codexSandboxNeedsProbe(config, platform)) {
|
|
76
|
+
const sandboxFailure = await probeCodexSandbox(config, deps);
|
|
77
|
+
if (sandboxFailure) {
|
|
78
|
+
return { status: "failed", reason: truncate(sandboxFailure), checked_at: now().toISOString() };
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return { status: "passed", checked_at: now().toISOString() };
|
|
82
|
+
}
|
|
83
|
+
/** Withholds `browser-use` when the probe failed; any other verdict keeps the configured list. */
|
|
84
|
+
function capabilitiesAfterBrowserProbe(capabilities, probe) {
|
|
85
|
+
return probe.status === "failed"
|
|
86
|
+
? capabilities.filter((capability) => capability !== exports.BROWSER_CAPABILITY)
|
|
87
|
+
: capabilities;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Codex runs agent shell commands under its own OS sandbox (seatbelt on
|
|
91
|
+
* macOS, Landlock/seccomp on Linux) unless the operator bypassed it. Every
|
|
92
|
+
* advertised runtime must be able to run a browser task: the control plane,
|
|
93
|
+
* not this machine, picks which adapter a claimed task runs under.
|
|
94
|
+
*/
|
|
95
|
+
function codexSandboxNeedsProbe(config, platform) {
|
|
96
|
+
if (config.sandboxMode !== "host" || platform === "win32")
|
|
97
|
+
return false;
|
|
98
|
+
const runtimes = new Set([config.agentType, ...(config.runtimes ?? [])]);
|
|
99
|
+
return runtimes.has("codex") && !codexSandboxBypassed(config.codexExtraArgs);
|
|
100
|
+
}
|
|
101
|
+
function codexSandboxBypassed(args) {
|
|
102
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
103
|
+
const arg = args[index];
|
|
104
|
+
if (arg === "--dangerously-bypass-approvals-and-sandbox")
|
|
105
|
+
return true;
|
|
106
|
+
const value = arg === "--sandbox" || arg === "-s"
|
|
107
|
+
? args[index + 1]
|
|
108
|
+
: arg.startsWith("--sandbox=") || arg.startsWith("-s=")
|
|
109
|
+
? arg.slice(arg.indexOf("=") + 1)
|
|
110
|
+
: (arg === "-c" || arg === "--config") && /^sandbox_mode=/.test(args[index + 1] ?? "")
|
|
111
|
+
? args[index + 1].slice("sandbox_mode=".length)
|
|
112
|
+
: undefined;
|
|
113
|
+
if (value?.replace(/["']/g, "") === "danger-full-access")
|
|
114
|
+
return true;
|
|
115
|
+
}
|
|
116
|
+
return false;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* `codex sandbox` arguments reproducing the policy a host session runs under:
|
|
120
|
+
* the operator's explicit policy when NAVARCH_CODEX_EXTRA_ARGS has one,
|
|
121
|
+
* otherwise Navarch's generated worktree-guard profile, otherwise the user's
|
|
122
|
+
* own Codex configuration (guard off). `--ignore-user-config` and approval
|
|
123
|
+
* settings only affect `codex exec`, so they are not forwarded.
|
|
124
|
+
*/
|
|
125
|
+
function codexSandboxProbeArgs(config, roots) {
|
|
126
|
+
if ((0, codex_cjs_1.hasExplicitPermissionPolicy)(config.codexExtraArgs)) {
|
|
127
|
+
const forwarded = [];
|
|
128
|
+
const args = config.codexExtraArgs;
|
|
129
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
130
|
+
const arg = args[index];
|
|
131
|
+
const next = args[index + 1];
|
|
132
|
+
if ((arg === "-c" || arg === "--config" || arg === "-p" || arg === "--profile") && next !== undefined) {
|
|
133
|
+
forwarded.push(arg, next);
|
|
134
|
+
index += 1;
|
|
135
|
+
}
|
|
136
|
+
else if (arg.startsWith("--config=")) {
|
|
137
|
+
forwarded.push("-c", arg.slice("--config=".length));
|
|
138
|
+
}
|
|
139
|
+
else if ((arg === "--sandbox" || arg === "-s") && next !== undefined) {
|
|
140
|
+
forwarded.push("-c", `sandbox_mode=${JSON.stringify(next)}`);
|
|
141
|
+
index += 1;
|
|
142
|
+
}
|
|
143
|
+
else if (arg.startsWith("--sandbox=") || arg.startsWith("-s=")) {
|
|
144
|
+
forwarded.push("-c", `sandbox_mode=${JSON.stringify(arg.slice(arg.indexOf("=") + 1))}`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return forwarded;
|
|
148
|
+
}
|
|
149
|
+
if (!config.worktreeGuard)
|
|
150
|
+
return [];
|
|
151
|
+
const workspaceRoot = node_path_1.default.join(roots.probeRoot, "workspace");
|
|
152
|
+
return (0, worktree_guard_cjs_1.codexWorktreeGuardArgs)({
|
|
153
|
+
workDir: node_path_1.default.join(workspaceRoot, "session"),
|
|
154
|
+
worktreePath: roots.worktreePath,
|
|
155
|
+
repositoryPath: node_path_1.default.join(workspaceRoot, "repo"),
|
|
156
|
+
workspaceRoot,
|
|
157
|
+
extraRoots: [...config.guardExtraRoots, ...roots.readRoots],
|
|
158
|
+
}).filter((arg) => arg !== "--ignore-user-config");
|
|
159
|
+
}
|
|
160
|
+
const PROBE_SCRIPT = `
|
|
161
|
+
process.stdout.write(${JSON.stringify(PROBE_START_MARKER)} + "\\n");
|
|
162
|
+
(async () => {
|
|
163
|
+
const { chromium } = require(process.env.NAVARCH_PROBE_PLAYWRIGHT);
|
|
164
|
+
const browser = await chromium.launch({ headless: true, timeout: 20000 });
|
|
165
|
+
try {
|
|
166
|
+
const page = await browser.newPage();
|
|
167
|
+
await page.setContent("<title>navarch-probe</title>");
|
|
168
|
+
} finally {
|
|
169
|
+
await browser.close();
|
|
170
|
+
}
|
|
171
|
+
process.stdout.write(${JSON.stringify(PROBE_OK_MARKER)} + "\\n");
|
|
172
|
+
})().catch((error) => {
|
|
173
|
+
process.stderr.write(String((error && error.stack) || error));
|
|
174
|
+
process.exit(1);
|
|
175
|
+
});
|
|
176
|
+
`;
|
|
177
|
+
async function probeCodexSandbox(config, deps) {
|
|
178
|
+
const probeRoot = await node_fs_1.promises.mkdtemp(node_path_1.default.join(deps.tmpDir ?? node_os_1.default.tmpdir(), "navarch-browser-probe-"));
|
|
179
|
+
try {
|
|
180
|
+
const worktreePath = node_path_1.default.join(probeRoot, "workspace", "worktree");
|
|
181
|
+
await node_fs_1.promises.mkdir(worktreePath, { recursive: true });
|
|
182
|
+
const scriptPath = node_path_1.default.join(worktreePath, "probe.cjs");
|
|
183
|
+
await node_fs_1.promises.writeFile(scriptPath, PROBE_SCRIPT, "utf8");
|
|
184
|
+
const playwrightDir = await node_fs_1.promises.realpath(deps.playwrightDir?.() ?? node_path_1.default.dirname(require.resolve("playwright/package.json")));
|
|
185
|
+
const args = codexSandboxProbeArgs(config, {
|
|
186
|
+
probeRoot,
|
|
187
|
+
worktreePath,
|
|
188
|
+
// Node and the runtime's Playwright install (with its playwright-core
|
|
189
|
+
// sibling) must be readable for the probe to start; the browser cache
|
|
190
|
+
// itself is a guard read root already.
|
|
191
|
+
readRoots: [node_path_1.default.dirname(await node_fs_1.promises.realpath(process.execPath)), node_path_1.default.dirname(playwrightDir)],
|
|
192
|
+
});
|
|
193
|
+
const env = { NAVARCH_PROBE_PLAYWRIGHT: playwrightDir };
|
|
194
|
+
for (const key of ["HOME", "PATH", "TMPDIR", "CODEX_HOME", "PLAYWRIGHT_BROWSERS_PATH", "XDG_CACHE_HOME"]) {
|
|
195
|
+
if (process.env[key])
|
|
196
|
+
env[key] = process.env[key];
|
|
197
|
+
}
|
|
198
|
+
const result = await (deps.run ?? runToCompletion)(config.codexBin, ["sandbox", ...args, "--", process.execPath, scriptPath], { cwd: worktreePath, env, timeoutMs: SANDBOX_PROBE_TIMEOUT_MS });
|
|
199
|
+
if (result.code === 0 && result.stdout.includes(PROBE_OK_MARKER))
|
|
200
|
+
return null;
|
|
201
|
+
const output = `${result.stderr}\n${result.stdout}`;
|
|
202
|
+
if (!result.stdout.includes(PROBE_START_MARKER)) {
|
|
203
|
+
return ("Could not verify a browser inside the Codex sandbox: `codex sandbox` did not run the probe " +
|
|
204
|
+
`(${result.timedOut ? "timed out" : output.trim() ? summarizeLaunchFailure(output) : `exit ${result.code}`}). ` +
|
|
205
|
+
"Update the Codex CLI, or run browser tasks with another agent.");
|
|
206
|
+
}
|
|
207
|
+
const detail = result.timedOut ? "the launch timed out." : summarizeLaunchFailure(output);
|
|
208
|
+
const machPort = /bootstrap_check_in|MachPortRendezvous|mach_port_rendezvous/i.test(output);
|
|
209
|
+
return (`Headless Chromium cannot start inside the Codex sandbox: ${detail} ` +
|
|
210
|
+
(machPort
|
|
211
|
+
? "Codex's macOS sandbox denies Chromium's Mach port registration. "
|
|
212
|
+
: "") +
|
|
213
|
+
"Run browser tasks with another agent (for example Claude Code), use NAVARCH_SANDBOX_MODE=docker, " +
|
|
214
|
+
"or run Codex without its sandbox.");
|
|
215
|
+
}
|
|
216
|
+
catch (error) {
|
|
217
|
+
log.warn(`codex sandbox browser probe could not run: ${String(error)}`);
|
|
218
|
+
return `Could not verify a browser inside the Codex sandbox: ${summarizeLaunchFailure(String(error))}`;
|
|
219
|
+
}
|
|
220
|
+
finally {
|
|
221
|
+
await node_fs_1.promises.rm(probeRoot, { recursive: true, force: true }).catch(() => undefined);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/** The one line of a Chromium/Playwright launch failure an operator can act on. */
|
|
225
|
+
function summarizeLaunchFailure(output) {
|
|
226
|
+
const lines = output.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
|
227
|
+
const fatal = lines.find((line) => /FATAL:|bootstrap_check_in/.test(line));
|
|
228
|
+
if (fatal) {
|
|
229
|
+
// "[pid=1][err] [1001/210406.593317:FATAL:base/apple/mach_port_rendezvous_mac.cc:159] Check failed: ..."
|
|
230
|
+
return fatal.replace(/^.*FATAL:[^\]]*\]\s*/, "");
|
|
231
|
+
}
|
|
232
|
+
const missing = lines.find((line) => /Executable doesn't exist/.test(line));
|
|
233
|
+
if (missing)
|
|
234
|
+
return "Chromium is not installed for the runtime's Playwright.";
|
|
235
|
+
const message = lines.find((line) => /Error:/.test(line)) ?? lines[0] ?? "unknown error";
|
|
236
|
+
return message.replace(/^.*?Error:\s*/, "");
|
|
237
|
+
}
|
|
238
|
+
function truncate(text) {
|
|
239
|
+
return text.length > MAX_REASON_LENGTH ? `${text.slice(0, MAX_REASON_LENGTH - 1)}…` : text;
|
|
240
|
+
}
|
|
241
|
+
function withTimeout(promise, ms, message) {
|
|
242
|
+
let timer;
|
|
243
|
+
return Promise.race([
|
|
244
|
+
promise,
|
|
245
|
+
new Promise((_, reject) => {
|
|
246
|
+
timer = setTimeout(() => reject(new Error(message)), ms);
|
|
247
|
+
}),
|
|
248
|
+
]).finally(() => clearTimeout(timer));
|
|
249
|
+
}
|
|
250
|
+
function runToCompletion(bin, args, options) {
|
|
251
|
+
return new Promise((resolve) => {
|
|
252
|
+
let stdout = "";
|
|
253
|
+
let stderr = "";
|
|
254
|
+
let timedOut = false;
|
|
255
|
+
const child = (0, node_child_process_1.spawn)(bin, args, { cwd: options.cwd, env: options.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
256
|
+
const timer = setTimeout(() => {
|
|
257
|
+
timedOut = true;
|
|
258
|
+
child.kill("SIGKILL");
|
|
259
|
+
}, options.timeoutMs);
|
|
260
|
+
child.stdout.on("data", (chunk) => {
|
|
261
|
+
if (stdout.length < 64_000)
|
|
262
|
+
stdout += chunk.toString();
|
|
263
|
+
});
|
|
264
|
+
child.stderr.on("data", (chunk) => {
|
|
265
|
+
if (stderr.length < 64_000)
|
|
266
|
+
stderr += chunk.toString();
|
|
267
|
+
});
|
|
268
|
+
child.on("error", (error) => {
|
|
269
|
+
clearTimeout(timer);
|
|
270
|
+
resolve({ code: null, stdout, stderr: `${stderr}\n${String(error)}`, timedOut });
|
|
271
|
+
});
|
|
272
|
+
child.on("close", (code) => {
|
|
273
|
+
clearTimeout(timer);
|
|
274
|
+
resolve({ code, stdout, stderr, timedOut });
|
|
275
|
+
});
|
|
276
|
+
});
|
|
277
|
+
}
|
package/dist/cli.cjs
CHANGED
|
@@ -19,6 +19,7 @@ const update_coordinator_cjs_1 = require("./update-coordinator.cjs");
|
|
|
19
19
|
const supervisor_cjs_1 = require("./supervisor.cjs");
|
|
20
20
|
const worktree_janitor_cjs_1 = require("./worktree-janitor.cjs");
|
|
21
21
|
const browser_evidence_cjs_1 = require("./browser-evidence.cjs");
|
|
22
|
+
const browser_probe_cjs_1 = require("./browser-probe.cjs");
|
|
22
23
|
const log = (0, logger_cjs_1.createLogger)("cli");
|
|
23
24
|
const PACKAGE_NAME = "@sagentlab/navarch-runtime";
|
|
24
25
|
/** Accepted `--agent` ids as usage metavariables, generated from one source of truth. */
|
|
@@ -235,8 +236,22 @@ async function connectCommand(flags) {
|
|
|
235
236
|
async function startCommand(flags) {
|
|
236
237
|
const baseConfig = configFromFlags(flags);
|
|
237
238
|
const identity = await (0, machine_store_cjs_1.resolveMachineIdentity)(baseConfig.configDir, baseConfig.apiBase);
|
|
238
|
-
const
|
|
239
|
-
warnUnknownRuntimes(
|
|
239
|
+
const configuredConfig = configFromFlags(flags, identity);
|
|
240
|
+
warnUnknownRuntimes(configuredConfig);
|
|
241
|
+
// Probe before the first heartbeat or claim so browser-use is never
|
|
242
|
+
// advertised on a machine whose agents cannot start a browser. The verdict
|
|
243
|
+
// holds for this process; restarting the runtime probes again.
|
|
244
|
+
const browserProbe = await (0, browser_probe_cjs_1.probeBrowserCapability)(configuredConfig);
|
|
245
|
+
if (browserProbe.status === "failed") {
|
|
246
|
+
log.warn(`not advertising browser-use: ${browserProbe.reason}`);
|
|
247
|
+
}
|
|
248
|
+
else if (browserProbe.status === "passed") {
|
|
249
|
+
log.info("browser probe passed; advertising browser-use");
|
|
250
|
+
}
|
|
251
|
+
const config = {
|
|
252
|
+
...configuredConfig,
|
|
253
|
+
capabilities: (0, browser_probe_cjs_1.capabilitiesAfterBrowserProbe)(configuredConfig.capabilities, browserProbe),
|
|
254
|
+
};
|
|
240
255
|
const api = new api_cjs_1.NavarchApiClient({ baseUrl: identity.api_base, token: identity.token });
|
|
241
256
|
const capacity = new capacity_cjs_1.CapacityTracker(config.maxSessions);
|
|
242
257
|
const runtimeShutdown = new AbortController();
|
|
@@ -279,7 +294,7 @@ async function startCommand(flags) {
|
|
|
279
294
|
// capacity slot.
|
|
280
295
|
runtimeShutdown.abort();
|
|
281
296
|
void capacity.waitForIdle().then(() => process.exit(CREDENTIALS_REJECTED_EXIT_CODE));
|
|
282
|
-
});
|
|
297
|
+
}, browserProbe);
|
|
283
298
|
updateCoordinatorRef.current = new update_coordinator_cjs_1.RuntimeUpdateCoordinator({
|
|
284
299
|
config,
|
|
285
300
|
claimLoop,
|
|
@@ -377,6 +392,8 @@ async function doctorCommand(flags) {
|
|
|
377
392
|
console.log(`worktree_guard: ${config.worktreeGuard ? "on" : "off"}`);
|
|
378
393
|
console.log(`runtimes: ${config.runtimes?.join(", ") ?? config.agentType}`);
|
|
379
394
|
console.log(`docker: ${dockerOk ? "available" : "NOT AVAILABLE (docker-backed sessions will fail)"}`);
|
|
395
|
+
const browserProbe = await (0, browser_probe_cjs_1.probeBrowserCapability)(config);
|
|
396
|
+
console.log(`browser: ${browserProbe.status.toUpperCase()}${browserProbe.reason ? ` — ${browserProbe.reason}` : ""}`);
|
|
380
397
|
if (!identity) {
|
|
381
398
|
console.log(`machine: not registered — run \`${invocation("register")}\``);
|
|
382
399
|
return;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.COMPLETION_TRANSIENT_RETRY_DELAYS_MS = void 0;
|
|
4
|
+
exports.isTransientCompletionError = isTransientCompletionError;
|
|
5
|
+
exports.completeWithTransientRetry = completeWithTransientRetry;
|
|
6
|
+
const api_cjs_1 = require("./api.cjs");
|
|
7
|
+
/**
|
|
8
|
+
* Backoff before each re-send of a completion that failed in transit. About
|
|
9
|
+
* two minutes in total: long enough to ride out a Wi-Fi drop or an edge blip,
|
|
10
|
+
* short enough that the lease is still live when the report finally lands.
|
|
11
|
+
*/
|
|
12
|
+
exports.COMPLETION_TRANSIENT_RETRY_DELAYS_MS = [2_000, 5_000, 15_000, 30_000, 60_000];
|
|
13
|
+
/** Statuses the edge or a gateway answers for an unreachable origin; the completion route never chose them. */
|
|
14
|
+
const TRANSIENT_STATUSES = new Set([408, 429, 502, 503, 504, 520, 521, 522, 523, 524]);
|
|
15
|
+
/**
|
|
16
|
+
* A failure that says nothing about the completion itself: the request never
|
|
17
|
+
* got an answer from the completion route. A plain 500 is the route's own
|
|
18
|
+
* answer and keeps the existing crash path.
|
|
19
|
+
*/
|
|
20
|
+
function isTransientCompletionError(err) {
|
|
21
|
+
return (err instanceof api_cjs_1.NavarchTransportError ||
|
|
22
|
+
(err instanceof api_cjs_1.NavarchApiError && TRANSIENT_STATUSES.has(err.status)));
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Sends a lease completion, re-sending it while the control plane is
|
|
26
|
+
* unreachable. Without this, one dropped connection turned a finished session
|
|
27
|
+
* (pull request already open) into `Session crashed:`, which spent an
|
|
28
|
+
* infrastructure retry and discarded the report and evidence the review loop
|
|
29
|
+
* needed (carova#310). A re-send whose earlier copy did land gets the
|
|
30
|
+
* `already_released` 409, which callers already treat as success.
|
|
31
|
+
*/
|
|
32
|
+
async function completeWithTransientRetry(send, options = {}) {
|
|
33
|
+
const delays = options.delaysMs ?? exports.COMPLETION_TRANSIENT_RETRY_DELAYS_MS;
|
|
34
|
+
const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
35
|
+
for (let attempt = 0;; attempt += 1) {
|
|
36
|
+
try {
|
|
37
|
+
await send();
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
const delayMs = delays[attempt];
|
|
42
|
+
if (delayMs === undefined || !isTransientCompletionError(err))
|
|
43
|
+
throw err;
|
|
44
|
+
options.onRetry?.(attempt + 1, delayMs, err);
|
|
45
|
+
await sleep(delayMs);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
package/dist/config.cjs
CHANGED
|
@@ -169,6 +169,7 @@ function loadRuntimeConfig(env = process.env) {
|
|
|
169
169
|
// Reads only the quota numbers providers already write to disk, so it is on
|
|
170
170
|
// by default; operators who would rather share nothing can opt out.
|
|
171
171
|
reportUsageLimits: !["off", "false", "0"].includes(env.NAVARCH_USAGE_LIMITS ?? ""),
|
|
172
|
+
browserProbe: !["off", "false", "0"].includes(env.NAVARCH_BROWSER_PROBE ?? ""),
|
|
172
173
|
codexHome: defaultCodexHome(env),
|
|
173
174
|
claudeConfigPath: defaultClaudeConfigPath(env),
|
|
174
175
|
};
|
package/dist/heartbeat-loop.cjs
CHANGED
|
@@ -34,6 +34,7 @@ class MachineHeartbeatLoop {
|
|
|
34
34
|
onResult;
|
|
35
35
|
onHealthy;
|
|
36
36
|
onCredentialsRejected;
|
|
37
|
+
browserProbe;
|
|
37
38
|
timer = null;
|
|
38
39
|
heartbeatInFlight = false;
|
|
39
40
|
heartbeatPending = false;
|
|
@@ -48,7 +49,9 @@ class MachineHeartbeatLoop {
|
|
|
48
49
|
* then: without this the runtime would warn every interval forever while
|
|
49
50
|
* claiming nothing, which is indistinguishable from an idle agent.
|
|
50
51
|
*/
|
|
51
|
-
onCredentialsRejected
|
|
52
|
+
onCredentialsRejected,
|
|
53
|
+
/** Cached for the process lifetime; a restart re-probes. */
|
|
54
|
+
browserProbe) {
|
|
52
55
|
this.api = api;
|
|
53
56
|
this.machineId = machineId;
|
|
54
57
|
this.config = config;
|
|
@@ -57,6 +60,7 @@ class MachineHeartbeatLoop {
|
|
|
57
60
|
this.onResult = onResult;
|
|
58
61
|
this.onHealthy = onHealthy;
|
|
59
62
|
this.onCredentialsRejected = onCredentialsRejected;
|
|
63
|
+
this.browserProbe = browserProbe;
|
|
60
64
|
}
|
|
61
65
|
start() {
|
|
62
66
|
if (this.timer)
|
|
@@ -101,6 +105,7 @@ class MachineHeartbeatLoop {
|
|
|
101
105
|
const result = await this.api.machineHeartbeat(this.machineId, {
|
|
102
106
|
available_capacity: this.draining ? 0 : this.capacity.available(),
|
|
103
107
|
capabilities: this.config.capabilities,
|
|
108
|
+
...(this.browserProbe ? { browser_probe: this.browserProbe } : {}),
|
|
104
109
|
...(agentUsage ? { agent_usage: agentUsage } : {}),
|
|
105
110
|
// Explicit null when unpinned, so the control plane clears a model
|
|
106
111
|
// this machine reported before its pin was removed.
|
package/dist/session.cjs
CHANGED
|
@@ -25,6 +25,7 @@ const delivery_cjs_1 = require("./delivery.cjs");
|
|
|
25
25
|
const worktree_guard_cjs_1 = require("./worktree-guard.cjs");
|
|
26
26
|
const adapter_watchdog_cjs_1 = require("./adapter-watchdog.cjs");
|
|
27
27
|
const lease_heartbeat_cjs_1 = require("./lease-heartbeat.cjs");
|
|
28
|
+
const completion_retry_cjs_1 = require("./completion-retry.cjs");
|
|
28
29
|
const adapter_capacity_cjs_1 = require("./adapter-capacity.cjs");
|
|
29
30
|
const agent_auth_cjs_1 = require("./agent-auth.cjs");
|
|
30
31
|
const browser_evidence_cjs_1 = require("./browser-evidence.cjs");
|
|
@@ -730,7 +731,9 @@ async function runClaimedSession(deps, claimed, sessionId, lifecycle) {
|
|
|
730
731
|
transcript_url: completion.transcript_url,
|
|
731
732
|
};
|
|
732
733
|
try {
|
|
733
|
-
await api.completeLease(leaseId, completion)
|
|
734
|
+
await (0, completion_retry_cjs_1.completeWithTransientRetry)(() => api.completeLease(leaseId, completion), {
|
|
735
|
+
onRetry: (attempt, delayMs, err) => log.warn(`completion for ${leaseId} did not reach the control plane (${(0, redact_cjs_1.redactText)(String(err), knownSecrets)}); re-sending in ${delayMs / 1000}s (${attempt}).`),
|
|
736
|
+
});
|
|
734
737
|
if (deliveredPullRequest && completion.status === "completed" && gitWorktree.restoredHostedCheckpoint) {
|
|
735
738
|
// The pull request carries the hosted attempt's work now; a later
|
|
736
739
|
// hosted attempt must not resume the stale snapshot.
|
|
@@ -766,7 +769,7 @@ async function runClaimedSession(deps, claimed, sessionId, lifecycle) {
|
|
|
766
769
|
log.error(`completion for ${leaseId} was rejected as ${rejection.code}, which is not remediable; failing the lease.`);
|
|
767
770
|
}
|
|
768
771
|
try {
|
|
769
|
-
await api.completeLease(leaseId, {
|
|
772
|
+
await (0, completion_retry_cjs_1.completeWithTransientRetry)(() => api.completeLease(leaseId, {
|
|
770
773
|
...completion,
|
|
771
774
|
status: "failed",
|
|
772
775
|
report: remediable
|
|
@@ -774,7 +777,7 @@ async function runClaimedSession(deps, claimed, sessionId, lifecycle) {
|
|
|
774
777
|
: `${redactedRejection}\n\n---\n\n${completion.report}`,
|
|
775
778
|
failure_summary: redactedRejection,
|
|
776
779
|
exit_status: "failed",
|
|
777
|
-
});
|
|
780
|
+
}));
|
|
778
781
|
}
|
|
779
782
|
catch (fallbackErr) {
|
|
780
783
|
// Same race as the primary completion above: if something else released
|
package/package.json
CHANGED