@sagentlab/navarch-runtime 0.1.50 → 0.1.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -3
- package/dist/activity-watchdog.cjs +17 -6
- package/dist/adapter-capacity.cjs +9 -2
- package/dist/adapter-watchdog.cjs +5 -2
- package/dist/adapters/opencode.cjs +89 -20
- package/dist/cli.cjs +36 -30
- package/dist/config.cjs +1 -0
- package/dist/exit-conditions.cjs +15 -3
- package/dist/sandbox.cjs +2 -0
- package/dist/session.cjs +19 -4
- package/dist/upload.cjs +3 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -360,6 +360,7 @@ unchanged across the deployment.
|
|
|
360
360
|
| `NAVARCH_LEASE_HEARTBEAT_INTERVAL_MS` | `300000` | Per-lease heartbeat interval; must stay well under the 60-minute lease TTL (schema-design.md §4). |
|
|
361
361
|
| `NAVARCH_SESSION_TIMEOUT_MS` | `0` (disabled) | Optional absolute runtime budget across agent turns in a claimed session. |
|
|
362
362
|
| `NAVARCH_SESSION_IDLE_TIMEOUT_MS` | `2700000` (45 min) | Stop after a continuously observed window with no stdout/stderr or process-tree CPU activity. `0` disables idle detection. |
|
|
363
|
+
| `NAVARCH_OPENCODE_STARTUP_TIMEOUT_MS` | `120000` (2 min) | Stop OpenCode when it produces no stdout/stderr during startup. This catches provider auth/quota failures that leave its CLI process alive. `0` disables the startup deadline. |
|
|
363
364
|
| `NAVARCH_WORKTREE_CLEANUP_INTERVAL_MS` | `3600000` (1 hour) | How often the runtime scans for session worktrees abandoned by a hard exit or host restart. |
|
|
364
365
|
| `NAVARCH_WORKTREE_STALE_AFTER_MS` | `86400000` (24 hours) | Minimum inactivity age before an abandoned session worktree is removed. Active sessions are always protected. |
|
|
365
366
|
| `NAVARCH_GIT_AUTHOR_NAME` / `NAVARCH_GIT_AUTHOR_EMAIL` | `sagentlab` / `z@sagentlab.com` | Git identity forced into session commits so host-level personal config is not inherited; override both for a project-authorized bot. |
|
|
@@ -368,7 +369,7 @@ unchanged across the deployment.
|
|
|
368
369
|
| `NAVARCH_SANDBOX_EGRESS_NETWORK` | _(unset)_ | Docker network that enforces a profile's egress allowlist. Docker cannot filter by domain itself, so an allowlist profile without this fails closed to `--network=none` and records the denial. |
|
|
369
370
|
| `NAVARCH_DOCKER_IMAGE` | `ghcr.io/sagentlab/navarch-sandbox-agent:0.1.0` | Version-pinned per-session image with Node 20, git, GitHub CLI, ripgrep, jq, SSH, and Claude Code 2.1.218. Override with an image tag or digest you control. |
|
|
370
371
|
| `NAVARCH_AGENT` | saved choice, then `claude-code` | Local choice of agent adapter: `claude-code`, `codex`, `gemini`, `opencode`, or `acp`. Overrides the choice saved by `connect`/`register`; `start --agent` has highest priority. |
|
|
371
|
-
| `NAVARCH_RUNTIMES` |
|
|
372
|
+
| `NAVARCH_RUNTIMES` | resolved local agent choice | Comma list of installed/authenticated adapters advertised to dispatch. The control plane chooses among these per project/task. |
|
|
372
373
|
| `NAVARCH_UPDATE_CHANNEL` | `stable` | Release channel advertised by the worker (`stable` or `canary`); the server-managed machine channel remains authoritative. |
|
|
373
374
|
| `NAVARCH_AUTO_UPDATE` | on under `supervise` | Set `off`, `false`, or `0` to report releases without staging or activating them. Automatic activation is always off under plain `start`. |
|
|
374
375
|
| `NAVARCH_CLAUDE_BIN` | `claude` | Path/name of the Claude Code CLI binary. |
|
|
@@ -600,6 +601,16 @@ to OpenCode's remote-server config with OAuth disabled and environment-backed
|
|
|
600
601
|
headers. Missing accounting fields stay absent rather than becoming a
|
|
601
602
|
fabricated zero.
|
|
602
603
|
|
|
604
|
+
OpenCode also runs with `--print-logs --log-level ERROR` by default (see its
|
|
605
|
+
[CLI flags](https://opencode.ai/docs/cli/#global-flags)), so provider errors
|
|
606
|
+
that are missing from assistant messages still reach the runtime. A rate or
|
|
607
|
+
usage limit in a provider error event or stream-error log stops the host
|
|
608
|
+
process or Docker container immediately. The runtime checkpoints local work,
|
|
609
|
+
records the error, and releases the lease with a failed session outcome.
|
|
610
|
+
Pending guidance does not restart that session. New claims on this machine
|
|
611
|
+
pause until the reported reset time (including relative resets such as
|
|
612
|
+
“in 23 days”), or for 15 minutes when no reset time is available.
|
|
613
|
+
|
|
603
614
|
The ACP adapter speaks the standard [Agent Client Protocol](https://agentclientprotocol.com)
|
|
604
615
|
v1 JSON-RPC transport over stdio. It creates one protocol session in the task
|
|
605
616
|
worktree, converts Navarch's HTTP/stdio MCP entries, consumes semantic message,
|
|
@@ -679,7 +690,7 @@ contract used by `api.cts`, including:
|
|
|
679
690
|
distinct from the per-lease heartbeat, needed because `machines.last_heartbeat_at`
|
|
680
691
|
/`status` must update even when no task is claimed.
|
|
681
692
|
3. **`POST /api/dispatch/:leaseId/transcript-upload-url`** — a signed
|
|
682
|
-
|
|
693
|
+
R2 upload URL for the session's redacted transcript. The
|
|
683
694
|
bucket is private; the legacy `public_url` response field is only an
|
|
684
695
|
attested locator returned at completion, not an anonymous read URL.
|
|
685
696
|
|
|
@@ -781,7 +792,7 @@ secrets absent from disk after exit"):
|
|
|
781
792
|
already-reassigned lease) — `session.cts` aborts the running adapter
|
|
782
793
|
process and still attempts a best-effort `complete()` call, which the
|
|
783
794
|
real server may reject; that path is untested against real semantics.
|
|
784
|
-
- Transcript upload against a real signed
|
|
795
|
+
- Transcript upload against a real signed R2 URL.
|
|
785
796
|
- The end-to-end "Connect an agent to a project" flow against a real
|
|
786
797
|
Supabase instance: an owner minting a token from `ConnectAgentPanel`
|
|
787
798
|
(or the onboarding wizard's Agent step), `navarch-runtime connect`
|
|
@@ -16,6 +16,7 @@ class ActivityWatchdog {
|
|
|
16
16
|
stopped = false;
|
|
17
17
|
polling = false;
|
|
18
18
|
pollTimer;
|
|
19
|
+
startupTimer;
|
|
19
20
|
hardTimer;
|
|
20
21
|
constructor(options) {
|
|
21
22
|
this.options = options;
|
|
@@ -24,16 +25,23 @@ class ActivityWatchdog {
|
|
|
24
25
|
if (options.idleTimeoutMs > 0) {
|
|
25
26
|
this.pollTimer = setInterval(() => void this.poll(), Math.min(30_000, options.idleTimeoutMs / 4));
|
|
26
27
|
}
|
|
28
|
+
if ((options.startupTimeoutMs ?? 0) > 0) {
|
|
29
|
+
this.startupTimer = setTimeout(() => this.expire("startup"), options.startupTimeoutMs);
|
|
30
|
+
}
|
|
27
31
|
if (options.timeoutMs > 0)
|
|
28
32
|
this.armHardLimit();
|
|
29
33
|
}
|
|
30
34
|
activity() {
|
|
31
|
-
if (!this.stopped)
|
|
35
|
+
if (!this.stopped) {
|
|
32
36
|
this.lastActivity = this.now();
|
|
37
|
+
clearTimeout(this.startupTimer);
|
|
38
|
+
this.startupTimer = undefined;
|
|
39
|
+
}
|
|
33
40
|
}
|
|
34
41
|
stop() {
|
|
35
42
|
this.stopped = true;
|
|
36
43
|
clearInterval(this.pollTimer);
|
|
44
|
+
clearTimeout(this.startupTimer);
|
|
37
45
|
clearTimeout(this.hardTimer);
|
|
38
46
|
}
|
|
39
47
|
armHardLimit() {
|
|
@@ -61,8 +69,11 @@ class ActivityWatchdog {
|
|
|
61
69
|
this.observed = false;
|
|
62
70
|
return;
|
|
63
71
|
}
|
|
72
|
+
// Process-tree activity keeps the ordinary idle window alive, but only
|
|
73
|
+
// actual adapter output proves that a CLI got through startup. Some
|
|
74
|
+
// providers leave a quiet process behind after an auth/quota failure.
|
|
64
75
|
if (!this.observed || signature !== this.signature)
|
|
65
|
-
this.
|
|
76
|
+
this.lastActivity = this.now();
|
|
66
77
|
this.signature = signature;
|
|
67
78
|
this.observed = true;
|
|
68
79
|
if (this.now() - this.lastActivity >= this.options.idleTimeoutMs)
|
|
@@ -79,15 +90,15 @@ class ActivityWatchdog {
|
|
|
79
90
|
}
|
|
80
91
|
exports.ActivityWatchdog = ActivityWatchdog;
|
|
81
92
|
/** No command arguments or environment values are collected from other jobs. */
|
|
82
|
-
async function processActivity(pid, containerName) {
|
|
93
|
+
async function processActivity(pid, containerName, ignoreRootCpu = false) {
|
|
83
94
|
if (!containerName && pid === undefined)
|
|
84
95
|
return null;
|
|
85
96
|
const output = await new Promise((resolve) => {
|
|
86
97
|
(0, node_child_process_1.execFile)(containerName ? "docker" : "ps", containerName ? ["top", containerName, "-eo", "pid=,ppid=,time="] : ["-e", "-o", "pid=,ppid=,time="], { timeout: 5_000, maxBuffer: 4 * 1024 * 1024 }, (error, stdout) => resolve(error ? null : stdout));
|
|
87
98
|
});
|
|
88
|
-
return output === null ? null : processTreeSignature(output, containerName ? undefined : pid);
|
|
99
|
+
return output === null ? null : processTreeSignature(output, containerName ? undefined : pid, ignoreRootCpu);
|
|
89
100
|
}
|
|
90
|
-
function processTreeSignature(output, rootPid) {
|
|
101
|
+
function processTreeSignature(output, rootPid, ignoreRootCpu = false) {
|
|
91
102
|
const rows = output.split("\n").flatMap((line) => {
|
|
92
103
|
const match = /^\s*(\d+)\s+(\d+)\s+([\d:.\-]+)\s*$/.exec(line);
|
|
93
104
|
return match ? [{ pid: Number(match[1]), parent: Number(match[2]), cpu: match[3] }] : [];
|
|
@@ -106,5 +117,5 @@ function processTreeSignature(output, rootPid) {
|
|
|
106
117
|
}
|
|
107
118
|
}
|
|
108
119
|
return rows.filter((row) => included.has(row.pid)).sort((a, b) => a.pid - b.pid)
|
|
109
|
-
.map((row) => `${row.pid}:${row.cpu}`).join(",");
|
|
120
|
+
.map((row) => `${row.pid}:${ignoreRootCpu && row.pid === rootPid ? "root" : row.cpu}`).join(",");
|
|
110
121
|
}
|
|
@@ -14,9 +14,9 @@ const MAX_RESET_SEARCH_MINUTES = 48 * 60;
|
|
|
14
14
|
function detectAdapterCapacityLimit(result, nowMs = Date.now()) {
|
|
15
15
|
const parsed = (0, exit_conditions_cjs_1.parseClaudeJsonResult)(result.stdout) ??
|
|
16
16
|
(0, exit_conditions_cjs_1.parseClaudeJsonResult)(result.stderr);
|
|
17
|
-
if (parsed?.api_error_status !== 429)
|
|
17
|
+
if (!result.capacityError && parsed?.api_error_status !== 429)
|
|
18
18
|
return null;
|
|
19
|
-
const detail = parsed
|
|
19
|
+
const detail = result.capacityError ?? parsed?.result ?? "";
|
|
20
20
|
if (!/\b(?:session|usage|rate)\s+limit\b|\btoo many requests\b/i.test(detail)) {
|
|
21
21
|
return null;
|
|
22
22
|
}
|
|
@@ -32,6 +32,13 @@ function detectAdapterCapacityLimit(result, nowMs = Date.now()) {
|
|
|
32
32
|
* the platform's IANA timezone database.
|
|
33
33
|
*/
|
|
34
34
|
function parseResetTime(detail, nowMs) {
|
|
35
|
+
const relative = detail.match(/\bresets?\s+in\s+(\d+(?:\.\d+)?)\s*(seconds?|minutes?|hours?|days?)\b/i);
|
|
36
|
+
if (relative) {
|
|
37
|
+
const units = { second: 1000, minute: 60_000, hour: 3_600_000, day: 86_400_000 };
|
|
38
|
+
const duration = Number(relative[1]) * units[relative[2].toLowerCase().replace(/s$/, "")];
|
|
39
|
+
if (Number.isFinite(duration) && duration > 0)
|
|
40
|
+
return nowMs + duration + RESET_GRACE_MS;
|
|
41
|
+
}
|
|
35
42
|
const match = detail.match(/\bresets?\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)\s*\(([^)]+)\)/i);
|
|
36
43
|
if (!match)
|
|
37
44
|
return null;
|
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.runWithActivityWatchdog = runWithActivityWatchdog;
|
|
4
4
|
const activity_watchdog_cjs_1 = require("./activity-watchdog.cjs");
|
|
5
5
|
/** Shared policy for every host/Docker/ACP adapter; adapters still own cancellation. */
|
|
6
|
-
async function runWithActivityWatchdog(adapter, options, idleTimeoutMs) {
|
|
6
|
+
async function runWithActivityWatchdog(adapter, options, idleTimeoutMs, startupTimeoutMs = 0) {
|
|
7
7
|
const controller = new AbortController();
|
|
8
8
|
const onAbort = () => controller.abort();
|
|
9
9
|
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
@@ -13,8 +13,11 @@ async function runWithActivityWatchdog(adapter, options, idleTimeoutMs) {
|
|
|
13
13
|
let timeoutReason;
|
|
14
14
|
const watchdog = new activity_watchdog_cjs_1.ActivityWatchdog({
|
|
15
15
|
idleTimeoutMs,
|
|
16
|
+
startupTimeoutMs,
|
|
16
17
|
timeoutMs: options.timeoutMs,
|
|
17
|
-
|
|
18
|
+
// OpenCode keeps doing root-process housekeeping after provider failures.
|
|
19
|
+
// Descendant work and stdout/stderr still count, but that churn does not.
|
|
20
|
+
probe: () => (0, activity_watchdog_cjs_1.processActivity)(pid, options.dockerExec?.containerName, adapter.agentType === "opencode" && !options.dockerExec),
|
|
18
21
|
onTimeout: (reason) => {
|
|
19
22
|
timeoutReason = reason;
|
|
20
23
|
controller.abort();
|
|
@@ -7,10 +7,10 @@ const node_fs_1 = require("node:fs");
|
|
|
7
7
|
const config_cjs_1 = require("../config.cjs");
|
|
8
8
|
const kill_agent_cjs_1 = require("../kill-agent.cjs");
|
|
9
9
|
/**
|
|
10
|
-
* Headless OpenCode adapter, fixture-pinned to the OpenCode
|
|
11
|
-
* contract:
|
|
10
|
+
* Headless OpenCode adapter, fixture-pinned to the OpenCode JSON/config
|
|
11
|
+
* contract shared by V1 and V2 (verified against 2.0.6):
|
|
12
12
|
*
|
|
13
|
-
* opencode run "<prompt>" --format json [--model provider/model]
|
|
13
|
+
* opencode run "<prompt>" --format json [--model provider/model[#variant]]
|
|
14
14
|
*
|
|
15
15
|
* OpenCode merges configuration from several machine and project locations,
|
|
16
16
|
* so each Navarch session gets an isolated XDG/config directory, disables
|
|
@@ -32,14 +32,18 @@ async function runOpenCodeAdapter(options) {
|
|
|
32
32
|
const env = { ...options.env };
|
|
33
33
|
await prepareOpenCodeConfig(options.mcpConfigPath, env);
|
|
34
34
|
const args = ["run", options.prompt];
|
|
35
|
+
// Provider failures may only appear in logs, absent from assistant events.
|
|
36
|
+
if (!hasArg(options.extraArgs, "--print-logs"))
|
|
37
|
+
args.push("--print-logs");
|
|
38
|
+
if (!hasArg(options.extraArgs, "--log-level"))
|
|
39
|
+
args.push("--log-level", "ERROR");
|
|
35
40
|
if (!hasArg(options.extraArgs, "--format"))
|
|
36
41
|
args.push("--format", "json");
|
|
37
42
|
args.push(...options.extraArgs);
|
|
38
43
|
if (options.model && options.model !== "default" && !hasArg(options.extraArgs, "--model", "-m")) {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
args.push("--variant", options.reasoningEffort);
|
|
44
|
+
// OpenCode V2 selects a reasoning variant through the model reference
|
|
45
|
+
// (`provider/model#variant`) rather than a separate flag.
|
|
46
|
+
args.push("--model", options.reasoningEffort ? `${options.model}#${options.reasoningEffort}` : options.model);
|
|
43
47
|
}
|
|
44
48
|
const runOptions = { ...options, env };
|
|
45
49
|
const raw = runOptions.dockerExec
|
|
@@ -80,23 +84,22 @@ async function prepareOpenCodeConfig(mcpConfigPath, env) {
|
|
|
80
84
|
mcp[name] = {
|
|
81
85
|
type: "remote",
|
|
82
86
|
url,
|
|
83
|
-
|
|
87
|
+
disabled: false,
|
|
84
88
|
oauth: false,
|
|
85
89
|
...(Object.keys(headers).length > 0 ? { headers } : {}),
|
|
86
90
|
};
|
|
87
91
|
}
|
|
88
92
|
const config = {
|
|
89
93
|
$schema: "https://opencode.ai/config.json",
|
|
90
|
-
|
|
94
|
+
update: "disable",
|
|
91
95
|
share: "disabled",
|
|
92
|
-
|
|
93
|
-
mcp,
|
|
94
|
-
|
|
95
|
-
"*": "allow",
|
|
96
|
-
external_directory: "deny",
|
|
97
|
-
question: "deny",
|
|
98
|
-
|
|
99
|
-
},
|
|
96
|
+
plugins: [],
|
|
97
|
+
mcp: { servers: mcp },
|
|
98
|
+
permissions: [
|
|
99
|
+
{ action: "*", resource: "*", effect: "allow" },
|
|
100
|
+
{ action: "external_directory", resource: "*", effect: "deny" },
|
|
101
|
+
{ action: "question", resource: "*", effect: "deny" },
|
|
102
|
+
],
|
|
100
103
|
};
|
|
101
104
|
const configPath = `${mcpConfigPath}.opencode.json`;
|
|
102
105
|
const configDir = `${mcpConfigPath}.opencode-config`;
|
|
@@ -186,6 +189,42 @@ function parseOpenCodeEvents(stdout) {
|
|
|
186
189
|
function nonNegativeMetric(value) {
|
|
187
190
|
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
|
|
188
191
|
}
|
|
192
|
+
/** Inspect only provider error envelopes/logs, never tool output or assistant text. */
|
|
193
|
+
function capacityMonitor(onLimit) {
|
|
194
|
+
const pending = { stdout: "", stderr: "" };
|
|
195
|
+
const inspect = (line, stream) => {
|
|
196
|
+
let detail = "";
|
|
197
|
+
try {
|
|
198
|
+
const event = JSON.parse(line);
|
|
199
|
+
if (event.type === "error" || event.type === "session.error") {
|
|
200
|
+
const error = event.error ?? event.properties?.error;
|
|
201
|
+
detail = typeof error === "string" ? error : error?.data?.message ?? error?.message ?? "";
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
catch {
|
|
205
|
+
// OpenCode's log format varies by version. Require a stream-error log
|
|
206
|
+
// to avoid confusing a tool's own HTTP 429 with provider exhaustion.
|
|
207
|
+
if (stream === "stderr" && /\bERROR\b/.test(line) && /stream error/i.test(line)) {
|
|
208
|
+
detail = line;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
if (typeof detail === "string" && /\b(?:rate|usage|session)\s+limit\b|\btoo many requests\b/i.test(detail)) {
|
|
212
|
+
onLimit(detail);
|
|
213
|
+
}
|
|
214
|
+
};
|
|
215
|
+
return {
|
|
216
|
+
push(stream, chunk) {
|
|
217
|
+
const lines = (pending[stream] + chunk).split(/\r?\n/);
|
|
218
|
+
pending[stream] = lines.pop() ?? "";
|
|
219
|
+
for (const line of lines)
|
|
220
|
+
inspect(line, stream);
|
|
221
|
+
},
|
|
222
|
+
flush() {
|
|
223
|
+
inspect(pending.stdout, "stdout");
|
|
224
|
+
inspect(pending.stderr, "stderr");
|
|
225
|
+
},
|
|
226
|
+
};
|
|
227
|
+
}
|
|
189
228
|
async function runOnHost(options, args) {
|
|
190
229
|
return new Promise((resolve) => {
|
|
191
230
|
let stdout = "";
|
|
@@ -197,6 +236,11 @@ async function runOnHost(options, args) {
|
|
|
197
236
|
cwd: options.cwd,
|
|
198
237
|
env: { ...process.env, ...options.env },
|
|
199
238
|
});
|
|
239
|
+
let capacityError;
|
|
240
|
+
const monitor = capacityMonitor((detail) => {
|
|
241
|
+
capacityError ??= detail;
|
|
242
|
+
(0, kill_agent_cjs_1.killAgent)(child);
|
|
243
|
+
});
|
|
200
244
|
child.stdin?.end();
|
|
201
245
|
options.onProcessStarted?.(child.pid);
|
|
202
246
|
const timer = options.timeoutMs > 0 ? setTimeout(() => {
|
|
@@ -213,21 +257,25 @@ async function runOnHost(options, args) {
|
|
|
213
257
|
child.stdout.on("data", (data) => {
|
|
214
258
|
options.onActivity?.();
|
|
215
259
|
stdout += data.toString();
|
|
260
|
+
monitor.push("stdout", data.toString());
|
|
216
261
|
});
|
|
217
262
|
child.stderr.on("data", (data) => {
|
|
218
263
|
options.onActivity?.();
|
|
219
264
|
stderr += data.toString();
|
|
265
|
+
monitor.push("stderr", data.toString());
|
|
220
266
|
});
|
|
221
267
|
child.on("error", (error) => {
|
|
222
268
|
clearTimeout(timer);
|
|
223
269
|
options.signal?.removeEventListener("abort", onAbort);
|
|
224
270
|
stderr += `\n${String(error)}`;
|
|
225
|
-
|
|
271
|
+
monitor.flush();
|
|
272
|
+
resolve({ exitCode: null, timedOut, killedByLeaseLoss, stdout, stderr, capacityError });
|
|
226
273
|
});
|
|
227
274
|
child.on("close", (code) => {
|
|
228
275
|
clearTimeout(timer);
|
|
229
276
|
options.signal?.removeEventListener("abort", onAbort);
|
|
230
|
-
|
|
277
|
+
monitor.flush();
|
|
278
|
+
resolve({ exitCode: code, timedOut, killedByLeaseLoss, stdout, stderr, capacityError });
|
|
231
279
|
});
|
|
232
280
|
});
|
|
233
281
|
}
|
|
@@ -236,8 +284,19 @@ async function runViaDocker(options, args) {
|
|
|
236
284
|
const quoted = [options.bin, ...args].map(shellQuote).join(" ");
|
|
237
285
|
const command = `[ -f /tmp/session.env ] && . /tmp/session.env; cd repo 2>/dev/null; ${quoted}`;
|
|
238
286
|
let killedByLeaseLoss = false;
|
|
287
|
+
let capacityError;
|
|
288
|
+
const controller = new AbortController();
|
|
289
|
+
const streamed = { stdout: false, stderr: false };
|
|
290
|
+
const monitor = capacityMonitor((detail) => {
|
|
291
|
+
if (capacityError)
|
|
292
|
+
return;
|
|
293
|
+
capacityError = detail;
|
|
294
|
+
controller.abort();
|
|
295
|
+
runner.run("docker", ["kill", containerName], { timeoutMs: 10_000 }).catch(() => undefined);
|
|
296
|
+
});
|
|
239
297
|
const onAbort = () => {
|
|
240
298
|
killedByLeaseLoss = true;
|
|
299
|
+
controller.abort();
|
|
241
300
|
runner.run("docker", ["kill", containerName], { timeoutMs: 10_000 }).catch(() => undefined);
|
|
242
301
|
};
|
|
243
302
|
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
@@ -247,10 +306,19 @@ async function runViaDocker(options, args) {
|
|
|
247
306
|
const forwardedEnv = Object.keys(options.env).flatMap((name) => ["--env", name]);
|
|
248
307
|
const result = await runner.run("docker", ["exec", ...forwardedEnv, containerName, "sh", "-c", command], { timeoutMs: options.timeoutMs,
|
|
249
308
|
onActivity: options.onActivity,
|
|
250
|
-
signal:
|
|
309
|
+
signal: controller.signal,
|
|
310
|
+
onStdout: (chunk) => { streamed.stdout = true; monitor.push("stdout", chunk); },
|
|
311
|
+
onStderr: (chunk) => { streamed.stderr = true; monitor.push("stderr", chunk); },
|
|
251
312
|
env: { ...process.env, ...options.env },
|
|
252
313
|
});
|
|
314
|
+
// Also inspect runners that only return buffered output.
|
|
315
|
+
if (!streamed.stdout)
|
|
316
|
+
monitor.push("stdout", result.stdout);
|
|
317
|
+
if (!streamed.stderr)
|
|
318
|
+
monitor.push("stderr", result.stderr);
|
|
319
|
+
monitor.flush();
|
|
253
320
|
return {
|
|
321
|
+
capacityError,
|
|
254
322
|
exitCode: result.code,
|
|
255
323
|
timedOut: false,
|
|
256
324
|
killedByLeaseLoss,
|
|
@@ -260,6 +328,7 @@ async function runViaDocker(options, args) {
|
|
|
260
328
|
}
|
|
261
329
|
catch (error) {
|
|
262
330
|
return {
|
|
331
|
+
capacityError,
|
|
263
332
|
exitCode: null,
|
|
264
333
|
timedOut: false,
|
|
265
334
|
killedByLeaseLoss,
|
package/dist/cli.cjs
CHANGED
|
@@ -61,11 +61,17 @@ function superviseInvocation(configDir) {
|
|
|
61
61
|
return `${invocation("supervise")} --config-dir ${configDir}`;
|
|
62
62
|
}
|
|
63
63
|
/** Resolve CLI-local configuration without mutating the parent shell environment. */
|
|
64
|
-
function configFromFlags(flags) {
|
|
65
|
-
const
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
64
|
+
function configFromFlags(flags, identity) {
|
|
65
|
+
const env = { ...process.env };
|
|
66
|
+
if (flags["config-dir"])
|
|
67
|
+
env.NAVARCH_CONFIG_DIR = node_path_1.default.resolve(flags["config-dir"]);
|
|
68
|
+
// Resolve the preferred adapter before deriving the advertised runtimes and
|
|
69
|
+
// applying host guards. Keep an explicit NAVARCH_RUNTIMES list intact.
|
|
70
|
+
const agentType = agentFromFlag(flags) ?? (env.NAVARCH_AGENT || identity?.agent_type);
|
|
71
|
+
if (agentType)
|
|
72
|
+
env.NAVARCH_AGENT = agentType;
|
|
73
|
+
const config = (0, config_cjs_1.loadRuntimeConfig)(env);
|
|
74
|
+
return { ...config, model: modelFromFlags(flags, config.agentType, identity) };
|
|
69
75
|
}
|
|
70
76
|
function agentFromFlag(flags) {
|
|
71
77
|
const value = flags.agent;
|
|
@@ -202,14 +208,10 @@ async function connectCommand(flags) {
|
|
|
202
208
|
async function startCommand(flags) {
|
|
203
209
|
const baseConfig = configFromFlags(flags);
|
|
204
210
|
const identity = await (0, machine_store_cjs_1.resolveMachineIdentity)(baseConfig.configDir, baseConfig.apiBase);
|
|
205
|
-
|
|
206
|
-
// wins, followed by NAVARCH_AGENT, the choice saved at connect/register
|
|
207
|
-
// time, and finally the backwards-compatible Claude Code default.
|
|
208
|
-
const agentType = agentFromFlag(flags) ??
|
|
209
|
-
(process.env.NAVARCH_AGENT ? baseConfig.agentType : identity.agent_type ?? baseConfig.agentType);
|
|
210
|
-
const config = { ...baseConfig, agentType, model: modelFromFlags(flags, agentType, identity) };
|
|
211
|
+
const config = configFromFlags(flags, identity);
|
|
211
212
|
const api = new api_cjs_1.NavarchApiClient({ baseUrl: identity.api_base, token: identity.token });
|
|
212
213
|
const capacity = new capacity_cjs_1.CapacityTracker(config.maxSessions);
|
|
214
|
+
const runtimeShutdown = new AbortController();
|
|
213
215
|
const activeSessionIds = new Set();
|
|
214
216
|
const worktreeJanitor = new worktree_janitor_cjs_1.WorktreeJanitor({
|
|
215
217
|
workspaceRoot: config.workspaceRoot,
|
|
@@ -219,7 +221,7 @@ async function startCommand(flags) {
|
|
|
219
221
|
});
|
|
220
222
|
const claimLoop = new claim_loop_cjs_1.ClaimLoop(api, config, capacity, (claimed, sessionId) => {
|
|
221
223
|
activeSessionIds.add(sessionId);
|
|
222
|
-
return (0, session_cjs_1.runSession)({ api, config }, claimed, sessionId)
|
|
224
|
+
return (0, session_cjs_1.runSession)({ api, config, shutdownSignal: runtimeShutdown.signal }, claimed, sessionId)
|
|
223
225
|
.finally(() => activeSessionIds.delete(sessionId));
|
|
224
226
|
});
|
|
225
227
|
const bootId = (0, node_crypto_1.randomUUID)();
|
|
@@ -243,8 +245,11 @@ async function startCommand(flags) {
|
|
|
243
245
|
: `machine credentials for ${identity.name} were rejected by ${identity.api_base}; it can no longer claim tasks.`);
|
|
244
246
|
log.error(`Re-enroll this machine with a fresh "Connect an agent" token, then run ` +
|
|
245
247
|
`\`${superviseInvocation(config.configDir)}\` again.`);
|
|
246
|
-
//
|
|
247
|
-
//
|
|
248
|
+
// Removal/revocation is terminal: stop detached agent process groups so
|
|
249
|
+
// a provider-side hang cannot keep consuming local resources forever.
|
|
250
|
+
// runSession checkpoints or retains the worktree before releasing its
|
|
251
|
+
// capacity slot.
|
|
252
|
+
runtimeShutdown.abort();
|
|
248
253
|
void capacity.waitForIdle().then(() => process.exit(CREDENTIALS_REJECTED_EXIT_CODE));
|
|
249
254
|
});
|
|
250
255
|
updateCoordinatorRef.current = new update_coordinator_cjs_1.RuntimeUpdateCoordinator({
|
|
@@ -279,19 +284,17 @@ async function startCommand(flags) {
|
|
|
279
284
|
process.on("SIGTERM", shutdown);
|
|
280
285
|
}
|
|
281
286
|
async function superviseCommand(flags) {
|
|
282
|
-
const
|
|
283
|
-
const identity = await (0, machine_store_cjs_1.resolveMachineIdentity)(
|
|
287
|
+
const baseConfig = configFromFlags(flags);
|
|
288
|
+
const identity = await (0, machine_store_cjs_1.resolveMachineIdentity)(baseConfig.configDir, baseConfig.apiBase);
|
|
289
|
+
const config = configFromFlags(flags, identity);
|
|
284
290
|
// Resolve and pin the adapter before spawning the worker. The supervisor
|
|
285
291
|
// passes machine credentials through the environment, so the child no
|
|
286
292
|
// longer reads machine.json for identity fields (including agent_type).
|
|
287
293
|
// Without an explicit worker argument, any saved non-Claude selection
|
|
288
294
|
// would therefore fall back to Claude Code.
|
|
289
|
-
const
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
const model = modelFromFlags(flags, agentType, identity);
|
|
293
|
-
if (model)
|
|
294
|
-
workerArgs.push("--model", model);
|
|
295
|
+
const workerArgs = ["--agent", config.agentType];
|
|
296
|
+
if (config.model)
|
|
297
|
+
workerArgs.push("--model", config.model);
|
|
295
298
|
// Pin the identity for this supervisor's lifetime. Without this snapshot, a
|
|
296
299
|
// replacement worker rereads machine.json after an automatic update and can
|
|
297
300
|
// silently become a different machine if another terminal reused the same
|
|
@@ -311,8 +314,17 @@ async function doctorCommand(flags) {
|
|
|
311
314
|
// start (config.cts). Report that as the diagnosis instead of rethrowing it
|
|
312
315
|
// as a bare CLI error.
|
|
313
316
|
let config;
|
|
317
|
+
let identity;
|
|
314
318
|
try {
|
|
315
319
|
config = configFromFlags(flags);
|
|
320
|
+
try {
|
|
321
|
+
identity = await (0, machine_store_cjs_1.resolveMachineIdentity)(config.configDir, config.apiBase);
|
|
322
|
+
}
|
|
323
|
+
catch {
|
|
324
|
+
// Unregistered machines can still inspect their environment configuration.
|
|
325
|
+
}
|
|
326
|
+
if (identity)
|
|
327
|
+
config = configFromFlags(flags, identity);
|
|
316
328
|
}
|
|
317
329
|
catch (err) {
|
|
318
330
|
console.log(`config: UNUSABLE — ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -329,18 +341,12 @@ async function doctorCommand(flags) {
|
|
|
329
341
|
console.log(`worktree_guard: ${config.worktreeGuard ? "on" : "off"}`);
|
|
330
342
|
console.log(`runtimes: ${config.runtimes?.join(", ") ?? config.agentType}`);
|
|
331
343
|
console.log(`docker: ${dockerOk ? "available" : "NOT AVAILABLE (docker-backed sessions will fail)"}`);
|
|
332
|
-
|
|
333
|
-
try {
|
|
334
|
-
identity = await (0, machine_store_cjs_1.resolveMachineIdentity)(config.configDir, config.apiBase);
|
|
335
|
-
}
|
|
336
|
-
catch {
|
|
344
|
+
if (!identity) {
|
|
337
345
|
console.log(`machine: not registered — run \`${invocation("register")}\``);
|
|
338
346
|
return;
|
|
339
347
|
}
|
|
340
|
-
const agentType = agentFromFlag(flags) ??
|
|
341
|
-
(process.env.NAVARCH_AGENT ? config.agentType : identity.agent_type ?? config.agentType);
|
|
342
348
|
console.log(`machine: ${identity.name} (${identity.machine_id})`);
|
|
343
|
-
console.log(`agent: ${agentType}`);
|
|
349
|
+
console.log(`agent: ${config.agentType}`);
|
|
344
350
|
}
|
|
345
351
|
function helpText() {
|
|
346
352
|
return `navarch-runtime — Navarch machine-side session manager
|
package/dist/config.cjs
CHANGED
|
@@ -108,6 +108,7 @@ function loadRuntimeConfig(env = process.env) {
|
|
|
108
108
|
leaseHeartbeatIntervalMs,
|
|
109
109
|
sessionTimeoutMs: envRuntimeLimit(env, "NAVARCH_SESSION_TIMEOUT_MS", 0),
|
|
110
110
|
sessionIdleTimeoutMs: envRuntimeLimit(env, "NAVARCH_SESSION_IDLE_TIMEOUT_MS", 45 * 60 * 1000),
|
|
111
|
+
opencodeStartupTimeoutMs: envRuntimeLimit(env, "NAVARCH_OPENCODE_STARTUP_TIMEOUT_MS", 2 * 60 * 1000),
|
|
111
112
|
worktreeCleanupIntervalMs: envInt(env, "NAVARCH_WORKTREE_CLEANUP_INTERVAL_MS", 60 * 60 * 1000),
|
|
112
113
|
worktreeStaleAfterMs: envInt(env, "NAVARCH_WORKTREE_STALE_AFTER_MS", 24 * 60 * 60 * 1000),
|
|
113
114
|
agentType,
|
package/dist/exit-conditions.cjs
CHANGED
|
@@ -229,13 +229,25 @@ function mapExitCondition(result) {
|
|
|
229
229
|
evidenceUrls,
|
|
230
230
|
};
|
|
231
231
|
}
|
|
232
|
+
if (result.capacityError) {
|
|
233
|
+
return {
|
|
234
|
+
leaseOutcome: "failed",
|
|
235
|
+
exitStatus: "failed",
|
|
236
|
+
reportSummary: summarize(`Session stopped: provider capacity exhausted. ${result.capacityError}`) +
|
|
237
|
+
(result.checkpointPath ? ` Local recovery checkpoint: ${result.checkpointPath}` : "") +
|
|
238
|
+
(result.retainedWorktreePath ? ` Recovery worktree retained: ${result.retainedWorktreePath}` : ""),
|
|
239
|
+
evidenceUrls,
|
|
240
|
+
};
|
|
241
|
+
}
|
|
232
242
|
if (result.timedOut) {
|
|
233
243
|
return {
|
|
234
244
|
leaseOutcome: "failed",
|
|
235
245
|
exitStatus: "killed",
|
|
236
|
-
reportSummary: (result.timeoutReason === "
|
|
237
|
-
? "Session stopped:
|
|
238
|
-
:
|
|
246
|
+
reportSummary: (result.timeoutReason === "startup"
|
|
247
|
+
? "Session stopped: the agent produced no output within the configured startup window."
|
|
248
|
+
: result.timeoutReason === "idle"
|
|
249
|
+
? "Session stopped: no output or process-tree CPU activity within the configured idle window."
|
|
250
|
+
: "Session killed: exceeded the configured max session duration.") +
|
|
239
251
|
(result.checkpointPath ? ` Local recovery checkpoint: ${result.checkpointPath}` : "") +
|
|
240
252
|
(result.retainedWorktreePath ? ` Recovery worktree retained: ${result.retainedWorktreePath}` : ""),
|
|
241
253
|
evidenceUrls,
|
package/dist/sandbox.cjs
CHANGED
|
@@ -40,10 +40,12 @@ exports.nodeCommandRunner = {
|
|
|
40
40
|
child.stdout?.on("data", (d) => {
|
|
41
41
|
opts.onActivity?.();
|
|
42
42
|
stdout += d.toString();
|
|
43
|
+
opts.onStdout?.(d.toString());
|
|
43
44
|
});
|
|
44
45
|
child.stderr?.on("data", (d) => {
|
|
45
46
|
opts.onActivity?.();
|
|
46
47
|
stderr += d.toString();
|
|
48
|
+
opts.onStderr?.(d.toString());
|
|
47
49
|
});
|
|
48
50
|
child.on("error", (err) => {
|
|
49
51
|
if (settled)
|
package/dist/session.cjs
CHANGED
|
@@ -211,6 +211,14 @@ async function runClaimedSession(deps, claimed, sessionId, lifecycle) {
|
|
|
211
211
|
let activeAbortController = null;
|
|
212
212
|
let leaseLost = false;
|
|
213
213
|
let leaseGone = false;
|
|
214
|
+
const abortForRuntimeShutdown = () => {
|
|
215
|
+
// Machine removal/revocation also makes lease completion impossible.
|
|
216
|
+
// Treat it like a terminal lease loss so the worktree is checkpointed
|
|
217
|
+
// and no remediation turn or authenticated API call is attempted.
|
|
218
|
+
leaseLost = true;
|
|
219
|
+
leaseGone = true;
|
|
220
|
+
activeAbortController?.abort();
|
|
221
|
+
};
|
|
214
222
|
let sessionClosing = false;
|
|
215
223
|
let retainWorktree = false;
|
|
216
224
|
let heartbeatInFlight = null;
|
|
@@ -221,7 +229,7 @@ async function runClaimedSession(deps, claimed, sessionId, lifecycle) {
|
|
|
221
229
|
return;
|
|
222
230
|
try {
|
|
223
231
|
result.checkpointPath = await gitWorktree.saveCheckpoint();
|
|
224
|
-
log.warn(`session ${leaseId} stopped for ${leaseLost ? "lease loss" : result.timeoutReason ?? "duration"}; recovery checkpoint: ${result.checkpointPath}`);
|
|
232
|
+
log.warn(`session ${leaseId} stopped for ${leaseLost ? "lease loss" : result.capacityError ? "provider capacity" : result.timeoutReason ?? "duration"}; recovery checkpoint: ${result.checkpointPath}`);
|
|
225
233
|
}
|
|
226
234
|
catch (error) {
|
|
227
235
|
retainWorktree = true;
|
|
@@ -392,6 +400,9 @@ async function runClaimedSession(deps, claimed, sessionId, lifecycle) {
|
|
|
392
400
|
activeAbortController?.abort();
|
|
393
401
|
});
|
|
394
402
|
const heartbeatTimer = setInterval(() => void pollLease(), config.leaseHeartbeatIntervalMs);
|
|
403
|
+
deps.shutdownSignal?.addEventListener("abort", abortForRuntimeShutdown);
|
|
404
|
+
if (deps.shutdownSignal?.aborted)
|
|
405
|
+
abortForRuntimeShutdown();
|
|
395
406
|
lifecycle.enteredMainSessionTry = true;
|
|
396
407
|
try {
|
|
397
408
|
await gitWorktree.prepare();
|
|
@@ -427,7 +438,7 @@ async function runClaimedSession(deps, claimed, sessionId, lifecycle) {
|
|
|
427
438
|
// notification and start the first turn with the corrected prompt.
|
|
428
439
|
pendingGuidance = [];
|
|
429
440
|
activeAbortController = new AbortController();
|
|
430
|
-
if (leaseLost)
|
|
441
|
+
if (leaseLost || deps.shutdownSignal?.aborted)
|
|
431
442
|
activeAbortController.abort();
|
|
432
443
|
const runPrompt = nextPrompt ??
|
|
433
444
|
(deliveredGuidance.length > (bundle.guidance?.length ?? 0)
|
|
@@ -459,7 +470,7 @@ async function runClaimedSession(deps, claimed, sessionId, lifecycle) {
|
|
|
459
470
|
cwd: gitWorktree.worktreePath,
|
|
460
471
|
dockerExec: sandbox ? { containerName: sandbox.name, runner: sandbox_cjs_1.nodeCommandRunner } : undefined,
|
|
461
472
|
signal: activeAbortController.signal,
|
|
462
|
-
}, task.idle_timeout_ms ?? config.sessionIdleTimeoutMs ?? 45 * 60_000);
|
|
473
|
+
}, task.idle_timeout_ms ?? config.sessionIdleTimeoutMs ?? 45 * 60_000, runtime === "opencode" ? config.opencodeStartupTimeoutMs : 0);
|
|
463
474
|
activeAbortController = null;
|
|
464
475
|
if (turnResult.timedOut || leaseLost)
|
|
465
476
|
await checkpointForRecovery(turnResult);
|
|
@@ -470,6 +481,7 @@ async function runClaimedSession(deps, claimed, sessionId, lifecycle) {
|
|
|
470
481
|
attempts.push(turnResult);
|
|
471
482
|
const capacityLimit = (0, adapter_capacity_cjs_1.detectAdapterCapacityLimit)(turnResult);
|
|
472
483
|
if (capacityLimit) {
|
|
484
|
+
await checkpointForRecovery(turnResult);
|
|
473
485
|
sessionOutcome.claimCooldownUntil = Math.max(sessionOutcome.claimCooldownUntil ?? 0, capacityLimit.retryAtMs);
|
|
474
486
|
}
|
|
475
487
|
// Close the small race between a naturally completed turn and the next
|
|
@@ -484,7 +496,7 @@ async function runClaimedSession(deps, claimed, sessionId, lifecycle) {
|
|
|
484
496
|
log.warn(`session ${leaseId} stopped without completion because its lease is no longer active.`);
|
|
485
497
|
return sessionOutcome;
|
|
486
498
|
}
|
|
487
|
-
if (!leaseLost && !turnResult.timedOut && pendingGuidance.length > 0)
|
|
499
|
+
if (!capacityLimit && !leaseLost && !turnResult.timedOut && pendingGuidance.length > 0)
|
|
488
500
|
continue;
|
|
489
501
|
const result = {
|
|
490
502
|
...turnResult,
|
|
@@ -711,6 +723,7 @@ async function runClaimedSession(deps, claimed, sessionId, lifecycle) {
|
|
|
711
723
|
.catch((completeErr) => log.warn(`complete() after crash also failed: ${String(completeErr)}`));
|
|
712
724
|
}
|
|
713
725
|
finally {
|
|
726
|
+
deps.shutdownSignal?.removeEventListener("abort", abortForRuntimeShutdown);
|
|
714
727
|
sessionClosing = true;
|
|
715
728
|
clearInterval(heartbeatTimer);
|
|
716
729
|
heartbeatGuard.stop();
|
|
@@ -794,6 +807,8 @@ function verificationFailureReport(taskType, report) {
|
|
|
794
807
|
}
|
|
795
808
|
/** Rejection codes another agent turn in the same worktree can plausibly fix. */
|
|
796
809
|
const REMEDIABLE_REJECTION_CODES = new Set([
|
|
810
|
+
"review_fix_required",
|
|
811
|
+
"review_fix_unavailable",
|
|
797
812
|
"pr_required",
|
|
798
813
|
// A video-deliverable build that completed without registering its render:
|
|
799
814
|
// the file usually already exists in the worktree, so the remediation turn
|
package/dist/upload.cjs
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.uploadTranscript = uploadTranscript;
|
|
4
4
|
/**
|
|
5
|
-
* PUTs an already-redacted transcript to the
|
|
5
|
+
* PUTs an already-redacted transcript to the presigned R2 URL the
|
|
6
6
|
* control plane hands back (schema-design.md §4 sessions.transcript_url;
|
|
7
7
|
* implementation-plan.md WP-07). Kept as its own module so the storage
|
|
8
8
|
* mechanism is a one-function swap if it ever changes.
|
|
@@ -16,9 +16,8 @@ async function uploadTranscript(uploadUrl, content, fetchImpl = fetch) {
|
|
|
16
16
|
body: content,
|
|
17
17
|
});
|
|
18
18
|
if (!response.ok) {
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
// from a bucket misconfiguration. Without it a 403 is unattributable.
|
|
19
|
+
// Include the bounded storage response to distinguish expired signatures
|
|
20
|
+
// from bucket misconfiguration.
|
|
22
21
|
throw new Error(`Transcript upload failed with ${response.status}: ${await readErrorBody(response)}`);
|
|
23
22
|
}
|
|
24
23
|
}
|
package/package.json
CHANGED