@runuai/host 0.9.77 → 0.9.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/images/standard/container/uai-init +76 -4
- package/lib/machine-gateway-tunnel.ts +166 -0
- package/lib/mcp-gateway.ts +14 -3
- package/lib/orchestrator.ts +23 -1
- package/lib/task-environment/machine-task-up.ts +6 -0
- package/lib/task-environment/machine.ts +25 -0
- package/lib/task-environment/types.ts +8 -0
- package/lib/vnode-pressure.ts +103 -0
- package/package.json +1 -1
- package/src/cli.ts +138 -1
- package/src/index.ts +22 -0
- package/src/main.ts +6 -0
|
@@ -419,9 +419,25 @@ install_folder_deps() {
|
|
|
419
419
|
|
|
420
420
|
reject_workspace_runtime_shadow || exit 78
|
|
421
421
|
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
422
|
+
# ADR-122: dependency installs leave the task-start critical path. The loop
|
|
423
|
+
# below runs in a DETACHED background process so uai-init (and with it
|
|
424
|
+
# task-up) returns as soon as the cheap, provable work is done — the editor
|
|
425
|
+
# and the agents start immediately, warm or cold. Progress is observable in
|
|
426
|
+
# the workspace itself:
|
|
427
|
+
#
|
|
428
|
+
# $WORKSPACE/.uai/init/deps-status running | done | failed | authority-violation
|
|
429
|
+
# $WORKSPACE/.uai/init/deps.log the full install transcript
|
|
430
|
+
#
|
|
431
|
+
# The runtime-authority posture narrows deliberately: the synchronous
|
|
432
|
+
# workspace-shadow proofs above and below still exit 78 (task-up
|
|
433
|
+
# quarantines), but a shadow that APPEARS MID-INSTALL can no longer stop the
|
|
434
|
+
# task — agents are already running by then, and an agent could author the
|
|
435
|
+
# same shadow a second later anyway. The runner still refuses to run any
|
|
436
|
+
# further dependency command under a shadowed root and records
|
|
437
|
+
# `authority-violation`; the shared asdf volume stays read-only-proven
|
|
438
|
+
# regardless.
|
|
439
|
+
run_workspace_installs() {
|
|
440
|
+
local seen_runtime_projects="|" project_slug folder git_marker
|
|
425
441
|
while IFS= read -r project_slug || [ -n "$project_slug" ]; do
|
|
426
442
|
if [[ ! "$project_slug" =~ ^[a-z0-9-]{1,64}$ ]]; then
|
|
427
443
|
log "warning: unsafe runtime-project allowlist entry — skipping"
|
|
@@ -441,11 +457,67 @@ if [ -d "$WORKSPACE" ] && [ -f "$RUNTIME_PROJECTS" ] \
|
|
|
441
457
|
log "warning: allowlisted project $project_slug is not a safe Git worktree — skipping"
|
|
442
458
|
continue
|
|
443
459
|
fi
|
|
444
|
-
|
|
460
|
+
if ! reject_workspace_runtime_shadow; then
|
|
461
|
+
printf 'authority-violation\n' > "$deps_status_file"
|
|
462
|
+
return 78
|
|
463
|
+
fi
|
|
445
464
|
# Run in a subshell so a `cd` (or a failing command under the relaxed
|
|
446
465
|
# error mode) in one folder never leaks into the next.
|
|
447
466
|
( install_folder_deps "$folder" )
|
|
448
467
|
done < "$RUNTIME_PROJECTS"
|
|
468
|
+
return 0
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
if [ -d "$WORKSPACE" ] && [ -f "$RUNTIME_PROJECTS" ] \
|
|
472
|
+
&& [ ! -L "$RUNTIME_PROJECTS" ]; then
|
|
473
|
+
deps_state_dir="$WORKSPACE/.uai/init"
|
|
474
|
+
deps_status_file="$deps_state_dir/deps-status"
|
|
475
|
+
deps_log_file="$deps_state_dir/deps.log"
|
|
476
|
+
deps_lock_dir="$deps_state_dir/deps.lock.d"
|
|
477
|
+
mkdir -p "$deps_state_dir"
|
|
478
|
+
|
|
479
|
+
# One runner at a time. uai-init is re-runnable (task-up retry, recovery),
|
|
480
|
+
# and each re-run SHOULD refresh dependencies — but never concurrently.
|
|
481
|
+
# mkdir is the portable atomic lock; a pid that no longer answers marks a
|
|
482
|
+
# crashed runner whose lock is stale and may be stolen.
|
|
483
|
+
deps_spawn=1
|
|
484
|
+
if mkdir "$deps_lock_dir" 2>/dev/null; then
|
|
485
|
+
# Hold the lock as ourselves until the runner's pid replaces it, so a
|
|
486
|
+
# concurrent uai-init never reads an empty pid file as a stale lock.
|
|
487
|
+
printf '%s\n' "$$" > "$deps_lock_dir/pid"
|
|
488
|
+
else
|
|
489
|
+
deps_holder=$(cat "$deps_lock_dir/pid" 2>/dev/null || true)
|
|
490
|
+
if [ -n "$deps_holder" ] && kill -0 "$deps_holder" 2>/dev/null; then
|
|
491
|
+
log "dependency install already running (pid $deps_holder) — leaving it be"
|
|
492
|
+
deps_spawn=0
|
|
493
|
+
else
|
|
494
|
+
rm -rf "$deps_lock_dir"
|
|
495
|
+
if mkdir "$deps_lock_dir" 2>/dev/null; then
|
|
496
|
+
printf '%s\n' "$$" > "$deps_lock_dir/pid"
|
|
497
|
+
else
|
|
498
|
+
log "warning: could not acquire the dependency install lock — skipping installs"
|
|
499
|
+
deps_spawn=0
|
|
500
|
+
fi
|
|
501
|
+
fi
|
|
502
|
+
fi
|
|
503
|
+
if [ "$deps_spawn" -eq 1 ]; then
|
|
504
|
+
printf 'running\n' > "$deps_status_file"
|
|
505
|
+
log "installing workspace dependencies in the background (status: $deps_status_file, log: $deps_log_file)"
|
|
506
|
+
(
|
|
507
|
+
trap 'rm -rf "$deps_lock_dir"' EXIT
|
|
508
|
+
deps_rc=0
|
|
509
|
+
run_workspace_installs || deps_rc=$?
|
|
510
|
+
if [ "$deps_rc" -eq 0 ]; then
|
|
511
|
+
printf 'done\n' > "$deps_status_file"
|
|
512
|
+
log "workspace dependency install complete"
|
|
513
|
+
elif [ "$deps_rc" -ne 78 ]; then
|
|
514
|
+
printf 'failed\n' > "$deps_status_file"
|
|
515
|
+
log "workspace dependency install failed (exit $deps_rc)"
|
|
516
|
+
fi
|
|
517
|
+
) >>"$deps_log_file" 2>&1 </dev/null &
|
|
518
|
+
printf '%s\n' "$!" > "$deps_lock_dir/pid"
|
|
519
|
+
disown
|
|
520
|
+
fi
|
|
449
521
|
elif [ ! -d "$WORKSPACE" ]; then
|
|
450
522
|
log "workspace $WORKSPACE missing — scratchpad/no-project task, skipping installs"
|
|
451
523
|
else
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ADR-121: the MCP gateway reverse tunnel for machine tasks.
|
|
3
|
+
*
|
|
4
|
+
* Containers reach the host's MCP gateway via container topology
|
|
5
|
+
* (host.docker.internal, the vmnet gateway IP). A machine — an EC2
|
|
6
|
+
* instance behind its own network — can never route to the host, so the
|
|
7
|
+
* host pushes the gateway TO the machine instead: one persistent
|
|
8
|
+
* `ssh -N -R 127.0.0.1:<port>:127.0.0.1:<port>` per machine task, using
|
|
9
|
+
* the same per-task key every other machine operation rides. Task configs
|
|
10
|
+
* then point MCP servers at the machine's own loopback
|
|
11
|
+
* (setupMcpTaskConfig's machine branch).
|
|
12
|
+
*
|
|
13
|
+
* Supervision: the tunnel respawns with a short backoff while ensured —
|
|
14
|
+
* re-resolving the machine address each spawn, since it changes across
|
|
15
|
+
* stop/start — and stops when the task stops, tears down, or the process
|
|
16
|
+
* exits. Failure is quiet-but-logged: MCP servers failing to connect is
|
|
17
|
+
* the visible symptom, and the agent-facing degradation is already
|
|
18
|
+
* per-server.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { spawn, type ChildProcess } from "node:child_process";
|
|
22
|
+
|
|
23
|
+
import { MCP_GATEWAY_PORT } from "./mcp-gateway";
|
|
24
|
+
|
|
25
|
+
const RESPAWN_DELAY_MS = 3_000;
|
|
26
|
+
/** After this many consecutive sub-5s deaths, slow the respawn cadence so a
|
|
27
|
+
* permanently unreachable machine does not spin ssh forever. */
|
|
28
|
+
const FAST_DEATH_THRESHOLD = 5;
|
|
29
|
+
const SLOW_RESPAWN_DELAY_MS = 60_000;
|
|
30
|
+
|
|
31
|
+
export interface MachineTunnelSource {
|
|
32
|
+
gatewayTunnelSshArgs(
|
|
33
|
+
machinePort: number,
|
|
34
|
+
hostPort: number,
|
|
35
|
+
): Promise<string[]>;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface TunnelState {
|
|
39
|
+
generation: number;
|
|
40
|
+
child: ChildProcess | null;
|
|
41
|
+
timer: ReturnType<typeof setTimeout> | null;
|
|
42
|
+
fastDeaths: number;
|
|
43
|
+
spawnedAt: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const tunnels = new Map<string, TunnelState>();
|
|
47
|
+
|
|
48
|
+
/** Idempotently keep a gateway tunnel alive for this machine task. */
|
|
49
|
+
export function ensureMachineGatewayTunnel(
|
|
50
|
+
taskId: string,
|
|
51
|
+
source: MachineTunnelSource,
|
|
52
|
+
): void {
|
|
53
|
+
const existing = tunnels.get(taskId);
|
|
54
|
+
if (existing && (existing.child !== null || existing.timer !== null)) {
|
|
55
|
+
return; // live or already scheduled
|
|
56
|
+
}
|
|
57
|
+
const state: TunnelState = existing ?? {
|
|
58
|
+
generation: 0,
|
|
59
|
+
child: null,
|
|
60
|
+
timer: null,
|
|
61
|
+
fastDeaths: 0,
|
|
62
|
+
spawnedAt: 0,
|
|
63
|
+
};
|
|
64
|
+
state.generation += 1;
|
|
65
|
+
tunnels.set(taskId, state);
|
|
66
|
+
if (existing === undefined) {
|
|
67
|
+
console.log(`[machine-tunnel] ${taskId}: ensuring gateway tunnel`);
|
|
68
|
+
}
|
|
69
|
+
void spawnTunnel(taskId, source, state, state.generation);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function spawnTunnel(
|
|
73
|
+
taskId: string,
|
|
74
|
+
source: MachineTunnelSource,
|
|
75
|
+
state: TunnelState,
|
|
76
|
+
generation: number,
|
|
77
|
+
): Promise<void> {
|
|
78
|
+
if (tunnels.get(taskId) !== state || state.generation !== generation) return;
|
|
79
|
+
let args: string[];
|
|
80
|
+
try {
|
|
81
|
+
args = await source.gatewayTunnelSshArgs(
|
|
82
|
+
MCP_GATEWAY_PORT,
|
|
83
|
+
MCP_GATEWAY_PORT,
|
|
84
|
+
);
|
|
85
|
+
} catch (err) {
|
|
86
|
+
// Machine not reachable right now (stopped, address settling) — retry
|
|
87
|
+
// on the slow cadence; a stop/teardown clears us before it matters.
|
|
88
|
+
scheduleRespawn(taskId, source, state, generation, SLOW_RESPAWN_DELAY_MS);
|
|
89
|
+
if (state.fastDeaths === 0) {
|
|
90
|
+
console.warn(
|
|
91
|
+
`[machine-tunnel] ${taskId}: target unavailable (${err instanceof Error ? err.message : err}) — retrying`,
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
state.fastDeaths += 1;
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
state.spawnedAt = Date.now();
|
|
98
|
+
const child = spawn("ssh", args, { stdio: ["ignore", "ignore", "pipe"] });
|
|
99
|
+
state.child = child;
|
|
100
|
+
let stderrTail = "";
|
|
101
|
+
child.stderr?.setEncoding("utf8");
|
|
102
|
+
child.stderr?.on("data", (chunk: string) => {
|
|
103
|
+
stderrTail = (stderrTail + chunk).slice(-2048);
|
|
104
|
+
});
|
|
105
|
+
child.once("exit", (code) => {
|
|
106
|
+
if (tunnels.get(taskId) !== state || state.generation !== generation) {
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
state.child = null;
|
|
110
|
+
const lifetimeMs = Date.now() - state.spawnedAt;
|
|
111
|
+
if (lifetimeMs < 5_000) state.fastDeaths += 1;
|
|
112
|
+
else state.fastDeaths = 0;
|
|
113
|
+
const slow = state.fastDeaths >= FAST_DEATH_THRESHOLD;
|
|
114
|
+
if (state.fastDeaths === FAST_DEATH_THRESHOLD) {
|
|
115
|
+
console.warn(
|
|
116
|
+
`[machine-tunnel] ${taskId}: tunnel keeps dying (exit ${code ?? "signal"}${
|
|
117
|
+
stderrTail.trim() ? `: ${stderrTail.trim().slice(0, 200)}` : ""
|
|
118
|
+
}) — slowing retries`,
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
scheduleRespawn(
|
|
122
|
+
taskId,
|
|
123
|
+
source,
|
|
124
|
+
state,
|
|
125
|
+
generation,
|
|
126
|
+
slow ? SLOW_RESPAWN_DELAY_MS : RESPAWN_DELAY_MS,
|
|
127
|
+
);
|
|
128
|
+
});
|
|
129
|
+
child.once("error", () => {
|
|
130
|
+
// exit fires too; the handler above owns respawn.
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function scheduleRespawn(
|
|
135
|
+
taskId: string,
|
|
136
|
+
source: MachineTunnelSource,
|
|
137
|
+
state: TunnelState,
|
|
138
|
+
generation: number,
|
|
139
|
+
delayMs: number,
|
|
140
|
+
): void {
|
|
141
|
+
if (tunnels.get(taskId) !== state || state.generation !== generation) return;
|
|
142
|
+
state.timer = setTimeout(() => {
|
|
143
|
+
state.timer = null;
|
|
144
|
+
void spawnTunnel(taskId, source, state, generation);
|
|
145
|
+
}, delayMs);
|
|
146
|
+
state.timer.unref?.();
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Tear the tunnel down (task stop/teardown/GC). Idempotent. */
|
|
150
|
+
export function stopMachineGatewayTunnel(taskId: string): void {
|
|
151
|
+
const state = tunnels.get(taskId);
|
|
152
|
+
if (!state) return;
|
|
153
|
+
tunnels.delete(taskId);
|
|
154
|
+
state.generation += 1; // invalidate in-flight spawns
|
|
155
|
+
if (state.timer) clearTimeout(state.timer);
|
|
156
|
+
if (state.child) {
|
|
157
|
+
state.child.kill("SIGTERM");
|
|
158
|
+
state.child = null;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Test-only view. */
|
|
163
|
+
export function machineGatewayTunnelActive(taskId: string): boolean {
|
|
164
|
+
const state = tunnels.get(taskId);
|
|
165
|
+
return Boolean(state && (state.child !== null || state.timer !== null));
|
|
166
|
+
}
|
package/lib/mcp-gateway.ts
CHANGED
|
@@ -854,11 +854,14 @@ function gatewayIdentity(value, slug) {
|
|
|
854
854
|
}
|
|
855
855
|
|
|
856
856
|
function sameTaskGateway(oldUrl, desiredUrl, slug) {
|
|
857
|
+
// Ownership is proven by the /t/<taskId>.<token>/<slug> path, not the
|
|
858
|
+
// origin: the gateway host is topology-dependent (container gateway IP,
|
|
859
|
+
// host.docker.internal, machine loopback behind a reverse tunnel) and
|
|
860
|
+
// migrating a task across topologies is exactly when the rewrite runs.
|
|
857
861
|
const oldIdentity = gatewayIdentity(oldUrl, slug);
|
|
858
862
|
const desiredIdentity = gatewayIdentity(desiredUrl, slug);
|
|
859
863
|
return oldIdentity !== null &&
|
|
860
864
|
desiredIdentity !== null &&
|
|
861
|
-
oldIdentity.origin === desiredIdentity.origin &&
|
|
862
865
|
oldIdentity.taskId === desiredIdentity.taskId;
|
|
863
866
|
}
|
|
864
867
|
|
|
@@ -1063,7 +1066,8 @@ export async function setupMcpTaskConfig(
|
|
|
1063
1066
|
connections: TaskMcpConnection[],
|
|
1064
1067
|
engineKinds: string[],
|
|
1065
1068
|
codexHomes: readonly string[] = ["/home/node/.codex"],
|
|
1066
|
-
environment?: Pick<TaskEnvironmentHandle, "exec"
|
|
1069
|
+
environment?: Pick<TaskEnvironmentHandle, "exec"> &
|
|
1070
|
+
Partial<Pick<TaskEnvironmentHandle, "descriptor">>,
|
|
1067
1071
|
): Promise<boolean> {
|
|
1068
1072
|
// No early return on empty: the claude adapter passes
|
|
1069
1073
|
// `--mcp-config /workspace/.mcp.json` unconditionally (ADR-057), so the
|
|
@@ -1084,7 +1088,14 @@ export async function setupMcpTaskConfig(
|
|
|
1084
1088
|
throw new Error("invalid Codex MCP connection slug");
|
|
1085
1089
|
}
|
|
1086
1090
|
const acl = ensureTaskGatewayAcl(taskId, connections);
|
|
1087
|
-
|
|
1091
|
+
// ADR-121: a machine dials the gateway through its ssh REVERSE tunnel —
|
|
1092
|
+
// the host's loopback appears at the machine's own 127.0.0.1:PORT
|
|
1093
|
+
// (machine-gateway-tunnel.ts). host.docker.internal / vmnet addresses
|
|
1094
|
+
// are container topology a machine can never route.
|
|
1095
|
+
const gatewayHost =
|
|
1096
|
+
environment?.descriptor?.locator.provider === "machine"
|
|
1097
|
+
? "127.0.0.1"
|
|
1098
|
+
: await taskGatewayHost();
|
|
1088
1099
|
const urlFor = (slug: string): string =>
|
|
1089
1100
|
`http://${gatewayHost}:${MCP_GATEWAY_PORT}/t/${acl.token}/${slug}`;
|
|
1090
1101
|
|
package/lib/orchestrator.ts
CHANGED
|
@@ -90,6 +90,7 @@ import {
|
|
|
90
90
|
type SkillExec,
|
|
91
91
|
writeAgentSkills,
|
|
92
92
|
} from "./skills";
|
|
93
|
+
import { ensureMachineGatewayTunnel } from "./machine-gateway-tunnel";
|
|
93
94
|
import {
|
|
94
95
|
CONTAINER_CLI_PATH,
|
|
95
96
|
agentCliEnv,
|
|
@@ -1655,7 +1656,14 @@ export class Orchestrator {
|
|
|
1655
1656
|
return environment;
|
|
1656
1657
|
})();
|
|
1657
1658
|
const tracked = created.then((environment) => {
|
|
1658
|
-
if (this.isActiveChannel(channel))
|
|
1659
|
+
if (this.isActiveChannel(channel)) {
|
|
1660
|
+
channel.agentHandle = environment;
|
|
1661
|
+
// Every path that touches a machine workspace settles the handle
|
|
1662
|
+
// here — including durable-session reattach after a host restart,
|
|
1663
|
+
// which spawns no sessions and runs no config writers. Ensuring the
|
|
1664
|
+
// gateway tunnel at the funnel keeps it alive across all of them.
|
|
1665
|
+
ensureMachineGatewayTunnelFor(channel.taskId, environment);
|
|
1666
|
+
}
|
|
1659
1667
|
return environment;
|
|
1660
1668
|
});
|
|
1661
1669
|
channel.agentEnvironment = tracked;
|
|
@@ -5641,6 +5649,20 @@ async function quarantineWritableAppleRuntimeContainers(options: {
|
|
|
5641
5649
|
* machine tasks (their workspace is not host-reachable). Best-effort like the
|
|
5642
5650
|
* host-side writer.
|
|
5643
5651
|
*/
|
|
5652
|
+
/** Keep the MCP gateway reverse tunnel alive for a machine channel. The
|
|
5653
|
+
* handle exposes tunnel argv only on the machine provider; anything else is
|
|
5654
|
+
* a quiet no-op. */
|
|
5655
|
+
function ensureMachineGatewayTunnelFor(
|
|
5656
|
+
taskId: string,
|
|
5657
|
+
handle: TaskEnvironmentHandle<unknown> | null,
|
|
5658
|
+
): void {
|
|
5659
|
+
const argv = handle?.gatewayTunnelSshArgs;
|
|
5660
|
+
if (!argv) return;
|
|
5661
|
+
ensureMachineGatewayTunnel(taskId, {
|
|
5662
|
+
gatewayTunnelSshArgs: argv.bind(handle),
|
|
5663
|
+
});
|
|
5664
|
+
}
|
|
5665
|
+
|
|
5644
5666
|
async function materializeAgentCli(
|
|
5645
5667
|
task: typeof schema.hostTasks.$inferSelect,
|
|
5646
5668
|
roster: RosterAgent[],
|
|
@@ -448,6 +448,12 @@ function withTaskDownResult(
|
|
|
448
448
|
...(handle.inspectRoute
|
|
449
449
|
? { inspectRoute: () => handle.inspectRoute!() }
|
|
450
450
|
: {}),
|
|
451
|
+
...(handle.gatewayTunnelSshArgs
|
|
452
|
+
? {
|
|
453
|
+
gatewayTunnelSshArgs: (machinePort: number, hostPort: number) =>
|
|
454
|
+
handle.gatewayTunnelSshArgs!(machinePort, hostPort),
|
|
455
|
+
}
|
|
456
|
+
: {}),
|
|
451
457
|
};
|
|
452
458
|
}
|
|
453
459
|
|
|
@@ -504,6 +504,31 @@ class MachineTaskEnvironmentHandle implements TaskEnvironmentHandle {
|
|
|
504
504
|
};
|
|
505
505
|
}
|
|
506
506
|
|
|
507
|
+
/** Full ssh argv for the persistent REVERSE gateway tunnel: the host's
|
|
508
|
+
* loopback gateway appears at the machine's own 127.0.0.1:<machinePort>.
|
|
509
|
+
* Resolved per call — cloud machines change address across stop/start, so
|
|
510
|
+
* each (re)spawn of the tunnel re-resolves the dial target. */
|
|
511
|
+
async gatewayTunnelSshArgs(
|
|
512
|
+
machinePort: number,
|
|
513
|
+
hostPort: number,
|
|
514
|
+
): Promise<string[]> {
|
|
515
|
+
const target = await this.#target();
|
|
516
|
+
return [
|
|
517
|
+
"-o", "BatchMode=yes",
|
|
518
|
+
"-o", "StrictHostKeyChecking=accept-new",
|
|
519
|
+
"-o", "ConnectTimeout=10",
|
|
520
|
+
"-o", "ExitOnForwardFailure=yes",
|
|
521
|
+
"-o", "ServerAliveInterval=15",
|
|
522
|
+
"-o", "ServerAliveCountMax=3",
|
|
523
|
+
"-i", target.keyPath,
|
|
524
|
+
"-p", String(target.port ?? 22),
|
|
525
|
+
"-l", this.#value.sshUser,
|
|
526
|
+
target.address,
|
|
527
|
+
"-N",
|
|
528
|
+
"-R", `127.0.0.1:${machinePort}:127.0.0.1:${hostPort}`,
|
|
529
|
+
];
|
|
530
|
+
}
|
|
531
|
+
|
|
507
532
|
async status(): Promise<TaskEnvironmentStatus> {
|
|
508
533
|
const info = await this.#describeOwn();
|
|
509
534
|
switch (info.state) {
|
|
@@ -220,6 +220,14 @@ export interface TaskEnvironmentHandle<TTeardownResult = void>
|
|
|
220
220
|
| { kind: "unavailable" }
|
|
221
221
|
| { kind: "unknown"; detail: string }
|
|
222
222
|
>;
|
|
223
|
+
/** ADR-121, machine provider only: full ssh argv for the persistent
|
|
224
|
+
* REVERSE gateway tunnel that makes the host's MCP gateway answer on the
|
|
225
|
+
* environment's own loopback. Container providers omit it — their tasks
|
|
226
|
+
* route to the gateway via container topology instead. */
|
|
227
|
+
gatewayTunnelSshArgs?(
|
|
228
|
+
machinePort: number,
|
|
229
|
+
hostPort: number,
|
|
230
|
+
): Promise<string[]>;
|
|
223
231
|
}
|
|
224
232
|
|
|
225
233
|
export interface TaskEnvironmentProvisionRequest<TInput, TCredentials> {
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ADR-118 follow-up: the macOS vnode-table pressure sensor.
|
|
3
|
+
*
|
|
4
|
+
* Apple's Virtualization.framework virtiofs pins one host vnode per distinct
|
|
5
|
+
* guest-touched file for the life of the task VM (~12k per task after
|
|
6
|
+
* ADR-118 moved the pnpm store off virtiofs; ~130k+ before it). The table
|
|
7
|
+
* ceiling (`kern.maxvnodes`) is a boot-time default the host cannot raise
|
|
8
|
+
* without root, and AT the ceiling unrelated host software starts failing
|
|
9
|
+
* with ENFILE — the suspected 2026-08-24 kernel-panic precursor.
|
|
10
|
+
*
|
|
11
|
+
* Three surfaces ride this module:
|
|
12
|
+
* - `readVnodePressure()` — the sensor (cached; null off-macOS).
|
|
13
|
+
* - `vnodeAdmissionProblem()` — task-up guard: refuse to start ANOTHER
|
|
14
|
+
* apple-container VM above the threshold instead of letting the kernel
|
|
15
|
+
* hit the wall. Existing tasks keep running.
|
|
16
|
+
* - `logVnodePressure()` — periodic observability, warn-level near the
|
|
17
|
+
* ceiling so host logs explain refusals before they happen.
|
|
18
|
+
*
|
|
19
|
+
* Raising the ceiling needs root once: `sudo uai-host tune` (see cli.ts)
|
|
20
|
+
* installs a LaunchDaemon that reasserts the sysctls at every boot —
|
|
21
|
+
* sysctl writes alone are ephemeral.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { execFile } from "node:child_process";
|
|
25
|
+
import { promisify } from "node:util";
|
|
26
|
+
|
|
27
|
+
const exec = promisify(execFile);
|
|
28
|
+
|
|
29
|
+
/** Refuse NEW apple task VMs above this fraction of the vnode ceiling. */
|
|
30
|
+
const ADMISSION_THRESHOLD = 0.9;
|
|
31
|
+
/** Warn-log above this fraction. */
|
|
32
|
+
const WARN_THRESHOLD = 0.8;
|
|
33
|
+
const CACHE_MS = 15_000;
|
|
34
|
+
|
|
35
|
+
export interface VnodePressure {
|
|
36
|
+
current: number;
|
|
37
|
+
max: number;
|
|
38
|
+
ratio: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
let cached: { at: number; value: VnodePressure | null } | null = null;
|
|
42
|
+
|
|
43
|
+
export async function readVnodePressure(): Promise<VnodePressure | null> {
|
|
44
|
+
if (process.platform !== "darwin") return null;
|
|
45
|
+
if (cached && Date.now() - cached.at < CACHE_MS) return cached.value;
|
|
46
|
+
let value: VnodePressure | null = null;
|
|
47
|
+
try {
|
|
48
|
+
const { stdout } = await exec(
|
|
49
|
+
"/usr/sbin/sysctl",
|
|
50
|
+
["-n", "kern.num_vnodes", "kern.maxvnodes"],
|
|
51
|
+
{ timeout: 5_000 },
|
|
52
|
+
);
|
|
53
|
+
const parts = stdout.trim().split(/\s+/).map(Number);
|
|
54
|
+
const current = parts[0] ?? Number.NaN;
|
|
55
|
+
const max = parts[1] ?? Number.NaN;
|
|
56
|
+
if (Number.isFinite(current) && Number.isFinite(max) && max > 0) {
|
|
57
|
+
value = { current, max, ratio: current / max };
|
|
58
|
+
}
|
|
59
|
+
} catch {
|
|
60
|
+
// Sensor failure is never a task failure; admission simply has no data.
|
|
61
|
+
}
|
|
62
|
+
cached = { at: Date.now(), value };
|
|
63
|
+
return value;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Non-null = refuse to start another apple-container task VM right now. */
|
|
67
|
+
export async function vnodeAdmissionProblem(): Promise<string | null> {
|
|
68
|
+
const pressure = await readVnodePressure();
|
|
69
|
+
if (!pressure || pressure.ratio < ADMISSION_THRESHOLD) return null;
|
|
70
|
+
return (
|
|
71
|
+
`the macOS vnode table is at ${Math.round(pressure.ratio * 100)}% ` +
|
|
72
|
+
`(${pressure.current}/${pressure.max}); starting another task VM risks ` +
|
|
73
|
+
"system-wide file-table exhaustion. Close a running task, or raise the " +
|
|
74
|
+
"ceiling once with: sudo uai-host tune"
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
let lastLoggedBand: "ok" | "warn" | "critical" = "ok";
|
|
79
|
+
|
|
80
|
+
/** Log on band TRANSITIONS only, so a host sitting near the ceiling does not
|
|
81
|
+
* spam its own log every interval. */
|
|
82
|
+
export async function logVnodePressure(): Promise<void> {
|
|
83
|
+
const pressure = await readVnodePressure();
|
|
84
|
+
if (!pressure) return;
|
|
85
|
+
const band =
|
|
86
|
+
pressure.ratio >= ADMISSION_THRESHOLD
|
|
87
|
+
? "critical"
|
|
88
|
+
: pressure.ratio >= WARN_THRESHOLD
|
|
89
|
+
? "warn"
|
|
90
|
+
: "ok";
|
|
91
|
+
if (band === lastLoggedBand) return;
|
|
92
|
+
lastLoggedBand = band;
|
|
93
|
+
const detail = `${pressure.current}/${pressure.max} (${Math.round(pressure.ratio * 100)}%)`;
|
|
94
|
+
if (band === "critical") {
|
|
95
|
+
console.warn(
|
|
96
|
+
`[vnode] table at ${detail} — new apple task VMs will be refused; run: sudo uai-host tune`,
|
|
97
|
+
);
|
|
98
|
+
} else if (band === "warn") {
|
|
99
|
+
console.warn(`[vnode] table filling: ${detail}`);
|
|
100
|
+
} else {
|
|
101
|
+
console.log(`[vnode] table pressure back to normal: ${detail}`);
|
|
102
|
+
}
|
|
103
|
+
}
|
package/package.json
CHANGED
package/src/cli.ts
CHANGED
|
@@ -22,7 +22,7 @@ import "./load-env";
|
|
|
22
22
|
|
|
23
23
|
import { spawn, spawnSync } from "node:child_process";
|
|
24
24
|
import { createHash, randomBytes } from "node:crypto";
|
|
25
|
-
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
25
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
26
26
|
import { homedir, hostname } from "node:os";
|
|
27
27
|
import { dirname, join, resolve } from "node:path";
|
|
28
28
|
|
|
@@ -137,6 +137,8 @@ async function main(): Promise<void> {
|
|
|
137
137
|
return cmdLogs(follow);
|
|
138
138
|
case "runtime":
|
|
139
139
|
return cmdRuntime(rest);
|
|
140
|
+
case "tune":
|
|
141
|
+
return cmdTune();
|
|
140
142
|
case "update":
|
|
141
143
|
return cmdUpdate(rest);
|
|
142
144
|
case "rollback":
|
|
@@ -498,6 +500,139 @@ async function boundedResponseJson(
|
|
|
498
500
|
|
|
499
501
|
// --- runtime ---------------------------------------------------------------
|
|
500
502
|
|
|
503
|
+
// --- tune (ADR-118 follow-up: vnode/file-table ceilings) ---------------------
|
|
504
|
+
|
|
505
|
+
const TUNE_DAEMON_LABEL = "com.runuai.tune";
|
|
506
|
+
const TUNE_DAEMON_PLIST = `/Library/LaunchDaemons/${TUNE_DAEMON_LABEL}.plist`;
|
|
507
|
+
/** Floors, not targets — an operator-raised ceiling is never lowered. Each
|
|
508
|
+
* apple-container task VM pins host vnodes for its life (~12k after ADR-118,
|
|
509
|
+
* far more during dependency churn), and the boot defaults sit low enough
|
|
510
|
+
* that two heavy tasks have filled the table live (2026-08-25). ~2M vnodes
|
|
511
|
+
* is roughly half a GB of kernel memory IF fully populated. */
|
|
512
|
+
const TUNE_MAXVNODES_FLOOR = 2_097_152;
|
|
513
|
+
const TUNE_MAXFILES_FLOOR = 2_097_152;
|
|
514
|
+
const TUNE_MAXFILESPERPROC_FLOOR = 1_048_576;
|
|
515
|
+
|
|
516
|
+
function readSysctlNumber(name: string): number | null {
|
|
517
|
+
const res = spawnSync("/usr/sbin/sysctl", ["-n", name], {
|
|
518
|
+
encoding: "utf8",
|
|
519
|
+
});
|
|
520
|
+
const value = Number((res.stdout ?? "").trim());
|
|
521
|
+
return res.status === 0 && Number.isFinite(value) ? value : null;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
function tunePlistContent(settings: Array<[string, number]>): string {
|
|
525
|
+
const args = ["/usr/sbin/sysctl", ...settings.map(([k, v]) => `${k}=${v}`)]
|
|
526
|
+
.map((a) => ` <string>${a}</string>`)
|
|
527
|
+
.join("\n");
|
|
528
|
+
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
529
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
530
|
+
<plist version="1.0">
|
|
531
|
+
<dict>
|
|
532
|
+
<key>Label</key>
|
|
533
|
+
<string>${TUNE_DAEMON_LABEL}</string>
|
|
534
|
+
<key>ProgramArguments</key>
|
|
535
|
+
<array>
|
|
536
|
+
${args}
|
|
537
|
+
</array>
|
|
538
|
+
<key>RunAtLoad</key>
|
|
539
|
+
<true/>
|
|
540
|
+
</dict>
|
|
541
|
+
</plist>
|
|
542
|
+
`;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
async function cmdTune(): Promise<void> {
|
|
546
|
+
if (process.platform !== "darwin") {
|
|
547
|
+
console.error(red("tune is macOS-only (apple-container task VMs)"));
|
|
548
|
+
process.exitCode = 1;
|
|
549
|
+
return;
|
|
550
|
+
}
|
|
551
|
+
const current = {
|
|
552
|
+
maxvnodes: readSysctlNumber("kern.maxvnodes"),
|
|
553
|
+
maxfiles: readSysctlNumber("kern.maxfiles"),
|
|
554
|
+
maxfilesperproc: readSysctlNumber("kern.maxfilesperproc"),
|
|
555
|
+
};
|
|
556
|
+
if (
|
|
557
|
+
current.maxvnodes === null ||
|
|
558
|
+
current.maxfiles === null ||
|
|
559
|
+
current.maxfilesperproc === null
|
|
560
|
+
) {
|
|
561
|
+
console.error(red("could not read the current kernel limits via sysctl"));
|
|
562
|
+
process.exitCode = 1;
|
|
563
|
+
return;
|
|
564
|
+
}
|
|
565
|
+
// Order matters at apply time: maxfilesperproc must stay <= maxfiles.
|
|
566
|
+
const settings: Array<[string, number]> = [
|
|
567
|
+
["kern.maxfiles", Math.max(current.maxfiles, TUNE_MAXFILES_FLOOR)],
|
|
568
|
+
[
|
|
569
|
+
"kern.maxfilesperproc",
|
|
570
|
+
Math.max(current.maxfilesperproc, TUNE_MAXFILESPERPROC_FLOOR),
|
|
571
|
+
],
|
|
572
|
+
["kern.maxvnodes", Math.max(current.maxvnodes, TUNE_MAXVNODES_FLOOR)],
|
|
573
|
+
];
|
|
574
|
+
const desiredPlist = tunePlistContent(settings);
|
|
575
|
+
const plistCurrent = existsSync(TUNE_DAEMON_PLIST)
|
|
576
|
+
? readFileSync(TUNE_DAEMON_PLIST, "utf8")
|
|
577
|
+
: null;
|
|
578
|
+
const changes = settings.filter(
|
|
579
|
+
([key, value]) => value !== current[key.slice("kern.".length) as keyof typeof current],
|
|
580
|
+
);
|
|
581
|
+
if (changes.length === 0 && plistCurrent === desiredPlist) {
|
|
582
|
+
const summary = settings.map(([k, v]) => `${k}=${v}`).join(" ");
|
|
583
|
+
console.log(`already tuned (${summary}); boot persistence in place`);
|
|
584
|
+
return;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
if (typeof process.getuid !== "function" || process.getuid() !== 0) {
|
|
588
|
+
console.log(`${bold("uai-host tune")} will (needs sudo):`);
|
|
589
|
+
for (const [key, value] of changes) {
|
|
590
|
+
console.log(
|
|
591
|
+
` raise ${key}: ${current[key.slice("kern.".length) as keyof typeof current]} -> ${value}`,
|
|
592
|
+
);
|
|
593
|
+
}
|
|
594
|
+
console.log(
|
|
595
|
+
` install ${TUNE_DAEMON_PLIST} so the limits survive reboots`,
|
|
596
|
+
);
|
|
597
|
+
console.log(`\nrun: ${cyan("sudo uai-host tune")}`);
|
|
598
|
+
process.exitCode = 1;
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
for (const [key, value] of settings) {
|
|
603
|
+
const res = spawnSync("/usr/sbin/sysctl", [`${key}=${value}`], {
|
|
604
|
+
encoding: "utf8",
|
|
605
|
+
});
|
|
606
|
+
if (res.status !== 0) {
|
|
607
|
+
console.error(
|
|
608
|
+
red(`sysctl ${key}=${value} failed: ${(res.stderr ?? "").trim()}`),
|
|
609
|
+
);
|
|
610
|
+
process.exitCode = 1;
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
writeFileSync(TUNE_DAEMON_PLIST, desiredPlist, { mode: 0o644 });
|
|
615
|
+
spawnSync("/usr/sbin/chown", ["root:wheel", TUNE_DAEMON_PLIST]);
|
|
616
|
+
// Re-bootstrap so launchd owns the daemon under its current definition.
|
|
617
|
+
spawnSync("/bin/launchctl", ["bootout", `system/${TUNE_DAEMON_LABEL}`]);
|
|
618
|
+
const bootstrap = spawnSync(
|
|
619
|
+
"/bin/launchctl",
|
|
620
|
+
["bootstrap", "system", TUNE_DAEMON_PLIST],
|
|
621
|
+
{ encoding: "utf8" },
|
|
622
|
+
);
|
|
623
|
+
if (bootstrap.status !== 0) {
|
|
624
|
+
console.error(
|
|
625
|
+
red(
|
|
626
|
+
`limits applied, but launchctl bootstrap failed (${(bootstrap.stderr ?? "").trim()}) — they will not survive a reboot`,
|
|
627
|
+
),
|
|
628
|
+
);
|
|
629
|
+
process.exitCode = 1;
|
|
630
|
+
return;
|
|
631
|
+
}
|
|
632
|
+
const summary = settings.map(([k, v]) => `${k}=${v}`).join(" ");
|
|
633
|
+
console.log(`tuned: ${summary} (persisted via ${TUNE_DAEMON_PLIST})`);
|
|
634
|
+
}
|
|
635
|
+
|
|
501
636
|
async function cmdRuntime(rest: string[]): Promise<void> {
|
|
502
637
|
if (rest[0] === "begin-install") {
|
|
503
638
|
return cmdBeginRuntimeInstall(rest.slice(1));
|
|
@@ -1882,6 +2017,8 @@ function printHelp(): void {
|
|
|
1882
2017
|
status connection, service info, active tasks (same as the UI)
|
|
1883
2018
|
logs [--follow] tail the service log
|
|
1884
2019
|
runtime recheck probe Docker again and refresh cloud capabilities
|
|
2020
|
+
tune (macOS, sudo) raise kernel vnode/file-table ceilings for
|
|
2021
|
+
apple-container task VMs and persist them across boots
|
|
1885
2022
|
setup --cloud <wss-url> --enroll <token>
|
|
1886
2023
|
claim this machine via an enrollment token from the web
|
|
1887
2024
|
app (mints + writes the host credential); on an
|
package/src/index.ts
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
removeTaskCliSecretStrict,
|
|
18
18
|
storeTaskCliSecret,
|
|
19
19
|
} from "../lib/agent-cli";
|
|
20
|
+
import { stopMachineGatewayTunnel } from "../lib/machine-gateway-tunnel";
|
|
20
21
|
import {
|
|
21
22
|
clearRefresh,
|
|
22
23
|
injectIntoContainer,
|
|
@@ -69,10 +70,12 @@ import {
|
|
|
69
70
|
containerRuntimeTeardownProblem,
|
|
70
71
|
ensureContainerRuntimeForTask,
|
|
71
72
|
initializeContainerRuntime,
|
|
73
|
+
pinnedContainerRuntimeProvider,
|
|
72
74
|
reprobeContainerRuntimeMachineIdentity,
|
|
73
75
|
suspendContainerRuntime,
|
|
74
76
|
waitForContainerRuntimeOperational,
|
|
75
77
|
} from "../lib/container-runtime";
|
|
78
|
+
import { vnodeAdmissionProblem } from "../lib/vnode-pressure";
|
|
76
79
|
import {
|
|
77
80
|
provisionTaskEnvironment,
|
|
78
81
|
reconstructPersistedTaskEnvironment,
|
|
@@ -263,6 +266,21 @@ export const hostCommands: HostCommands = {
|
|
|
263
266
|
if (readyAdmissionFailure) return readyAdmissionFailure;
|
|
264
267
|
const readyRuntimeFailure = runtimeUnavailable();
|
|
265
268
|
if (readyRuntimeFailure) return readyRuntimeFailure;
|
|
269
|
+
// ADR-118 follow-up: each apple-container task is its own VM, and each
|
|
270
|
+
// VM pins host vnodes for its life. Refuse to start ANOTHER one when the
|
|
271
|
+
// table is nearly full — a clear, retryable refusal now beats ENFILE
|
|
272
|
+
// storms (and the suspected panic path) for the whole machine later.
|
|
273
|
+
if (pinnedContainerRuntimeProvider() === "apple-container") {
|
|
274
|
+
const vnodeProblem = await vnodeAdmissionProblem();
|
|
275
|
+
if (vnodeProblem) {
|
|
276
|
+
return {
|
|
277
|
+
ok: false,
|
|
278
|
+
code: HostErrorCode.HostUnavailable,
|
|
279
|
+
message: vnodeProblem,
|
|
280
|
+
retryable: true,
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
}
|
|
266
284
|
const orchestrator = getOrchestrator();
|
|
267
285
|
return orchestrator.runTaskLifecycle(input.task.id, async () => {
|
|
268
286
|
// A direct duplicate teardown may still be draining operations that
|
|
@@ -966,9 +984,13 @@ export const hostCommands: HostCommands = {
|
|
|
966
984
|
!orphanGc &&
|
|
967
985
|
environment.descriptor.locator.provider === "machine"
|
|
968
986
|
) {
|
|
987
|
+
stopMachineGatewayTunnel(input.taskId);
|
|
969
988
|
await environment.stop();
|
|
970
989
|
return taskDownResultForInput(input, { status: "stopped" });
|
|
971
990
|
}
|
|
991
|
+
if (environment.descriptor.locator.provider === "machine") {
|
|
992
|
+
stopMachineGatewayTunnel(input.taskId);
|
|
993
|
+
}
|
|
972
994
|
return taskDownResultForInput(input, await environment.teardown());
|
|
973
995
|
});
|
|
974
996
|
if (result.ok) {
|
package/src/main.ts
CHANGED
|
@@ -110,6 +110,7 @@ import {
|
|
|
110
110
|
publishActivationPhase,
|
|
111
111
|
} from "../lib/container-runtime";
|
|
112
112
|
import { reconstructPersistedTaskEnvironment } from "../lib/task-environment";
|
|
113
|
+
import { logVnodePressure } from "../lib/vnode-pressure";
|
|
113
114
|
import { resolveAppleTunnelRoute } from "./apple-tunnel-route";
|
|
114
115
|
import {
|
|
115
116
|
configureBundledContainerRuntime,
|
|
@@ -516,6 +517,11 @@ void connect();
|
|
|
516
517
|
// Local browser UI (ADR-028) — same single process, alongside the WSS client.
|
|
517
518
|
// Best-effort: a UI bind failure must not take the host service down.
|
|
518
519
|
void startLocalUi();
|
|
520
|
+
// ADR-118 follow-up: vnode-pressure observability (macOS/apple-container
|
|
521
|
+
// hosts). Logs only on band transitions; the sensor caches its sysctl read.
|
|
522
|
+
const vnodePressureTimer = setInterval(() => void logVnodePressure(), 60_000);
|
|
523
|
+
vnodePressureTimer.unref?.();
|
|
524
|
+
void logVnodePressure();
|
|
519
525
|
|
|
520
526
|
async function startLocalUi(): Promise<void> {
|
|
521
527
|
try {
|