agent-relay-orchestrator 0.62.2 → 0.63.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/spawn.ts CHANGED
@@ -1,1477 +1 @@
1
- import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
2
- import { homedir } from "node:os";
3
- import { dirname, isAbsolute, join, relative, resolve } from "node:path";
4
- import { artifactProxyBaseUrl } from "./artifact-proxy";
5
- import type { OrchestratorConfig } from "./config";
6
- import type { ManagedAgentReport, ManagedSessionExitDiagnostics } from "./relay";
7
- import { resolveSpawnWorkspace, workspacesRoot } from "./workspace-probe";
8
- import type { AgentLifecycle, WorkspaceMetadata, WorkspaceMode } from "agent-relay-sdk";
9
- import { errMessage, extractClaudeModelUnavailableMessage } from "agent-relay-sdk";
10
- import { isPidAlive, parseProcStateIsZombie } from "agent-relay-sdk/process-utils";
11
- import { shellEscape } from "agent-relay-sdk/shell-utils";
12
- import { tmuxCommand, tmuxHasSession } from "agent-relay-sdk/tmux-utils";
13
- import { sanitizeFsName } from "agent-relay-sdk/fs-name";
14
-
15
- export interface SpawnOptions {
16
- provider: "claude" | "codex";
17
- cwd: string;
18
- rig?: string;
19
- model?: string;
20
- effort?: string;
21
- profile?: string; workspaceMode?: WorkspaceMode; lifecycle?: AgentLifecycle;
22
- workspace?: WorkspaceMetadata;
23
- /** Untracked files/dirs to symlink from main into an isolated worktree (relay's global workspace config). */
24
- workspaceSymlinks?: string[];
25
- agentProfile?: Record<string, unknown>;
26
- label?: string;
27
- agentId?: string;
28
- approvalMode: string;
29
- prompt?: string;
30
- systemPromptAppend?: string;
31
- env?: Record<string, string>;
32
- tags?: string[];
33
- capabilities?: string[];
34
- providerArgs?: string[];
35
- policyName?: string;
36
- spawnRequestId?: string;
37
- automationId?: string;
38
- automationRunId?: string;
39
- /** How the spawn was requested (`mcp` = an agent via the MCP surface, else dashboard/CLI). Drives
40
- * the origin tag so an MCP-spawned worker isn't mislabeled `dashboard-spawned` (#330). */
41
- requestedVia?: string;
42
- }
43
-
44
- interface SessionInfo {
45
- name: string;
46
- sessionName: string;
47
- pid: number;
48
- alive: boolean;
49
- supervisor: SessionSupervisor["type"];
50
- systemdUnit?: string;
51
- terminalSession?: string;
52
- terminalAvailable: boolean;
53
- logFile: string;
54
- }
55
-
56
- interface TerminalGuestSession {
57
- session: string;
58
- mode: "guest";
59
- provider: string;
60
- running: boolean;
61
- interactive: boolean;
62
- expiresAt: number;
63
- }
64
-
65
- export interface TerminalSnapshot {
66
- session: string;
67
- content: string;
68
- running: boolean;
69
- // `running` only means the tmux pane still exists — tmux keeps a pane after the
70
- // process inside it exits. `agentAlive` is the real liveness of the agent process
71
- // (pid / systemd unit) so the dashboard can flag an orphaned, stale terminal.
72
- agentAlive: boolean;
73
- cols?: number;
74
- rows?: number;
75
- cursorX?: number;
76
- cursorY?: number;
77
- capturedAt: number;
78
- }
79
-
80
- type TerminalInputToken =
81
- | { type: "literal"; value: string }
82
- | { type: "key"; value: string };
83
-
84
- interface TerminalInputResult {
85
- session: string;
86
- running: boolean;
87
- sent: number;
88
- capturedAt: number;
89
- }
90
-
91
- export interface SessionRecord {
92
- name: string;
93
- pid: number;
94
- supervisor?: SessionSupervisor;
95
- provider: string;
96
- model?: string;
97
- effort?: string;
98
- profile?: string; workspaceMode?: WorkspaceMode; lifecycle?: AgentLifecycle;
99
- workspace?: WorkspaceMetadata;
100
- label?: string;
101
- cwd: string;
102
- logFile: string;
103
- runnerInfoFile?: string;
104
- agentId: string;
105
- approvalMode: string;
106
- policyName?: string;
107
- spawnRequestId?: string;
108
- automationId?: string;
109
- automationRunId?: string;
110
- startedAt: number;
111
- }
112
-
113
- interface SessionSupervisor {
114
- type: "process" | "systemd";
115
- unit?: string;
116
- launchScript?: string;
117
- }
118
-
119
- interface SpawnedRunner {
120
- pid: number;
121
- supervisor: SessionSupervisor;
122
- }
123
-
124
- interface RunnerInfo {
125
- agentId: string;
126
- runnerId: string;
127
- provider: string;
128
- controlUrl: string;
129
- tmuxSession?: string;
130
- tmuxSocket?: string;
131
- pid?: number;
132
- startedAt?: number;
133
- }
134
-
135
- interface TerminalAttachSpec {
136
- mode: "guest";
137
- provider: string;
138
- cwd: string;
139
- command: string[];
140
- env?: Record<string, string>;
141
- title?: string;
142
- ttlMs?: number;
143
- }
144
-
145
- const LOG_DIR = join(homedir(), ".agent-relay", "logs");
146
- const STATE_FILE = join(homedir(), ".agent-relay", "orchestrator-sessions.json");
147
- const SESSION_DIR = join(homedir(), ".agent-relay", "sessions");
148
- const RUNNER_INFO_DIR = join(homedir(), ".agent-relay", "runners");
149
- const GUEST_TTL_MS = 60 * 60 * 1000;
150
- const GUEST_STATE_FILE = join(homedir(), ".agent-relay", "orchestrator-guests.json");
151
- const terminalGuests = new Map<string, { expiresAt: number }>();
152
- let guestStateHydrated = false;
153
-
154
- export function isWithinBaseDir(path: string, baseDir: string): boolean {
155
- const base = resolve(baseDir);
156
- const target = resolve(path);
157
- const rel = relative(base, target);
158
- return rel === "" || (!!rel && !rel.startsWith("..") && !isAbsolute(rel));
159
- }
160
-
161
- export function sessionName(config: OrchestratorConfig, provider: string, label: string, uniqueId?: string): string {
162
- const clean = sanitizeFsName(label, { replacement: "-", lowercase: true });
163
- const suffix = uniqueId ? `-${sanitizeFsName(uniqueId, { replacement: "-", lowercase: true }).slice(-8)}` : "";
164
- return `${config.tmuxPrefix}-${provider}-${clean}${suffix}`;
165
- }
166
-
167
- export function defaultSpawnLabel(now = Date.now()): string {
168
- return `session-${now}`;
169
- }
170
-
171
- export function buildRunnerCommand(opts: SpawnOptions, config: OrchestratorConfig): string[] {
172
- const repoLauncher = resolve(import.meta.dir, "../../runner/src/index.ts");
173
- const installedLauncher = resolve(import.meta.dir, "../../agent-relay-runner/src/index.ts");
174
- const bun = process.env.AGENT_RELAY_BUN_BIN
175
- || (process.platform === "darwin" && existsSync("/opt/homebrew/bin/bun") ? "/opt/homebrew/bin/bun" : "bun");
176
- const launcher = existsSync(repoLauncher)
177
- ? [bun, "run", repoLauncher, opts.provider]
178
- : existsSync(installedLauncher)
179
- ? [bun, "run", installedLauncher, opts.provider]
180
- : [`${opts.provider}-relay`, opts.provider];
181
- const args = [
182
- ...launcher,
183
- "--headless",
184
- "--cwd", opts.cwd,
185
- "--relay-url", config.relayUrl,
186
- "--approval", opts.approvalMode || "guarded",
187
- ];
188
- if (opts.rig) args.push("--rig", opts.rig);
189
- if (opts.model) args.push("--model", opts.model);
190
- if (opts.effort) args.push("--effort", opts.effort);
191
- if (opts.profile) args.push("--profile", opts.profile);
192
- if (opts.label) args.push("--label", opts.label);
193
- if (opts.agentId) args.push("--agent-id", opts.agentId);
194
- if (opts.prompt) args.push("--prompt", opts.prompt);
195
- if (opts.systemPromptAppend) args.push("--system-prompt-append", opts.systemPromptAppend);
196
- if (opts.tags?.length) args.push("--tags", opts.tags.join(","));
197
- if (opts.capabilities?.length) args.push("--caps", opts.capabilities.join(","));
198
- if (opts.providerArgs?.length) args.push("--", ...opts.providerArgs);
199
- return args;
200
- }
201
-
202
- export function buildEnv(opts: SpawnOptions & { label: string; agentId: string }, config: OrchestratorConfig, logFile?: string, tmuxSession?: string): Record<string, string> {
203
- const currentPath = process.env.PATH || "";
204
- const extraPaths = [
205
- join(homedir(), ".local", "bin"),
206
- join(homedir(), ".bun", "bin"),
207
- join(homedir(), ".npm-global", "bin"),
208
- ];
209
- const fullPath = [...extraPaths, ...currentPath.split(":").filter(Boolean)]
210
- .filter((v, i, a) => a.indexOf(v) === i)
211
- .join(":");
212
-
213
- return {
214
- ...process.env as Record<string, string>,
215
- ...(config.token ? { AGENT_RELAY_TOKEN: config.token } : {}),
216
- ...config.env,
217
- ...agentProfileEnv(opts.agentProfile),
218
- ...(opts.env || {}),
219
- PATH: fullPath,
220
- AGENT_RELAY_URL: config.relayUrl,
221
- AGENT_RELAY_ORCHESTRATOR_URL: `http://127.0.0.1:${config.apiPort}`,
222
- AGENT_RELAY_ARTIFACT_URL: artifactProxyBaseUrl(config),
223
- AGENT_RELAY_APPROVAL: opts.approvalMode || "guarded",
224
- ...(opts.profile ? { AGENT_RELAY_AGENT_PROFILE: opts.profile } : {}),
225
- ...(opts.agentProfile ? { AGENT_RELAY_AGENT_PROFILE_JSON: JSON.stringify(opts.agentProfile) } : {}),
226
- // #330 — tag by TRUE origin. An MCP spawn (an agent spawning a helper) is `agent-spawned`, not
227
- // `dashboard-spawned`; the old blanket `dashboard-spawned` mislabeled every headless spawn as
228
- // dashboard-originated. Dashboard/CLI/automation spawns (no `requestedVia: "mcp"`) keep the
229
- // `dashboard-spawned` tag the smoke test and UI filter on.
230
- AGENT_RELAY_TAGS: [...new Set(["headless", opts.requestedVia === "mcp" ? "agent-spawned" : "dashboard-spawned", config.hostname, ...(opts.tags ?? [])])].join(","),
231
- AGENT_RELAY_CAPS: [...new Set(opts.capabilities ?? [])].join(","),
232
- AGENT_RELAY_CAPABILITIES: [...new Set(opts.capabilities ?? [])].join(","),
233
- AGENT_RELAY_HEADLESS: "1",
234
- ...(logFile ? { AGENT_RELAY_LOG_FILE: logFile } : {}),
235
- ...(tmuxSession ? { AGENT_RELAY_TMUX_SESSION: tmuxSession } : {}),
236
- ...(opts.label ? { AGENT_RELAY_LABEL: opts.label } : {}),
237
- ...(opts.policyName ? { AGENT_RELAY_POLICY: opts.policyName } : {}),
238
- ...(opts.spawnRequestId ? { AGENT_RELAY_SPAWN_REQUEST_ID: opts.spawnRequestId } : {}),
239
- AGENT_RELAY_LIFECYCLE: opts.lifecycle ?? "persistent", AGENT_RELAY_WORKSPACE_MODE: opts.workspaceMode ?? "inherit",
240
- ...(opts.workspace ? { AGENT_RELAY_WORKSPACE_JSON: JSON.stringify(opts.workspace) } : {}),
241
- ...(opts.automationId ? { AGENT_RELAY_AUTOMATION_ID: opts.automationId } : {}),
242
- ...(opts.automationRunId ? { AGENT_RELAY_AUTOMATION_RUN_ID: opts.automationRunId } : {}),
243
- };
244
- }
245
-
246
- function agentProfileEnv(profile: Record<string, unknown> | undefined): Record<string, string> {
247
- const raw = profile?.env;
248
- if (!raw || typeof raw !== "object" || Array.isArray(raw)) return {};
249
- return Object.fromEntries(Object.entries(raw).filter((entry): entry is [string, string] => typeof entry[1] === "string"));
250
- }
251
-
252
- function logFilePath(name: string): string {
253
- return join(LOG_DIR, `${name}.log`);
254
- }
255
-
256
- function runnerInfoPath(name: string): string {
257
- const safe = sanitizeFsName(name, { replacement: "-", trimEdge: true, fallback: "runner" });
258
- return join(RUNNER_INFO_DIR, `${safe}.json`);
259
- }
260
-
261
- function ensureLogDir(): void {
262
- mkdirSync(LOG_DIR, { recursive: true });
263
- }
264
-
265
- function ensureSessionDir(): void {
266
- mkdirSync(SESSION_DIR, { recursive: true, mode: 0o700 });
267
- }
268
-
269
- function ensureRunnerInfoDir(): void {
270
- mkdirSync(RUNNER_INFO_DIR, { recursive: true, mode: 0o700 });
271
- }
272
-
273
- function saveState(records: SessionRecord[]): void {
274
- mkdirSync(join(homedir(), ".agent-relay"), { recursive: true });
275
- // Atomic write: a crash mid-write would otherwise leave truncated JSON and
276
- // loadState would silently return [], losing every tracked session.
277
- const tmp = `${STATE_FILE}.tmp`;
278
- writeFileSync(tmp, JSON.stringify(records, null, 2) + "\n");
279
- renameSync(tmp, STATE_FILE);
280
- }
281
-
282
- function loadState(): SessionRecord[] {
283
- try {
284
- return JSON.parse(readFileSync(STATE_FILE, "utf8"));
285
- } catch {
286
- return [];
287
- }
288
- }
289
-
290
- function addSessionRecord(record: SessionRecord): void {
291
- const records = loadState().filter((r) => r.name !== record.name);
292
- records.push(record);
293
- saveState(records);
294
- }
295
-
296
- function removeSessionRecord(name: string): void {
297
- saveState(loadState().filter((r) => r.name !== name));
298
- }
299
-
300
- // Zombie-aware liveness primitives are shared with the runner via the SDK.
301
- // Re-exported so existing `./spawn` consumers (and tests) keep resolving them.
302
- export { isPidAlive, parseProcStateIsZombie };
303
-
304
- function sessionSupervisor(record?: Pick<SessionRecord, "supervisor">): SessionSupervisor {
305
- return record?.supervisor ?? { type: "process" };
306
- }
307
-
308
- function isSessionRecordAlive(record: SessionRecord): boolean {
309
- const supervisor = sessionSupervisor(record);
310
- if (supervisor.type === "systemd" && supervisor.unit) {
311
- const pid = systemdMainPid(supervisor.unit);
312
- return pid > 0 && isPidAlive(pid);
313
- }
314
- return isPidAlive(record.pid);
315
- }
316
-
317
- function currentSessionPid(record: SessionRecord): number {
318
- const supervisor = sessionSupervisor(record);
319
- if (supervisor.type === "systemd" && supervisor.unit) {
320
- const pid = systemdMainPid(supervisor.unit);
321
- if (pid > 0) return pid;
322
- }
323
- return record.pid;
324
- }
325
-
326
- function sessionReportFields(record: Pick<SessionRecord, "name" | "supervisor" | "runnerInfoFile" | "agentId" | "provider">): Pick<ManagedAgentReport, "sessionName" | "tmuxSession" | "supervisor" | "systemdUnit" | "terminalSession" | "terminalAvailable"> {
327
- const supervisor = sessionSupervisor(record);
328
- const terminalAvailable = tmuxHasSession(record.name, readRunnerInfo(record)?.tmuxSocket);
329
- return {
330
- sessionName: record.name,
331
- tmuxSession: record.name,
332
- supervisor: supervisor.type,
333
- ...(supervisor.type === "systemd" && supervisor.unit ? { systemdUnit: supervisor.unit } : {}),
334
- terminalSession: record.name,
335
- terminalAvailable,
336
- };
337
- }
338
-
339
- export async function spawnAgent(
340
- opts: SpawnOptions,
341
- config: OrchestratorConfig,
342
- ): Promise<ManagedAgentReport> {
343
- const label = opts.label || defaultSpawnLabel();
344
- const agentId = opts.agentId || managedAgentId(config, opts.provider, label);
345
- const name = sessionName(config, opts.provider, label, opts.spawnRequestId ?? agentId);
346
-
347
- if (!existsSync(opts.cwd)) {
348
- throw new Error(`cwd does not exist: ${opts.cwd}`);
349
- }
350
- if (!isWithinBaseDir(opts.cwd, config.baseDir)) {
351
- throw new Error(`cwd must be within base directory: ${config.baseDir}`);
352
- }
353
-
354
- const resolvedWorkspace = await resolveSpawnWorkspace({
355
- ...opts,
356
- label,
357
- workspaceSymlinks: opts.workspaceSymlinks,
358
- workspaceRoot: workspacesRoot(config.baseDir),
359
- });
360
- const spawnOpts = { ...opts, label, agentId, cwd: resolvedWorkspace.cwd, workspace: resolvedWorkspace.workspace };
361
-
362
- const command = buildRunnerCommand(spawnOpts, config);
363
-
364
- ensureLogDir();
365
- ensureRunnerInfoDir();
366
- const logFile = logFilePath(name);
367
- const runnerInfoFile = runnerInfoPath(name);
368
- rmSync(runnerInfoFile, { force: true });
369
- const env = buildEnv({ ...spawnOpts, env: { ...(spawnOpts.env ?? {}), AGENT_RELAY_RUNNER_INFO_FILE: runnerInfoFile } }, config, logFile, name);
370
- const logFd = openSync(logFile, "w");
371
-
372
- console.error(`[orchestrator] Spawning ${opts.provider} agent: ${name}`);
373
- console.error(`[orchestrator] cwd: ${opts.cwd}`);
374
- console.error(`[orchestrator] command: ${command.join(" ")}`);
375
- console.error(`[orchestrator] log: ${logFile}`);
376
-
377
- closeSync(logFd);
378
-
379
- const runner = spawnRunner(name, command, spawnOpts.cwd, env, logFile);
380
-
381
- addSessionRecord({
382
- name,
383
- pid: runner.pid,
384
- supervisor: runner.supervisor,
385
- provider: spawnOpts.provider,
386
- model: spawnOpts.model,
387
- effort: spawnOpts.effort,
388
- profile: spawnOpts.profile,
389
- workspaceMode: spawnOpts.workspaceMode, lifecycle: spawnOpts.lifecycle ?? "persistent",
390
- workspace: spawnOpts.workspace,
391
- label,
392
- cwd: spawnOpts.cwd,
393
- logFile,
394
- runnerInfoFile,
395
- agentId,
396
- approvalMode: spawnOpts.approvalMode,
397
- policyName: spawnOpts.policyName,
398
- spawnRequestId: spawnOpts.spawnRequestId,
399
- automationId: spawnOpts.automationId,
400
- automationRunId: spawnOpts.automationRunId,
401
- startedAt: Date.now(),
402
- });
403
-
404
- return {
405
- agentId,
406
- provider: spawnOpts.provider,
407
- model: spawnOpts.model,
408
- effort: spawnOpts.effort,
409
- profile: spawnOpts.profile,
410
- workspaceMode: spawnOpts.workspaceMode, lifecycle: spawnOpts.lifecycle ?? "persistent",
411
- workspace: spawnOpts.workspace,
412
- ...sessionReportFields({ name, supervisor: runner.supervisor, runnerInfoFile, agentId, provider: spawnOpts.provider }),
413
- cwd: spawnOpts.cwd,
414
- label,
415
- approvalMode: spawnOpts.approvalMode || "guarded",
416
- policyName: spawnOpts.policyName,
417
- spawnRequestId: spawnOpts.spawnRequestId,
418
- automationRunId: spawnOpts.automationRunId,
419
- pid: runner.pid,
420
- startedAt: Date.now(),
421
- };
422
- }
423
-
424
- export async function createTerminalGuest(
425
- input: { agentId?: string; policyName?: string; spawnRequestId?: string; tmuxSession?: string },
426
- config: OrchestratorConfig,
427
- ): Promise<TerminalGuestSession> {
428
- cleanupExpiredTerminalGuests();
429
- const record = findSessionRecord(input);
430
- if (!record || !isSessionRecordAlive(record)) throw new Error("managed runner session not found");
431
- const runner = readRunnerInfo(record);
432
- if (!runner?.controlUrl) throw new Error("runner control URL is unavailable; restart the agent to enable terminal attach");
433
- const spec = await fetchTerminalAttachSpec(runner.controlUrl);
434
- validateAttachSpec(spec, config);
435
- const session = guestSessionName(config, spec.provider, record.agentId);
436
- killTmuxSession(session);
437
- const expiresAt = Date.now() + Math.min(Math.max(spec.ttlMs ?? GUEST_TTL_MS, 60_000), 4 * GUEST_TTL_MS);
438
- const shellCmd = spec.command.map(shellEscape).join(" ");
439
- const tmuxArgs = ["new-session", "-d", "-s", session, "-x", "200", "-y", "50"];
440
- for (const [key, value] of Object.entries(spec.env ?? {}).sort(([a], [b]) => a.localeCompare(b))) {
441
- if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) tmuxArgs.push("-e", `${key}=${value}`);
442
- }
443
- tmuxArgs.push("-c", spec.cwd, shellCmd);
444
- const result = Bun.spawnSync(["tmux", ...tmuxArgs], {
445
- stdin: "ignore",
446
- stdout: "pipe",
447
- stderr: "pipe",
448
- });
449
- if (result.exitCode !== 0) {
450
- const stderr = result.stderr.toString().trim();
451
- throw new Error(stderr || `tmux guest creation failed with exit code ${result.exitCode}`);
452
- }
453
- terminalGuests.set(session, { expiresAt });
454
- saveGuestState();
455
- return { session, mode: "guest", provider: spec.provider, running: true, interactive: true, expiresAt };
456
- }
457
-
458
- export function stopTerminalGuest(session: string, config: OrchestratorConfig): { session: string; stopped: boolean } {
459
- if (!isGuestSessionName(session, config)) throw new Error("terminal session is not a guest session");
460
- const running = tmuxHasSession(session);
461
- if (running) killTmuxSession(session);
462
- terminalGuests.delete(session);
463
- saveGuestState();
464
- return { session, stopped: running };
465
- }
466
-
467
- export function selectSessionRecord(records: SessionRecord[], input: { agentId?: string; policyName?: string; spawnRequestId?: string; tmuxSession?: string }): SessionRecord | undefined {
468
- if (input.tmuxSession) return records.find((record) => record.name === input.tmuxSession);
469
-
470
- if (input.spawnRequestId) {
471
- return records.find((record) =>
472
- record.spawnRequestId === input.spawnRequestId &&
473
- (!input.policyName || record.policyName === input.policyName)
474
- );
475
- }
476
-
477
- if (input.agentId) {
478
- return records.find((record) =>
479
- record.agentId === input.agentId &&
480
- (!input.policyName || record.policyName === input.policyName)
481
- );
482
- }
483
-
484
- // Policy-only lookup: a respawn leaves multiple records for one policy. The
485
- // live session is the most recent one, so pick the highest startedAt rather
486
- // than the first match (which is the stale, already-replaced session).
487
- if (input.policyName) {
488
- const policyName = input.policyName;
489
- return records
490
- .filter((record) => record.policyName === policyName)
491
- .reduce<SessionRecord | undefined>((latest, record) => (
492
- !latest || record.startedAt > latest.startedAt ? record : latest
493
- ), undefined);
494
- }
495
- return undefined;
496
- }
497
-
498
- function findSessionRecord(input: { agentId?: string; policyName?: string; spawnRequestId?: string; tmuxSession?: string }): SessionRecord | undefined {
499
- return selectSessionRecord(loadState(), input);
500
- }
501
-
502
- function readRunnerInfo(record: Pick<SessionRecord, "runnerInfoFile" | "agentId" | "provider">): RunnerInfo | null {
503
- if (!record.runnerInfoFile) return null;
504
- try {
505
- const parsed = JSON.parse(readFileSync(record.runnerInfoFile, "utf8"));
506
- if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
507
- const info = parsed as Record<string, unknown>;
508
- if (typeof info.controlUrl !== "string" || !info.controlUrl.startsWith("http://127.0.0.1:")) return null;
509
- return {
510
- agentId: typeof info.agentId === "string" ? info.agentId : record.agentId,
511
- runnerId: typeof info.runnerId === "string" ? info.runnerId : "",
512
- provider: typeof info.provider === "string" ? info.provider : record.provider,
513
- controlUrl: info.controlUrl,
514
- tmuxSession: typeof info.tmuxSession === "string" ? info.tmuxSession : undefined,
515
- tmuxSocket: typeof info.tmuxSocket === "string" ? info.tmuxSocket : undefined,
516
- pid: typeof info.pid === "number" ? info.pid : undefined,
517
- startedAt: typeof info.startedAt === "number" ? info.startedAt : undefined,
518
- };
519
- } catch {
520
- return null;
521
- }
522
- }
523
-
524
- async function fetchTerminalAttachSpec(controlUrl: string): Promise<TerminalAttachSpec> {
525
- const res = await fetch(`${controlUrl}/terminal/attach-spec`, { signal: AbortSignal.timeout(5_000) });
526
- const body = await res.json().catch(() => null) as unknown;
527
- if (!res.ok) {
528
- const message = body && typeof body === "object" && !Array.isArray(body) && typeof (body as { error?: unknown }).error === "string"
529
- ? (body as { error: string }).error
530
- : `runner attach-spec failed with ${res.status}`;
531
- throw new Error(message);
532
- }
533
- if (!body || typeof body !== "object" || Array.isArray(body)) throw new Error("runner attach-spec response must be an object");
534
- return body as TerminalAttachSpec;
535
- }
536
-
537
- function validateAttachSpec(spec: TerminalAttachSpec, config: OrchestratorConfig): void {
538
- if (spec.mode !== "guest") throw new Error("runner attach-spec mode must be guest");
539
- if (typeof spec.provider !== "string" || !spec.provider.trim()) throw new Error("runner attach-spec provider required");
540
- if (typeof spec.cwd !== "string" || !isWithinBaseDir(spec.cwd, config.baseDir)) throw new Error("runner attach-spec cwd must be within base directory");
541
- if (!Array.isArray(spec.command) || spec.command.length === 0 || spec.command.some((item) => typeof item !== "string" || !item)) {
542
- throw new Error("runner attach-spec command must be a non-empty string array");
543
- }
544
- if (spec.env !== undefined && (!spec.env || typeof spec.env !== "object" || Array.isArray(spec.env) || Object.values(spec.env).some((value) => typeof value !== "string"))) {
545
- throw new Error("runner attach-spec env must be a string record");
546
- }
547
- }
548
-
549
- function guestSessionName(config: OrchestratorConfig, provider: string, agentId: string): string {
550
- const cleanProvider = sanitizeFsName(provider, { replacement: "-", lowercase: true, fallback: "provider" });
551
- const cleanAgent = sanitizeFsName(agentId, { replacement: "-", lowercase: true, maxLen: 48, fallback: "agent" });
552
- return `${config.tmuxPrefix}-guest-${cleanProvider}-${cleanAgent}-${crypto.randomUUID().slice(0, 8)}`;
553
- }
554
-
555
- function isGuestSessionName(session: string, config: OrchestratorConfig): boolean {
556
- return session.startsWith(`${config.tmuxPrefix}-guest-`);
557
- }
558
-
559
- interface GuestRecord {
560
- session: string;
561
- expiresAt: number;
562
- }
563
-
564
- interface LiveGuestSession {
565
- session: string;
566
- createdAtMs: number;
567
- }
568
-
569
- /** Flatten the in-memory guest registry to a persistable, deterministic list. */
570
- export function serializeGuests(guests: Map<string, { expiresAt: number }>): GuestRecord[] {
571
- return [...guests.entries()]
572
- .map(([session, { expiresAt }]) => ({ session, expiresAt }))
573
- .sort((a, b) => a.session.localeCompare(b.session));
574
- }
575
-
576
- /** Tolerant inverse of serializeGuests — drops malformed entries instead of throwing. */
577
- export function deserializeGuests(raw: unknown): Map<string, { expiresAt: number }> {
578
- const map = new Map<string, { expiresAt: number }>();
579
- if (!Array.isArray(raw)) return map;
580
- for (const entry of raw) {
581
- if (!entry || typeof entry !== "object") continue;
582
- const { session, expiresAt } = entry as Record<string, unknown>;
583
- if (typeof session === "string" && session && typeof expiresAt === "number" && Number.isFinite(expiresAt)) {
584
- map.set(session, { expiresAt });
585
- }
586
- }
587
- return map;
588
- }
589
-
590
- function saveGuestState(): void {
591
- try {
592
- mkdirSync(join(homedir(), ".agent-relay"), { recursive: true });
593
- const tmp = `${GUEST_STATE_FILE}.tmp`;
594
- writeFileSync(tmp, JSON.stringify(serializeGuests(terminalGuests), null, 2) + "\n");
595
- renameSync(tmp, GUEST_STATE_FILE);
596
- } catch {
597
- // Persistence is best-effort: a write failure must never break guest creation.
598
- // The periodic reaper's tmux age-based fallback still bounds orphan lifetime.
599
- }
600
- }
601
-
602
- /**
603
- * Rehydrate the in-memory guest registry from disk so guest TTLs survive an
604
- * orchestrator restart. Call once at boot before the first reap.
605
- */
606
- export function hydrateTerminalGuests(): void {
607
- if (guestStateHydrated) return;
608
- guestStateHydrated = true;
609
- try {
610
- const persisted = deserializeGuests(JSON.parse(readFileSync(GUEST_STATE_FILE, "utf8")));
611
- for (const [session, value] of persisted) {
612
- if (!terminalGuests.has(session)) terminalGuests.set(session, value);
613
- }
614
- } catch {
615
- // No persisted state (first boot or unreadable) — the age-based fallback in
616
- // reapTerminalGuests still cleans any orphaned guest tmux sessions.
617
- }
618
- }
619
-
620
- /** Live `<prefix>-guest-*` tmux sessions with their creation time (ms). */
621
- function listGuestTmuxSessions(config: OrchestratorConfig): LiveGuestSession[] {
622
- const result = Bun.spawnSync(["tmux", "list-sessions", "-F", "#{session_name}\t#{session_created}"], {
623
- stdin: "ignore",
624
- stdout: "pipe",
625
- stderr: "ignore",
626
- });
627
- if (result.exitCode !== 0) return []; // no tmux server / no sessions
628
- const sessions: LiveGuestSession[] = [];
629
- for (const line of result.stdout.toString().split("\n")) {
630
- const tab = line.indexOf("\t");
631
- if (tab < 0) continue;
632
- const session = line.slice(0, tab);
633
- if (!isGuestSessionName(session, config)) continue;
634
- const createdSec = Number(line.slice(tab + 1).trim());
635
- sessions.push({ session, createdAtMs: Number.isFinite(createdSec) ? createdSec * 1000 : 0 });
636
- }
637
- return sessions;
638
- }
639
-
640
- /**
641
- * Decide which live guest sessions to reap. Pure so the TTL policy is testable
642
- * without tmux or fs:
643
- * - tracked + past its recorded expiry → reap
644
- * - untracked (metadata lost across a restart) + older than the fallback TTL → reap
645
- */
646
- export function selectExpiredGuests(
647
- tracked: Map<string, { expiresAt: number }>,
648
- liveGuests: LiveGuestSession[],
649
- now: number,
650
- fallbackTtlMs = GUEST_TTL_MS,
651
- ): string[] {
652
- const toReap = new Set<string>();
653
- for (const { session, createdAtMs } of liveGuests) {
654
- const record = tracked.get(session);
655
- if (record) {
656
- if (record.expiresAt <= now) toReap.add(session);
657
- } else if (now - createdAtMs >= fallbackTtlMs) {
658
- toReap.add(session);
659
- }
660
- }
661
- return [...toReap];
662
- }
663
-
664
- /**
665
- * Kill guest tmux sessions whose TTL has elapsed, independent of any new guest
666
- * creation, and prune tracked entries whose tmux session is already gone. Runs
667
- * at boot and on a periodic timer (see orchestrator index).
668
- */
669
- export function reapTerminalGuests(config: OrchestratorConfig, now = Date.now()): string[] {
670
- const live = listGuestTmuxSessions(config);
671
- const liveNames = new Set(live.map((g) => g.session));
672
- const reaped = selectExpiredGuests(terminalGuests, live, now);
673
- for (const session of reaped) {
674
- killTmuxSession(session);
675
- terminalGuests.delete(session);
676
- }
677
- // Drop tracked guests with no live tmux session (manually killed, or reaped
678
- // above) so the registry can't grow without bound.
679
- let pruned = false;
680
- for (const session of [...terminalGuests.keys()]) {
681
- if (!liveNames.has(session)) {
682
- terminalGuests.delete(session);
683
- pruned = true;
684
- }
685
- }
686
- if (reaped.length || pruned) saveGuestState();
687
- return reaped;
688
- }
689
-
690
- function cleanupExpiredTerminalGuests(): void {
691
- const now = Date.now();
692
- let changed = false;
693
- for (const [session, guest] of terminalGuests.entries()) {
694
- if (guest.expiresAt > now) continue;
695
- killTmuxSession(session);
696
- terminalGuests.delete(session);
697
- changed = true;
698
- }
699
- if (changed) saveGuestState();
700
- }
701
-
702
- function killTmuxSession(session: string): void {
703
- Bun.spawnSync(["tmux", "kill-session", "-t", session], {
704
- stdin: "ignore",
705
- stdout: "ignore",
706
- stderr: "ignore",
707
- });
708
- }
709
-
710
- function spawnRunner(name: string, command: string[], cwd: string, env: Record<string, string>, logFile: string): SpawnedRunner {
711
- if (shouldUseSystemdSupervisor()) {
712
- try {
713
- return spawnSystemdRunner(name, command, cwd, env, logFile);
714
- } catch (error) {
715
- console.error(`[orchestrator] systemd runner supervisor unavailable for ${name}: ${errMessage(error)}`);
716
- console.error("[orchestrator] Falling back to process child; this agent will not survive orchestrator service restart.");
717
- }
718
- }
719
-
720
- const launchScript = launchScriptPath(name);
721
- ensureSessionDir();
722
- writeFileSync(launchScript, buildLaunchScript(command, cwd, env), { mode: 0o700 });
723
- chmodSync(launchScript, 0o700);
724
-
725
- const logFd = openSync(logFile, "a");
726
- try {
727
- const proc = Bun.spawn([launchScript], {
728
- cwd,
729
- env,
730
- stdin: "ignore",
731
- stdout: logFd,
732
- stderr: logFd,
733
- });
734
- return { pid: proc.pid, supervisor: { type: "process", launchScript } };
735
- } finally {
736
- closeSync(logFd);
737
- }
738
- }
739
-
740
- function shouldUseSystemdSupervisor(): boolean {
741
- if (process.platform !== "linux") return false;
742
- if (process.env.AGENT_RELAY_DISABLE_SYSTEMD_SUPERVISOR === "1") return false;
743
- if (process.env.AGENT_RELAY_FORCE_SYSTEMD_SUPERVISOR === "1") return true;
744
- const result = Bun.spawnSync(["systemctl", "--user", "show-environment"], {
745
- stdin: "ignore",
746
- stdout: "ignore",
747
- stderr: "ignore",
748
- });
749
- return result.exitCode === 0;
750
- }
751
-
752
- function spawnSystemdRunner(name: string, command: string[], cwd: string, env: Record<string, string>, logFile: string): SpawnedRunner {
753
- const unit = systemdUnitName(name);
754
- const launchScript = launchScriptPath(name);
755
- ensureSessionDir();
756
- writeFileSync(launchScript, buildLaunchScript(command, cwd, env), { mode: 0o700 });
757
- chmodSync(launchScript, 0o700);
758
-
759
- Bun.spawnSync(["systemctl", "--user", "stop", `${unit}.service`], {
760
- stdin: "ignore",
761
- stdout: "ignore",
762
- stderr: "ignore",
763
- });
764
-
765
- const result = Bun.spawnSync([
766
- "systemd-run",
767
- "--user",
768
- `--unit=${unit}`,
769
- "--collect",
770
- "--property=KillMode=control-group",
771
- `--property=StandardOutput=append:${logFile}`,
772
- `--property=StandardError=append:${logFile}`,
773
- launchScript,
774
- ], {
775
- stdin: "ignore",
776
- stdout: "pipe",
777
- stderr: "pipe",
778
- });
779
- if (result.exitCode !== 0) {
780
- const stderr = result.stderr.toString().trim();
781
- throw new Error(stderr || `systemd-run failed with exit code ${result.exitCode}`);
782
- }
783
-
784
- const pid = waitForSystemdMainPid(unit, 2_000);
785
- if (!pid) throw new Error(`systemd unit ${unit}.service started without a MainPID`);
786
- return { pid, supervisor: { type: "systemd", unit, launchScript } };
787
- }
788
-
789
- export function systemdUnitName(session: string): string {
790
- const safe = sanitizeFsName(session, { replacement: "-", trimEdge: true, fallback: "agent" });
791
- return `agent-relay-managed-${safe}`.slice(0, 180);
792
- }
793
-
794
- function launchScriptPath(session: string): string {
795
- const safe = sanitizeFsName(session, { replacement: "-", trimEdge: true, fallback: "agent" });
796
- return join(SESSION_DIR, `${safe}.sh`);
797
- }
798
-
799
- export function buildLaunchScript(command: string[], cwd: string, env: Record<string, string>): string {
800
- const exports = Object.entries(env)
801
- .filter(([key, value]) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(key) && value !== undefined)
802
- .sort(([a], [b]) => a.localeCompare(b))
803
- .map(([key, value]) => `export ${key}=${shellEscape(String(value))}`);
804
- return [
805
- "#!/usr/bin/env bash",
806
- "set -euo pipefail",
807
- ...exports,
808
- `cd ${shellEscape(cwd)}`,
809
- `exec ${command.map(shellEscape).join(" ")}`,
810
- "",
811
- ].join("\n");
812
- }
813
-
814
- function waitForSystemdMainPid(unit: string, timeoutMs: number): number {
815
- const deadline = Date.now() + timeoutMs;
816
- while (Date.now() < deadline) {
817
- const pid = systemdMainPid(unit);
818
- if (pid > 0 && isPidAlive(pid)) return pid;
819
- Bun.sleepSync(50);
820
- }
821
- return 0;
822
- }
823
-
824
- function systemdMainPid(unit: string): number {
825
- const result = Bun.spawnSync(["systemctl", "--user", "show", `${unit}.service`, "-p", "MainPID", "--value"], {
826
- stdin: "ignore",
827
- stdout: "pipe",
828
- stderr: "ignore",
829
- });
830
- if (result.exitCode !== 0) return 0;
831
- const pid = Number(result.stdout.toString().trim());
832
- return Number.isFinite(pid) ? pid : 0;
833
- }
834
-
835
- function systemdUnitDiagnostics(unit: string): NonNullable<ManagedSessionExitDiagnostics["systemd"]> {
836
- const result = Bun.spawnSync([
837
- "systemctl", "--user", "show", `${unit}.service`,
838
- "-p", "ActiveState",
839
- "-p", "SubState",
840
- "-p", "Result",
841
- "-p", "ExecMainCode",
842
- "-p", "ExecMainStatus",
843
- "-p", "MainPID",
844
- ], {
845
- stdin: "ignore",
846
- stdout: "pipe",
847
- stderr: "pipe",
848
- });
849
- if (result.exitCode !== 0) {
850
- return {
851
- unit,
852
- unavailable: result.stderr.toString().trim() || `systemctl show exited with ${result.exitCode}`,
853
- };
854
- }
855
- const props = new Map<string, string>();
856
- for (const line of result.stdout.toString().split("\n")) {
857
- const index = line.indexOf("=");
858
- if (index <= 0) continue;
859
- props.set(line.slice(0, index), line.slice(index + 1));
860
- }
861
- const mainPid = Number(props.get("MainPID"));
862
- return {
863
- unit,
864
- activeState: props.get("ActiveState") || undefined,
865
- subState: props.get("SubState") || undefined,
866
- result: props.get("Result") || undefined,
867
- execMainCode: props.get("ExecMainCode") || undefined,
868
- execMainStatus: props.get("ExecMainStatus") || undefined,
869
- mainPid: Number.isFinite(mainPid) && mainPid > 0 ? mainPid : undefined,
870
- };
871
- }
872
-
873
- function logFileDiagnostics(logFile: string): Pick<ManagedSessionExitDiagnostics, "logBytes" | "logEmpty" | "logTail"> & { logUnavailable?: string } {
874
- try {
875
- const stat = statSync(logFile);
876
- if (stat.size === 0) return { logBytes: 0, logEmpty: true, logTail: [] };
877
- const content = readFileSync(logFile, "utf8");
878
- return {
879
- logBytes: stat.size,
880
- logEmpty: false,
881
- logTail: logLines(content).slice(-20),
882
- };
883
- } catch (error) {
884
- return {
885
- logUnavailable: errMessage(error),
886
- };
887
- }
888
- }
889
-
890
- function describeSessionExit(record: SessionRecord, diagnostics: Omit<ManagedSessionExitDiagnostics, "lastError">): string {
891
- if (record.provider === "claude") {
892
- const modelUnavailable = extractClaudeModelUnavailableMessage((diagnostics.logTail ?? []).join("\n"));
893
- if (modelUnavailable) return modelUnavailable;
894
- }
895
- const seconds = Math.max(0, Math.round(diagnostics.runtimeMs / 1000));
896
- const parts = [`managed ${record.provider} session ${record.name} exited after ${seconds}s`];
897
- if (diagnostics.systemd?.unavailable) {
898
- parts.push(`systemd status unavailable: ${diagnostics.systemd.unavailable}`);
899
- } else if (diagnostics.systemd) {
900
- const state = [diagnostics.systemd.activeState, diagnostics.systemd.subState].filter(Boolean).join("/") || "unknown";
901
- const result = diagnostics.systemd.result || "unknown";
902
- const exit = [diagnostics.systemd.execMainCode, diagnostics.systemd.execMainStatus].filter(Boolean).join("/") || "unknown";
903
- parts.push(`systemd ${diagnostics.systemd.unit}.service state=${state} result=${result} exit=${exit}`);
904
- }
905
- if (diagnostics.logEmpty) {
906
- parts.push("stdout/stderr log is empty");
907
- } else if (diagnostics.logBytes === undefined) {
908
- parts.push("stdout/stderr log unavailable");
909
- }
910
- if (!diagnostics.runnerInfoPresent) parts.push("runner info was not written");
911
- return parts.join("; ");
912
- }
913
-
914
- export function diagnoseSessionExit(input: { agentId?: string; policyName?: string; spawnRequestId?: string; tmuxSession?: string }): ManagedSessionExitDiagnostics | null {
915
- const record = findSessionRecord(input);
916
- if (!record) return null;
917
- const detectedAt = Date.now();
918
- const supervisor = sessionSupervisor(record);
919
- const currentPid = currentSessionPid(record);
920
- const terminalAvailable = tmuxHasSession(record.name, readRunnerInfo(record)?.tmuxSocket);
921
- const log = logFileDiagnostics(record.logFile);
922
- const runnerInfoPresent = record.runnerInfoFile ? existsSync(record.runnerInfoFile) : false;
923
- const unavailable = [
924
- ...(log.logUnavailable ? [`stdout/stderr log unavailable: ${log.logUnavailable}`] : []),
925
- ...(log.logEmpty ? ["stdout/stderr log empty"] : []),
926
- ...(!runnerInfoPresent ? ["runner info unavailable"] : []),
927
- ];
928
- const base: Omit<ManagedSessionExitDiagnostics, "lastError"> = {
929
- agentId: record.agentId,
930
- provider: record.provider as "claude" | "codex",
931
- workspaceMode: record.workspaceMode,
932
- workspace: record.workspace ?? (record.workspaceMode ? { mode: "shared", requestedMode: record.workspaceMode } : undefined),
933
- sessionName: record.name,
934
- tmuxSession: record.name,
935
- cwd: record.cwd,
936
- label: record.label,
937
- policyName: record.policyName,
938
- spawnRequestId: record.spawnRequestId,
939
- automationRunId: record.automationRunId,
940
- supervisor: supervisor.type,
941
- ...(supervisor.type === "systemd" && supervisor.unit ? { systemdUnit: supervisor.unit } : {}),
942
- terminalSession: record.name,
943
- terminalAvailable,
944
- pid: record.pid,
945
- currentPid,
946
- startedAt: record.startedAt,
947
- detectedAt,
948
- runtimeMs: Math.max(0, detectedAt - record.startedAt),
949
- logFile: record.logFile,
950
- logBytes: log.logBytes,
951
- logEmpty: log.logEmpty,
952
- logTail: log.logTail,
953
- runnerInfoFile: record.runnerInfoFile,
954
- runnerInfoPresent,
955
- ...(supervisor.type === "systemd" && supervisor.unit ? { systemd: systemdUnitDiagnostics(supervisor.unit) } : {}),
956
- ...(unavailable.length ? { unavailable } : {}),
957
- };
958
- return {
959
- ...base,
960
- lastError: describeSessionExit(record, base),
961
- };
962
- }
963
-
964
- function stopSystemdUnit(unit: string): void {
965
- Bun.spawnSync(["systemctl", "--user", "stop", `${unit}.service`], {
966
- stdin: "ignore",
967
- stdout: "ignore",
968
- stderr: "ignore",
969
- });
970
- }
971
-
972
- function killSystemdUnit(unit: string): void {
973
- Bun.spawnSync(["systemctl", "--user", "kill", "--kill-whom=all", "--signal=SIGKILL", `${unit}.service`], {
974
- stdin: "ignore",
975
- stdout: "ignore",
976
- stderr: "ignore",
977
- });
978
- }
979
-
980
- function cleanupSupervisor(supervisor: SessionSupervisor): void {
981
- if (supervisor.type === "systemd" && supervisor.unit) stopSystemdUnit(supervisor.unit);
982
- if (supervisor.launchScript) rmSync(supervisor.launchScript, { force: true });
983
- }
984
-
985
- function cleanupSessionRecord(record: SessionRecord): void {
986
- cleanupSupervisor(sessionSupervisor(record));
987
- if (record.runnerInfoFile) rmSync(record.runnerInfoFile, { force: true });
988
- }
989
-
990
- export async function stopSession(name: string, config: OrchestratorConfig, reason: string, graceful = true, timeoutMs?: number): Promise<{ stopped: boolean; wasRunning: boolean }> {
991
- if (!name.startsWith(`${config.tmuxPrefix}-`)) throw new Error("session is not managed by this orchestrator");
992
-
993
- const records = loadState();
994
- const record = records.find((r) => r.name === name);
995
- if (!record || !isSessionRecordAlive(record)) {
996
- if (record) cleanupSessionRecord(record);
997
- removeSessionRecord(name);
998
- return { stopped: false, wasRunning: false };
999
- }
1000
-
1001
- const pid = currentSessionPid(record);
1002
- console.error(`[orchestrator] Stopping session ${name} (pid ${pid}): ${reason}`);
1003
-
1004
- const supervisor = sessionSupervisor(record);
1005
- const gracefulTimeoutMs = sessionStopTimeoutMs(graceful, timeoutMs);
1006
- if (supervisor.type === "systemd" && supervisor.unit) {
1007
- stopSystemdUnit(supervisor.unit);
1008
- const deadline = Date.now() + gracefulTimeoutMs;
1009
- while (Date.now() < deadline && isSessionRecordAlive(record)) {
1010
- await Bun.sleep(200);
1011
- }
1012
- if (isSessionRecordAlive(record)) {
1013
- killSystemdUnit(supervisor.unit);
1014
- const killDeadline = Date.now() + 2_000;
1015
- while (Date.now() < killDeadline && isSessionRecordAlive(record)) {
1016
- await Bun.sleep(100);
1017
- }
1018
- }
1019
- if (isSessionRecordAlive(record)) return { stopped: false, wasRunning: true };
1020
- cleanupSessionRecord(record);
1021
- removeSessionRecord(name);
1022
- return { stopped: true, wasRunning: true };
1023
- }
1024
-
1025
- if (graceful) {
1026
- try { process.kill(pid, "SIGTERM"); } catch {}
1027
- const deadline = Date.now() + gracefulTimeoutMs;
1028
- while (Date.now() < deadline && isPidAlive(pid)) {
1029
- await Bun.sleep(200);
1030
- }
1031
- }
1032
-
1033
- if (isPidAlive(pid)) {
1034
- try { process.kill(pid, "SIGKILL"); } catch {}
1035
- const deadline = Date.now() + 2_000;
1036
- while (Date.now() < deadline && isPidAlive(pid)) {
1037
- await Bun.sleep(100);
1038
- }
1039
- }
1040
-
1041
- // Never report success while the process is still alive: deleting the session
1042
- // record here would orphan a running process with no handle to stop it again.
1043
- if (isPidAlive(pid)) {
1044
- console.error(`[orchestrator] Session ${name} (pid ${pid}) survived SIGKILL; keeping record for retry`);
1045
- return { stopped: false, wasRunning: true };
1046
- }
1047
-
1048
- cleanupSessionRecord(record);
1049
- removeSessionRecord(name);
1050
- return { stopped: true, wasRunning: true };
1051
- }
1052
-
1053
- function sessionStopTimeoutMs(graceful: boolean, timeoutMs?: number): number {
1054
- if (!graceful) return 2_000;
1055
- if (!Number.isSafeInteger(timeoutMs) || !timeoutMs || timeoutMs <= 0) return 10_000;
1056
- return Math.min(timeoutMs, 60_000);
1057
- }
1058
-
1059
- export function captureSession(
1060
- name: string,
1061
- config: OrchestratorConfig,
1062
- lines = 100,
1063
- options: { raw?: boolean } = {},
1064
- ): { session: string; lines: string[]; running: boolean } {
1065
- if (!name.startsWith(`${config.tmuxPrefix}-`)) throw new Error("session is not managed by this orchestrator");
1066
-
1067
- const records = loadState();
1068
- const record = records.find((r) => r.name === name);
1069
- const logFile = record?.logFile ?? logFilePath(name);
1070
- const running = record ? isSessionRecordAlive(record) : false;
1071
-
1072
- let content: string;
1073
- try {
1074
- content = readFileSync(logFile, "utf8");
1075
- } catch {
1076
- return { session: name, lines: [], running };
1077
- }
1078
-
1079
- const allLines = logLines(content, !options.raw);
1080
- const safeLines = Math.min(Math.max(lines, 1), 1000);
1081
- return {
1082
- session: name,
1083
- lines: allLines.slice(-safeLines),
1084
- running,
1085
- };
1086
- }
1087
-
1088
- // Shared with the runner's logger via the SDK helper, so reader + writer
1089
- // resolve the same session-mirror filename for a given agent id.
1090
- function safeMirrorLogName(value: string): string {
1091
- return sanitizeFsName(value, { replacement: "_", maxLen: 180 });
1092
- }
1093
-
1094
- // Read the clean, ANSI-free session-mirror diagnostics log for a managed agent.
1095
- // Accepts either the tmux session name or the agent id; the mirror log is keyed by
1096
- // agent id. Returns the same shape as captureSession so the proxy is uniform.
1097
- export function captureSessionMirror(
1098
- name: string,
1099
- _config: OrchestratorConfig,
1100
- lines = 200,
1101
- ): { session: string; lines: string[]; running: boolean; mirror: true } {
1102
- const records = loadState();
1103
- const record = records.find((r) => r.name === name) ?? records.find((r) => r.agentId === name);
1104
- const agentId = record?.agentId ?? name;
1105
- const running = record ? isSessionRecordAlive(record) : false;
1106
- // The mirror log lives in the same directory as the provider log (both written
1107
- // by the same user on this host). Derive from the record's logFile when known so
1108
- // it tracks any per-session log relocation.
1109
- const logDir = record?.logFile ? dirname(record.logFile) : process.env.AGENT_RELAY_LOG_DIR || LOG_DIR;
1110
- const mirrorPath = join(logDir, `session-mirror-${safeMirrorLogName(agentId)}.log`);
1111
- let content: string;
1112
- try {
1113
- content = readFileSync(mirrorPath, "utf8");
1114
- } catch {
1115
- return { session: name, lines: [], running, mirror: true };
1116
- }
1117
- const allLines = content.split(/\r?\n/).filter(Boolean);
1118
- const safeLines = Math.min(Math.max(lines, 1), 2000);
1119
- return { session: name, lines: allLines.slice(-safeLines), running, mirror: true };
1120
- }
1121
-
1122
- export function captureTerminal(name: string, config: OrchestratorConfig): TerminalSnapshot {
1123
- if (!name.startsWith(`${config.tmuxPrefix}-`)) throw new Error("session is not managed by this orchestrator");
1124
-
1125
- const agentAlive = isSessionAlive(name);
1126
- const socketName = tmuxSocketForSession(name);
1127
- const running = tmuxHasSession(name, socketName);
1128
- if (!running) {
1129
- return { session: name, content: "", running: false, agentAlive, capturedAt: Date.now() };
1130
- }
1131
-
1132
- const size = tmuxPaneSize(name, socketName);
1133
- const { content, cursor } = captureConsistent(
1134
- () => captureContent(name, socketName),
1135
- () => tmuxCursorPos(name, socketName),
1136
- );
1137
-
1138
- return {
1139
- session: name,
1140
- content,
1141
- running: true,
1142
- agentAlive,
1143
- ...size,
1144
- ...cursor,
1145
- capturedAt: Date.now(),
1146
- };
1147
- }
1148
-
1149
- // Capture cursor and content *consistently*. They come from separate tmux invocations,
1150
- // so if the pane scrolls between them (e.g. tool output streaming while the agent
1151
- // "thinks"), cursorY ends up off-by-one against the captured grid — the parked cursor
1152
- // then lands a row off and the TUI's next relative redraw stacks a stale statusline row
1153
- // (bottom-box ghost). Read content, then cursor, then content again; accept only when the
1154
- // two content reads bracket the cursor read unchanged, which proves the cursor reflects
1155
- // that exact grid. Fall through with the latest capture if the pane never holds still.
1156
- export function captureConsistent(
1157
- readContent: () => string,
1158
- readCursor: () => { cursorX?: number; cursorY?: number },
1159
- maxAttempts = 4,
1160
- ): { content: string; cursor: { cursorX?: number; cursorY?: number } } {
1161
- let content = readContent();
1162
- let cursor = readCursor();
1163
- for (let attempt = 0; attempt < maxAttempts; attempt++) {
1164
- const recheck = readContent();
1165
- if (recheck === content) break;
1166
- content = recheck;
1167
- cursor = readCursor();
1168
- }
1169
- return { content, cursor };
1170
- }
1171
-
1172
- function captureContent(name: string, socketName?: string): string {
1173
- const result = Bun.spawnSync(tmuxCommand(socketName, "capture-pane", "-p", "-e", "-S", "-1000", "-t", name), {
1174
- stdin: "ignore",
1175
- stdout: "pipe",
1176
- stderr: "pipe",
1177
- });
1178
- if (result.exitCode !== 0) {
1179
- const stderr = result.stderr.toString().trim();
1180
- throw new Error(stderr || `tmux capture-pane failed with exit code ${result.exitCode}`);
1181
- }
1182
- return result.stdout.toString();
1183
- }
1184
-
1185
- export function terminalInputTokens(data: string): TerminalInputToken[] {
1186
- const tokens: TerminalInputToken[] = [];
1187
- let literal = "";
1188
- const flushLiteral = () => {
1189
- if (!literal) return;
1190
- tokens.push({ type: "literal", value: literal });
1191
- literal = "";
1192
- };
1193
- const escapeSequences: Array<[string, string]> = [
1194
- ["\x1b[A", "Up"],
1195
- ["\x1b[B", "Down"],
1196
- ["\x1b[C", "Right"],
1197
- ["\x1b[D", "Left"],
1198
- ["\x1b[H", "Home"],
1199
- ["\x1b[F", "End"],
1200
- ["\x1b[3~", "Delete"],
1201
- ];
1202
-
1203
- for (let index = 0; index < data.length;) {
1204
- const match = escapeSequences.find(([sequence]) => data.startsWith(sequence, index));
1205
- if (match) {
1206
- flushLiteral();
1207
- tokens.push({ type: "key", value: match[1] });
1208
- index += match[0].length;
1209
- continue;
1210
- }
1211
-
1212
- const ch = data[index]!;
1213
- if (ch === "\r" || ch === "\n") {
1214
- flushLiteral();
1215
- tokens.push({ type: "key", value: "Enter" });
1216
- } else if (ch === "\t") {
1217
- flushLiteral();
1218
- tokens.push({ type: "key", value: "Tab" });
1219
- } else if (ch === "\u0003") {
1220
- flushLiteral();
1221
- tokens.push({ type: "key", value: "C-c" });
1222
- } else if (ch === "\u007f" || ch === "\b") {
1223
- flushLiteral();
1224
- tokens.push({ type: "key", value: "BSpace" });
1225
- } else if (ch === "\x1b") {
1226
- flushLiteral();
1227
- tokens.push({ type: "key", value: "Escape" });
1228
- } else if (ch >= " " || ch > "\x7f") {
1229
- literal += ch;
1230
- }
1231
- index += 1;
1232
- }
1233
-
1234
- flushLiteral();
1235
- return tokens;
1236
- }
1237
-
1238
- // Validation contract shared by the HTTP terminal routes and the websocket terminal
1239
- // frames (orchestrator/src/api.ts). Both transports MUST enforce the same envelope —
1240
- // keep these the single source of truth (see #143). Pure: no tmux, safe to unit-test.
1241
- const TERMINAL_INPUT_MAX = 4096;
1242
-
1243
- export function validateTerminalInputData(input: unknown): string {
1244
- if (!input || typeof input !== "object" || Array.isArray(input)) throw new Error("terminal input body must be an object");
1245
- const data = (input as { data?: unknown }).data;
1246
- if (typeof data !== "string") throw new Error("terminal input data must be a string");
1247
- if (data.length > TERMINAL_INPUT_MAX) throw new Error(`terminal input exceeds ${TERMINAL_INPUT_MAX} characters`);
1248
- return data;
1249
- }
1250
-
1251
- export function validateTerminalResize(input: unknown): { cols: number; rows: number } {
1252
- if (!input || typeof input !== "object" || Array.isArray(input)) throw new Error("resize body must be an object");
1253
- const cols = (input as { cols?: unknown }).cols;
1254
- const rows = (input as { rows?: unknown }).rows;
1255
- // typeof narrows for the bounds comparison; Number.isFinite additionally rejects
1256
- // NaN/Infinity — without it NaN slips past the bounds check below (every NaN
1257
- // comparison is false), the exact malformed-resize frame the websocket path used to
1258
- // forward via Number(frame.cols).
1259
- if (typeof cols !== "number" || typeof rows !== "number" || !Number.isFinite(cols) || !Number.isFinite(rows)) {
1260
- throw new Error("cols and rows must be numbers");
1261
- }
1262
- if (cols < 10 || cols > 500 || rows < 5 || rows > 200) throw new Error("cols must be 10-500, rows must be 5-200");
1263
- return { cols: Math.round(cols), rows: Math.round(rows) };
1264
- }
1265
-
1266
- export function sendTerminalInput(name: string, config: OrchestratorConfig, input: unknown): TerminalInputResult {
1267
- if (!name.startsWith(`${config.tmuxPrefix}-`)) throw new Error("session is not managed by this orchestrator");
1268
- const socketName = tmuxSocketForSession(name);
1269
- if (!tmuxHasSession(name, socketName)) throw new Error("terminal session is not running");
1270
- const data = validateTerminalInputData(input);
1271
-
1272
- const tokens = terminalInputTokens(data);
1273
- for (const token of tokens) {
1274
- const args = token.type === "literal"
1275
- ? tmuxCommand(socketName, "send-keys", "-t", name, "-l", token.value)
1276
- : tmuxCommand(socketName, "send-keys", "-t", name, token.value);
1277
- const result = Bun.spawnSync(args, {
1278
- stdin: "ignore",
1279
- stdout: "pipe",
1280
- stderr: "pipe",
1281
- });
1282
- if (result.exitCode !== 0) {
1283
- const stderr = result.stderr.toString().trim();
1284
- throw new Error(stderr || `tmux send-keys failed with exit code ${result.exitCode}`);
1285
- }
1286
- }
1287
-
1288
- return {
1289
- session: name,
1290
- running: true,
1291
- sent: tokens.length,
1292
- capturedAt: Date.now(),
1293
- };
1294
- }
1295
-
1296
- export function resizeTerminal(name: string, config: OrchestratorConfig, input: unknown): { session: string; cols: number; rows: number } {
1297
- if (!name.startsWith(`${config.tmuxPrefix}-`)) throw new Error("session is not managed by this orchestrator");
1298
- const socketName = tmuxSocketForSession(name);
1299
- if (!tmuxHasSession(name, socketName)) throw new Error("terminal session is not running");
1300
- const clamped = validateTerminalResize(input);
1301
- const result = Bun.spawnSync(tmuxCommand(socketName, "resize-window", "-t", name, "-x", String(clamped.cols), "-y", String(clamped.rows)), {
1302
- stdin: "ignore",
1303
- stdout: "pipe",
1304
- stderr: "pipe",
1305
- });
1306
- if (result.exitCode !== 0) {
1307
- const stderr = result.stderr.toString().trim();
1308
- throw new Error(stderr || `tmux resize-window failed with exit code ${result.exitCode}`);
1309
- }
1310
-
1311
- return { session: name, ...clamped };
1312
- }
1313
-
1314
- export function tmuxSocketForSession(name: string): string | undefined {
1315
- const record = loadState().find((item) => item.name === name);
1316
- return record ? readRunnerInfo(record)?.tmuxSocket : undefined;
1317
- }
1318
-
1319
- // Shared tmux helpers; tmuxCommand re-exported for ./terminal-stream.
1320
- export { tmuxCommand };
1321
-
1322
- // Lightweight liveness for the live terminal stream's backfill metadata — avoids a full
1323
- // capture-pane just to learn whether the pane/agent are still up.
1324
- export function sessionLiveness(name: string): { running: boolean; agentAlive: boolean } {
1325
- const socketName = tmuxSocketForSession(name);
1326
- return { running: tmuxHasSession(name, socketName), agentAlive: isSessionAlive(name) };
1327
- }
1328
-
1329
- function tmuxPaneSize(name: string, socketName?: string): { cols?: number; rows?: number } {
1330
- const result = Bun.spawnSync(tmuxCommand(socketName, "display-message", "-p", "-t", name, "#{pane_width} #{pane_height}"), {
1331
- stdin: "ignore",
1332
- stdout: "pipe",
1333
- stderr: "ignore",
1334
- });
1335
- if (result.exitCode !== 0) return {};
1336
- const [colsRaw, rowsRaw] = result.stdout.toString().trim().split(/\s+/, 2);
1337
- const cols = Number(colsRaw);
1338
- const rows = Number(rowsRaw);
1339
- return {
1340
- ...(Number.isFinite(cols) && cols > 0 ? { cols } : {}),
1341
- ...(Number.isFinite(rows) && rows > 0 ? { rows } : {}),
1342
- };
1343
- }
1344
-
1345
- function tmuxCursorPos(name: string, socketName?: string): { cursorX?: number; cursorY?: number } {
1346
- const result = Bun.spawnSync(tmuxCommand(socketName, "display-message", "-p", "-t", name, "#{cursor_x} #{cursor_y}"), {
1347
- stdin: "ignore",
1348
- stdout: "pipe",
1349
- stderr: "ignore",
1350
- });
1351
- if (result.exitCode !== 0) return {};
1352
- const [xRaw, yRaw] = result.stdout.toString().trim().split(/\s+/, 2);
1353
- const cursorX = Number(xRaw);
1354
- const cursorY = Number(yRaw);
1355
- return {
1356
- ...(Number.isFinite(cursorX) ? { cursorX } : {}),
1357
- ...(Number.isFinite(cursorY) ? { cursorY } : {}),
1358
- };
1359
- }
1360
-
1361
- export function logLines(content: string, sanitize = true): string[] {
1362
- const text = sanitize ? sanitizeLogText(content) : content.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
1363
- return text
1364
- .split("\n")
1365
- .map((line) => line.trimEnd())
1366
- .filter((line) => line.trim().length > 0);
1367
- }
1368
-
1369
- export function sanitizeLogText(content: string): string {
1370
- return content
1371
- .replace(/\x1B\][^\x07\x1B]*(?:\x07|\x1B\\)/g, "")
1372
- .replace(/\x1B[PX^_][\s\S]*?\x1B\\/g, "")
1373
- .replace(/\x1B\[(\d*)C/g, (_match, count: string) => " ".repeat(Math.min(Number(count || "1"), 120)))
1374
- .replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, "")
1375
- .replace(/\x1B[()#%*+\-.\/ ][ -~]/g, "")
1376
- .replace(/\x1B[ -/]*[@-~]/g, "")
1377
- .replace(/\x9B[0-?]*[ -/]*[@-~]/g, "")
1378
- .replace(/\x1B.?/g, "")
1379
- .replace(/\r\n/g, "\n")
1380
- .replace(/\r/g, "\n")
1381
- .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "");
1382
- }
1383
-
1384
- export function listSessions(prefix: string): SessionInfo[] {
1385
- return loadState()
1386
- .filter((r) => r.name.startsWith(`${prefix}-`))
1387
- .map((r) => {
1388
- const supervisor = sessionSupervisor(r);
1389
- return {
1390
- name: r.name,
1391
- sessionName: r.name,
1392
- pid: currentSessionPid(r),
1393
- alive: isSessionRecordAlive(r),
1394
- supervisor: supervisor.type,
1395
- ...(supervisor.type === "systemd" && supervisor.unit ? { systemdUnit: supervisor.unit } : {}),
1396
- terminalSession: r.name,
1397
- terminalAvailable: tmuxHasSession(r.name, readRunnerInfo(r)?.tmuxSocket),
1398
- logFile: r.logFile,
1399
- };
1400
- });
1401
- }
1402
-
1403
- export function isSessionAlive(name: string): boolean {
1404
- const record = loadState().find((r) => r.name === name);
1405
- return record ? isSessionRecordAlive(record) : false;
1406
- }
1407
-
1408
- export function refreshManagedAgentReport(agent: ManagedAgentReport): ManagedAgentReport {
1409
- const record = findSessionRecord({
1410
- tmuxSession: agent.sessionName ?? agent.tmuxSession,
1411
- agentId: agent.agentId,
1412
- policyName: agent.policyName,
1413
- spawnRequestId: agent.spawnRequestId,
1414
- });
1415
- if (!record) return agent;
1416
- return {
1417
- ...agent,
1418
- workspaceMode: record.workspaceMode,
1419
- workspace: record.workspace ?? agent.workspace ?? (record.workspaceMode ? { mode: "shared", requestedMode: record.workspaceMode } : undefined),
1420
- pid: currentSessionPid(record),
1421
- ...sessionReportFields(record),
1422
- };
1423
- }
1424
-
1425
- export async function recoverExistingSessions(
1426
- config: OrchestratorConfig,
1427
- ): Promise<ManagedAgentReport[]> {
1428
- const records = loadState().filter((r) => r.name.startsWith(`${config.tmuxPrefix}-`));
1429
- const managed: ManagedAgentReport[] = [];
1430
- const alive: SessionRecord[] = [];
1431
-
1432
- for (const record of records) {
1433
- if (!isSessionRecordAlive(record)) {
1434
- console.error(`[orchestrator] Stale session: ${record.name} (pid ${record.pid} dead) — removing`);
1435
- cleanupSessionRecord(record);
1436
- continue;
1437
- }
1438
-
1439
- const pid = currentSessionPid(record);
1440
- const updatedRecord = { ...record, pid };
1441
- alive.push(updatedRecord);
1442
- managed.push({
1443
- agentId: record.agentId,
1444
- provider: record.provider as "claude" | "codex",
1445
- workspaceMode: record.workspaceMode,
1446
- workspace: record.workspace ?? (record.workspaceMode ? { mode: "shared", requestedMode: record.workspaceMode } : undefined),
1447
- ...sessionReportFields(updatedRecord),
1448
- cwd: record.cwd,
1449
- label: record.label,
1450
- approvalMode: record.approvalMode || "guarded",
1451
- policyName: record.policyName,
1452
- spawnRequestId: record.spawnRequestId,
1453
- automationRunId: record.automationRunId,
1454
- pid,
1455
- startedAt: record.startedAt,
1456
- });
1457
-
1458
- console.error(`[orchestrator] Recovered existing session: ${record.name} (pid ${record.pid})`);
1459
- }
1460
-
1461
- // Merge rather than overwrite: only replace the records this recovery actually
1462
- // inspected, so a session added concurrently (or owned by another prefix) is
1463
- // not erased by writing back a pre-filtered snapshot.
1464
- const processedNames = new Set(records.map((r) => r.name));
1465
- const untouched = loadState().filter((r) => !processedNames.has(r.name));
1466
- saveState([...untouched, ...alive]);
1467
- return managed;
1468
- }
1469
-
1470
- function managedAgentId(config: OrchestratorConfig, provider: string, label: string): string {
1471
- const cleanHost = sanitizeFsName(config.hostname, { replacement: "-", lowercase: true });
1472
- const cleanLabel = sanitizeFsName(label, { replacement: "-", lowercase: true });
1473
- return `${cleanHost}-${provider}-${cleanLabel}-${crypto.randomUUID().slice(0, 8)}`;
1474
- }
1475
-
1476
- // Shared shell-quoting; re-exported so `./spawn` consumers + tests resolve it.
1477
- export { shellEscape };
1
+ export * from "./spawn/index";