@parall/daemon 1.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.ts +62 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +81 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +108 -0
- package/dist/runtimes.d.ts +12 -0
- package/dist/runtimes.d.ts.map +1 -0
- package/dist/runtimes.js +55 -0
- package/dist/supervisor.d.ts +50 -0
- package/dist/supervisor.d.ts.map +1 -0
- package/dist/supervisor.js +415 -0
- package/package.json +38 -0
- package/src/config.ts +142 -0
- package/src/index.ts +130 -0
- package/src/runtimes.ts +71 -0
- package/src/supervisor.ts +480 -0
package/src/index.ts
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { ParallClient } from "@parall/sdk";
|
|
4
|
+
import { resolveClaudeDaemonConfig, type ClaudeDaemonConfig } from "./config.js";
|
|
5
|
+
import { DaemonSupervisor, sleepCancellable, type DaemonLogger } from "./supervisor.js";
|
|
6
|
+
|
|
7
|
+
function createLogger(prefix: string): DaemonLogger {
|
|
8
|
+
return {
|
|
9
|
+
info: (msg: string) => console.log(`[${prefix}] ${msg}`),
|
|
10
|
+
warn: (msg: string) => console.warn(`[${prefix}] ${msg}`),
|
|
11
|
+
error: (msg: string) => console.error(`[${prefix}] ${msg}`),
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function formatError(reason: unknown): string {
|
|
16
|
+
if (reason instanceof Error) {
|
|
17
|
+
return reason.stack ?? reason.message;
|
|
18
|
+
}
|
|
19
|
+
return String(reason);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Outer keepalive: runs `supervisor.run()` in a loop, restarting it with
|
|
24
|
+
* exponential backoff if it rejects. The daemon process only exits when
|
|
25
|
+
* the abort signal fires (SIGINT/SIGTERM) — anything else is treated as
|
|
26
|
+
* a transient failure that we recover from in-process.
|
|
27
|
+
*
|
|
28
|
+
* Why this exists in addition to entrypoint.sh's `while :; do parall-daemon; done`:
|
|
29
|
+
* - In-process restart preserves the `client` (= same TCP/TLS pool) and
|
|
30
|
+
* skips the ~hundreds-of-ms cost of Node startup + module loading per
|
|
31
|
+
* cycle, which matters when the API is flapping.
|
|
32
|
+
* - The shell wrapper is the "process really crashed" backstop —
|
|
33
|
+
* segfault, OOM, uncaughtException, etc. The two layers are
|
|
34
|
+
* complementary: in-process for soft failures, shell for hard failures.
|
|
35
|
+
*
|
|
36
|
+
* Operators can disable the in-process loop by setting
|
|
37
|
+
* PRLL_DAEMON_SUPERVISOR_RESTART_BACKOFF_MS=0 — main() will then exit on
|
|
38
|
+
* any supervisor.run() rejection and rely entirely on the shell wrapper +
|
|
39
|
+
* K8s for restart.
|
|
40
|
+
*/
|
|
41
|
+
async function runForever(
|
|
42
|
+
config: ClaudeDaemonConfig,
|
|
43
|
+
client: ParallClient,
|
|
44
|
+
log: DaemonLogger,
|
|
45
|
+
signal: AbortSignal,
|
|
46
|
+
): Promise<void> {
|
|
47
|
+
let attempt = 0;
|
|
48
|
+
while (!signal.aborted) {
|
|
49
|
+
const supervisor = new DaemonSupervisor(config, client, log);
|
|
50
|
+
try {
|
|
51
|
+
await supervisor.run(signal);
|
|
52
|
+
// Clean exit (signal aborted) — done.
|
|
53
|
+
await supervisor.stop();
|
|
54
|
+
return;
|
|
55
|
+
} catch (err) {
|
|
56
|
+
// supervisor.run() rejected. The supervisor's per-tick handlers
|
|
57
|
+
// already swallow individual failures, so the only paths that reach
|
|
58
|
+
// here are (a) bootstrap fail-fast mode and (b) genuine programmer
|
|
59
|
+
// bugs. Restart anyway — operators rely on this daemon as the only
|
|
60
|
+
// thing keeping per-agent children alive on this host.
|
|
61
|
+
log.error(`supervisor crashed: ${String(err)}`);
|
|
62
|
+
try {
|
|
63
|
+
await supervisor.stop();
|
|
64
|
+
} catch (stopErr) {
|
|
65
|
+
log.warn(`supervisor.stop() after crash threw: ${String(stopErr)}`);
|
|
66
|
+
}
|
|
67
|
+
if (config.supervisorRestartBackoffMs === 0) {
|
|
68
|
+
log.error(
|
|
69
|
+
"supervisor keepalive disabled (PRLL_DAEMON_SUPERVISOR_RESTART_BACKOFF_MS=0) — exiting",
|
|
70
|
+
);
|
|
71
|
+
throw err;
|
|
72
|
+
}
|
|
73
|
+
const delay = Math.min(
|
|
74
|
+
config.supervisorRestartBackoffMs * Math.pow(2, attempt),
|
|
75
|
+
config.supervisorRestartBackoffMaxMs,
|
|
76
|
+
);
|
|
77
|
+
attempt += 1;
|
|
78
|
+
log.warn(`restarting supervisor in ${delay}ms (attempt ${attempt})`);
|
|
79
|
+
const slept = await sleepCancellable(delay, signal);
|
|
80
|
+
if (!slept) return;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function main(): Promise<void> {
|
|
86
|
+
const config = resolveClaudeDaemonConfig(process.env);
|
|
87
|
+
const log = createLogger("daemon");
|
|
88
|
+
|
|
89
|
+
log.info(
|
|
90
|
+
`boot: api=${config.apiUrl} pollMs=${config.pollIntervalMs} heartbeatMs=${config.heartbeatIntervalMs} agentBin=${config.agentBin}`,
|
|
91
|
+
);
|
|
92
|
+
log.info(
|
|
93
|
+
`keepalive: bootstrapBackoffMs=${config.bootstrapBackoffMs} supervisorRestartBackoffMs=${config.supervisorRestartBackoffMs}`,
|
|
94
|
+
);
|
|
95
|
+
|
|
96
|
+
// The daemon talks to the API as a Machine — the bearer is mck_*.
|
|
97
|
+
// No orgId is configured here; per-agent subprocesses get their
|
|
98
|
+
// own org_id via the spawn env.
|
|
99
|
+
const client = new ParallClient({
|
|
100
|
+
baseUrl: config.apiUrl,
|
|
101
|
+
token: config.apiKey,
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
const abortController = new AbortController();
|
|
105
|
+
const onSignal = (sig: NodeJS.Signals) => {
|
|
106
|
+
log.info(`received ${sig} — initiating shutdown`);
|
|
107
|
+
abortController.abort();
|
|
108
|
+
};
|
|
109
|
+
process.on("SIGINT", () => onSignal("SIGINT"));
|
|
110
|
+
process.on("SIGTERM", () => onSignal("SIGTERM"));
|
|
111
|
+
|
|
112
|
+
// Defensive: unexpected async failures are logged explicitly. Rejections
|
|
113
|
+
// stay in-process so the supervisor loop can recover; uncaught exceptions
|
|
114
|
+
// exit so entrypoint.sh's shell-level keepalive restarts from a clean VM.
|
|
115
|
+
process.on("unhandledRejection", (reason) => {
|
|
116
|
+
log.error(`unhandledRejection: ${formatError(reason)}`);
|
|
117
|
+
});
|
|
118
|
+
process.on("uncaughtException", (err) => {
|
|
119
|
+
log.error(`uncaughtException: ${formatError(err)}`);
|
|
120
|
+
process.exitCode = 1;
|
|
121
|
+
process.exit(1);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
await runForever(config, client, log, abortController.signal);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
main().catch((err) => {
|
|
128
|
+
console.error(`[daemon] fatal: ${formatError(err)}`);
|
|
129
|
+
process.exitCode = 1;
|
|
130
|
+
});
|
package/src/runtimes.ts
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
export interface RuntimeAdapter {
|
|
2
|
+
bin: string;
|
|
3
|
+
buildEnv(baseEnv: NodeJS.ProcessEnv, agentId: string, orgId: string, apiKey: string, dirs: AgentDirs): NodeJS.ProcessEnv;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
export interface AgentDirs {
|
|
7
|
+
stateDir: string;
|
|
8
|
+
workspaceDir: string;
|
|
9
|
+
claudeHome: string;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
const claudeCodeAdapter: RuntimeAdapter = {
|
|
13
|
+
bin: "parall-claude-agent",
|
|
14
|
+
buildEnv(baseEnv, agentId, orgId, apiKey, dirs) {
|
|
15
|
+
const env = { ...baseEnv };
|
|
16
|
+
env.PRLL_API_KEY = apiKey;
|
|
17
|
+
env.PRLL_ORG_ID = orgId;
|
|
18
|
+
env.AGENT_ID = agentId;
|
|
19
|
+
env.PRLL_AGENT_ID = agentId;
|
|
20
|
+
env.PRLL_CLAUDE_HOME = dirs.claudeHome;
|
|
21
|
+
env.PRLL_CLAUDE_STATE_DIR = dirs.stateDir;
|
|
22
|
+
env.PRLL_CLAUDE_WORKSPACE_DIR = dirs.workspaceDir;
|
|
23
|
+
delete env.PRLL_DAEMON_MODE;
|
|
24
|
+
return env;
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
const codexAdapter: RuntimeAdapter = {
|
|
29
|
+
bin: "parall-codex-agent",
|
|
30
|
+
buildEnv(baseEnv, agentId, orgId, apiKey, dirs) {
|
|
31
|
+
const env = { ...baseEnv };
|
|
32
|
+
env.PRLL_API_KEY = apiKey;
|
|
33
|
+
env.PRLL_ORG_ID = orgId;
|
|
34
|
+
env.AGENT_ID = agentId;
|
|
35
|
+
env.PRLL_AGENT_ID = agentId;
|
|
36
|
+
env.PRLL_CODEX_STATE_DIR = dirs.stateDir;
|
|
37
|
+
env.PRLL_CODEX_WORKSPACE_DIR = dirs.workspaceDir;
|
|
38
|
+
delete env.PRLL_DAEMON_MODE;
|
|
39
|
+
return env;
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const defaultAdapter: RuntimeAdapter = {
|
|
44
|
+
bin: "parall-agent",
|
|
45
|
+
buildEnv(baseEnv, agentId, orgId, apiKey, dirs) {
|
|
46
|
+
const env = { ...baseEnv };
|
|
47
|
+
env.PRLL_API_KEY = apiKey;
|
|
48
|
+
env.PRLL_ORG_ID = orgId;
|
|
49
|
+
env.AGENT_ID = agentId;
|
|
50
|
+
env.PRLL_AGENT_ID = agentId;
|
|
51
|
+
env.PRLL_STATE_DIR = dirs.stateDir;
|
|
52
|
+
env.PRLL_WORKSPACE_DIR = dirs.workspaceDir;
|
|
53
|
+
delete env.PRLL_DAEMON_MODE;
|
|
54
|
+
return env;
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const RUNTIME_ADAPTERS: Record<string, RuntimeAdapter> = {
|
|
59
|
+
"claude-code": claudeCodeAdapter,
|
|
60
|
+
"codex": codexAdapter,
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
export function getRuntimeAdapter(runtimeType: string): RuntimeAdapter {
|
|
64
|
+
return RUNTIME_ADAPTERS[runtimeType] ?? defaultAdapter;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function assertAgentKey(apiKey: string): void {
|
|
68
|
+
if (apiKey.startsWith("mck_")) {
|
|
69
|
+
throw new Error("BUG: machine key leaked to child process — expected agk_, got mck_");
|
|
70
|
+
}
|
|
71
|
+
}
|
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
import { spawn, type ChildProcess } from "node:child_process";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import * as path from "node:path";
|
|
4
|
+
import { ParallClient, ParallWs, type AttachedAgent, type LaunchCredentialResponse, type MachineAgentAttachedData, type MachineAgentDetachedData, type MachineStopData } from "@parall/sdk";
|
|
5
|
+
import {
|
|
6
|
+
type ClaudeDaemonConfig,
|
|
7
|
+
agentClaudeCredentialsFileFor,
|
|
8
|
+
agentClaudeHomeFor,
|
|
9
|
+
agentStateDirFor,
|
|
10
|
+
agentWorkspaceDirFor,
|
|
11
|
+
sharedClaudeCredentialsFileFor,
|
|
12
|
+
} from "./config.js";
|
|
13
|
+
import { assertAgentKey, getRuntimeAdapter, type AgentDirs } from "./runtimes.js";
|
|
14
|
+
|
|
15
|
+
export interface DaemonLogger {
|
|
16
|
+
info(msg: string): void;
|
|
17
|
+
warn(msg: string): void;
|
|
18
|
+
error(msg: string): void;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Sleep that wakes early on abort. Returns true if the full delay elapsed,
|
|
23
|
+
* false if aborted. Used by bootstrap retry and the outer keepalive in
|
|
24
|
+
* `runForever` so SIGTERM during a long backoff doesn't stall shutdown.
|
|
25
|
+
*/
|
|
26
|
+
function sleepCancellable(ms: number, signal: AbortSignal): Promise<boolean> {
|
|
27
|
+
if (signal.aborted) return Promise.resolve(false);
|
|
28
|
+
return new Promise<boolean>((resolve) => {
|
|
29
|
+
const timer = setTimeout(() => {
|
|
30
|
+
signal.removeEventListener("abort", onAbort);
|
|
31
|
+
resolve(true);
|
|
32
|
+
}, ms);
|
|
33
|
+
const onAbort = () => {
|
|
34
|
+
clearTimeout(timer);
|
|
35
|
+
resolve(false);
|
|
36
|
+
};
|
|
37
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export { sleepCancellable };
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Per-agent runtime state held by the supervisor. Exactly one of these
|
|
45
|
+
* exists per attached agent for the lifetime of an attachment; on detach
|
|
46
|
+
* the entry is removed and the child SIGTERM'd.
|
|
47
|
+
*/
|
|
48
|
+
interface ChildState {
|
|
49
|
+
agentId: string;
|
|
50
|
+
orgId: string;
|
|
51
|
+
/** Runtime type from the agent profile (e.g. "claude-code", "codex"). */
|
|
52
|
+
runtimeType: string;
|
|
53
|
+
child: ChildProcess | null;
|
|
54
|
+
credential: LaunchCredentialResponse | null;
|
|
55
|
+
restartAttempts: number;
|
|
56
|
+
restartTimer: NodeJS.Timeout | null;
|
|
57
|
+
shuttingDown: boolean;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Supervisor orchestrates one mck_ machine bearer into N
|
|
62
|
+
* `parall-claude-agent` subprocesses, one per AttachedAgent.
|
|
63
|
+
*
|
|
64
|
+
* Lifecycle is WS event-driven:
|
|
65
|
+
* 1. Bootstrap: confirm machine identity with retry.
|
|
66
|
+
* 2. Full reconcile: list attached agents, diff with children, spawn/kill.
|
|
67
|
+
* 3. Connect WS: receive machine.agent.attached / .detached / .stop
|
|
68
|
+
* events for incremental updates; full reconcile on every reconnect
|
|
69
|
+
* (machine.hello) to catch events missed while disconnected.
|
|
70
|
+
*/
|
|
71
|
+
export class DaemonSupervisor {
|
|
72
|
+
private readonly children = new Map<string, ChildState>();
|
|
73
|
+
private ws: ParallWs | null = null;
|
|
74
|
+
private running = false;
|
|
75
|
+
private machineOrgId: string | null = null;
|
|
76
|
+
private stopResolve: (() => void) | null = null;
|
|
77
|
+
|
|
78
|
+
constructor(
|
|
79
|
+
private readonly config: ClaudeDaemonConfig,
|
|
80
|
+
private readonly client: ParallClient,
|
|
81
|
+
private readonly log: DaemonLogger,
|
|
82
|
+
) {}
|
|
83
|
+
|
|
84
|
+
/** Start the supervisor. Returns a promise that resolves on `stop()`. */
|
|
85
|
+
async run(signal: AbortSignal): Promise<void> {
|
|
86
|
+
if (this.running) throw new Error("supervisor already running");
|
|
87
|
+
this.running = true;
|
|
88
|
+
|
|
89
|
+
const onAbort = () => {
|
|
90
|
+
this.stop().catch((err) => this.log.error(`stop() failed: ${String(err)}`));
|
|
91
|
+
};
|
|
92
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
93
|
+
|
|
94
|
+
try {
|
|
95
|
+
if (!(await this.bootstrapWithRetry(signal))) {
|
|
96
|
+
signal.removeEventListener("abort", onAbort);
|
|
97
|
+
this.running = false;
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
} catch (err) {
|
|
101
|
+
signal.removeEventListener("abort", onAbort);
|
|
102
|
+
this.running = false;
|
|
103
|
+
throw err;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
await this.fullReconcile();
|
|
107
|
+
|
|
108
|
+
this.ws = new ParallWs({
|
|
109
|
+
getTicket: () => this.client.getMachineWsTicket(),
|
|
110
|
+
wsUrl: this.config.wsUrl,
|
|
111
|
+
reconnect: true,
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
this.ws.on("machine.hello", (_data) => {
|
|
115
|
+
this.log.info("machine WS connected (machine.hello)");
|
|
116
|
+
void this.fullReconcile();
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
this.ws.on("machine.agent.attached", (data: MachineAgentAttachedData) => {
|
|
120
|
+
this.log.info(`WS: agent ${data.agent_id} attached`);
|
|
121
|
+
void this.handleAgentAttached(data.agent_id);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
this.ws.on("machine.agent.detached", (data: MachineAgentDetachedData) => {
|
|
125
|
+
this.log.info(`WS: agent ${data.agent_id} detached`);
|
|
126
|
+
void this.handleAgentDetached(data.agent_id);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
this.ws.on("machine.stop", (data: MachineStopData) => {
|
|
130
|
+
this.log.info(`WS: machine.stop received (reason=${data.reason ?? "none"})`);
|
|
131
|
+
void this.stop();
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
this.ws.onStateChange((state) => {
|
|
135
|
+
if (state === "disconnected" || state === "reconnecting") {
|
|
136
|
+
this.log.warn(`machine WS state: ${state}`);
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
await this.ws.connect();
|
|
141
|
+
|
|
142
|
+
await new Promise<void>((resolve) => {
|
|
143
|
+
this.stopResolve = resolve;
|
|
144
|
+
});
|
|
145
|
+
signal.removeEventListener("abort", onAbort);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Disconnect WS, cancel timers, SIGTERM all children, await exit. */
|
|
149
|
+
async stop(): Promise<void> {
|
|
150
|
+
if (!this.running) return;
|
|
151
|
+
this.running = false;
|
|
152
|
+
|
|
153
|
+
if (this.ws) {
|
|
154
|
+
this.ws.disconnect();
|
|
155
|
+
this.ws = null;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const exits: Promise<void>[] = [];
|
|
159
|
+
for (const state of this.children.values()) {
|
|
160
|
+
state.shuttingDown = true;
|
|
161
|
+
if (state.restartTimer) {
|
|
162
|
+
clearTimeout(state.restartTimer);
|
|
163
|
+
state.restartTimer = null;
|
|
164
|
+
}
|
|
165
|
+
exits.push(this.terminateChild(state));
|
|
166
|
+
}
|
|
167
|
+
await Promise.allSettled(exits);
|
|
168
|
+
this.children.clear();
|
|
169
|
+
this.log.info("daemon supervisor stopped");
|
|
170
|
+
if (this.stopResolve) {
|
|
171
|
+
this.stopResolve();
|
|
172
|
+
this.stopResolve = null;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// ---- Bootstrap (resilient identity probe) ----
|
|
177
|
+
|
|
178
|
+
private async bootstrapWithRetry(signal: AbortSignal): Promise<boolean> {
|
|
179
|
+
let attempt = 0;
|
|
180
|
+
while (this.running && !signal.aborted) {
|
|
181
|
+
try {
|
|
182
|
+
const machine = await this.client.getMachineSelf();
|
|
183
|
+
this.machineOrgId = machine.org_id;
|
|
184
|
+
this.log.info(
|
|
185
|
+
`daemon online — machine_id=${machine.id} org=${machine.org_id} label=${machine.label}`,
|
|
186
|
+
);
|
|
187
|
+
return true;
|
|
188
|
+
} catch (err) {
|
|
189
|
+
if (this.config.bootstrapBackoffMs === 0) {
|
|
190
|
+
this.log.error(`getMachineSelf failed: ${String(err)} (fail-fast mode)`);
|
|
191
|
+
throw err;
|
|
192
|
+
}
|
|
193
|
+
const delay = Math.min(
|
|
194
|
+
this.config.bootstrapBackoffMs * Math.pow(2, attempt),
|
|
195
|
+
this.config.bootstrapBackoffMaxMs,
|
|
196
|
+
);
|
|
197
|
+
attempt += 1;
|
|
198
|
+
this.log.warn(
|
|
199
|
+
`getMachineSelf failed (attempt ${attempt}): ${String(err)} — retrying in ${delay}ms`,
|
|
200
|
+
);
|
|
201
|
+
const slept = await sleepCancellable(delay, signal);
|
|
202
|
+
if (!slept) return false;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// ---- Full reconcile (HTTP-based, used on boot + WS reconnect) ----
|
|
209
|
+
// Safe to run concurrently with WS event handlers: JS single-threaded
|
|
210
|
+
// event loop guarantees no mid-statement interleaving, and both
|
|
211
|
+
// handleAgentAttached/Detached guard on children.has()/get() so a WS
|
|
212
|
+
// event between the HTTP fetch and the spawn/kill loop is a no-op.
|
|
213
|
+
|
|
214
|
+
private async fullReconcile(): Promise<void> {
|
|
215
|
+
if (!this.running) return;
|
|
216
|
+
|
|
217
|
+
let attached: AttachedAgent[];
|
|
218
|
+
try {
|
|
219
|
+
attached = await this.client.listAttachedAgents();
|
|
220
|
+
} catch (err) {
|
|
221
|
+
this.log.warn(`fullReconcile: listAttachedAgents failed: ${String(err)}`);
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const seen = new Set<string>();
|
|
226
|
+
for (const a of attached) {
|
|
227
|
+
const userId = a.user?.id ?? a.profile.user_id;
|
|
228
|
+
if (!userId) {
|
|
229
|
+
this.log.warn(`skipping attached entry with no user_id (profile=${JSON.stringify(a.profile)})`);
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
if (a.user && a.user.status !== "active") {
|
|
233
|
+
this.log.info(`agent ${userId} not active (status=${a.user.status}) — skipping`);
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
seen.add(userId);
|
|
237
|
+
|
|
238
|
+
const existing = this.children.get(userId);
|
|
239
|
+
if (!existing) {
|
|
240
|
+
const orgId = this.machineOrgId;
|
|
241
|
+
if (!orgId) {
|
|
242
|
+
this.log.warn(`agent ${userId}: no org_id available yet; skipping`);
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
await this.spawnAgent(userId, orgId, a);
|
|
246
|
+
} else if (!existing.child && !existing.restartTimer && !existing.shuttingDown) {
|
|
247
|
+
await this.restartChildNow(existing, "reconcile found no live child");
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
for (const [userId, state] of this.children) {
|
|
252
|
+
if (!seen.has(userId)) {
|
|
253
|
+
this.log.info(`agent ${userId} detached (reconcile) — terminating subprocess`);
|
|
254
|
+
state.shuttingDown = true;
|
|
255
|
+
if (state.restartTimer) {
|
|
256
|
+
clearTimeout(state.restartTimer);
|
|
257
|
+
state.restartTimer = null;
|
|
258
|
+
}
|
|
259
|
+
await this.terminateChild(state);
|
|
260
|
+
this.children.delete(userId);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// ---- WS event handlers (incremental) ----
|
|
266
|
+
|
|
267
|
+
private async handleAgentAttached(agentId: string): Promise<void> {
|
|
268
|
+
if (this.children.has(agentId)) return;
|
|
269
|
+
|
|
270
|
+
let attached: AttachedAgent[];
|
|
271
|
+
try {
|
|
272
|
+
attached = await this.client.listAttachedAgents();
|
|
273
|
+
} catch (err) {
|
|
274
|
+
this.log.warn(`handleAgentAttached: listAttachedAgents failed: ${String(err)}`);
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
const entry = attached.find((a) => (a.user?.id ?? a.profile.user_id) === agentId);
|
|
279
|
+
if (!entry) {
|
|
280
|
+
this.log.warn(`handleAgentAttached: agent ${agentId} not found in attached list`);
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
if (entry.user && entry.user.status !== "active") {
|
|
284
|
+
this.log.info(`agent ${agentId} not active (status=${entry.user.status}) — skipping`);
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
const orgId = this.machineOrgId;
|
|
289
|
+
if (!orgId) {
|
|
290
|
+
this.log.warn(`agent ${agentId}: no org_id available yet; skipping`);
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
await this.spawnAgent(agentId, orgId, entry);
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
private async handleAgentDetached(agentId: string): Promise<void> {
|
|
297
|
+
const state = this.children.get(agentId);
|
|
298
|
+
if (!state) return;
|
|
299
|
+
|
|
300
|
+
this.log.info(`agent ${agentId} detached — terminating subprocess`);
|
|
301
|
+
state.shuttingDown = true;
|
|
302
|
+
if (state.restartTimer) {
|
|
303
|
+
clearTimeout(state.restartTimer);
|
|
304
|
+
state.restartTimer = null;
|
|
305
|
+
}
|
|
306
|
+
await this.terminateChild(state);
|
|
307
|
+
this.children.delete(agentId);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// ---- Spawn / restart ----
|
|
311
|
+
|
|
312
|
+
private async restartChildNow(state: ChildState, reason: string): Promise<void> {
|
|
313
|
+
if (!this.running || state.shuttingDown || state.child || state.restartTimer) {
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
try {
|
|
317
|
+
state.credential = await this.client.mintLaunchCredential(state.agentId);
|
|
318
|
+
state.restartAttempts = 0;
|
|
319
|
+
this.log.info(`agent ${state.agentId}: restarting child (${reason})`);
|
|
320
|
+
this.startChild(state);
|
|
321
|
+
} catch (err) {
|
|
322
|
+
this.log.warn(`agent ${state.agentId}: restart mint failed (${reason}): ${String(err)}`);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
private async spawnAgent(agentId: string, orgId: string, attached: AttachedAgent): Promise<void> {
|
|
327
|
+
let credential: LaunchCredentialResponse;
|
|
328
|
+
try {
|
|
329
|
+
credential = await this.client.mintLaunchCredential(agentId);
|
|
330
|
+
} catch (err) {
|
|
331
|
+
this.log.error(`mintLaunchCredential ${agentId} failed: ${String(err)}`);
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
const stateDir = agentStateDirFor(this.config.rootStateDir, agentId);
|
|
335
|
+
const workspaceDir = agentWorkspaceDirFor(this.config.rootStateDir, agentId);
|
|
336
|
+
const claudeHome = agentClaudeHomeFor(this.config.rootClaudeHome, agentId);
|
|
337
|
+
try {
|
|
338
|
+
fs.mkdirSync(stateDir, { recursive: true });
|
|
339
|
+
fs.mkdirSync(workspaceDir, { recursive: true });
|
|
340
|
+
fs.mkdirSync(claudeHome, { recursive: true });
|
|
341
|
+
this.ensureSharedCredentialLink(claudeHome, agentId);
|
|
342
|
+
} catch (err) {
|
|
343
|
+
this.log.warn(`mkdir agent dirs (${agentId}) failed: ${String(err)}`);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
const runtimeType = attached.profile.runtime_type ?? "claude-code";
|
|
347
|
+
const state: ChildState = {
|
|
348
|
+
agentId,
|
|
349
|
+
orgId,
|
|
350
|
+
runtimeType,
|
|
351
|
+
child: null,
|
|
352
|
+
credential,
|
|
353
|
+
restartAttempts: 0,
|
|
354
|
+
restartTimer: null,
|
|
355
|
+
shuttingDown: false,
|
|
356
|
+
};
|
|
357
|
+
this.children.set(agentId, state);
|
|
358
|
+
this.startChild(state);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
private startChild(state: ChildState): void {
|
|
362
|
+
if (state.shuttingDown || !this.running) return;
|
|
363
|
+
if (!state.credential) {
|
|
364
|
+
this.log.error(`startChild ${state.agentId}: no credential — bug`);
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
assertAgentKey(state.credential.api_key);
|
|
369
|
+
const adapter = getRuntimeAdapter(state.runtimeType);
|
|
370
|
+
const dirs: AgentDirs = {
|
|
371
|
+
stateDir: agentStateDirFor(this.config.rootStateDir, state.agentId),
|
|
372
|
+
workspaceDir: agentWorkspaceDirFor(this.config.rootStateDir, state.agentId),
|
|
373
|
+
claudeHome: agentClaudeHomeFor(this.config.rootClaudeHome, state.agentId),
|
|
374
|
+
};
|
|
375
|
+
const env = adapter.buildEnv(
|
|
376
|
+
{ ...process.env, PRLL_API_URL: this.config.apiUrl },
|
|
377
|
+
state.agentId, state.orgId, state.credential.api_key, dirs,
|
|
378
|
+
);
|
|
379
|
+
|
|
380
|
+
this.log.info(`spawning agent ${state.agentId} runtime=${state.runtimeType} bin=${adapter.bin} (attempt ${state.restartAttempts + 1})`);
|
|
381
|
+
const child = spawn(adapter.bin, [], {
|
|
382
|
+
env,
|
|
383
|
+
stdio: ["ignore", "inherit", "inherit"],
|
|
384
|
+
detached: false,
|
|
385
|
+
});
|
|
386
|
+
state.child = child;
|
|
387
|
+
|
|
388
|
+
let childSettled = false;
|
|
389
|
+
const settleChild = (event: "close" | "error", code: number | null, signal: NodeJS.Signals | null, err?: Error) => {
|
|
390
|
+
if (childSettled) return;
|
|
391
|
+
childSettled = true;
|
|
392
|
+
if (state.child !== child) return;
|
|
393
|
+
const wasShutting = state.shuttingDown;
|
|
394
|
+
state.child = null;
|
|
395
|
+
if (err) {
|
|
396
|
+
this.log.error(`agent ${state.agentId} child ${event}: ${String(err)}${wasShutting ? " (shutting down)" : ""}`);
|
|
397
|
+
} else {
|
|
398
|
+
this.log.info(
|
|
399
|
+
`agent ${state.agentId} exited code=${code ?? "null"} signal=${signal ?? "null"}${wasShutting ? " (shutting down)" : ""}`,
|
|
400
|
+
);
|
|
401
|
+
}
|
|
402
|
+
if (wasShutting || !this.running) return;
|
|
403
|
+
const delay = Math.min(
|
|
404
|
+
this.config.restartBackoffMs * Math.pow(2, state.restartAttempts),
|
|
405
|
+
this.config.restartBackoffMaxMs,
|
|
406
|
+
);
|
|
407
|
+
state.restartAttempts += 1;
|
|
408
|
+
this.log.warn(`agent ${state.agentId} will restart in ${delay}ms`);
|
|
409
|
+
state.restartTimer = setTimeout(() => {
|
|
410
|
+
state.restartTimer = null;
|
|
411
|
+
this.startChild(state);
|
|
412
|
+
}, delay);
|
|
413
|
+
};
|
|
414
|
+
|
|
415
|
+
child.once("error", (err) => settleChild("error", null, null, err));
|
|
416
|
+
child.once("close", (code, signal) => settleChild("close", code, signal));
|
|
417
|
+
|
|
418
|
+
setTimeout(() => {
|
|
419
|
+
if (state.child === child) {
|
|
420
|
+
state.restartAttempts = 0;
|
|
421
|
+
}
|
|
422
|
+
}, Math.max(this.config.restartBackoffMs, 30_000));
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
private async terminateChild(state: ChildState): Promise<void> {
|
|
426
|
+
const child = state.child;
|
|
427
|
+
if (!child) return;
|
|
428
|
+
return new Promise<void>((resolve) => {
|
|
429
|
+
const onExit = () => resolve();
|
|
430
|
+
child.once("exit", onExit);
|
|
431
|
+
try {
|
|
432
|
+
child.kill("SIGTERM");
|
|
433
|
+
} catch (err) {
|
|
434
|
+
this.log.warn(`SIGTERM ${state.agentId} threw: ${String(err)}`);
|
|
435
|
+
child.off("exit", onExit);
|
|
436
|
+
resolve();
|
|
437
|
+
return;
|
|
438
|
+
}
|
|
439
|
+
const hardKill = setTimeout(() => {
|
|
440
|
+
try {
|
|
441
|
+
child.kill("SIGKILL");
|
|
442
|
+
} catch {
|
|
443
|
+
/* already gone */
|
|
444
|
+
}
|
|
445
|
+
}, 10_000);
|
|
446
|
+
child.once("exit", () => clearTimeout(hardKill));
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
private ensureSharedCredentialLink(agentClaudeHome: string, agentId: string): void {
|
|
451
|
+
const sharedCredentials = path.resolve(sharedClaudeCredentialsFileFor(this.config.rootClaudeHome));
|
|
452
|
+
const agentCredentials = agentClaudeCredentialsFileFor(agentClaudeHome);
|
|
453
|
+
const agentCredentialsDir = path.dirname(agentCredentials);
|
|
454
|
+
|
|
455
|
+
fs.mkdirSync(path.dirname(sharedCredentials), { recursive: true });
|
|
456
|
+
fs.mkdirSync(agentCredentialsDir, { recursive: true });
|
|
457
|
+
|
|
458
|
+
try {
|
|
459
|
+
const existing = fs.lstatSync(agentCredentials);
|
|
460
|
+
if (existing.isSymbolicLink()) {
|
|
461
|
+
const currentTarget = fs.readlinkSync(agentCredentials);
|
|
462
|
+
if (path.resolve(agentCredentialsDir, currentTarget) === sharedCredentials) {
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
fs.unlinkSync(agentCredentials);
|
|
466
|
+
} else if (existing.isDirectory()) {
|
|
467
|
+
this.log.warn(`agent ${agentId}: credential path is a directory, cannot link ${agentCredentials}`);
|
|
468
|
+
return;
|
|
469
|
+
} else {
|
|
470
|
+
fs.unlinkSync(agentCredentials);
|
|
471
|
+
}
|
|
472
|
+
} catch (err) {
|
|
473
|
+
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
|
474
|
+
throw err;
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
fs.symlinkSync(sharedCredentials, agentCredentials);
|
|
479
|
+
}
|
|
480
|
+
}
|