@parall/daemon 1.27.0

package/dist/supervisor.js

@@ -0,0 +1,415 @@
+import { spawn } from "node:child_process";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { ParallWs } from "@parall/sdk";
+import { agentClaudeCredentialsFileFor, agentClaudeHomeFor, agentStateDirFor, agentWorkspaceDirFor, sharedClaudeCredentialsFileFor, } from "./config.js";
+import { assertAgentKey, getRuntimeAdapter } from "./runtimes.js";
+/**
+ * Sleep that wakes early on abort. Returns true if the full delay elapsed,
+ * false if aborted. Used by bootstrap retry and the outer keepalive in
+ * `runForever` so SIGTERM during a long backoff doesn't stall shutdown.
+ */
+function sleepCancellable(ms, signal) {
+    if (signal.aborted)
+        return Promise.resolve(false);
+    return new Promise((resolve) => {
+        const timer = setTimeout(() => {
+            signal.removeEventListener("abort", onAbort);
+            resolve(true);
+        }, ms);
+        const onAbort = () => {
+            clearTimeout(timer);
+            resolve(false);
+        };
+        signal.addEventListener("abort", onAbort, { once: true });
+    });
+}
+export { sleepCancellable };
+/**
+ * Supervisor orchestrates one mck_ machine bearer into N
+ * `parall-claude-agent` subprocesses, one per AttachedAgent.
+ *
+ * Lifecycle is WS event-driven:
+ *   1. Bootstrap: confirm machine identity with retry.
+ *   2. Full reconcile: list attached agents, diff with children, spawn/kill.
+ *   3. Connect WS: receive machine.agent.attached / .detached / .stop
+ *      events for incremental updates; full reconcile on every reconnect
+ *      (machine.hello) to catch events missed while disconnected.
+ */
+export class DaemonSupervisor {
+    config;
+    client;
+    log;
+    children = new Map();
+    ws = null;
+    running = false;
+    machineOrgId = null;
+    stopResolve = null;
+    constructor(config, client, log) {
+        this.config = config;
+        this.client = client;
+        this.log = log;
+    }
+    /** Start the supervisor. Returns a promise that resolves on `stop()`. */
+    async run(signal) {
+        if (this.running)
+            throw new Error("supervisor already running");
+        this.running = true;
+        const onAbort = () => {
+            this.stop().catch((err) => this.log.error(`stop() failed: ${String(err)}`));
+        };
+        signal.addEventListener("abort", onAbort, { once: true });
+        try {
+            if (!(await this.bootstrapWithRetry(signal))) {
+                signal.removeEventListener("abort", onAbort);
+                this.running = false;
+                return;
+            }
+        }
+        catch (err) {
+            signal.removeEventListener("abort", onAbort);
+            this.running = false;
+            throw err;
+        }
+        await this.fullReconcile();
+        this.ws = new ParallWs({
+            getTicket: () => this.client.getMachineWsTicket(),
+            wsUrl: this.config.wsUrl,
+            reconnect: true,
+        });
+        this.ws.on("machine.hello", (_data) => {
+            this.log.info("machine WS connected (machine.hello)");
+            void this.fullReconcile();
+        });
+        this.ws.on("machine.agent.attached", (data) => {
+            this.log.info(`WS: agent ${data.agent_id} attached`);
+            void this.handleAgentAttached(data.agent_id);
+        });
+        this.ws.on("machine.agent.detached", (data) => {
+            this.log.info(`WS: agent ${data.agent_id} detached`);
+            void this.handleAgentDetached(data.agent_id);
+        });
+        this.ws.on("machine.stop", (data) => {
+            this.log.info(`WS: machine.stop received (reason=${data.reason ?? "none"})`);
+            void this.stop();
+        });
+        this.ws.onStateChange((state) => {
+            if (state === "disconnected" || state === "reconnecting") {
+                this.log.warn(`machine WS state: ${state}`);
+            }
+        });
+        await this.ws.connect();
+        await new Promise((resolve) => {
+            this.stopResolve = resolve;
+        });
+        signal.removeEventListener("abort", onAbort);
+    }
+    /** Disconnect WS, cancel timers, SIGTERM all children, await exit. */
+    async stop() {
+        if (!this.running)
+            return;
+        this.running = false;
+        if (this.ws) {
+            this.ws.disconnect();
+            this.ws = null;
+        }
+        const exits = [];
+        for (const state of this.children.values()) {
+            state.shuttingDown = true;
+            if (state.restartTimer) {
+                clearTimeout(state.restartTimer);
+                state.restartTimer = null;
+            }
+            exits.push(this.terminateChild(state));
+        }
+        await Promise.allSettled(exits);
+        this.children.clear();
+        this.log.info("daemon supervisor stopped");
+        if (this.stopResolve) {
+            this.stopResolve();
+            this.stopResolve = null;
+        }
+    }
+    // ---- Bootstrap (resilient identity probe) ----
+    async bootstrapWithRetry(signal) {
+        let attempt = 0;
+        while (this.running && !signal.aborted) {
+            try {
+                const machine = await this.client.getMachineSelf();
+                this.machineOrgId = machine.org_id;
+                this.log.info(`daemon online — machine_id=${machine.id} org=${machine.org_id} label=${machine.label}`);
+                return true;
+            }
+            catch (err) {
+                if (this.config.bootstrapBackoffMs === 0) {
+                    this.log.error(`getMachineSelf failed: ${String(err)} (fail-fast mode)`);
+                    throw err;
+                }
+                const delay = Math.min(this.config.bootstrapBackoffMs * Math.pow(2, attempt), this.config.bootstrapBackoffMaxMs);
+                attempt += 1;
+                this.log.warn(`getMachineSelf failed (attempt ${attempt}): ${String(err)} — retrying in ${delay}ms`);
+                const slept = await sleepCancellable(delay, signal);
+                if (!slept)
+                    return false;
+            }
+        }
+        return false;
+    }
+    // ---- Full reconcile (HTTP-based, used on boot + WS reconnect) ----
+    // Safe to run concurrently with WS event handlers: JS single-threaded
+    // event loop guarantees no mid-statement interleaving, and both
+    // handleAgentAttached/Detached guard on children.has()/get() so a WS
+    // event between the HTTP fetch and the spawn/kill loop is a no-op.
+    async fullReconcile() {
+        if (!this.running)
+            return;
+        let attached;
+        try {
+            attached = await this.client.listAttachedAgents();
+        }
+        catch (err) {
+            this.log.warn(`fullReconcile: listAttachedAgents failed: ${String(err)}`);
+            return;
+        }
+        const seen = new Set();
+        for (const a of attached) {
+            const userId = a.user?.id ?? a.profile.user_id;
+            if (!userId) {
+                this.log.warn(`skipping attached entry with no user_id (profile=${JSON.stringify(a.profile)})`);
+                continue;
+            }
+            if (a.user && a.user.status !== "active") {
+                this.log.info(`agent ${userId} not active (status=${a.user.status}) — skipping`);
+                continue;
+            }
+            seen.add(userId);
+            const existing = this.children.get(userId);
+            if (!existing) {
+                const orgId = this.machineOrgId;
+                if (!orgId) {
+                    this.log.warn(`agent ${userId}: no org_id available yet; skipping`);
+                    continue;
+                }
+                await this.spawnAgent(userId, orgId, a);
+            }
+            else if (!existing.child && !existing.restartTimer && !existing.shuttingDown) {
+                await this.restartChildNow(existing, "reconcile found no live child");
+            }
+        }
+        for (const [userId, state] of this.children) {
+            if (!seen.has(userId)) {
+                this.log.info(`agent ${userId} detached (reconcile) — terminating subprocess`);
+                state.shuttingDown = true;
+                if (state.restartTimer) {
+                    clearTimeout(state.restartTimer);
+                    state.restartTimer = null;
+                }
+                await this.terminateChild(state);
+                this.children.delete(userId);
+            }
+        }
+    }
+    // ---- WS event handlers (incremental) ----
+    async handleAgentAttached(agentId) {
+        if (this.children.has(agentId))
+            return;
+        let attached;
+        try {
+            attached = await this.client.listAttachedAgents();
+        }
+        catch (err) {
+            this.log.warn(`handleAgentAttached: listAttachedAgents failed: ${String(err)}`);
+            return;
+        }
+        const entry = attached.find((a) => (a.user?.id ?? a.profile.user_id) === agentId);
+        if (!entry) {
+            this.log.warn(`handleAgentAttached: agent ${agentId} not found in attached list`);
+            return;
+        }
+        if (entry.user && entry.user.status !== "active") {
+            this.log.info(`agent ${agentId} not active (status=${entry.user.status}) — skipping`);
+            return;
+        }
+        const orgId = this.machineOrgId;
+        if (!orgId) {
+            this.log.warn(`agent ${agentId}: no org_id available yet; skipping`);
+            return;
+        }
+        await this.spawnAgent(agentId, orgId, entry);
+    }
+    async handleAgentDetached(agentId) {
+        const state = this.children.get(agentId);
+        if (!state)
+            return;
+        this.log.info(`agent ${agentId} detached — terminating subprocess`);
+        state.shuttingDown = true;
+        if (state.restartTimer) {
+            clearTimeout(state.restartTimer);
+            state.restartTimer = null;
+        }
+        await this.terminateChild(state);
+        this.children.delete(agentId);
+    }
+    // ---- Spawn / restart ----
+    async restartChildNow(state, reason) {
+        if (!this.running || state.shuttingDown || state.child || state.restartTimer) {
+            return;
+        }
+        try {
+            state.credential = await this.client.mintLaunchCredential(state.agentId);
+            state.restartAttempts = 0;
+            this.log.info(`agent ${state.agentId}: restarting child (${reason})`);
+            this.startChild(state);
+        }
+        catch (err) {
+            this.log.warn(`agent ${state.agentId}: restart mint failed (${reason}): ${String(err)}`);
+        }
+    }
+    async spawnAgent(agentId, orgId, attached) {
+        let credential;
+        try {
+            credential = await this.client.mintLaunchCredential(agentId);
+        }
+        catch (err) {
+            this.log.error(`mintLaunchCredential ${agentId} failed: ${String(err)}`);
+            return;
+        }
+        const stateDir = agentStateDirFor(this.config.rootStateDir, agentId);
+        const workspaceDir = agentWorkspaceDirFor(this.config.rootStateDir, agentId);
+        const claudeHome = agentClaudeHomeFor(this.config.rootClaudeHome, agentId);
+        try {
+            fs.mkdirSync(stateDir, { recursive: true });
+            fs.mkdirSync(workspaceDir, { recursive: true });
+            fs.mkdirSync(claudeHome, { recursive: true });
+            this.ensureSharedCredentialLink(claudeHome, agentId);
+        }
+        catch (err) {
+            this.log.warn(`mkdir agent dirs (${agentId}) failed: ${String(err)}`);
+        }
+        const runtimeType = attached.profile.runtime_type ?? "claude-code";
+        const state = {
+            agentId,
+            orgId,
+            runtimeType,
+            child: null,
+            credential,
+            restartAttempts: 0,
+            restartTimer: null,
+            shuttingDown: false,
+        };
+        this.children.set(agentId, state);
+        this.startChild(state);
+    }
+    startChild(state) {
+        if (state.shuttingDown || !this.running)
+            return;
+        if (!state.credential) {
+            this.log.error(`startChild ${state.agentId}: no credential — bug`);
+            return;
+        }
+        assertAgentKey(state.credential.api_key);
+        const adapter = getRuntimeAdapter(state.runtimeType);
+        const dirs = {
+            stateDir: agentStateDirFor(this.config.rootStateDir, state.agentId),
+            workspaceDir: agentWorkspaceDirFor(this.config.rootStateDir, state.agentId),
+            claudeHome: agentClaudeHomeFor(this.config.rootClaudeHome, state.agentId),
+        };
+        const env = adapter.buildEnv({ ...process.env, PRLL_API_URL: this.config.apiUrl }, state.agentId, state.orgId, state.credential.api_key, dirs);
+        this.log.info(`spawning agent ${state.agentId} runtime=${state.runtimeType} bin=${adapter.bin} (attempt ${state.restartAttempts + 1})`);
+        const child = spawn(adapter.bin, [], {
+            env,
+            stdio: ["ignore", "inherit", "inherit"],
+            detached: false,
+        });
+        state.child = child;
+        let childSettled = false;
+        const settleChild = (event, code, signal, err) => {
+            if (childSettled)
+                return;
+            childSettled = true;
+            if (state.child !== child)
+                return;
+            const wasShutting = state.shuttingDown;
+            state.child = null;
+            if (err) {
+                this.log.error(`agent ${state.agentId} child ${event}: ${String(err)}${wasShutting ? " (shutting down)" : ""}`);
+            }
+            else {
+                this.log.info(`agent ${state.agentId} exited code=${code ?? "null"} signal=${signal ?? "null"}${wasShutting ? " (shutting down)" : ""}`);
+            }
+            if (wasShutting || !this.running)
+                return;
+            const delay = Math.min(this.config.restartBackoffMs * Math.pow(2, state.restartAttempts), this.config.restartBackoffMaxMs);
+            state.restartAttempts += 1;
+            this.log.warn(`agent ${state.agentId} will restart in ${delay}ms`);
+            state.restartTimer = setTimeout(() => {
+                state.restartTimer = null;
+                this.startChild(state);
+            }, delay);
+        };
+        child.once("error", (err) => settleChild("error", null, null, err));
+        child.once("close", (code, signal) => settleChild("close", code, signal));
+        setTimeout(() => {
+            if (state.child === child) {
+                state.restartAttempts = 0;
+            }
+        }, Math.max(this.config.restartBackoffMs, 30_000));
+    }
+    async terminateChild(state) {
+        const child = state.child;
+        if (!child)
+            return;
+        return new Promise((resolve) => {
+            const onExit = () => resolve();
+            child.once("exit", onExit);
+            try {
+                child.kill("SIGTERM");
+            }
+            catch (err) {
+                this.log.warn(`SIGTERM ${state.agentId} threw: ${String(err)}`);
+                child.off("exit", onExit);
+                resolve();
+                return;
+            }
+            const hardKill = setTimeout(() => {
+                try {
+                    child.kill("SIGKILL");
+                }
+                catch {
+                    /* already gone */
+                }
+            }, 10_000);
+            child.once("exit", () => clearTimeout(hardKill));
+        });
+    }
+    ensureSharedCredentialLink(agentClaudeHome, agentId) {
+        const sharedCredentials = path.resolve(sharedClaudeCredentialsFileFor(this.config.rootClaudeHome));
+        const agentCredentials = agentClaudeCredentialsFileFor(agentClaudeHome);
+        const agentCredentialsDir = path.dirname(agentCredentials);
+        fs.mkdirSync(path.dirname(sharedCredentials), { recursive: true });
+        fs.mkdirSync(agentCredentialsDir, { recursive: true });
+        try {
+            const existing = fs.lstatSync(agentCredentials);
+            if (existing.isSymbolicLink()) {
+                const currentTarget = fs.readlinkSync(agentCredentials);
+                if (path.resolve(agentCredentialsDir, currentTarget) === sharedCredentials) {
+                    return;
+                }
+                fs.unlinkSync(agentCredentials);
+            }
+            else if (existing.isDirectory()) {
+                this.log.warn(`agent ${agentId}: credential path is a directory, cannot link ${agentCredentials}`);
+                return;
+            }
+            else {
+                fs.unlinkSync(agentCredentials);
+            }
+        }
+        catch (err) {
+            if (err.code !== "ENOENT") {
+                throw err;
+            }
+        }
+        fs.symlinkSync(sharedCredentials, agentCredentials);
+    }
+}

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@parall/daemon",
+  "version": "1.27.0",
+  "description": "Per-host supervisor for daemon-mode Parall Machines — fans one mck_ bearer out into N agent-runtime subprocesses, one per attached agent",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/parall-hq/parall-mono",
+    "directory": "ts/daemon"
+  },
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "bin": {
+    "parall-daemon": "./dist/index.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@parall/sdk": "1.27.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "typescript": "^5.7.0"
+  },
+  "scripts": {
+    "build": "tsc -b",
+    "start": "node dist/index.js"
+  }
+}

package/src/config.ts

@@ -0,0 +1,142 @@
+import * as os from "node:os";
+import * as path from "node:path";
+
+/**
+ * Daemon-mode env. The daemon authenticates to the Parall API with an
+ * `mck_*` bearer token (PRLL_API_KEY) and supervises N per-agent
+ * `parall-claude-agent` subprocesses on this host.
+ *
+ * Compared to claude-agent's config:
+ *   - `apiKey` is an mck_ bearer (Machine-scoped), not an agk_
+ *   - there is no PRLL_ORG_ID — the Machine row's org_id is implicit;
+ *     each spawned per-agent subprocess is given its own PRLL_ORG_ID
+ *     resolved from the AttachedAgent's profile / org context
+ *   - per-agent state lives at `<rootStateDir>/agents/<agent_id>/`
+ */
+export type ClaudeDaemonConfig = {
+  apiUrl: string;
+  /** mck_*-prefixed Machine bearer. */
+  apiKey: string;
+  /** @deprecated Superseded by RuntimeAdapter pattern (runtimes.ts). Kept for backward compat / fallback. */
+  agentBin: string;
+  /** Root for per-agent state dirs. Each agent gets `<rootStateDir>/agents/<agent_id>`. */
+  rootStateDir: string;
+  /** Shared host home root. Per-agent HOME dirs live under `<rootClaudeHome>/agents/<agent_id>`. */
+  rootClaudeHome: string;
+  /** Optional WS URL override for the machine control-plane WebSocket. */
+  wsUrl?: string;
+  /** @deprecated Superseded by WS event-driven model; kept for backward compat. */
+  pollIntervalMs: number;
+  /** @deprecated Superseded by WS event-driven model; kept for backward compat. */
+  heartbeatIntervalMs: number;
+  /** Backoff base after a subprocess crash, in ms. */
+  restartBackoffMs: number;
+  /** Hard cap on parallel restart attempts per agent before giving up for a cycle. */
+  restartBackoffMaxMs: number;
+  /**
+   * Bootstrap retry: when `getMachineSelf` fails on startup (network blip,
+   * mck_ key not yet propagated, server warming up), wait this long and
+   * retry instead of crashing the daemon. Exponential backoff up to
+   * `bootstrapBackoffMaxMs`. Set 0 to disable (fail-fast on first error).
+   */
+  bootstrapBackoffMs: number;
+  bootstrapBackoffMaxMs: number;
+  /**
+   * Outer supervisor keepalive: if `supervisor.run()` rejects with an
+   * unexpected error, wait this long and reinstantiate. Same exp-backoff
+   * shape as bootstrap. Set 0 to disable (let main() exit, rely on K8s/PID-1).
+   */
+  supervisorRestartBackoffMs: number;
+  supervisorRestartBackoffMaxMs: number;
+};
+
+function requireEnv(env: NodeJS.ProcessEnv, name: string): string {
+  const value = env[name]?.trim();
+  if (!value) {
+    throw new Error(`Missing required env var: ${name}`);
+  }
+  return value;
+}
+
+function resolvePath(value: string): string {
+  return path.isAbsolute(value) ? value : path.resolve(process.cwd(), value);
+}
+
+function parseMs(value: string | undefined, fallback: number): number {
+  if (!value) return fallback;
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? n : fallback;
+}
+
+/** Like parseMs but allows 0 (operator opt-out). */
+function parseMsAllowZero(value: string | undefined, fallback: number): number {
+  if (value === undefined) return fallback;
+  const n = Number(value);
+  return Number.isFinite(n) && n >= 0 ? n : fallback;
+}
+
+export function resolveClaudeDaemonConfig(env: NodeJS.ProcessEnv = process.env): ClaudeDaemonConfig {
+  const apiUrl = requireEnv(env, "PRLL_API_URL");
+  const apiKey = requireEnv(env, "PRLL_API_KEY");
+  if (!apiKey.startsWith("mck_")) {
+    // Fatal startup validation: the daemon must never run with an agent or
+    // human key because child launch credentials are minted from this bearer.
+    throw new Error(
+      `PRLL_API_KEY does not look like a Machine bearer (expected prefix "mck_"). ` +
+        `Daemon mode requires a machine-scoped key issued via POST /machines/{id}/keys.`,
+    );
+  }
+  const rootClaudeHome = resolvePath(env.PRLL_CLAUDE_HOME?.trim() || env.HOME || os.homedir());
+  const rootStateDir = resolvePath(env.PRLL_CLAUDE_STATE_DIR?.trim() || path.join(rootClaudeHome, ".parall-agent"));
+
+  return {
+    apiUrl,
+    apiKey,
+    agentBin: env.PRLL_CLAUDE_AGENT_BIN?.trim() || "parall-claude-agent",
+    rootStateDir,
+    rootClaudeHome,
+    wsUrl: env.PRLL_WS_URL?.trim() || undefined,
+    pollIntervalMs: parseMs(env.PRLL_DAEMON_POLL_INTERVAL_MS, 30_000),
+    heartbeatIntervalMs: parseMs(env.PRLL_DAEMON_HEARTBEAT_INTERVAL_MS, 30_000),
+    restartBackoffMs: parseMs(env.PRLL_DAEMON_RESTART_BACKOFF_MS, 5_000),
+    restartBackoffMaxMs: parseMs(env.PRLL_DAEMON_RESTART_BACKOFF_MAX_MS, 5 * 60_000),
+    bootstrapBackoffMs: parseMsAllowZero(env.PRLL_DAEMON_BOOTSTRAP_BACKOFF_MS, 2_000),
+    bootstrapBackoffMaxMs: parseMs(env.PRLL_DAEMON_BOOTSTRAP_BACKOFF_MAX_MS, 60_000),
+    supervisorRestartBackoffMs: parseMsAllowZero(env.PRLL_DAEMON_SUPERVISOR_RESTART_BACKOFF_MS, 5_000),
+    supervisorRestartBackoffMaxMs: parseMs(env.PRLL_DAEMON_SUPERVISOR_RESTART_BACKOFF_MAX_MS, 5 * 60_000),
+  };
+}
+
+function assertSafeAgentId(agentId: string): string {
+  if (!/^[A-Za-z0-9_-]+$/.test(agentId)) {
+    throw new Error(`Invalid agentId for filesystem path: ${agentId}`);
+  }
+  return agentId;
+}
+
+/** Per-agent state dir under the shared host volume. */
+export function agentStateDirFor(rootStateDir: string, agentId: string): string {
+  return path.join(rootStateDir, "agents", assertSafeAgentId(agentId));
+}
+
+/** Per-agent HOME dir. Claude Code stores project/session state under
+ * `${HOME}/.claude`, so each agent gets its own HOME root while the daemon
+ * links shared OAuth credentials into that `.claude` directory. */
+export function agentClaudeHomeFor(rootClaudeHome: string, agentId: string): string {
+  return path.join(rootClaudeHome, "agents", assertSafeAgentId(agentId));
+}
+
+/** Shared Claude Code OAuth credential written by server-side runtime auth. */
+export function sharedClaudeCredentialsFileFor(rootClaudeHome: string): string {
+  return path.join(rootClaudeHome, ".claude", ".credentials.json");
+}
+
+/** Per-agent credential location inside that agent's isolated HOME. */
+export function agentClaudeCredentialsFileFor(agentClaudeHome: string): string {
+  return path.join(agentClaudeHome, ".claude", ".credentials.json");
+}
+
+/** Per-agent workspace dir where the agent runs git commands. */
+export function agentWorkspaceDirFor(rootStateDir: string, agentId: string): string {
+  return path.join(rootStateDir, "agents", assertSafeAgentId(agentId), "workspace");
+}