@parall/daemon 1.27.0 → 1.28.1

package/src/supervisor.ts

@@ -1,480 +0,0 @@
-import { spawn, type ChildProcess } from "node:child_process";
-import * as fs from "node:fs";
-import * as path from "node:path";
-import { ParallClient, ParallWs, type AttachedAgent, type LaunchCredentialResponse, type MachineAgentAttachedData, type MachineAgentDetachedData, type MachineStopData } from "@parall/sdk";
-import {
-  type ClaudeDaemonConfig,
-  agentClaudeCredentialsFileFor,
-  agentClaudeHomeFor,
-  agentStateDirFor,
-  agentWorkspaceDirFor,
-  sharedClaudeCredentialsFileFor,
-} from "./config.js";
-import { assertAgentKey, getRuntimeAdapter, type AgentDirs } from "./runtimes.js";
-
-export interface DaemonLogger {
-  info(msg: string): void;
-  warn(msg: string): void;
-  error(msg: string): void;
-}
-
-/**
- * Sleep that wakes early on abort. Returns true if the full delay elapsed,
- * false if aborted. Used by bootstrap retry and the outer keepalive in
- * `runForever` so SIGTERM during a long backoff doesn't stall shutdown.
- */
-function sleepCancellable(ms: number, signal: AbortSignal): Promise<boolean> {
-  if (signal.aborted) return Promise.resolve(false);
-  return new Promise<boolean>((resolve) => {
-    const timer = setTimeout(() => {
-      signal.removeEventListener("abort", onAbort);
-      resolve(true);
-    }, ms);
-    const onAbort = () => {
-      clearTimeout(timer);
-      resolve(false);
-    };
-    signal.addEventListener("abort", onAbort, { once: true });
-  });
-}
-
-export { sleepCancellable };
-
-/**
- * Per-agent runtime state held by the supervisor. Exactly one of these
- * exists per attached agent for the lifetime of an attachment; on detach
- * the entry is removed and the child SIGTERM'd.
- */
-interface ChildState {
-  agentId: string;
-  orgId: string;
-  /** Runtime type from the agent profile (e.g. "claude-code", "codex"). */
-  runtimeType: string;
-  child: ChildProcess | null;
-  credential: LaunchCredentialResponse | null;
-  restartAttempts: number;
-  restartTimer: NodeJS.Timeout | null;
-  shuttingDown: boolean;
-}
-
-/**
- * Supervisor orchestrates one mck_ machine bearer into N
- * `parall-claude-agent` subprocesses, one per AttachedAgent.
- *
- * Lifecycle is WS event-driven:
- *   1. Bootstrap: confirm machine identity with retry.
- *   2. Full reconcile: list attached agents, diff with children, spawn/kill.
- *   3. Connect WS: receive machine.agent.attached / .detached / .stop
- *      events for incremental updates; full reconcile on every reconnect
- *      (machine.hello) to catch events missed while disconnected.
- */
-export class DaemonSupervisor {
-  private readonly children = new Map<string, ChildState>();
-  private ws: ParallWs | null = null;
-  private running = false;
-  private machineOrgId: string | null = null;
-  private stopResolve: (() => void) | null = null;
-
-  constructor(
-    private readonly config: ClaudeDaemonConfig,
-    private readonly client: ParallClient,
-    private readonly log: DaemonLogger,
-  ) {}
-
-  /** Start the supervisor. Returns a promise that resolves on `stop()`. */
-  async run(signal: AbortSignal): Promise<void> {
-    if (this.running) throw new Error("supervisor already running");
-    this.running = true;
-
-    const onAbort = () => {
-      this.stop().catch((err) => this.log.error(`stop() failed: ${String(err)}`));
-    };
-    signal.addEventListener("abort", onAbort, { once: true });
-
-    try {
-      if (!(await this.bootstrapWithRetry(signal))) {
-        signal.removeEventListener("abort", onAbort);
-        this.running = false;
-        return;
-      }
-    } catch (err) {
-      signal.removeEventListener("abort", onAbort);
-      this.running = false;
-      throw err;
-    }
-
-    await this.fullReconcile();
-
-    this.ws = new ParallWs({
-      getTicket: () => this.client.getMachineWsTicket(),
-      wsUrl: this.config.wsUrl,
-      reconnect: true,
-    });
-
-    this.ws.on("machine.hello", (_data) => {
-      this.log.info("machine WS connected (machine.hello)");
-      void this.fullReconcile();
-    });
-
-    this.ws.on("machine.agent.attached", (data: MachineAgentAttachedData) => {
-      this.log.info(`WS: agent ${data.agent_id} attached`);
-      void this.handleAgentAttached(data.agent_id);
-    });
-
-    this.ws.on("machine.agent.detached", (data: MachineAgentDetachedData) => {
-      this.log.info(`WS: agent ${data.agent_id} detached`);
-      void this.handleAgentDetached(data.agent_id);
-    });
-
-    this.ws.on("machine.stop", (data: MachineStopData) => {
-      this.log.info(`WS: machine.stop received (reason=${data.reason ?? "none"})`);
-      void this.stop();
-    });
-
-    this.ws.onStateChange((state) => {
-      if (state === "disconnected" || state === "reconnecting") {
-        this.log.warn(`machine WS state: ${state}`);
-      }
-    });
-
-    await this.ws.connect();
-
-    await new Promise<void>((resolve) => {
-      this.stopResolve = resolve;
-    });
-    signal.removeEventListener("abort", onAbort);
-  }
-
-  /** Disconnect WS, cancel timers, SIGTERM all children, await exit. */
-  async stop(): Promise<void> {
-    if (!this.running) return;
-    this.running = false;
-
-    if (this.ws) {
-      this.ws.disconnect();
-      this.ws = null;
-    }
-
-    const exits: Promise<void>[] = [];
-    for (const state of this.children.values()) {
-      state.shuttingDown = true;
-      if (state.restartTimer) {
-        clearTimeout(state.restartTimer);
-        state.restartTimer = null;
-      }
-      exits.push(this.terminateChild(state));
-    }
-    await Promise.allSettled(exits);
-    this.children.clear();
-    this.log.info("daemon supervisor stopped");
-    if (this.stopResolve) {
-      this.stopResolve();
-      this.stopResolve = null;
-    }
-  }
-
-  // ---- Bootstrap (resilient identity probe) ----
-
-  private async bootstrapWithRetry(signal: AbortSignal): Promise<boolean> {
-    let attempt = 0;
-    while (this.running && !signal.aborted) {
-      try {
-        const machine = await this.client.getMachineSelf();
-        this.machineOrgId = machine.org_id;
-        this.log.info(
-          `daemon online — machine_id=${machine.id} org=${machine.org_id} label=${machine.label}`,
-        );
-        return true;
-      } catch (err) {
-        if (this.config.bootstrapBackoffMs === 0) {
-          this.log.error(`getMachineSelf failed: ${String(err)} (fail-fast mode)`);
-          throw err;
-        }
-        const delay = Math.min(
-          this.config.bootstrapBackoffMs * Math.pow(2, attempt),
-          this.config.bootstrapBackoffMaxMs,
-        );
-        attempt += 1;
-        this.log.warn(
-          `getMachineSelf failed (attempt ${attempt}): ${String(err)} — retrying in ${delay}ms`,
-        );
-        const slept = await sleepCancellable(delay, signal);
-        if (!slept) return false;
-      }
-    }
-    return false;
-  }
-
-  // ---- Full reconcile (HTTP-based, used on boot + WS reconnect) ----
-  // Safe to run concurrently with WS event handlers: JS single-threaded
-  // event loop guarantees no mid-statement interleaving, and both
-  // handleAgentAttached/Detached guard on children.has()/get() so a WS
-  // event between the HTTP fetch and the spawn/kill loop is a no-op.
-
-  private async fullReconcile(): Promise<void> {
-    if (!this.running) return;
-
-    let attached: AttachedAgent[];
-    try {
-      attached = await this.client.listAttachedAgents();
-    } catch (err) {
-      this.log.warn(`fullReconcile: listAttachedAgents failed: ${String(err)}`);
-      return;
-    }
-
-    const seen = new Set<string>();
-    for (const a of attached) {
-      const userId = a.user?.id ?? a.profile.user_id;
-      if (!userId) {
-        this.log.warn(`skipping attached entry with no user_id (profile=${JSON.stringify(a.profile)})`);
-        continue;
-      }
-      if (a.user && a.user.status !== "active") {
-        this.log.info(`agent ${userId} not active (status=${a.user.status}) — skipping`);
-        continue;
-      }
-      seen.add(userId);
-
-      const existing = this.children.get(userId);
-      if (!existing) {
-        const orgId = this.machineOrgId;
-        if (!orgId) {
-          this.log.warn(`agent ${userId}: no org_id available yet; skipping`);
-          continue;
-        }
-        await this.spawnAgent(userId, orgId, a);
-      } else if (!existing.child && !existing.restartTimer && !existing.shuttingDown) {
-        await this.restartChildNow(existing, "reconcile found no live child");
-      }
-    }
-
-    for (const [userId, state] of this.children) {
-      if (!seen.has(userId)) {
-        this.log.info(`agent ${userId} detached (reconcile) — terminating subprocess`);
-        state.shuttingDown = true;
-        if (state.restartTimer) {
-          clearTimeout(state.restartTimer);
-          state.restartTimer = null;
-        }
-        await this.terminateChild(state);
-        this.children.delete(userId);
-      }
-    }
-  }
-
-  // ---- WS event handlers (incremental) ----
-
-  private async handleAgentAttached(agentId: string): Promise<void> {
-    if (this.children.has(agentId)) return;
-
-    let attached: AttachedAgent[];
-    try {
-      attached = await this.client.listAttachedAgents();
-    } catch (err) {
-      this.log.warn(`handleAgentAttached: listAttachedAgents failed: ${String(err)}`);
-      return;
-    }
-
-    const entry = attached.find((a) => (a.user?.id ?? a.profile.user_id) === agentId);
-    if (!entry) {
-      this.log.warn(`handleAgentAttached: agent ${agentId} not found in attached list`);
-      return;
-    }
-    if (entry.user && entry.user.status !== "active") {
-      this.log.info(`agent ${agentId} not active (status=${entry.user.status}) — skipping`);
-      return;
-    }
-
-    const orgId = this.machineOrgId;
-    if (!orgId) {
-      this.log.warn(`agent ${agentId}: no org_id available yet; skipping`);
-      return;
-    }
-    await this.spawnAgent(agentId, orgId, entry);
-  }
-
-  private async handleAgentDetached(agentId: string): Promise<void> {
-    const state = this.children.get(agentId);
-    if (!state) return;
-
-    this.log.info(`agent ${agentId} detached — terminating subprocess`);
-    state.shuttingDown = true;
-    if (state.restartTimer) {
-      clearTimeout(state.restartTimer);
-      state.restartTimer = null;
-    }
-    await this.terminateChild(state);
-    this.children.delete(agentId);
-  }
-
-  // ---- Spawn / restart ----
-
-  private async restartChildNow(state: ChildState, reason: string): Promise<void> {
-    if (!this.running || state.shuttingDown || state.child || state.restartTimer) {
-      return;
-    }
-    try {
-      state.credential = await this.client.mintLaunchCredential(state.agentId);
-      state.restartAttempts = 0;
-      this.log.info(`agent ${state.agentId}: restarting child (${reason})`);
-      this.startChild(state);
-    } catch (err) {
-      this.log.warn(`agent ${state.agentId}: restart mint failed (${reason}): ${String(err)}`);
-    }
-  }
-
-  private async spawnAgent(agentId: string, orgId: string, attached: AttachedAgent): Promise<void> {
-    let credential: LaunchCredentialResponse;
-    try {
-      credential = await this.client.mintLaunchCredential(agentId);
-    } catch (err) {
-      this.log.error(`mintLaunchCredential ${agentId} failed: ${String(err)}`);
-      return;
-    }
-    const stateDir = agentStateDirFor(this.config.rootStateDir, agentId);
-    const workspaceDir = agentWorkspaceDirFor(this.config.rootStateDir, agentId);
-    const claudeHome = agentClaudeHomeFor(this.config.rootClaudeHome, agentId);
-    try {
-      fs.mkdirSync(stateDir, { recursive: true });
-      fs.mkdirSync(workspaceDir, { recursive: true });
-      fs.mkdirSync(claudeHome, { recursive: true });
-      this.ensureSharedCredentialLink(claudeHome, agentId);
-    } catch (err) {
-      this.log.warn(`mkdir agent dirs (${agentId}) failed: ${String(err)}`);
-    }
-
-    const runtimeType = attached.profile.runtime_type ?? "claude-code";
-    const state: ChildState = {
-      agentId,
-      orgId,
-      runtimeType,
-      child: null,
-      credential,
-      restartAttempts: 0,
-      restartTimer: null,
-      shuttingDown: false,
-    };
-    this.children.set(agentId, state);
-    this.startChild(state);
-  }
-
-  private startChild(state: ChildState): void {
-    if (state.shuttingDown || !this.running) return;
-    if (!state.credential) {
-      this.log.error(`startChild ${state.agentId}: no credential — bug`);
-      return;
-    }
-
-    assertAgentKey(state.credential.api_key);
-    const adapter = getRuntimeAdapter(state.runtimeType);
-    const dirs: AgentDirs = {
-      stateDir: agentStateDirFor(this.config.rootStateDir, state.agentId),
-      workspaceDir: agentWorkspaceDirFor(this.config.rootStateDir, state.agentId),
-      claudeHome: agentClaudeHomeFor(this.config.rootClaudeHome, state.agentId),
-    };
-    const env = adapter.buildEnv(
-      { ...process.env, PRLL_API_URL: this.config.apiUrl },
-      state.agentId, state.orgId, state.credential.api_key, dirs,
-    );
-
-    this.log.info(`spawning agent ${state.agentId} runtime=${state.runtimeType} bin=${adapter.bin} (attempt ${state.restartAttempts + 1})`);
-    const child = spawn(adapter.bin, [], {
-      env,
-      stdio: ["ignore", "inherit", "inherit"],
-      detached: false,
-    });
-    state.child = child;
-
-    let childSettled = false;
-    const settleChild = (event: "close" | "error", code: number | null, signal: NodeJS.Signals | null, err?: Error) => {
-      if (childSettled) return;
-      childSettled = true;
-      if (state.child !== child) return;
-      const wasShutting = state.shuttingDown;
-      state.child = null;
-      if (err) {
-        this.log.error(`agent ${state.agentId} child ${event}: ${String(err)}${wasShutting ? " (shutting down)" : ""}`);
-      } else {
-        this.log.info(
-          `agent ${state.agentId} exited code=${code ?? "null"} signal=${signal ?? "null"}${wasShutting ? " (shutting down)" : ""}`,
-        );
-      }
-      if (wasShutting || !this.running) return;
-      const delay = Math.min(
-        this.config.restartBackoffMs * Math.pow(2, state.restartAttempts),
-        this.config.restartBackoffMaxMs,
-      );
-      state.restartAttempts += 1;
-      this.log.warn(`agent ${state.agentId} will restart in ${delay}ms`);
-      state.restartTimer = setTimeout(() => {
-        state.restartTimer = null;
-        this.startChild(state);
-      }, delay);
-    };
-
-    child.once("error", (err) => settleChild("error", null, null, err));
-    child.once("close", (code, signal) => settleChild("close", code, signal));
-
-    setTimeout(() => {
-      if (state.child === child) {
-        state.restartAttempts = 0;
-      }
-    }, Math.max(this.config.restartBackoffMs, 30_000));
-  }
-
-  private async terminateChild(state: ChildState): Promise<void> {
-    const child = state.child;
-    if (!child) return;
-    return new Promise<void>((resolve) => {
-      const onExit = () => resolve();
-      child.once("exit", onExit);
-      try {
-        child.kill("SIGTERM");
-      } catch (err) {
-        this.log.warn(`SIGTERM ${state.agentId} threw: ${String(err)}`);
-        child.off("exit", onExit);
-        resolve();
-        return;
-      }
-      const hardKill = setTimeout(() => {
-        try {
-          child.kill("SIGKILL");
-        } catch {
-          /* already gone */
-        }
-      }, 10_000);
-      child.once("exit", () => clearTimeout(hardKill));
-    });
-  }
-
-  private ensureSharedCredentialLink(agentClaudeHome: string, agentId: string): void {
-    const sharedCredentials = path.resolve(sharedClaudeCredentialsFileFor(this.config.rootClaudeHome));
-    const agentCredentials = agentClaudeCredentialsFileFor(agentClaudeHome);
-    const agentCredentialsDir = path.dirname(agentCredentials);
-
-    fs.mkdirSync(path.dirname(sharedCredentials), { recursive: true });
-    fs.mkdirSync(agentCredentialsDir, { recursive: true });
-
-    try {
-      const existing = fs.lstatSync(agentCredentials);
-      if (existing.isSymbolicLink()) {
-        const currentTarget = fs.readlinkSync(agentCredentials);
-        if (path.resolve(agentCredentialsDir, currentTarget) === sharedCredentials) {
-          return;
-        }
-        fs.unlinkSync(agentCredentials);
-      } else if (existing.isDirectory()) {
-        this.log.warn(`agent ${agentId}: credential path is a directory, cannot link ${agentCredentials}`);
-        return;
-      } else {
-        fs.unlinkSync(agentCredentials);
-      }
-    } catch (err) {
-      if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
-        throw err;
-      }
-    }
-
-    fs.symlinkSync(sharedCredentials, agentCredentials);
-  }
-}