@doriandev/devcc 0.1.2

package/src/devcc/processes/ProcessManager.ts

@@ -0,0 +1,625 @@
+import { spawn, type ChildProcess } from "node:child_process"
+import { isAbsolute, resolve } from "node:path"
+import { TypedEmitter } from "../core/events.js"
+import type { LogEntry, LogStore } from "../logs/LogBuffer.js"
+import { DevccError, errorMessage } from "../utils/errors.js"
+import { probeTcpPort } from "../utils/exec.js"
+import type { ScopedLogger } from "../utils/logger.js"
+import { planStartOrder, type StartPlan } from "./graph.js"
+import {
+  canTransition,
+  crashBackoffDelay,
+  createManagedProcess,
+  DEFAULT_CRASH_POLICY,
+  DEFAULT_READINESS,
+  describeCommand,
+  isActive,
+  shouldAutoRestart,
+  type ManagedProcess,
+  type ProcessDefinition,
+  type ProcessStatus,
+  type ReadinessCheck,
+} from "./ManagedProcess.js"
+
+export const DEFAULT_KILL_GRACE_MS = 5_000
+
+export type ProcessManagerEvents = {
+  /** Any change to a process snapshot (status, pid, exit code, ...). */
+  "process:changed": [ManagedProcess]
+  /** The set of known processes changed (added or removed). */
+  "process:list": [ManagedProcess[]]
+  "process:started": [ManagedProcess]
+  "process:exit": [ManagedProcess]
+  /** An automatic restart was scheduled after a crash. */
+  "process:retry": [ManagedProcess, number]
+  /** The readiness check passed (or timed out and was reported anyway). */
+  "process:ready": [ManagedProcess]
+  "process:error": [DevccError]
+}
+
+interface Runtime {
+  child: ChildProcess
+  /** Process group id we may signal, when `detached` spawning succeeded. */
+  pgid: number | null
+  killTimer: NodeJS.Timeout | null
+  exitPromise: Promise<void>
+  resolveExit: () => void
+}
+
+export interface ProcessManagerOptions {
+  logs: LogStore
+  logger: ScopedLogger
+  /** Default cwd for definitions that don't specify one. */
+  rootDir: string
+  env?: NodeJS.ProcessEnv
+}
+
+/**
+ * Owns every child process devcc launches.
+ *
+ * Design notes:
+ * - Commands are always `spawn(executable, args[])`; nothing is ever passed to a
+ *   shell, so arguments containing spaces or quotes are safe by construction.
+ * - Children are spawned `detached`, which makes each one a process-group leader.
+ *   Stopping signals the whole group (`kill(-pgid)`) so dev servers that fork
+ *   workers do not leave orphans behind.
+ * - Stopping escalates SIGTERM -> SIGKILL after a grace period.
+ */
+export class ProcessManager extends TypedEmitter<ProcessManagerEvents> {
+  private readonly definitions = new Map<string, ProcessDefinition>()
+  private readonly snapshots = new Map<string, ManagedProcess>()
+  private readonly runtimes = new Map<string, Runtime>()
+  /** Pending automatic-restart timers, keyed by process id. */
+  private readonly retryTimers = new Map<string, NodeJS.Timeout>()
+  /** In-flight readiness checks, so a stop can cancel the probe loop. */
+  private readonly readiness = new Map<string, { cancel: () => void; done: Promise<void> }>()
+  private readonly logs: LogStore
+  private readonly logger: ScopedLogger
+  private readonly rootDir: string
+  private readonly baseEnv: NodeJS.ProcessEnv
+  private disposed = false
+
+  constructor(options: ProcessManagerOptions) {
+    super()
+    this.logs = options.logs
+    this.logger = options.logger
+    this.rootDir = options.rootDir
+    this.baseEnv = options.env ?? process.env
+  }
+
+  /** Register (or replace, while stopped) a process definition. */
+  register(definition: ProcessDefinition): ManagedProcess {
+    const existing = this.snapshots.get(definition.id)
+    if (existing && isActive(existing.status)) {
+      this.definitions.set(definition.id, definition)
+      return existing
+    }
+
+    this.definitions.set(definition.id, definition)
+    const snapshot = createManagedProcess(definition)
+    this.snapshots.set(definition.id, snapshot)
+    this.logs.register(definition.id, definition.name)
+    this.emit("process:list", this.list())
+    return snapshot
+  }
+
+  registerAll(definitions: readonly ProcessDefinition[]): void {
+    for (const definition of definitions) this.register(definition)
+  }
+
+  /** Remove a stopped process. Refuses while it is still running. */
+  unregister(id: string): void {
+    const snapshot = this.snapshots.get(id)
+    if (snapshot && isActive(snapshot.status)) {
+      throw new DevccError({ scope: "process", message: `Cannot remove "${id}" while it is running` })
+    }
+    this.definitions.delete(id)
+    this.snapshots.delete(id)
+    this.logs.unregister(id)
+    this.emit("process:list", this.list())
+  }
+
+  list(): ManagedProcess[] {
+    return [...this.snapshots.values()]
+  }
+
+  get(id: string): ManagedProcess | undefined {
+    return this.snapshots.get(id)
+  }
+
+  getDefinition(id: string): ProcessDefinition | undefined {
+    return this.definitions.get(id)
+  }
+
+  isRunning(id: string): boolean {
+    const snapshot = this.snapshots.get(id)
+    return snapshot ? isActive(snapshot.status) : false
+  }
+
+  /**
+   * Start a process.
+   *
+   * A start the user asked for clears the crash budget; the automatic restart
+   * path passes `auto` so a crash loop cannot refill its own budget.
+   */
+  async start(id: string, options: { auto?: boolean } = {}): Promise<ManagedProcess> {
+    const definition = this.definitions.get(id)
+    const snapshot = this.snapshots.get(id)
+    if (!definition || !snapshot) {
+      throw new DevccError({ scope: "process", message: `Unknown process "${id}"` })
+    }
+    this.cancelRetry(id)
+    if (isActive(snapshot.status)) return snapshot
+    if (this.disposed) {
+      throw new DevccError({ scope: "process", message: "Shutting down; refusing to start new processes" })
+    }
+
+    this.update(id, {
+      status: "starting",
+      exitCode: undefined,
+      exitSignal: undefined,
+      lastError: undefined,
+      ...(options.auto ? {} : { crashRestarts: 0, crashRestartsExhausted: false }),
+    })
+
+    const cwd = definition.cwd ? resolveCwd(this.rootDir, definition.cwd) : this.rootDir
+    const buffer = this.logs.register(id, definition.name)
+    buffer.push(`$ ${describeCommand({ command: definition.command, args: definition.args })}`, "system")
+
+    let child: ChildProcess
+    try {
+      child = spawn(definition.command, definition.args ? [...definition.args] : [], {
+        cwd,
+        env: { ...this.baseEnv, ...definition.env, FORCE_COLOR: definition.env?.FORCE_COLOR ?? "0" },
+        stdio: ["ignore", "pipe", "pipe"],
+        // Own process group so we can signal the whole tree on stop.
+        detached: process.platform !== "win32",
+        windowsHide: true,
+      })
+    } catch (error) {
+      const message = `Failed to start ${definition.name}: ${errorMessage(error)}`
+      this.update(id, { status: "failed", lastError: message })
+      const devccError = new DevccError({ scope: "process", message, cause: error })
+      this.emit("process:error", devccError)
+      throw devccError
+    }
+
+    let resolveExit: () => void = () => {}
+    const exitPromise = new Promise<void>((resolve) => {
+      resolveExit = resolve
+    })
+
+    const runtime: Runtime = {
+      child,
+      pgid: child.pid !== undefined && process.platform !== "win32" ? child.pid : null,
+      killTimer: null,
+      exitPromise,
+      resolveExit,
+    }
+    this.runtimes.set(id, runtime)
+
+    child.stdout?.setEncoding("utf8")
+    child.stdout?.on("data", (chunk: string) => this.logs.appendChunk(id, chunk, "stdout"))
+    child.stderr?.setEncoding("utf8")
+    child.stderr?.on("data", (chunk: string) => this.logs.appendChunk(id, chunk, "stderr"))
+
+    child.once("error", (error: Error) => {
+      const message = `${definition.name}: ${error.message}`
+      this.logs.append(id, message, "stderr")
+      this.update(id, { status: "failed", lastError: message, pid: undefined })
+      this.finishRuntime(id)
+      this.emit("process:error", new DevccError({ scope: "process", message, cause: error }))
+    })
+
+    child.once("exit", (code, signal) => {
+      this.handleExit(id, code, signal)
+    })
+
+    this.update(id, {
+      pid: child.pid,
+      startedAt: new Date(),
+      stoppedAt: undefined,
+      ready: false,
+      readinessTimedOut: false,
+    })
+    this.logger.info("process started", { id, pid: child.pid, cwd })
+
+    if (definition.readiness) {
+      // Stay in `starting` until the check passes; `running` means usable.
+      this.beginReadinessCheck(id, definition.readiness)
+    } else {
+      this.markReady(id, false)
+    }
+
+    const started = this.snapshots.get(id)
+    if (started) this.emit("process:started", started)
+    return started ?? snapshot
+  }
+
+  /**
+   * Resolve once the process is ready (readiness passed or timed out), or
+   * immediately if it is not starting. Used to sequence dependent starts.
+   */
+  async waitForReady(id: string): Promise<ManagedProcess | undefined> {
+    const pending = this.readiness.get(id)
+    if (pending) await pending.done
+    return this.snapshots.get(id)
+  }
+
+  /**
+   * Start several processes honouring `dependsOn`: each wave waits for the
+   * previous one to become ready. Failures in one wave are reported and the
+   * next wave still starts, since an unrelated process should not be blocked.
+   */
+  async startAll(ids?: readonly string[]): Promise<StartPlan> {
+    const plan = planStartOrder([...this.definitions.values()], ids)
+
+    for (const edge of plan.missing) {
+      this.logs.append(edge.id, `dependsOn "${edge.dependsOn}" is not a known process; ignoring`, "stderr")
+    }
+    if (plan.cycle.length > 0) {
+      this.emit(
+        "process:error",
+        new DevccError({
+          scope: "process",
+          message: `Dependency cycle between ${plan.cycle.join(", ")}; not starting them`,
+        }),
+      )
+    }
+
+    for (const wave of plan.waves) {
+      const targets = wave.filter((id) => {
+        const snapshot = this.snapshots.get(id)
+        return snapshot && !isActive(snapshot.status)
+      })
+
+      await Promise.all(
+        targets.map(async (id) => {
+          try {
+            await this.start(id)
+          } catch (error) {
+            this.logger.warn("start failed", { id, error: errorMessage(error) })
+          }
+        }),
+      )
+      // Wait for the whole wave (including ones already running) before moving on.
+      await Promise.all(wave.map((id) => this.waitForReady(id)))
+    }
+
+    return plan
+  }
+
+  private beginReadinessCheck(id: string, check: ReadinessCheck): void {
+    this.cancelReadiness(id)
+
+    const timeoutMs = check.timeoutMs ?? DEFAULT_READINESS.timeoutMs
+    const intervalMs = check.intervalMs ?? DEFAULT_READINESS.intervalMs
+    let cancelled = false
+    let settle: () => void = () => {}
+    const done = new Promise<void>((resolve) => {
+      settle = resolve
+    })
+
+    const finish = (timedOut: boolean) => {
+      if (cancelled) return
+      cancelled = true
+      this.readiness.delete(id)
+      if (logListener) this.logs.off("log:append", logListener)
+      clearTimeout(timer)
+      if (this.isActiveRuntime(id)) this.markReady(id, timedOut)
+      settle()
+    }
+
+    // --- log pattern -------------------------------------------------------
+    let logListener: ((sourceId: string, entries: LogEntry[]) => void) | null = null
+    if (check.logPattern) {
+      let pattern: RegExp | null = null
+      try {
+        pattern = new RegExp(check.logPattern, "iu")
+      } catch {
+        this.logs.append(id, `readiness.logPattern is not a valid regex: ${check.logPattern}`, "stderr")
+      }
+      if (pattern) {
+        const matcher = pattern
+        logListener = (sourceId, entries) => {
+          if (sourceId !== id) return
+          if (entries.some((entry) => matcher.test(entry.text))) finish(false)
+        }
+        this.logs.on("log:append", logListener)
+      }
+    }
+
+    // --- port probe --------------------------------------------------------
+    const probe = async () => {
+      while (!cancelled) {
+        if (
+          check.port &&
+          (await probeTcpPort(check.port, check.host ?? "127.0.0.1", Math.min(intervalMs, 700)))
+        ) {
+          finish(false)
+          return
+        }
+        if (!check.port && !logListener) {
+          // Nothing to wait for after all.
+          finish(false)
+          return
+        }
+        await new Promise((resolve) => setTimeout(resolve, intervalMs))
+      }
+    }
+
+    const timer = setTimeout(() => {
+      this.logs.append(id, `Readiness check timed out after ${timeoutMs}ms; reporting as running`, "stderr")
+      finish(true)
+    }, timeoutMs)
+    timer.unref?.()
+
+    this.readiness.set(id, { cancel: () => finish(false), done })
+    void probe()
+  }
+
+  private cancelReadiness(id: string): void {
+    const pending = this.readiness.get(id)
+    if (!pending) return
+    this.readiness.delete(id)
+    pending.cancel()
+  }
+
+  private isActiveRuntime(id: string): boolean {
+    const snapshot = this.snapshots.get(id)
+    return this.runtimes.has(id) && snapshot?.status === "starting"
+  }
+
+  private markReady(id: string, timedOut: boolean): void {
+    this.update(id, { status: "running", ready: true, readinessTimedOut: timedOut })
+    const snapshot = this.snapshots.get(id)
+    if (snapshot) {
+      if (!timedOut && this.definitions.get(id)?.readiness) this.logs.append(id, "Ready", "system")
+      this.emit("process:ready", snapshot)
+    }
+  }
+
+  /**
+   * Stop a managed process and everything it spawned.
+   * Resolves once the child has actually exited (or the grace period elapsed).
+   */
+  async stop(id: string, graceMs?: number): Promise<void> {
+    const snapshot = this.snapshots.get(id)
+    const runtime = this.runtimes.get(id)
+    if (!snapshot) throw new DevccError({ scope: "process", message: `Unknown process "${id}"` })
+    if (snapshot.ownership !== "managed") {
+      throw new DevccError({
+        scope: "process",
+        message: `"${snapshot.name}" was detected, not launched by devcc - refusing to signal it`,
+      })
+    }
+    // A manual stop cancels any pending automatic restart and clears the
+    // crash budget: the user is now in charge of this process.
+    this.cancelRetry(id)
+    this.cancelReadiness(id)
+    this.update(id, { crashRestarts: 0, retryAt: undefined, crashRestartsExhausted: false })
+
+    if (!runtime || !isActive(snapshot.status)) return
+
+    this.update(id, { status: "stopping" })
+    this.logs.append(id, "Stopping…", "system")
+
+    const grace = graceMs ?? this.definitions.get(id)?.killGraceMs ?? DEFAULT_KILL_GRACE_MS
+    this.signal(runtime, "SIGTERM")
+
+    runtime.killTimer = setTimeout(() => {
+      if (this.runtimes.has(id)) {
+        this.logs.append(id, "Grace period elapsed, sending SIGKILL", "system")
+        this.signal(runtime, "SIGKILL")
+      }
+    }, grace)
+    runtime.killTimer.unref?.()
+
+    await runtime.exitPromise
+  }
+
+  async restart(id: string): Promise<ManagedProcess> {
+    const snapshot = this.snapshots.get(id)
+    if (!snapshot) throw new DevccError({ scope: "process", message: `Unknown process "${id}"` })
+    this.cancelRetry(id)
+    if (isActive(snapshot.status)) await this.stop(id)
+    const restarted = await this.start(id)
+    this.update(id, { restarts: (this.snapshots.get(id)?.restarts ?? 0) + 1 })
+    return this.snapshots.get(id) ?? restarted
+  }
+
+  /** Start every definition marked `autoStart`. Failures are reported, not thrown. */
+  async startAutoStart(): Promise<void> {
+    const targets = [...this.definitions.values()].filter((definition) => definition.autoStart)
+    if (targets.length === 0) return
+    await this.startAll(targets.map((definition) => definition.id))
+  }
+
+  /**
+   * Stop everything devcc launched. Called on shutdown; never touches processes
+   * that were only detected.
+   */
+  async stopAll(graceMs = DEFAULT_KILL_GRACE_MS): Promise<void> {
+    this.disposed = true
+    for (const id of [...this.retryTimers.keys()]) this.cancelRetry(id)
+    const running = this.list().filter((snapshot) => isActive(snapshot.status))
+    await Promise.all(
+      running.map(async (snapshot) => {
+        try {
+          await this.stop(snapshot.id, graceMs)
+        } catch (error) {
+          this.logger.warn("stop failed during shutdown", { id: snapshot.id, error: errorMessage(error) })
+        }
+      }),
+    )
+  }
+
+  private handleExit(id: string, code: number | null, signal: NodeJS.Signals | null): void {
+    this.cancelReadiness(id)
+    this.logs.flush(id)
+    const snapshot = this.snapshots.get(id)
+    const wasStopping = snapshot?.status === "stopping"
+    const kind = snapshot?.kind ?? "service"
+
+    const failed = !wasStopping && code !== 0 && code !== null
+    const nextStatus: ProcessStatus = failed ? "failed" : "stopped"
+
+    this.logs.append(
+      id,
+      signal ? `Exited via ${signal}` : `Exited with code ${code ?? "unknown"}`,
+      failed ? "stderr" : "system",
+    )
+
+    // A run that lasted long enough counts as healthy and clears the budget.
+    const definition = this.definitions.get(id)
+    const ranFor = snapshot?.startedAt ? Date.now() - snapshot.startedAt.getTime() : 0
+    const healthyAfter = definition?.healthyAfterMs ?? DEFAULT_CRASH_POLICY.healthyAfterMs
+    const wasHealthy = ranFor >= healthyAfter
+    const crashRestarts = failed ? (wasHealthy ? 0 : (snapshot?.crashRestarts ?? 0)) : 0
+
+    this.update(id, {
+      status: nextStatus,
+      pid: undefined,
+      exitCode: code ?? undefined,
+      exitSignal: signal ?? undefined,
+      stoppedAt: new Date(),
+      crashRestarts,
+      retryAt: undefined,
+      ready: false,
+      lastError: failed ? `${snapshot?.name ?? id} exited with code ${code ?? "?"}` : undefined,
+    })
+
+    this.finishRuntime(id)
+
+    const updated = this.snapshots.get(id)
+    if (updated) {
+      this.logger.info("process exited", { id, code, signal, kind })
+      this.emit("process:exit", updated)
+      if (failed) this.considerAutoRestart(id)
+      if (failed) {
+        this.emit(
+          "process:error",
+          new DevccError({
+            scope: "process",
+            message: `${updated.name} exited with code ${code ?? "?"}`,
+            detail: `Command: ${describeCommand(updated)}`,
+          }),
+        )
+      }
+    }
+  }
+
+  /**
+   * Schedule an automatic restart for a crashed process, if its definition opted
+   * in and it still has budget left. Backoff doubles per consecutive crash, so a
+   * process that is broken rather than flaky stops thrashing quickly.
+   */
+  private considerAutoRestart(id: string): void {
+    if (this.disposed) return
+    const definition = this.definitions.get(id)
+    const snapshot = this.snapshots.get(id)
+    if (!definition || !snapshot) return
+
+    if (!definition.restartOnCrash) return
+
+    if (!shouldAutoRestart(definition, snapshot)) {
+      if (!snapshot.crashRestartsExhausted) {
+        this.update(id, { crashRestartsExhausted: true })
+        this.logs.append(id, `Giving up after ${snapshot.crashRestarts} automatic restarts`, "system")
+        this.emit(
+          "process:error",
+          new DevccError({
+            scope: "process",
+            message: `${definition.name} keeps crashing; automatic restarts exhausted`,
+            detail: `Last exit code ${snapshot.exitCode ?? "?"}. Start it manually to try again.`,
+          }),
+        )
+      }
+      return
+    }
+
+    const attempt = snapshot.crashRestarts + 1
+    const delay = crashBackoffDelay(attempt, definition.crashBackoffMs, definition.maxCrashBackoffMs)
+    const retryAt = new Date(Date.now() + delay)
+
+    this.update(id, { crashRestarts: attempt, retryAt, crashRestartsExhausted: false })
+    this.logs.append(
+      id,
+      `Restarting automatically in ${Math.round(delay / 100) / 10}s (attempt ${attempt})`,
+      "system",
+    )
+
+    const updated = this.snapshots.get(id)
+    if (updated) this.emit("process:retry", updated, delay)
+
+    const timer = setTimeout(() => {
+      this.retryTimers.delete(id)
+      if (this.disposed) return
+      // The user may have started or removed it while we waited.
+      const current = this.snapshots.get(id)
+      if (!current || isActive(current.status)) return
+      this.update(id, { retryAt: undefined })
+      void this.start(id, { auto: true }).catch((error: unknown) => {
+        this.logger.warn("automatic restart failed", { id, error: errorMessage(error) })
+      })
+    }, delay)
+    timer.unref?.()
+    this.retryTimers.set(id, timer)
+  }
+
+  private cancelRetry(id: string): void {
+    const timer = this.retryTimers.get(id)
+    if (!timer) return
+    clearTimeout(timer)
+    this.retryTimers.delete(id)
+    this.update(id, { retryAt: undefined })
+  }
+
+  private finishRuntime(id: string): void {
+    const runtime = this.runtimes.get(id)
+    if (!runtime) return
+    if (runtime.killTimer) clearTimeout(runtime.killTimer)
+    this.runtimes.delete(id)
+    runtime.resolveExit()
+  }
+
+  /** Signal the child's process group when we have one, else the child alone. */
+  private signal(runtime: Runtime, signal: NodeJS.Signals): void {
+    const { child, pgid } = runtime
+    try {
+      if (pgid !== null) {
+        process.kill(-pgid, signal)
+        return
+      }
+    } catch (error) {
+      const code = (error as NodeJS.ErrnoException).code
+      // ESRCH: the group is already gone. Anything else: fall back to the child.
+      if (code === "ESRCH") return
+    }
+    try {
+      child.kill(signal)
+    } catch {
+      // The process is already gone.
+    }
+  }
+
+  private update(id: string, patch: Partial<ManagedProcess>): void {
+    const current = this.snapshots.get(id)
+    if (!current) return
+
+    if (patch.status && patch.status !== current.status && !canTransition(current.status, patch.status)) {
+      this.logger.debug("ignored illegal status transition", { id, from: current.status, to: patch.status })
+      const { status: _ignored, ...rest } = patch
+      if (Object.keys(rest).length === 0) return
+      patch = rest
+    }
+
+    const next: ManagedProcess = { ...current, ...patch }
+    this.snapshots.set(id, next)
+    this.emit("process:changed", next)
+  }
+}
+
+function resolveCwd(rootDir: string, cwd: string): string {
+  return isAbsolute(cwd) ? cwd : resolve(rootDir, cwd)
+}