@pushpalsdev/cli 1.0.35 → 1.0.36

package/dist/pushpals-cli.js

@@ -17,6 +17,258 @@ import {
 import { basename, delimiter, dirname, extname, join as join2, resolve as resolve4, win32 as pathWin32 } from "path";
 import { createInterface } from "readline";
 
+// ../shared/src/localbuddy_runtime.ts
+var TRUTHY = new Set(["1", "true", "yes", "on"]);
+var FALSY = new Set(["0", "false", "no", "off"]);
+
+// ../../scripts/start_runtime_services.ts
+var DEFAULT_SERVICE_MANAGER_POLL_MS = 1000;
+var DEFAULT_SERVICE_MANAGER_MAX_RESTART_ATTEMPTS = 4;
+var DEFAULT_SERVICE_MANAGER_STABLE_WINDOW_MS = 60000;
+var DEFAULT_SERVICE_MANAGER_BASE_BACKOFF_MS = 2000;
+var DEFAULT_SERVICE_MANAGER_MAX_BACKOFF_MS = 30000;
+function formatEmbeddedRuntimeHealthLines(health) {
+  if (!health)
+    return [];
+  const lines = [`[pushpals] embeddedRuntime=${health.state} detail=${health.detail}`];
+  if (health.action) {
+    lines.push(`[pushpals] embeddedRuntimeAction=${health.action}`);
+  }
+  return lines;
+}
+function computeServiceRestartBackoffMs(attempt) {
+  const boundedAttempt = Math.max(1, Math.floor(attempt));
+  const exponential = DEFAULT_SERVICE_MANAGER_BASE_BACKOFF_MS * Math.pow(2, boundedAttempt - 1);
+  return Math.max(DEFAULT_SERVICE_MANAGER_BASE_BACKOFF_MS, Math.min(DEFAULT_SERVICE_MANAGER_MAX_BACKOFF_MS, Math.floor(exponential)));
+}
+function shouldRestartService(attempts, maxAttempts = DEFAULT_SERVICE_MANAGER_MAX_RESTART_ATTEMPTS) {
+  const normalizedAttempts = Math.max(0, Math.floor(attempts));
+  const normalizedMax = Math.max(1, Math.floor(maxAttempts));
+  return normalizedAttempts < normalizedMax;
+}
+function pipeProcessStreamToLines(stream, onLine) {
+  if (!stream || typeof stream === "number" || typeof stream.getReader !== "function")
+    return;
+  const reader = stream.getReader();
+  const decoder = new TextDecoder;
+  let pending = "";
+  (async () => {
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done)
+          break;
+        pending += decoder.decode(value, { stream: true });
+        const lines = pending.split(/\r?\n/);
+        pending = lines.pop() ?? "";
+        for (const line of lines) {
+          const trimmed = line.trimEnd();
+          if (!trimmed)
+            continue;
+          onLine?.(trimmed);
+        }
+      }
+      const rest = decoder.decode();
+      if (rest)
+        pending += rest;
+      const tail = pending.trimEnd();
+      if (tail)
+        onLine?.(tail);
+    } catch {} finally {
+      reader.releaseLock();
+    }
+  })();
+}
+function spawnManagedService(spec) {
+  const env = { ...spec.env ?? {} };
+  const proc = Bun.spawn(spec.command, {
+    cwd: spec.cwd,
+    env,
+    stdout: "pipe",
+    stderr: "pipe"
+  });
+  pipeProcessStreamToLines(proc.stdout, spec.onStdoutLine);
+  pipeProcessStreamToLines(proc.stderr, spec.onStderrLine);
+  const service = {
+    name: spec.name,
+    proc,
+    command: [...spec.command],
+    cwd: spec.cwd,
+    env,
+    exited: false,
+    exitCode: null,
+    launchedAtMs: Date.now(),
+    logPath: spec.logPath
+  };
+  proc.exited.then((code) => {
+    service.exited = true;
+    service.exitCode = code;
+  });
+  return service;
+}
+
+class ServiceManager {
+  services = new Map;
+  launchSpecs = new Map;
+  stateByService = new Map;
+  degradedServiceReasons = new Map;
+  pollMs;
+  maxRestartAttempts;
+  stableWindowMs;
+  computeRestartBackoffMs;
+  degradedAction;
+  spawnService;
+  onHealthChange;
+  onServiceDegraded;
+  onEvent;
+  timer;
+  stopped = false;
+  constructor(options = {}) {
+    this.pollMs = Math.max(50, Math.floor(options.pollMs ?? DEFAULT_SERVICE_MANAGER_POLL_MS));
+    this.maxRestartAttempts = Math.max(1, Math.floor(options.maxRestartAttempts ?? DEFAULT_SERVICE_MANAGER_MAX_RESTART_ATTEMPTS));
+    this.stableWindowMs = Math.max(1000, Math.floor(options.stableWindowMs ?? DEFAULT_SERVICE_MANAGER_STABLE_WINDOW_MS));
+    this.computeRestartBackoffMs = options.computeRestartBackoffMs ?? computeServiceRestartBackoffMs;
+    this.degradedAction = options.degradedAction ?? "Inspect the affected service logs or restart the runtime after fixing the failure.";
+    this.spawnService = options.spawnService ?? spawnManagedService;
+    this.onHealthChange = options.onHealthChange;
+    this.onServiceDegraded = options.onServiceDegraded;
+    this.onEvent = options.onEvent;
+    this.timer = setInterval(() => this.tick(), this.pollMs);
+  }
+  startService(spec) {
+    this.launchSpecs.set(spec.name, {
+      ...spec,
+      command: [...spec.command],
+      env: { ...spec.env ?? {} }
+    });
+    const service = this.spawnService(spec);
+    this.services.set(spec.name, service);
+    return service;
+  }
+  getServices() {
+    return Array.from(this.services.values());
+  }
+  getService(name) {
+    return this.services.get(name) ?? null;
+  }
+  getHealth() {
+    if (this.degradedServiceReasons.size === 0)
+      return null;
+    const detail = Array.from(this.degradedServiceReasons.entries()).map(([name, reason]) => `${name}: ${reason}`).join(" | ");
+    return {
+      state: "degraded",
+      detail,
+      action: this.degradedAction
+    };
+  }
+  stop() {
+    if (this.stopped)
+      return;
+    this.stopped = true;
+    clearInterval(this.timer);
+    for (const state of this.stateByService.values()) {
+      if (!state.pendingRestartTimer)
+        continue;
+      clearTimeout(state.pendingRestartTimer);
+      state.pendingRestartTimer = null;
+    }
+    for (const service of this.services.values()) {
+      try {
+        const pid = service.proc.pid;
+        if (process.platform === "win32" && typeof pid === "number" && pid > 0) {
+          Bun.spawnSync(["taskkill", "/PID", String(pid), "/T", "/F"], {
+            stdin: "ignore",
+            stdout: "ignore",
+            stderr: "ignore"
+          });
+        } else {
+          service.proc.kill();
+        }
+      } catch {}
+    }
+  }
+  ensureState(name) {
+    const existing = this.stateByService.get(name);
+    if (existing)
+      return existing;
+    const created = {
+      attempts: 0,
+      nextRestartAtMs: 0,
+      lastRestartReason: "",
+      pendingRestartTimer: null
+    };
+    this.stateByService.set(name, created);
+    return created;
+  }
+  emitHealthChange() {
+    this.onHealthChange?.(this.getHealth());
+  }
+  emitEvent(level, line) {
+    this.onEvent?.(level, line);
+  }
+  tick() {
+    if (this.stopped)
+      return;
+    const now = Date.now();
+    for (const [name, service] of this.services.entries()) {
+      const launchSpec = this.launchSpecs.get(name);
+      if (!launchSpec)
+        continue;
+      const state = this.ensureState(name);
+      if (!service.exited) {
+        if (state.attempts > 0 && now - service.launchedAtMs >= this.stableWindowMs) {
+          state.attempts = 0;
+          state.nextRestartAtMs = 0;
+          state.lastRestartReason = "";
+        }
+        continue;
+      }
+      if (state.pendingRestartTimer)
+        continue;
+      if (state.nextRestartAtMs > now)
+        continue;
+      const reason = `exit code ${service.exitCode ?? "unknown"}`;
+      if (!shouldRestartService(state.attempts, this.maxRestartAttempts)) {
+        this.emitEvent("error", `Managed ${name} exited (${reason}) and reached restart limit (${state.attempts}/${this.maxRestartAttempts}).`);
+        this.launchSpecs.delete(name);
+        if (!this.degradedServiceReasons.has(name)) {
+          const degradationReason = `reached restart limit after ${reason} (${state.attempts}/${this.maxRestartAttempts})`;
+          this.degradedServiceReasons.set(name, degradationReason);
+          const health = this.getHealth();
+          if (health) {
+            this.onHealthChange?.(health);
+            this.onServiceDegraded?.(name, degradationReason, health);
+          }
+        }
+        continue;
+      }
+      const nextAttempt = state.attempts + 1;
+      state.lastRestartReason = reason;
+      const backoffMs = Math.max(1, Math.floor(this.computeRestartBackoffMs(nextAttempt)));
+      this.emitEvent("warn", `Managed ${name} exited (${reason}); restarting attempt ${nextAttempt}/${this.maxRestartAttempts} in ${backoffMs}ms.`);
+      state.nextRestartAtMs = now + backoffMs;
+      state.pendingRestartTimer = setTimeout(() => {
+        state.pendingRestartTimer = null;
+        state.nextRestartAtMs = 0;
+        if (this.stopped)
+          return;
+        const current = this.services.get(name);
+        if (!current || !current.exited)
+          return;
+        const spec = this.launchSpecs.get(name);
+        if (!spec)
+          return;
+        if (!shouldRestartService(state.attempts, this.maxRestartAttempts))
+          return;
+        state.attempts += 1;
+        const restarted = this.spawnService(spec);
+        this.services.set(name, restarted);
+        this.emitEvent("log", `Restarted managed ${name}.`);
+      }, backoffMs);
+    }
+  }
+}
+
 // ../shared/src/client_preflight.ts
 import { existsSync as existsSync2, readFileSync as readFileSync2 } from "fs";
 import { relative, resolve as resolve2 } from "path";
@@ -101,8 +353,8 @@ function normalizeLoopbackHttpUrl(value, fallbackPort) {
 // ../shared/src/config.ts
 var PROJECT_ROOT = resolve(import.meta.dir, "..", "..", "..");
 var DEFAULT_CONFIG_DIR = "configs";
-var TRUTHY = new Set(["1", "true", "yes", "on"]);
-var FALSY = new Set(["0", "false", "no", "off"]);
+var TRUTHY2 = new Set(["1", "true", "yes", "on"]);
+var FALSY2 = new Set(["0", "false", "no", "off"]);
 var DEFAULT_WORKERPALS_QUALITY_CRITIC_MIN_SCORE = 8;
 var DEFAULT_WORKERPALS_QUALITY_MAX_AUTO_REVISIONS = 1;
 var DEFAULT_WORKERPALS_FILE_MODIFYING_JOBS = ["task.execute"];
@@ -133,9 +385,9 @@ function parseBoolEnv(name) {
   const raw = (process.env[name] ?? "").trim().toLowerCase();
   if (!raw)
     return null;
-  if (TRUTHY.has(raw))
+  if (TRUTHY2.has(raw))
     return true;
-  if (FALSY.has(raw))
+  if (FALSY2.has(raw))
     return false;
   return null;
 }
@@ -192,9 +444,9 @@ function asBoolean(value, fallback) {
     return value;
   if (typeof value === "string") {
     const lowered = value.trim().toLowerCase();
-    if (TRUTHY.has(lowered))
+    if (TRUTHY2.has(lowered))
       return true;
-    if (FALSY.has(lowered))
+    if (FALSY2.has(lowered))
       return false;
   }
   return fallback;
@@ -1217,6 +1469,7 @@ var DEFAULT_RUNTIME_BOOT_TIMEOUT_MS = 90000;
 var DEFAULT_RUNTIME_BOOT_POLL_MS = 1000;
 var DEFAULT_SERVER_BOOT_TIMEOUT_MS = 20000;
 var DEFAULT_SERVICE_STABILITY_GRACE_MS = 4000;
+var EMBEDDED_SERVICE_RESTART_MAX_ATTEMPTS = 4;
 var WORKERPAL_STARTUP_READINESS_PROBE_MAX_MS = 15000;
 var EMBEDDED_RUNTIME_SAFETY_CAP_DISABLE_ENV = "PUSHPALS_DISABLE_EMBEDDED_SAFETY_CAPS";
 var EMBEDDED_RUNTIME_WINDOWS_SAFETY_CAPS = {
@@ -1292,6 +1545,9 @@ function formatWorkerExecutionReadinessLines(readiness) {
   }
   return lines;
 }
+function formatEmbeddedRuntimeHealthLines2(health) {
+  return formatEmbeddedRuntimeHealthLines(health);
+}
 function summarizeWorkerStatusRows(workers) {
   const onlineWorkers = workers.filter((worker) => Boolean(worker?.isOnline) && String(worker?.status ?? "").trim().toLowerCase() !== "offline");
   const idleWorkers = onlineWorkers.filter((worker) => Number(worker?.activeJobCount ?? 0) <= 0);
@@ -2282,71 +2538,6 @@ async function ensureRuntimeBinaries(runtimeRoot, runtimeTag) {
   console.log("[pushpals] Embedded runtime binaries are ready.");
   return runtimeBinaries;
 }
-function spawnRuntimeService(name, command, cwd, env, logPath, runtimeServicesLogPath) {
-  const header = `[pushpals] service=${name} command=${command.join(" ")} cwd=${cwd}`;
-  writeFileSync(logPath, `${header}
-`, "utf8");
-  if (runtimeServicesLogPath) {
-    appendRuntimeServicesLogLine(runtimeServicesLogPath, header);
-  }
-  const proc = Bun.spawn(command, {
-    cwd,
-    env,
-    stdout: "pipe",
-    stderr: "pipe"
-  });
-  const pipeToLog = async (stream, channel) => {
-    if (!stream)
-      return;
-    const reader = stream.getReader();
-    const decoder = new TextDecoder;
-    let pending = "";
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done)
-        break;
-      const chunk = decoder.decode(value, { stream: true });
-      if (!chunk)
-        continue;
-      pending += chunk;
-      const lines = pending.split(/\r?\n/);
-      pending = lines.pop() ?? "";
-      for (const line of lines) {
-        const serviceLine = `[${channel}] ${line}`;
-        appendFileSync(logPath, `${serviceLine}
-`, "utf8");
-        if (runtimeServicesLogPath) {
-          appendRuntimeServicesLogLine(runtimeServicesLogPath, `[${name}] ${serviceLine}`);
-        }
-      }
-    }
-    const rest = decoder.decode();
-    if (rest)
-      pending += rest;
-    if (pending.trim().length > 0) {
-      const serviceLine = `[${channel}] ${pending.trimEnd()}`;
-      appendFileSync(logPath, `${serviceLine}
-`, "utf8");
-      if (runtimeServicesLogPath) {
-        appendRuntimeServicesLogLine(runtimeServicesLogPath, `[${name}] ${serviceLine}`);
-      }
-    }
-  };
-  pipeToLog(proc.stdout, "stdout");
-  pipeToLog(proc.stderr, "stderr");
-  const service = {
-    name,
-    proc,
-    logPath,
-    exited: false,
-    exitCode: null
-  };
-  proc.exited.then((code) => {
-    service.exited = true;
-    service.exitCode = code;
-  });
-  return service;
-}
 function buildServiceStopCommand(pid, platform = process.platform) {
   if (platform === "win32" && typeof pid === "number" && pid > 0) {
     return ["taskkill", "/PID", String(pid), "/T", "/F"];
@@ -2560,6 +2751,12 @@ function quoteWindowsCmdArg(value) {
 function isOptionalEmbeddedService(name) {
   return name === "source_control_manager";
 }
+function computeEmbeddedServiceRestartBackoffMs(attempt) {
+  return computeServiceRestartBackoffMs(attempt);
+}
+function shouldRestartEmbeddedService(attempts, maxAttempts = EMBEDDED_SERVICE_RESTART_MAX_ATTEMPTS) {
+  return shouldRestartService(attempts, maxAttempts);
+}
 async function canSpawnCommand(command, cwd, env) {
   try {
     const proc = Bun.spawn(command, {
@@ -3482,7 +3679,6 @@ async function autoStartRuntimeServices(opts) {
   if (resolvedGitBinary) {
     applyResolvedGitBinaryToRuntimeEnv(runtimeEnv, resolvedGitBinary);
   }
-  const services = [];
   const startupStartedAt = Date.now();
   const startupPhases = [];
   const recordStartupPhase = (name, startedAt, status) => {
@@ -3521,23 +3717,70 @@ async function autoStartRuntimeServices(opts) {
   console.log(`[pushpals] service log (localbuddy)=${serviceLogPaths.localbuddy}`);
   console.log(`[pushpals] service log (remotebuddy)=${serviceLogPaths.remotebuddy}`);
   console.log(`[pushpals] service log (source_control_manager)=${serviceLogPaths.source_control_manager}`);
+  const serviceManager = new ServiceManager({
+    degradedAction: "Inspect the embedded service log or restart pushpals after fixing the runtime failure.",
+    onEvent: (level, line) => {
+      const cliLine = `[pushpals] ${line.replace(/^Managed /, "Embedded ").replace(/^Restarted managed /, "Restarted embedded ")}`;
+      if (level === "error") {
+        console.error(cliLine);
+      } else if (level === "warn") {
+        console.warn(cliLine);
+      } else {
+        console.log(cliLine);
+      }
+      appendRuntimeServicesLogLine(runtimeServicesLogPath, cliLine);
+    },
+    onHealthChange: (health) => {
+      for (const line of formatEmbeddedRuntimeHealthLines2(health)) {
+        console.error(line);
+        appendRuntimeServicesLogLine(runtimeServicesLogPath, line);
+      }
+    }
+  });
+  const launchService = (name, command) => {
+    const logPath = serviceLogPaths[name];
+    const header = `[pushpals] service=${name} command=${command.join(" ")} cwd=${opts.repoRoot}`;
+    writeFileSync(logPath, `${header}
+`, "utf8");
+    appendRuntimeServicesLogLine(runtimeServicesLogPath, header);
+    return serviceManager.startService({
+      name,
+      color: "",
+      command,
+      cwd: opts.repoRoot,
+      env: runtimeEnv,
+      logPath,
+      onStdoutLine: (line) => {
+        const serviceLine = `[stdout] ${line}`;
+        appendFileSync(logPath, `${serviceLine}
+`, "utf8");
+        appendRuntimeServicesLogLine(runtimeServicesLogPath, `[${name}] ${serviceLine}`);
+      },
+      onStderrLine: (line) => {
+        const serviceLine = `[stderr] ${line}`;
+        appendFileSync(logPath, `${serviceLine}
+`, "utf8");
+        appendRuntimeServicesLogLine(runtimeServicesLogPath, `[${name}] ${serviceLine}`);
+      }
+    });
+  };
   const serverHealthy = await probeServer(opts.serverUrl);
   if (!serverHealthy) {
     const serverPhaseStartedAt = Date.now();
     console.log("[pushpals] Starting embedded server...");
-    const serverService = spawnRuntimeService("server", [runtimeBinaries.server], opts.repoRoot, runtimeEnv, serviceLogPaths.server, runtimeServicesLogPath);
-    services.push(serverService);
-    console.log(`[pushpals] server log: ${serverService.logPath}`);
+    const serverService = launchService("server", [runtimeBinaries.server]);
+    const serverLogPath = serverService.logPath ?? serviceLogPaths.server;
+    console.log(`[pushpals] server log: ${serverLogPath}`);
     const serverDeadline = Date.now() + DEFAULT_SERVER_BOOT_TIMEOUT_MS;
     let serverIsReady = false;
     while (Date.now() < serverDeadline) {
       if (serverService.exited) {
-        const tail = readLogTail(serverService.logPath);
+        const tail = readLogTail(serverLogPath);
         appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] embedded server exited during bootstrap (code=${serverService.exitCode ?? "unknown"}).`);
         recordStartupPhase("server", serverPhaseStartedAt, "exited");
         emitStartupTimingSummary("failed", "server exited during bootstrap");
-        stopRuntimeServices(services);
-        throw new Error(`Embedded server exited during bootstrap (code=${serverService.exitCode ?? "unknown"}). ` + `See ${serverService.logPath}${tail ? `
+        stopRuntimeServices(serviceManager.getServices());
+        throw new Error(`Embedded server exited during bootstrap (code=${serverService.exitCode ?? "unknown"}). ` + `See ${serverLogPath}${tail ? `
 --- server log tail ---
 ${tail}` : ""}`);
       }
@@ -3548,12 +3791,12 @@ ${tail}` : ""}`);
       await Bun.sleep(DEFAULT_RUNTIME_BOOT_POLL_MS);
     }
     if (!serverIsReady) {
-      const tail = readLogTail(serverService.logPath);
+      const tail = readLogTail(serverLogPath);
       appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] embedded server did not become healthy within ${DEFAULT_SERVER_BOOT_TIMEOUT_MS}ms.`);
       recordStartupPhase("server", serverPhaseStartedAt, "timeout");
       emitStartupTimingSummary("failed", "server health timeout");
-      stopRuntimeServices(services);
-      throw new Error(`Embedded server did not become healthy within ${DEFAULT_SERVER_BOOT_TIMEOUT_MS}ms. ` + `See ${serverService.logPath}${tail ? `
+      stopRuntimeServices(serviceManager.getServices());
+      throw new Error(`Embedded server did not become healthy within ${DEFAULT_SERVER_BOOT_TIMEOUT_MS}ms. ` + `See ${serverLogPath}${tail ? `
 --- server log tail ---
 ${tail}` : ""}`);
     }
@@ -3567,9 +3810,8 @@ ${tail}` : ""}`);
   if (localBuddyEnabled) {
     const localBuddyPhaseStartedAt = Date.now();
     console.log("[pushpals] Starting embedded LocalBuddy...");
-    const localbuddyService = spawnRuntimeService("localbuddy", [runtimeBinaries.localbuddy], opts.repoRoot, runtimeEnv, serviceLogPaths.localbuddy, runtimeServicesLogPath);
-    services.push(localbuddyService);
-    console.log(`[pushpals] localbuddy log: ${localbuddyService.logPath}`);
+    const localbuddyService = launchService("localbuddy", [runtimeBinaries.localbuddy]);
+    console.log(`[pushpals] localbuddy log: ${localbuddyService.logPath ?? serviceLogPaths.localbuddy}`);
     recordStartupPhase("localbuddy", localBuddyPhaseStartedAt, "started");
   } else {
     recordStartupPhase("localbuddy", Date.now(), "skipped");
@@ -3578,13 +3820,13 @@ ${tail}` : ""}`);
   }
   const remoteBuddyPhaseStartedAt = Date.now();
   console.log("[pushpals] Starting embedded RemoteBuddy...");
-  const remotebuddyService = spawnRuntimeService("remotebuddy", [runtimeBinaries.remotebuddy], opts.repoRoot, runtimeEnv, serviceLogPaths.remotebuddy, runtimeServicesLogPath);
-  services.push(remotebuddyService);
-  console.log(`[pushpals] remotebuddy log: ${remotebuddyService.logPath}`);
+  const remotebuddyService = launchService("remotebuddy", [runtimeBinaries.remotebuddy]);
+  const remotebuddyLogPath = remotebuddyService.logPath ?? serviceLogPaths.remotebuddy;
+  console.log(`[pushpals] remotebuddy log: ${remotebuddyLogPath}`);
   recordStartupPhase("remotebuddy", remoteBuddyPhaseStartedAt, "started");
   let lastReportedRemoteBuddyAutonomyState = "unknown";
   const reportRemoteBuddyAutonomousEngineState = () => {
-    const autonomyState = readRemoteBuddyAutonomousEngineState(remotebuddyService.logPath);
+    const autonomyState = readRemoteBuddyAutonomousEngineState(remotebuddyLogPath);
     if (autonomyState === "unknown" || autonomyState === lastReportedRemoteBuddyAutonomyState) {
       return;
     }
@@ -3635,9 +3877,11 @@ ${tail}` : ""}`);
     } else if (scmRemoteStatus.status === "ok") {
       console.log(`[pushpals] Embedded SourceControlManager git=${scmGitProbe.detail}`);
       console.log("[pushpals] Starting embedded SourceControlManager...");
-      const sourceControlManagerService = spawnRuntimeService("source_control_manager", [runtimeBinaries.sourceControlManager, "--skip-clean-check"], opts.repoRoot, runtimeEnv, serviceLogPaths.source_control_manager, runtimeServicesLogPath);
-      services.push(sourceControlManagerService);
-      console.log(`[pushpals] source_control_manager log: ${sourceControlManagerService.logPath}`);
+      const sourceControlManagerService = launchService("source_control_manager", [
+        runtimeBinaries.sourceControlManager,
+        "--skip-clean-check"
+      ]);
+      console.log(`[pushpals] source_control_manager log: ${sourceControlManagerService.logPath ?? serviceLogPaths.source_control_manager}`);
       recordStartupPhase("source_control_manager", scmPhaseStartedAt, "started");
     } else {
       console.log(`[pushpals] Repo has no git remote "${opts.sourceControlManagerRemote}"; skipping embedded SourceControlManager.`);
@@ -3651,30 +3895,36 @@ ${tail}` : ""}`);
   }
   const deadline = Date.now() + DEFAULT_RUNTIME_BOOT_TIMEOUT_MS;
   const readinessPhaseStartedAt = Date.now();
+  const optionalServiceExitWarned = new Set;
   while (Date.now() < deadline) {
     reportRemoteBuddyAutonomousEngineState();
-    for (let i = services.length - 1;i >= 0; i--) {
-      const service = services[i];
+    for (const service of serviceManager.getServices()) {
       if (service.exited) {
         if (isOptionalEmbeddedService(service.name)) {
-          console.warn(`[pushpals] Embedded ${service.name} exited during startup (code=${service.exitCode ?? "unknown"}); continuing without SCM.`);
-          appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] embedded ${service.name} exited during startup (code=${service.exitCode ?? "unknown"}); continuing.`);
-          const tail2 = readLogTail(service.logPath);
-          if (tail2) {
-            console.warn(`[pushpals] ${service.name} log tail:
+          const runtimeServiceName2 = service.name;
+          const serviceLogPath2 = service.logPath ?? serviceLogPaths[runtimeServiceName2];
+          if (!optionalServiceExitWarned.has(runtimeServiceName2)) {
+            console.warn(`[pushpals] Embedded ${service.name} exited during startup (code=${service.exitCode ?? "unknown"}); startup will continue and host supervisor will attempt recovery.`);
+            appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] embedded ${service.name} exited during startup (code=${service.exitCode ?? "unknown"}); startup will continue and host supervisor will attempt recovery.`);
+            const tail2 = readLogTail(serviceLogPath2);
+            if (tail2) {
+              console.warn(`[pushpals] ${service.name} log tail:
 ${tail2}`);
-            appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] ${service.name} log tail:
+              appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] ${service.name} log tail:
 ${tail2}`);
+            }
+            optionalServiceExitWarned.add(runtimeServiceName2);
           }
-          services.splice(i, 1);
           continue;
         }
-        const tail = readLogTail(service.logPath);
+        const runtimeServiceName = service.name;
+        const serviceLogPath = service.logPath ?? serviceLogPaths[runtimeServiceName];
+        const tail = readLogTail(serviceLogPath);
         appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] embedded ${service.name} exited during startup (code=${service.exitCode ?? "unknown"}).`);
         recordStartupPhase("readiness", readinessPhaseStartedAt, "failed");
         emitStartupTimingSummary("failed", `${service.name} exited during startup`);
-        stopRuntimeServices(services);
-        throw new Error(`Embedded ${service.name} exited during startup (code=${service.exitCode ?? "unknown"}). ` + `See ${service.logPath}${tail ? `
+        stopRuntimeServices(serviceManager.getServices());
+        throw new Error(`Embedded ${service.name} exited during startup (code=${service.exitCode ?? "unknown"}). ` + `See ${serviceLogPath}${tail ? `
 --- ${service.name} log tail ---
 ${tail}` : ""}`);
       }
@@ -3686,29 +3936,34 @@ ${tail}` : ""}`);
       const stabilityDeadline = Date.now() + DEFAULT_SERVICE_STABILITY_GRACE_MS;
       while (Date.now() < stabilityDeadline) {
         reportRemoteBuddyAutonomousEngineState();
-        for (let i = services.length - 1;i >= 0; i--) {
-          const service = services[i];
+        for (const service of serviceManager.getServices()) {
           if (!service.exited)
             continue;
           if (isOptionalEmbeddedService(service.name)) {
-            const tail2 = readLogTail(service.logPath);
-            console.warn(`[pushpals] Embedded ${service.name} exited immediately after bootstrap (code=${service.exitCode ?? "unknown"}); continuing without SCM.`);
-            appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] embedded ${service.name} exited immediately after bootstrap (code=${service.exitCode ?? "unknown"}); continuing.`);
-            if (tail2) {
-              console.warn(`[pushpals] ${service.name} log tail:
+            const runtimeServiceName2 = service.name;
+            const serviceLogPath2 = service.logPath ?? serviceLogPaths[runtimeServiceName2];
+            if (!optionalServiceExitWarned.has(runtimeServiceName2)) {
+              const tail2 = readLogTail(serviceLogPath2);
+              console.warn(`[pushpals] Embedded ${service.name} exited immediately after bootstrap (code=${service.exitCode ?? "unknown"}); startup will continue and host supervisor will attempt recovery.`);
+              appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] embedded ${service.name} exited immediately after bootstrap (code=${service.exitCode ?? "unknown"}); startup will continue and host supervisor will attempt recovery.`);
+              if (tail2) {
+                console.warn(`[pushpals] ${service.name} log tail:
 ${tail2}`);
-              appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] ${service.name} log tail:
+                appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] ${service.name} log tail:
 ${tail2}`);
+              }
+              optionalServiceExitWarned.add(runtimeServiceName2);
             }
-            services.splice(i, 1);
             continue;
           }
-          const tail = readLogTail(service.logPath);
+          const runtimeServiceName = service.name;
+          const serviceLogPath = service.logPath ?? serviceLogPaths[runtimeServiceName];
+          const tail = readLogTail(serviceLogPath);
           appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] embedded ${service.name} exited immediately after bootstrap (code=${service.exitCode ?? "unknown"}).`);
           recordStartupPhase("readiness", readinessPhaseStartedAt, "failed");
           emitStartupTimingSummary("failed", `${service.name} exited immediately after bootstrap`);
-          stopRuntimeServices(services);
-          throw new Error(`Embedded ${service.name} exited immediately after bootstrap (code=${service.exitCode ?? "unknown"}). ` + `See ${service.logPath}${tail ? `
+          stopRuntimeServices(serviceManager.getServices());
+          throw new Error(`Embedded ${service.name} exited immediately after bootstrap (code=${service.exitCode ?? "unknown"}). ` + `See ${serviceLogPath}${tail ? `
 --- ${service.name} log tail ---
 ${tail}` : ""}`);
         }
@@ -3719,13 +3974,13 @@ ${tail}` : ""}`);
       emitStartupTimingSummary("ready");
       appendRuntimeServicesLogLine(runtimeServicesLogPath, "[pushpals] embedded runtime is ready.");
       return {
-        services,
+        serviceManager,
         pushpalsLogPath: runtimeServicesLogPath
       };
     }
     await Bun.sleep(DEFAULT_RUNTIME_BOOT_POLL_MS);
   }
-  stopRuntimeServices(services);
+  stopRuntimeServices(serviceManager.getServices());
   const remoteBuddyHealth = await probeRemoteBuddySessionConsumer(opts.serverUrl, opts.sessionId);
   if (!localBuddyEnabled && !remoteBuddyHealth.ok) {
     appendRuntimeServicesLogLine(runtimeServicesLogPath, `[pushpals] timed out waiting for RemoteBuddy session consumer readiness after ${DEFAULT_RUNTIME_BOOT_TIMEOUT_MS}ms (${remoteBuddyHealth.detail}).`);
@@ -4405,7 +4660,7 @@ async function main() {
     platform: `${process.platform}/${process.arch}`,
     repoRoot
   };
-  let autoStartedServices = [];
+  let autoStartedServiceManager = null;
   let pushpalsLogPath;
   let resolvedRuntimeTagForAutoStart = preparedRuntime.runtimeTag || parsed.runtimeTag || "";
   const cleanupWorkerpalWarmContainersIfNeeded = async (phase) => {
@@ -4464,17 +4719,24 @@ async function main() {
     }
     return readiness;
   };
+  const reportEmbeddedRuntimeHealth = () => {
+    const health = autoStartedServiceManager?.getHealth() ?? null;
+    for (const line of formatEmbeddedRuntimeHealthLines2(health)) {
+      console.log(line);
+    }
+    return health;
+  };
   const stopAutoStartedServices = () => {
-    if (autoStartedServices.length === 0)
+    if (!autoStartedServiceManager)
       return;
-    stopRuntimeServices(autoStartedServices);
-    autoStartedServices = [];
+    autoStartedServiceManager.stop();
+    autoStartedServiceManager = null;
   };
   const stopAutoStartedServicesGracefully = async (reason) => {
-    if (autoStartedServices.length === 0)
+    if (!autoStartedServiceManager)
       return;
-    const services = autoStartedServices;
-    autoStartedServices = [];
+    const serviceManager = autoStartedServiceManager;
+    autoStartedServiceManager = null;
     const shutdown = await requestLocalRuntimeShutdown(serverUrl, repoRoot, reason);
     if (shutdown.attempted && shutdown.accepted) {
       console.log("[pushpals] Local runtime shutdown accepted; waiting for services to exit...");
@@ -4484,6 +4746,8 @@ async function main() {
     } else if (shutdown.detail) {
       console.warn(`[pushpals] ${shutdown.detail}`);
     }
+    serviceManager.stop();
+    const services = serviceManager.getServices();
     await stopRuntimeServicesGracefully(services);
     await cleanupWorkerpalWarmContainersIfNeeded("cli shutdown");
     await cleanupPushPalsGitWorktreesIfNeeded("cli shutdown");
@@ -4531,7 +4795,7 @@ async function main() {
           startLocalBuddy: resolveCliLocalBuddyAutostart(parsed.runtimeOnly, Boolean(config.localbuddy.enabled)),
           baseEnv: workerpalDockerPrecheck.env
         });
-        autoStartedServices = startedRuntime.services;
+        autoStartedServiceManager = startedRuntime.serviceManager;
         pushpalsLogPath = startedRuntime.pushpalsLogPath;
         serverHealthy = await probeServer(serverUrl);
       } catch (err) {
@@ -4655,6 +4919,7 @@ async function main() {
   console.log(`[pushpals] pushpalsLog=${pushpalsLogPath ?? "unavailable"}`);
   console.log(`[pushpals] cliStateFile=${statePath ?? "unavailable"}`);
   reportWorkerExecutionReadinessFromSnapshot(startupWorkerExecutionReadiness);
+  reportEmbeddedRuntimeHealth();
   if (parsed.runtimeOnly) {
     console.log("[pushpals] runtimeOnly=true");
   } else {
@@ -4689,7 +4954,7 @@ ${line}
       try {
         monitoringHub?.stop();
       } catch {}
-      if (autoStartedServices.length > 0) {
+      if (autoStartedServiceManager) {
         console.log("[pushpals] Stopping embedded runtime services...");
       }
       await stopAutoStartedServicesGracefully("pushpals CLI exit");
@@ -4767,6 +5032,7 @@ ${line}
       console.log(`[pushpals] pushpalsLog=${pushpalsLogPath ?? "unavailable"}`);
       console.log(monitoringHubUrl ? `[pushpals] monitoringHubUrl=${monitoringHubUrl}` : "[pushpals] monitoringHubUrl=unavailable");
       await reportWorkerExecutionReadiness();
+      reportEmbeddedRuntimeHealth();
       rl.prompt();
       continue;
     }
@@ -4806,6 +5072,7 @@ export {
   waitForWorkerpalCapacity,
   startEmbeddedMonitoringHub,
   shouldRunEmbeddedRuntimeStartupPrechecks,
+  shouldRestartEmbeddedService,
   resolveWorkerExecutionReadiness,
   resolveWindowsWhereExecutableCandidatesForEnv,
   resolveWindowsShellExecutableCandidatesForEnv,
@@ -4832,6 +5099,7 @@ export {
   formatTimestampedCliLine,
   formatSessionEventLine,
   formatRuntimeStartupTimingSummary,
+  formatEmbeddedRuntimeHealthLines2 as formatEmbeddedRuntimeHealthLines,
   extractRemoteBuddySessionConsumerHealth,
   extractRemoteBuddyAutonomousEngineState,
   ensureWorkerpalDockerImageReady,
@@ -4840,6 +5108,7 @@ export {
   describeWorkerExecutionReadiness,
   createSessionEventReplayFilter,
   copyTrackedRepoPath,
+  computeEmbeddedServiceRestartBackoffMs,
   cleanupLingeringWorkerpalWarmContainers,
   cleanupLingeringPushPalsGitWorktrees,
   bundledMonitoringHubNeedsRefresh,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushpalsdev/cli",
-  "version": "1.0.35",
+  "version": "1.0.36",
   "description": "PushPals terminal CLI for LocalBuddy -> RemoteBuddy orchestration",
   "license": "MIT",
   "repository": {

package/runtime/sandbox/package.json

@@ -45,6 +45,7 @@
     "test:prompt-policy": "bun test tests/prompt-policy.enforcement.test.ts",
     "test:cli:integration": "bun test tests/cli.invocation-logging.test.ts tests/cli.runtime-bootstrap.test.ts tests/client.runtime-bootstrap.test.ts tests/shared.client-preflight.test.ts",
     "test:cli:e2e": "bun test ./tests/integration/cli.e2e.ts",
+    "test:start:e2e": "bun test ./tests/integration/start.e2e.ts",
     "test:root": "bun test tests",
     "test:protocol": "bun run tests/protocol.integration.ts",
     "test:integration": "python -u tests/integration/integration_controller.py --mode integration",