@chrysb/alphaclaw 0.2.2 → 0.3.0

package/lib/server/watchdog.js

@@ -0,0 +1,585 @@
+const {
+  kWatchdogCheckIntervalMs,
+  kWatchdogMaxRepairAttempts,
+  kWatchdogCrashLoopWindowMs,
+  kWatchdogCrashLoopThreshold,
+} = require("./constants");
+
+const kHealthStartupGraceMs = 30 * 1000;
+const kBootstrapHealthCheckMs = 5 * 1000;
+const kExpectedRestartWindowMs = 45 * 1000;
+
+const isTruthy = (value) =>
+  ["1", "true", "yes", "on"].includes(String(value || "").trim().toLowerCase());
+
+const parseHealthResult = (result) => {
+  if (!result?.ok) return { ok: false, reason: result?.stderr || "health command failed" };
+  const raw = String(result.stdout || "").trim();
+  if (!raw) return { ok: true };
+  try {
+    const parsed = JSON.parse(raw);
+    if (parsed?.ok === false || parsed?.status === "unhealthy") {
+      return { ok: false, reason: parsed?.error || "gateway unhealthy" };
+    }
+    return { ok: true, details: parsed };
+  } catch {
+    const firstBrace = raw.indexOf("{");
+    const lastBrace = raw.lastIndexOf("}");
+    if (firstBrace >= 0 && lastBrace > firstBrace) {
+      try {
+        const parsed = JSON.parse(raw.slice(firstBrace, lastBrace + 1));
+        if (parsed?.ok === false || parsed?.status === "unhealthy") {
+          return { ok: false, reason: parsed?.error || "gateway unhealthy" };
+        }
+        return { ok: true, details: parsed };
+      } catch {}
+    }
+  }
+  return { ok: true };
+};
+
+const createWatchdog = ({
+  clawCmd,
+  launchGatewayProcess,
+  insertWatchdogEvent,
+  notifier,
+  readEnvFile,
+  writeEnvFile,
+  reloadEnv,
+  resolveSetupUrl,
+}) => {
+  const state = {
+    lifecycle: "stopped",
+    health: "unknown",
+    uptimeStartedAt: null,
+    lastHealthCheckAt: null,
+    repairAttempts: 0,
+    crashTimestamps: [],
+    autoRepair: isTruthy(process.env.WATCHDOG_AUTO_REPAIR),
+    notificationsDisabled: isTruthy(process.env.WATCHDOG_NOTIFICATIONS_DISABLED),
+    operationInProgress: false,
+    gatewayStartedAt: null,
+    crashRecoveryActive: false,
+    expectedRestartInProgress: false,
+    expectedRestartUntilMs: 0,
+    pendingRecoveryNoticeSource: "",
+  };
+  let healthTimer = null;
+  let bootstrapHealthTimer = null;
+
+  const clearExpectedRestartWindow = () => {
+    state.expectedRestartInProgress = false;
+    state.expectedRestartUntilMs = 0;
+  };
+
+  const markExpectedRestartWindow = (durationMs = kExpectedRestartWindowMs) => {
+    const safeDuration = Math.max(5000, Number(durationMs) || kExpectedRestartWindowMs);
+    state.expectedRestartInProgress = true;
+    state.expectedRestartUntilMs = Date.now() + safeDuration;
+  };
+
+  const startRegularHealthChecks = () => {
+    if (healthTimer) return;
+    healthTimer = setInterval(() => {
+      void runHealthCheck();
+    }, kWatchdogCheckIntervalMs);
+    if (typeof healthTimer.unref === "function") healthTimer.unref();
+  };
+
+  const startBootstrapHealthChecks = () => {
+    if (bootstrapHealthTimer) return;
+    const runBootstrapCheck = async () => {
+      const healthy = await runHealthCheck();
+      // Bootstrap checks are only for the "initializing" phase. As soon as we
+      // either become healthy or transition into any non-unknown state
+      // (degraded/unhealthy/etc.), stop 5s polling and fall back to normal
+      // interval checks to avoid noisy health-check spam.
+      if (healthy || state.health !== "unknown") {
+        if (bootstrapHealthTimer) {
+          clearTimeout(bootstrapHealthTimer);
+          bootstrapHealthTimer = null;
+        }
+        startRegularHealthChecks();
+        return;
+      }
+      bootstrapHealthTimer = setTimeout(() => {
+        void runBootstrapCheck();
+      }, kBootstrapHealthCheckMs);
+      if (typeof bootstrapHealthTimer.unref === "function") {
+        bootstrapHealthTimer.unref();
+      }
+    };
+    void runBootstrapCheck();
+  };
+
+  const trimCrashWindow = () => {
+    const threshold = Date.now() - kWatchdogCrashLoopWindowMs;
+    state.crashTimestamps = state.crashTimestamps.filter((ts) => ts >= threshold);
+  };
+
+  const createCorrelationId = () =>
+    `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
+
+  const logEvent = (eventType, source, status, details = null, correlationId = "") => {
+    try {
+      insertWatchdogEvent({
+        eventType,
+        source,
+        status,
+        details,
+        correlationId,
+      });
+    } catch (err) {
+      console.error(`[watchdog] failed to log event: ${err.message}`);
+    }
+  };
+
+  const notify = async (message, correlationId = "") => {
+    if (state.notificationsDisabled) {
+      return { ok: false, skipped: true, reason: "notifications_disabled" };
+    }
+    if (!notifier?.notify) return { ok: false, reason: "notifier_unavailable" };
+    const result = await notifier.notify(message);
+    logEvent("notification", "watchdog", result.ok ? "ok" : "failed", result, correlationId);
+    return result;
+  };
+
+  const getWatchdogSetupUrl = () => {
+    try {
+      const base =
+        typeof resolveSetupUrl === "function" ? String(resolveSetupUrl() || "") : "";
+      if (base) return `${base.replace(/\/+$/, "")}/#/watchdog`;
+      const fallbackPort = Number.parseInt(String(process.env.PORT || "3000"), 10) || 3000;
+      return `http://localhost:${fallbackPort}/#/watchdog`;
+    } catch {
+      return "";
+    }
+  };
+
+  const withViewLogsSuffix = (line) => {
+    const setupUrl = getWatchdogSetupUrl();
+    if (!setupUrl) return line;
+    return `${line} - [View logs](${setupUrl})`;
+  };
+
+  const asInlineCode = (value) => `\`${String(value || "").replace(/`/g, "")}\``;
+
+  const notifyAutoRepairOutcome = async ({
+    source,
+    correlationId,
+    ok,
+    verifiedHealthy = null,
+    attempts = 0,
+  }) => {
+    if (source === "manual") return;
+    const title = ok
+      ? verifiedHealthy
+        ? "🟢 Auto-repair complete, gateway healthy"
+        : "🟡 Auto-repair started, awaiting health check"
+      : "🔴 Auto-repair failed";
+    await notify(
+      [
+        "🐺 *AlphaClaw Watchdog*",
+        withViewLogsSuffix(title),
+        `Trigger: ${asInlineCode(source)}`,
+        ...(attempts > 0 ? [`Attempt count: ${attempts}`] : []),
+      ].join("\n"),
+      correlationId,
+    );
+  };
+
+  const getSettings = () => ({
+    autoRepair: state.autoRepair,
+    notificationsEnabled: !state.notificationsDisabled,
+  });
+
+  const updateSettings = ({ autoRepair, notificationsEnabled } = {}) => {
+    const hasAutoRepair = typeof autoRepair === "boolean";
+    const hasNotificationsEnabled = typeof notificationsEnabled === "boolean";
+    if (!hasAutoRepair && !hasNotificationsEnabled) {
+      throw new Error("Expected autoRepair and/or notificationsEnabled boolean");
+    }
+    const envVars = readEnvFile();
+    if (hasAutoRepair) {
+      const existingIdx = envVars.findIndex((item) => item.key === "WATCHDOG_AUTO_REPAIR");
+      const nextValue = autoRepair ? "true" : "false";
+      if (existingIdx >= 0) {
+        envVars[existingIdx] = { ...envVars[existingIdx], value: nextValue };
+      } else {
+        envVars.push({ key: "WATCHDOG_AUTO_REPAIR", value: nextValue });
+      }
+    }
+    if (hasNotificationsEnabled) {
+      const existingIdx = envVars.findIndex(
+        (item) => item.key === "WATCHDOG_NOTIFICATIONS_DISABLED",
+      );
+      const nextValue = notificationsEnabled ? "false" : "true";
+      if (existingIdx >= 0) {
+        envVars[existingIdx] = { ...envVars[existingIdx], value: nextValue };
+      } else {
+        envVars.push({
+          key: "WATCHDOG_NOTIFICATIONS_DISABLED",
+          value: nextValue,
+        });
+      }
+    }
+    writeEnvFile(envVars);
+    reloadEnv();
+    state.autoRepair = isTruthy(process.env.WATCHDOG_AUTO_REPAIR);
+    state.notificationsDisabled = isTruthy(process.env.WATCHDOG_NOTIFICATIONS_DISABLED);
+    return getSettings();
+  };
+
+  const runRepair = async ({ source, correlationId, force = false }) => {
+    if (!force && !state.autoRepair) {
+      return { ok: false, skipped: true, reason: "auto_repair_disabled" };
+    }
+    if (state.operationInProgress) {
+      return { ok: false, skipped: true, reason: "operation_in_progress" };
+    }
+
+    state.operationInProgress = true;
+    try {
+      const result = await clawCmd("doctor --fix --yes", { quiet: true });
+      const ok = !!result?.ok;
+      logEvent("repair", source, ok ? "ok" : "failed", result, correlationId);
+      if (ok) {
+        let launchedGateway = false;
+        try {
+          const child = launchGatewayProcess();
+          launchedGateway = !!child;
+          if (launchedGateway) {
+            logEvent("restart", "repair", "ok", { pid: child.pid }, correlationId);
+          } else {
+            logEvent(
+              "restart",
+              "repair",
+              "failed",
+              { reason: "launchGatewayProcess returned no child" },
+              correlationId,
+            );
+          }
+        } catch (err) {
+          logEvent("restart", "repair", "failed", { error: err.message }, correlationId);
+        }
+        state.health = "unknown";
+        state.lifecycle = "running";
+        state.repairAttempts = 0;
+        state.crashTimestamps = [];
+        const verifiedHealthy = await runHealthCheck({
+          allowDuringOperation: true,
+          source: "repair_verify",
+          allowAutoRepair: false,
+        });
+        await notifyAutoRepairOutcome({
+          source,
+          correlationId,
+          ok: true,
+          verifiedHealthy,
+          attempts: state.repairAttempts,
+        });
+        if (!verifiedHealthy && source !== "manual") {
+          state.pendingRecoveryNoticeSource = source;
+        } else {
+          state.pendingRecoveryNoticeSource = "";
+        }
+        return { ok: true, verifiedHealthy, launchedGateway, result };
+      }
+
+      state.repairAttempts += 1;
+      state.health = "unhealthy";
+      await notifyAutoRepairOutcome({
+        source,
+        correlationId,
+        ok: false,
+        attempts: state.repairAttempts,
+      });
+      if (state.repairAttempts >= kWatchdogMaxRepairAttempts) {
+        await notify(
+          [
+            "🐺 *AlphaClaw Watchdog*",
+            "🔴 Auto-repair failed repeatedly",
+            `Attempts: ${state.repairAttempts}`,
+            withViewLogsSuffix("Auto-repair paused until manual action."),
+          ].join("\n"),
+          correlationId,
+        );
+      }
+      return { ok: false, result };
+    } finally {
+      state.operationInProgress = false;
+    }
+  };
+
+  const runHealthCheck = async ({
+    allowDuringOperation = false,
+    source = "health_timer",
+    allowAutoRepair = true,
+  } = {}) => {
+    if (state.expectedRestartInProgress && Date.now() >= state.expectedRestartUntilMs) {
+      clearExpectedRestartWindow();
+    }
+    if (state.operationInProgress && !allowDuringOperation) return false;
+    const gatewayStartedAtAtStart = state.gatewayStartedAt;
+    const correlationId = createCorrelationId();
+    state.lastHealthCheckAt = new Date().toISOString();
+    const result = await clawCmd("health --json", { quiet: true });
+    const parsed = parseHealthResult(result);
+    const staleAfterRestart =
+      gatewayStartedAtAtStart != null &&
+      state.gatewayStartedAt != null &&
+      state.gatewayStartedAt !== gatewayStartedAtAtStart;
+    const restartWindowActive =
+      state.expectedRestartInProgress && Date.now() < state.expectedRestartUntilMs;
+    if (staleAfterRestart) {
+      return false;
+    }
+    if (parsed.ok) {
+      const wasUnhealthy = state.health !== "healthy";
+      clearExpectedRestartWindow();
+      state.health = "healthy";
+      if (state.lifecycle !== "crash_loop") state.lifecycle = "running";
+      if (!state.uptimeStartedAt || wasUnhealthy) state.uptimeStartedAt = Date.now();
+      state.repairAttempts = 0;
+      state.crashRecoveryActive = false;
+      if (state.pendingRecoveryNoticeSource) {
+        const recoverySource = state.pendingRecoveryNoticeSource;
+        state.pendingRecoveryNoticeSource = "";
+        await notify(
+          [
+            "🐺 *AlphaClaw Watchdog*",
+            withViewLogsSuffix("🟢 Gateway healthy again"),
+            `Trigger: ${asInlineCode(recoverySource)}`,
+          ].join("\n"),
+          correlationId,
+        );
+      }
+      logEvent("health_check", source, "ok", parsed.details || result, correlationId);
+      return true;
+    }
+    if (restartWindowActive) {
+      logEvent(
+        "health_check",
+        source,
+        "ok",
+        {
+          reason: parsed.reason,
+          result,
+          skipped: true,
+          expectedRestartActive: true,
+          expectedRestartUntilMs: state.expectedRestartUntilMs,
+        },
+        correlationId,
+      );
+      return false;
+    }
+
+    const withinStartupGrace =
+      !!state.gatewayStartedAt &&
+      Date.now() - state.gatewayStartedAt < kHealthStartupGraceMs &&
+      state.lifecycle === "running" &&
+      !state.crashRecoveryActive;
+    if (withinStartupGrace) {
+      logEvent(
+        "health_check",
+        source,
+        "ok",
+        {
+          reason: parsed.reason,
+          result,
+          skipped: true,
+          startupGraceActive: true,
+          startupGraceMs: kHealthStartupGraceMs,
+        },
+        correlationId,
+      );
+      return false;
+    }
+
+    state.health = "degraded";
+    logEvent(
+      "health_check",
+      source,
+      "failed",
+      { reason: parsed.reason, result },
+      correlationId,
+    );
+    if (!state.autoRepair || !allowAutoRepair) return false;
+    await runRepair({ source, correlationId });
+    return false;
+  };
+
+  const restartAfterCrash = async (correlationId) => {
+    if (state.operationInProgress) return;
+    state.operationInProgress = true;
+    try {
+      const child = launchGatewayProcess();
+      if (child) {
+        logEvent("restart", "exit_event", "ok", { pid: child.pid }, correlationId);
+      } else {
+        logEvent(
+          "restart",
+          "exit_event",
+          "failed",
+          { reason: "launchGatewayProcess returned no child" },
+          correlationId,
+        );
+      }
+    } catch (err) {
+      logEvent("restart", "exit_event", "failed", { error: err.message }, correlationId);
+    } finally {
+      state.operationInProgress = false;
+    }
+  };
+
+  const onGatewayExit = ({ code, signal, expectedExit = false, stderrTail = [] } = {}) => {
+    const correlationId = createCorrelationId();
+    if (expectedExit) {
+      state.lifecycle = "restarting";
+      state.health = "unknown";
+      state.crashRecoveryActive = false;
+      markExpectedRestartWindow();
+      startBootstrapHealthChecks();
+      logEvent(
+        "restart",
+        "exit_event",
+        "ok",
+        { expectedExit: true, code: code ?? null, signal: signal ?? null },
+        correlationId,
+      );
+      return;
+    }
+
+    state.lifecycle = "crashed";
+    state.health = "unhealthy";
+    state.crashRecoveryActive = true;
+    state.crashTimestamps.push(Date.now());
+    trimCrashWindow();
+    logEvent(
+      "crash",
+      "exit_event",
+      "failed",
+      { code: code ?? null, signal: signal ?? null, stderrTail },
+      correlationId,
+    );
+
+    if (state.crashTimestamps.length >= kWatchdogCrashLoopThreshold) {
+      state.lifecycle = "crash_loop";
+      logEvent(
+        "crash_loop",
+        "exit_event",
+        "failed",
+        {
+          crashesInWindow: state.crashTimestamps.length,
+          windowMs: kWatchdogCrashLoopWindowMs,
+        },
+        correlationId,
+      );
+      void notify(
+        [
+          "🐺 *AlphaClaw Watchdog*",
+          withViewLogsSuffix(
+            state.autoRepair
+              ? "🔴 Crash loop detected, auto-repairing..."
+              : "🔴 Crash loop detected",
+          ),
+          `Crashes: ${state.crashTimestamps.length} in the last ${Math.floor(kWatchdogCrashLoopWindowMs / 1000)}s`,
+          `Last exit code: ${code ?? "unknown"}`,
+          ...(state.autoRepair ? [] : ["Auto-restart paused; manual action required."]),
+        ].join("\n"),
+        correlationId,
+      );
+      if (state.autoRepair) {
+        void runRepair({
+          source: "crash_loop",
+          correlationId,
+        });
+        return;
+      }
+      return;
+    }
+
+    void restartAfterCrash(correlationId);
+  };
+
+  const onGatewayLaunch = ({ startedAt = Date.now() } = {}) => {
+    state.lifecycle = "running";
+    state.health = "unknown";
+    state.crashRecoveryActive = false;
+    clearExpectedRestartWindow();
+    state.uptimeStartedAt = startedAt;
+    state.gatewayStartedAt = startedAt;
+    startBootstrapHealthChecks();
+  };
+
+  const onExpectedRestart = () => {
+    state.lifecycle = "restarting";
+    state.health = "unknown";
+    state.crashRecoveryActive = false;
+    markExpectedRestartWindow();
+    startBootstrapHealthChecks();
+  };
+
+  const triggerRepair = async () => {
+    const correlationId = createCorrelationId();
+    return runRepair({
+      source: "manual",
+      correlationId,
+      force: true,
+    });
+  };
+
+  const start = () => {
+    if (healthTimer || bootstrapHealthTimer) return;
+    state.lifecycle = "running";
+    state.health = "unknown";
+    state.uptimeStartedAt = Date.now();
+    state.gatewayStartedAt = Date.now();
+    startBootstrapHealthChecks();
+  };
+
+  const stop = () => {
+    if (bootstrapHealthTimer) {
+      clearTimeout(bootstrapHealthTimer);
+      bootstrapHealthTimer = null;
+    }
+    if (healthTimer) {
+      clearInterval(healthTimer);
+      healthTimer = null;
+    }
+    state.lifecycle = "stopped";
+  };
+
+  const getStatus = () => {
+    trimCrashWindow();
+    return {
+      lifecycle: state.lifecycle,
+      health: state.health,
+      uptimeMs: state.uptimeStartedAt ? Date.now() - state.uptimeStartedAt : 0,
+      uptimeStartedAt: state.uptimeStartedAt
+        ? new Date(state.uptimeStartedAt).toISOString()
+        : null,
+      lastHealthCheckAt: state.lastHealthCheckAt,
+      repairAttempts: state.repairAttempts,
+      autoRepair: state.autoRepair,
+      crashCountInWindow: state.crashTimestamps.length,
+      crashLoopThreshold: kWatchdogCrashLoopThreshold,
+      crashLoopWindowMs: kWatchdogCrashLoopWindowMs,
+      operationInProgress: state.operationInProgress,
+    };
+  };
+
+  return {
+    getStatus,
+    getSettings,
+    updateSettings,
+    triggerRepair,
+    onExpectedRestart,
+    onGatewayExit,
+    onGatewayLaunch,
+    start,
+    stop,
+  };
+};
+
+module.exports = { createWatchdog };