@chrysb/alphaclaw 0.8.1-beta.8 → 0.8.2

package/lib/server/watchdog.js

@@ -10,6 +10,7 @@ const {
 const kHealthStartupGraceMs = 30 * 1000;
 const kBootstrapHealthCheckMs = 5 * 1000;
 const kExpectedRestartWindowMs = 15 * 1000;
+const kGatewayHealthTimeoutMs = 5 * 1000;
 
 const isTruthy = (value) =>
   ["1", "true", "yes", "on"].includes(
@@ -18,32 +19,6 @@ const isTruthy = (value) =>
       .toLowerCase(),
   );
 
-const parseHealthResult = (result) => {
-  if (!result?.ok)
-    return { ok: false, reason: result?.stderr || "health command failed" };
-  const raw = String(result.stdout || "").trim();
-  if (!raw) return { ok: true };
-  try {
-    const parsed = JSON.parse(raw);
-    if (parsed?.ok === false || parsed?.status === "unhealthy") {
-      return { ok: false, reason: parsed?.error || "gateway unhealthy" };
-    }
-    return { ok: true, details: parsed };
-  } catch {
-    const firstBrace = raw.indexOf("{");
-    const lastBrace = raw.lastIndexOf("}");
-    if (firstBrace >= 0 && lastBrace > firstBrace) {
-      try {
-        const parsed = JSON.parse(raw.slice(firstBrace, lastBrace + 1));
-        if (parsed?.ok === false || parsed?.status === "unhealthy") {
-          return { ok: false, reason: parsed?.error || "gateway unhealthy" };
-        }
-        return { ok: true, details: parsed };
-      } catch {}
-    }
-  }
-  return { ok: true };
-};
 const isDuplicateGatewayLaunchExit = ({ code, stderrTail = [] } = {}) => {
   if (code !== 1) return false;
   const stderrText = (Array.isArray(stderrTail) ? stderrTail : [])
@@ -66,6 +41,7 @@ const createWatchdog = ({
   writeEnvFile,
   reloadEnv,
   resolveSetupUrl,
+  resolveGatewayHealthUrl = () => "",
 }) => {
   const state = {
     lifecycle: "stopped",
@@ -85,11 +61,26 @@ const createWatchdog = ({
     expectedRestartInProgress: false,
     expectedRestartUntilMs: 0,
     pendingRecoveryNoticeSource: "",
+    awaitingAutoRepairRecovery: false,
     startupConsecutiveHealthFailures: 0,
   };
   let healthTimer = null;
   let bootstrapHealthTimer = null;
   let degradedHealthTimer = null;
+  let activeIncidentKey = "";
+  let sentIncidentNotifications = new Set();
+
+  const openIncident = (incidentKey = "gateway") => {
+    const normalizedKey = String(incidentKey || "gateway");
+    if (activeIncidentKey === normalizedKey) return;
+    activeIncidentKey = normalizedKey;
+    sentIncidentNotifications = new Set();
+  };
+
+  const closeIncident = () => {
+    activeIncidentKey = "";
+    sentIncidentNotifications = new Set();
+  };
 
   const clearDegradedHealthCheckTimer = () => {
     if (!degradedHealthTimer) return;
@@ -209,6 +200,27 @@ const createWatchdog = ({
     return result;
   };
 
+  const notifyOncePerIncident = async (
+    notificationKey,
+    message,
+    correlationId = "",
+  ) => {
+    const key = String(notificationKey || "").trim();
+    if (!key) return notify(message, correlationId);
+    if (sentIncidentNotifications.has(key)) {
+      return {
+        ok: false,
+        skipped: true,
+        reason: "incident_notification_already_sent",
+      };
+    }
+    const result = await notify(message, correlationId);
+    if (result?.ok || result?.skipped) {
+      sentIncidentNotifications.add(key);
+    }
+    return result;
+  };
+
   const getWatchdogSetupUrl = () => {
     try {
       const base =
@@ -241,12 +253,19 @@ const createWatchdog = ({
     attempts = 0,
   }) => {
     if (source === "manual") return;
+    openIncident("gateway_recovery");
     const title = ok
       ? verifiedHealthy
         ? "🟢 Auto-repair complete, gateway healthy"
         : "🟡 Auto-repair started, awaiting health check"
       : "🔴 Auto-repair failed";
-    await notify(
+    const notificationKey = ok
+      ? verifiedHealthy
+        ? "auto_repair_complete"
+        : "auto_repair_awaiting_health"
+      : "auto_repair_failed";
+    await notifyOncePerIncident(
+      notificationKey,
       [
         "🐺 *AlphaClaw Watchdog*",
         withViewLogsSuffix(title),
@@ -262,6 +281,59 @@ const createWatchdog = ({
     notificationsEnabled: !state.notificationsDisabled,
   });
 
+  const probeGatewayHealth = async () => {
+    const healthUrl = String(resolveGatewayHealthUrl() || "").trim();
+    if (!healthUrl) {
+      return {
+        ok: false,
+        reason: "gateway health URL unavailable",
+      };
+    }
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), kGatewayHealthTimeoutMs);
+    try {
+      const response = await fetch(healthUrl, {
+        method: "GET",
+        headers: { Accept: "application/json" },
+        signal: controller.signal,
+      });
+      const rawBody = await response.text();
+      let parsedBody = null;
+      try {
+        parsedBody = rawBody ? JSON.parse(rawBody) : null;
+      } catch {}
+      if (!response.ok) {
+        return {
+          ok: false,
+          reason:
+            parsedBody?.error ||
+            `gateway health returned HTTP ${response.status}`,
+        };
+      }
+      if (parsedBody?.ok === false) {
+        return {
+          ok: false,
+          reason: parsedBody?.error || "gateway unhealthy",
+        };
+      }
+      return {
+        ok: true,
+        details: parsedBody,
+      };
+    } catch (error) {
+      const message =
+        error?.name === "AbortError"
+          ? `gateway health timed out after ${kGatewayHealthTimeoutMs}ms`
+          : error?.message || "gateway health request failed";
+      return {
+        ok: false,
+        reason: message,
+      };
+    } finally {
+      clearTimeout(timeoutId);
+    }
+  };
+
   const updateSettings = ({ autoRepair, notificationsEnabled } = {}) => {
     const hasAutoRepair = typeof autoRepair === "boolean";
     const hasNotificationsEnabled = typeof notificationsEnabled === "boolean";
@@ -309,6 +381,9 @@ const createWatchdog = ({
     if (!force && !state.autoRepair) {
       return { ok: false, skipped: true, reason: "auto_repair_disabled" };
     }
+    if (!force && state.awaitingAutoRepairRecovery) {
+      return { ok: false, skipped: true, reason: "awaiting_health_recovery" };
+    }
     if (state.operationInProgress) {
       return { ok: false, skipped: true, reason: "operation_in_progress" };
     }
@@ -353,6 +428,7 @@ const createWatchdog = ({
         state.lifecycle = "running";
         state.repairAttempts = 0;
         state.crashTimestamps = [];
+        state.awaitingAutoRepairRecovery = false;
         const verifiedHealthy = await runHealthCheck({
           allowDuringOperation: true,
           source: "repair_verify",
@@ -367,8 +443,10 @@ const createWatchdog = ({
         });
         if (!verifiedHealthy && source !== "manual") {
           state.pendingRecoveryNoticeSource = source;
+          state.awaitingAutoRepairRecovery = true;
         } else {
           state.pendingRecoveryNoticeSource = "";
+          state.awaitingAutoRepairRecovery = false;
         }
         return { ok: true, verifiedHealthy, launchedGateway, result };
       }
@@ -413,8 +491,7 @@ const createWatchdog = ({
     const gatewayStartedAtAtStart = state.gatewayStartedAt;
     const correlationId = createCorrelationId();
     state.lastHealthCheckAt = new Date().toISOString();
-    const result = await clawCmd("health --json", { quiet: true });
-    const parsed = parseHealthResult(result);
+    const parsed = await probeGatewayHealth();
     const staleAfterRestart =
       gatewayStartedAtAtStart != null &&
       state.gatewayStartedAt != null &&
@@ -428,6 +505,11 @@ const createWatchdog = ({
     if (parsed.ok) {
       const wasUnhealthy = state.health !== "healthy";
       const recoveredFromCrashLoop = state.lifecycle === "crash_loop";
+      const shouldNotifyRecovery =
+        !!activeIncidentKey ||
+        recoveredFromCrashLoop ||
+        !!state.pendingRecoveryNoticeSource ||
+        state.awaitingAutoRepairRecovery;
       state.startupConsecutiveHealthFailures = 0;
       clearDegradedHealthCheckTimer();
       clearExpectedRestartWindow();
@@ -437,29 +519,25 @@ const createWatchdog = ({
         state.uptimeStartedAt = Date.now();
       state.repairAttempts = 0;
       state.crashRecoveryActive = false;
-      if (recoveredFromCrashLoop) {
+      state.awaitingAutoRepairRecovery = false;
+      if (shouldNotifyRecovery) {
         logEvent(
           "recovery",
           source,
           "ok",
-          {
-            previousLifecycle: "crash_loop",
-            health: "healthy",
-          },
-          correlationId,
-        );
-        await notify(
           [
-            "🐺 *AlphaClaw Watchdog*",
-            withViewLogsSuffix("🟢 Gateway healthy again"),
-          ].join("\n"),
+            {
+              previousLifecycle: recoveredFromCrashLoop
+                ? "crash_loop"
+                : null,
+              previousRecoverySource: state.pendingRecoveryNoticeSource || null,
+              health: "healthy",
+            },
+          ][0],
           correlationId,
         );
-      }
-      if (state.pendingRecoveryNoticeSource) {
-        const recoverySource = state.pendingRecoveryNoticeSource;
-        state.pendingRecoveryNoticeSource = "";
-        await notify(
+        await notifyOncePerIncident(
+          "gateway_healthy_again",
           [
             "🐺 *AlphaClaw Watchdog*",
             withViewLogsSuffix("🟢 Gateway healthy again"),
@@ -467,11 +545,13 @@ const createWatchdog = ({
           correlationId,
         );
       }
+      state.pendingRecoveryNoticeSource = "";
+      closeIncident();
       logEvent(
         "health_check",
         source,
         "ok",
-        parsed.details || result,
+        parsed.details || { ok: true },
         correlationId,
       );
       return true;
@@ -485,7 +565,7 @@ const createWatchdog = ({
         "ok",
         {
           reason: parsed.reason,
-          result,
+          details: parsed.details || null,
           skipped: true,
           expectedRestartActive: true,
           expectedRestartUntilMs: state.expectedRestartUntilMs,
@@ -509,7 +589,7 @@ const createWatchdog = ({
         "ok",
         {
           reason: parsed.reason,
-          result,
+          details: parsed.details || null,
           skipped: true,
           startupGraceActive: true,
           startupGraceMs: kHealthStartupGraceMs,
@@ -531,7 +611,7 @@ const createWatchdog = ({
           "ok",
           {
             reason: parsed.reason,
-            result,
+            details: parsed.details || null,
             skipped: true,
             startupFailureRetryActive: true,
             startupConsecutiveFailures: state.startupConsecutiveHealthFailures,
@@ -551,10 +631,11 @@ const createWatchdog = ({
       "health_check",
       source,
       "failed",
-      { reason: parsed.reason, result },
+      { reason: parsed.reason, details: parsed.details || null },
       correlationId,
     );
     if (!state.autoRepair || !allowAutoRepair) return false;
+    if (state.awaitingAutoRepairRecovery) return false;
     await runRepair({ source, correlationId });
     return false;
   };
@@ -658,6 +739,7 @@ const createWatchdog = ({
 
     if (state.crashTimestamps.length >= kWatchdogCrashLoopThreshold) {
       state.lifecycle = "crash_loop";
+      openIncident("gateway_recovery");
       logEvent(
         "crash_loop",
         "exit_event",
@@ -668,7 +750,8 @@ const createWatchdog = ({
         },
         correlationId,
       );
-      void notify(
+      void notifyOncePerIncident(
+        "crash_loop_detected",
         [
           "🐺 *AlphaClaw Watchdog*",
           withViewLogsSuffix(
@@ -753,6 +836,9 @@ const createWatchdog = ({
     state.lifecycle = "stopped";
     state.uptimeStartedAt = null;
     state.startupConsecutiveHealthFailures = 0;
+    state.awaitingAutoRepairRecovery = false;
+    state.pendingRecoveryNoticeSource = "";
+    closeIncident();
   };
 
   const getStatus = () => {

package/lib/server.js

@@ -226,6 +226,7 @@ const watchdog = createWatchdog({
   writeEnvFile,
   reloadEnv,
   resolveSetupUrl,
+  resolveGatewayHealthUrl: () => `${getGatewayUrl()}/health`,
 });
 const watchdogTerminal = createWatchdogTerminalService({
   cwd: constants.OPENCLAW_DIR,
@@ -310,6 +311,7 @@ const { isAuthorizedRequest, gmailWatchService } = registerServerRoutes({
   deleteOauthCallback,
   markOauthCallbackUsed,
   watchdog,
+  watchdogNotifier,
   getRecentEvents,
   readLogTail,
   watchdogTerminal,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chrysb/alphaclaw",
-  "version": "0.8.1-beta.8",
+  "version": "0.8.2",
   "publishConfig": {
     "access": "public"
   },
@@ -23,10 +23,12 @@
   ],
   "scripts": {
     "start": "node bin/alphaclaw.js start",
+    "build:ui": "node scripts/build-ui.mjs",
     "test": "vitest run",
     "test:watch": "vitest",
     "test:watchdog": "vitest run tests/server/watchdog.test.js tests/server/watchdog-db.test.js tests/server/routes-watchdog.test.js",
-    "test:coverage": "vitest run --coverage"
+    "test:coverage": "vitest run --coverage",
+    "prepack": "npm run build:ui"
   },
   "dependencies": {
     "express": "^4.21.0",
@@ -35,9 +37,18 @@
     "ws": "^8.19.0"
   },
   "devDependencies": {
+    "@xterm/addon-fit": "^0.10.0",
+    "@xterm/xterm": "^5.5.0",
     "@vitest/coverage-v8": "^4.0.18",
+    "chart.js": "^4.5.1",
+    "esbuild": "^0.25.9",
+    "htm": "^3.1.1",
+    "marked": "^16.4.1",
+    "preact": "^10.27.2",
     "supertest": "^7.2.2",
-    "vitest": "^4.0.18"
+    "tailwindcss": "^3.4.17",
+    "vitest": "^4.0.18",
+    "wouter-preact": "^3.7.1"
   },
   "engines": {
     "node": ">=22.12.0"