@misterhuydo/sentinel 1.1.4 → 1.1.6

package/.cairn/.hint-lock

@@ -1 +1 @@
-2026-03-23T08:08:15.205Z
+2026-03-23T08:53:23.129Z

package/.cairn/session.json

@@ -1,6 +1,6 @@
 {
-  "message": "Auto-checkpoint at 2026-03-23T08:33:20.221Z",
-  "checkpoint_at": "2026-03-23T08:33:20.223Z",
+  "message": "Auto-checkpoint at 2026-03-23T08:59:04.141Z",
+  "checkpoint_at": "2026-03-23T08:59:04.142Z",
   "active_files": [],
   "notes": [],
   "mtime_snapshot": {}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@misterhuydo/sentinel",
-  "version": "1.1.4",
+  "version": "1.1.6",
   "description": "Sentinel — Autonomous DevOps Agent installer and manager",
   "bin": {
     "sentinel": "./bin/sentinel.js"

package/python/sentinel/notify.py

@@ -7,6 +7,7 @@ Calls never raise — failures are logged and silently dropped.
 
 import logging
 import re
+import time
 
 import requests
 
@@ -28,8 +29,88 @@ def is_rate_limited(text: str) -> bool:
     return bool(_RATE_LIMIT_RE.search(text))
 
 
+# ── Circuit breaker ────────────────────────────────────────────────────────────
+#
+# Prevents alert storms when Claude is persistently rate-limited.
+# Each `source` string gets its own independent circuit:
+#   CLOSED → normal; alerts pass through immediately
+#   OPEN   → suppressed; one re-alert every CIRCUIT_COOLDOWN_SECONDS
+#
+# On recovery (first non-rate-limited output after OPEN):
+#   → post "resolved" to Slack, close the circuit
+
+CIRCUIT_COOLDOWN_SECONDS = 3600   # 1 h between repeat alerts while open
+
+# source → {opened_at, last_alerted_at, count}
+_circuits: dict[str, dict] = {}
+
+
+def get_circuit_status() -> dict:
+    """
+    Return a snapshot of all open circuits.
+    Used by the `check_auth_status` Boss tool.
+
+    Returns:
+        { source: { state, opened_at, open_for_seconds, alert_count } }
+        Only open circuits are included; an empty dict means everything is healthy.
+    """
+    now = time.time()
+    return {
+        src: {
+            "state": "open",
+            "opened_at": c["opened_at"],
+            "open_for_seconds": int(now - c["opened_at"]),
+            "alert_count": c["count"],
+        }
+        for src, c in _circuits.items()
+    }
+
+
+def _open_or_repeat(bot_token: str, channel: str, source: str, output: str) -> None:
+    """Open circuit on first hit; re-alert after cooldown if still failing."""
+    now = time.time()
+    circuit = _circuits.get(source)
+
+    if circuit is None:
+        # First occurrence — open and alert immediately
+        _circuits[source] = {"opened_at": now, "last_alerted_at": now, "count": 1}
+        logger.error("Circuit opened for %s: %s", source, output[:200])
+        slack_alert(bot_token, channel, rate_limit_message(source, output))
+        return
+
+    circuit["count"] += 1
+    elapsed = now - circuit["last_alerted_at"]
+    if elapsed >= CIRCUIT_COOLDOWN_SECONDS:
+        # Still failing after cooldown — remind admins once per hour
+        circuit["last_alerted_at"] = now
+        open_mins = int((now - circuit["opened_at"]) / 60)
+        msg = (
+            f":warning: *Sentinel — Claude usage/auth problem still active ({source})*\n"
+            f"Still failing after {open_mins} minutes. Total occurrences: {circuit['count']}.\n"
+            f"Last error:\n```{output.strip()[:300]}```\n"
+            f"Run `check_auth_status` in Slack to see the full picture."
+        )
+        logger.error("Circuit still open for %s (count=%d)", source, circuit["count"])
+        slack_alert(bot_token, channel, msg)
+    # else: within cooldown window — suppress
+
+
+def _close_if_open(bot_token: str, channel: str, source: str) -> None:
+    """If circuit was open, close it and post a recovery alert."""
+    circuit = _circuits.pop(source, None)
+    if circuit is None:
+        return
+    duration_mins = int((time.time() - circuit["opened_at"]) / 60)
+    msg = (
+        f":white_check_mark: *Sentinel — Claude auth restored ({source})*\n"
+        f"Fixed after {duration_mins} min. Total failures during outage: {circuit['count']}."
+    )
+    logger.info("Circuit closed for %s after %d min, %d failures", source, duration_mins, circuit["count"])
+    slack_alert(bot_token, channel, msg)
+
+
 def rate_limit_message(source: str, raw: str) -> str:
-    """Produce a human-readable Slack alert for a rate-limit event."""
+    """Produce a human-readable Slack alert for a rate-limit / auth event (first occurrence)."""
     snippet = raw.strip()[:300].replace("\n", " ")
     return (
         f":warning: *Sentinel — Claude usage/auth problem ({source})*\n"
@@ -37,9 +118,10 @@ def rate_limit_message(source: str, raw: str) -> str:
         f"```{snippet}```\n"
         f"*What to check:*\n"
         f"• API key: verify `ANTHROPIC_API_KEY` in `sentinel.properties` is valid and has credit\n"
-        f"• Claude Pro: run `claude login` on the server to refresh OAuth\n"
-        f"• Both: at least one auth method must be working\n"
-        f"Sentinel will retry on the next poll cycle."
+        f"• Claude Pro: run `claude login` on the server to refresh the OAuth session\n"
+        f"• Both: Sentinel tries both methods — at least one must be working\n"
+        f"Repeat alerts will be suppressed for 1 hour. "
+        f"Run `check_auth_status` in Slack to see current state."
     )
 
 
@@ -77,12 +159,15 @@ def alert_if_rate_limited(
     output: str,
 ) -> bool:
     """
-    Check output for rate-limit / auth signals.
-    If found, post a Slack alert and return True.
+    Check output for rate-limit / auth signals and manage the circuit breaker.
+
+    - Rate limited  → open/keep-open circuit, alert (with cooldown suppression)
+    - Not limited   → close circuit if it was open (recovery alert), return False
+
+    Returns True if a rate-limit signal was found.
     """
     if not is_rate_limited(output):
+        _close_if_open(bot_token, channel, source)
         return False
-    msg = rate_limit_message(source, output)
-    logger.error("Claude rate-limit/auth failure in %s: %s", source, output[:200])
-    slack_alert(bot_token, channel, msg)
+    _open_or_repeat(bot_token, channel, source, output)
     return True

package/python/sentinel/sentinel_boss.py

@@ -137,6 +137,7 @@ reply with a short summary grouped by category:
 *Fix management*
 • `get_fix_details` — full details of a specific fix — "show fix abc123"
 • `list_pending_prs` — all open Sentinel PRs awaiting review — "list open PRs"
+• `check_auth_status` — Claude auth health, rate-limit circuit state, fix engine 24 h stats — "is Claude working?", "any rate limits?", "auth issues?"
 
 *Project & task delivery*
 • `list_projects` — all projects and repos Sentinel manages — "what projects do you manage?"
@@ -275,6 +276,16 @@ _TOOLS = [
         "description": "List all open Sentinel PRs awaiting admin review.",
         "input_schema": {"type": "object", "properties": {}},
     },
+    {
+        "name": "check_auth_status",
+        "description": (
+            "Check Claude authentication health, current rate-limit / usage-limit circuit state, "
+            "and fix engine stats for the last 24 h. "
+            "Use when someone asks: 'is Claude working?', 'any rate limits?', 'why aren't fixes running?', "
+            "'is the API key OK?', 'auth issues?', 'fix engine status'."
+        ),
+        "input_schema": {"type": "object", "properties": {}},
+    },
     {
         "name": "pause_sentinel",
         "description": (
@@ -798,6 +809,67 @@ async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=Non
             "sentinel_paused": Path("SENTINEL_PAUSE").exists(),
         })
 
+    if name == "check_auth_status":
+        import subprocess as _sp
+        from .notify import get_circuit_status
+        cfg = cfg_loader.sentinel
+
+        # Auth configuration
+        has_key      = bool(cfg.anthropic_api_key)
+        pro_for_tasks = cfg.claude_pro_for_tasks
+        if pro_for_tasks and has_key:
+            primary, fallback = "claude_pro_oauth", "api_key"
+        elif pro_for_tasks:
+            primary, fallback = "claude_pro_oauth", None
+        else:
+            primary, fallback = "api_key", "claude_pro_oauth" if not has_key else "claude_pro_oauth"
+
+        # Claude CLI liveness check
+        cli_ok, cli_version = False, ""
+        try:
+            r = _sp.run(
+                [cfg.claude_code_bin, "--version"],
+                capture_output=True, text=True, timeout=10,
+            )
+            if r.returncode == 0:
+                cli_ok     = True
+                cli_version = r.stdout.strip() or r.stderr.strip()
+        except Exception:
+            pass
+
+        # Circuit breaker snapshot — only open (unhealthy) circuits appear here
+        circuits = get_circuit_status()
+
+        # Fix engine stats (last 24 h)
+        recent = store.get_recent_fixes(hours=24)
+        counts = {"applied": 0, "failed": 0, "skipped": 0, "pending": 0}
+        last_success = None
+        for f in recent:
+            s = f.get("status", "")
+            if s in counts:
+                counts[s] += 1
+            if s == "applied" and not last_success:
+                last_success = f.get("timestamp", "")
+
+        overall = "healthy"
+        if circuits:
+            overall = "degraded — rate/auth limit active on: " + ", ".join(circuits)
+        elif not cli_ok:
+            overall = "warning — claude CLI not reachable"
+
+        return json.dumps({
+            "overall": overall,
+            "auth": {
+                "api_key_configured": has_key,
+                "claude_pro_for_tasks": pro_for_tasks,
+                "primary_method": primary,
+                "fallback_method": fallback,
+            },
+            "claude_cli": {"available": cli_ok, "version": cli_version},
+            "rate_limit_circuits": circuits,
+            "fix_engine_24h": {**counts, "last_successful_fix": last_success},
+        })
+
     if name == "create_issue":
         description = inputs["description"]
         target_repo = inputs.get("target_repo", "")
@@ -1938,7 +2010,7 @@ async def handle_message(
                 from .notify import rate_limit_message
                 alert_if_rate_limited(cfg.slack_bot_token, cfg.slack_channel,
                                       "sentinel_boss/api", err_str)
-            logger.warning("Boss: API key path failed (%s), trying CLI fallback", err_str[:80])
+            logger.warning("Boss: API key path failed (%s), trying CLI fallback", err_str)
 
     # 2nd priority: Claude Pro / OAuth via CLI (limited tools but no API key needed)
     cli_reply, cli_done = await _handle_with_cli(