@misterhuydo/sentinel 1.1.3 → 1.1.5

package/.cairn/.hint-lock

@@ -1 +1 @@
-2026-03-23T08:08:15.205Z
+2026-03-23T08:53:23.129Z

package/.cairn/session.json

@@ -1,6 +1,6 @@
 {
-  "message": "Auto-checkpoint at 2026-03-23T08:30:55.526Z",
-  "checkpoint_at": "2026-03-23T08:30:55.527Z",
+  "message": "Auto-checkpoint at 2026-03-23T08:53:48.152Z",
+  "checkpoint_at": "2026-03-23T08:53:48.153Z",
   "active_files": [],
   "notes": [],
   "mtime_snapshot": {}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@misterhuydo/sentinel",
-  "version": "1.1.3",
+  "version": "1.1.5",
   "description": "Sentinel — Autonomous DevOps Agent installer and manager",
   "bin": {
     "sentinel": "./bin/sentinel.js"

package/python/sentinel/fix_engine.py

@@ -99,42 +99,109 @@ def _validate_patch(patch: str) -> tuple[bool, str]:
     return True, ""
 
 
+_AUTH_ERROR_HINTS = (
+    "not logged in", "please run claude login", "authentication failed",
+    "api key is not set", "invalid x-api-key", "unauthorized", "please authenticate",
+    "unauthenticated", "auth_required", "no auth", "login required",
+)
+
+
+def _is_auth_error(output: str) -> bool:
+    low = output.lower()
+    return any(hint in low for hint in _AUTH_ERROR_HINTS)
+
+
+def _claude_cmd(bin_path: str, prompt: str) -> list[str]:
+    import os as _os
+    try:
+        skip = _os.getuid() != 0
+    except AttributeError:
+        skip = True  # Windows — always pass flag
+    if skip:
+        return [bin_path, "--dangerously-skip-permissions", "--print", prompt]
+    return [bin_path, "--print", prompt]
+
+
+def _run_claude_attempt(bin_path: str, prompt: str, env: dict) -> tuple[str, bool]:
+    """
+    Run claude CLI with the given env. Returns (output, timed_out).
+    Raises FileNotFoundError if binary is missing.
+    """
+    try:
+        result = subprocess.run(
+            _claude_cmd(bin_path, prompt),
+            capture_output=True, text=True, timeout=SUBPROCESS_TIMEOUT, env=env,
+        )
+        return (result.stdout or "") + (result.stderr or ""), False
+    except subprocess.TimeoutExpired:
+        return "", True
+
+
 def generate_fix(
     event: ErrorEvent,
     repo: RepoConfig,
     cfg: SentinelConfig,
     patches_dir: Path,
-) -> tuple[str, Path | None]:
+) -> tuple[str, Path | None, str]:
     """
     Generate a fix for the given error event.
 
     Returns:
-        (status, patch_path)
+        (status, patch_path, marker)
         status: "patch" | "skip" | "error"
+
+    Auth strategy — API key and Claude Pro (OAuth) are interchangeable:
+      Primary  : Claude Pro (OAuth) if claude_pro_for_tasks=True, else API key
+      Fallback : the other method, if primary fails with an auth error
+      On total auth failure: notify Slack admins + email report recipients
     """
-    # Issues have source like "issues/filename" — no rolling log file exists
+    import os as _os
+
+    marker   = f"sentinel-{event.fingerprint[:8]}"
     log_file = Path(cfg.workspace_dir) / "fetched" / f"{event.source}.log"
     if not log_file.exists():
         log_file = None
-    prompt = _build_prompt(event, repo, log_file)
+    prompt = _build_prompt(event, repo, log_file, marker)
 
     logger.info("Invoking Claude Code for %s (fp=%s)", event.source, event.fingerprint)
-    import os as _os
-    env = _os.environ.copy()
-    # Inject API key only when Claude Pro is NOT preferred for tasks
-    # (when claude_pro_for_tasks=True and API key is set, let claude CLI use OAuth/Pro)
-    if cfg.anthropic_api_key and not cfg.claude_pro_for_tasks:
-        env["ANTHROPIC_API_KEY"] = cfg.anthropic_api_key
+
+    base_env  = _os.environ.copy()
+    api_env   = {**base_env, "ANTHROPIC_API_KEY": cfg.anthropic_api_key} if cfg.anthropic_api_key else None
+    oauth_env = base_env  # relies on cached `claude login` session — no key injected
+
+    # Choose primary/fallback order based on config
+    if cfg.claude_pro_for_tasks and cfg.anthropic_api_key:
+        attempts = [("Claude Pro (OAuth)", oauth_env), ("API key", api_env)]
+    elif cfg.claude_pro_for_tasks:
+        attempts = [("Claude Pro (OAuth)", oauth_env)]
+    elif cfg.anthropic_api_key:
+        attempts = [("API key", api_env), ("Claude Pro (OAuth)", oauth_env)]
+    else:
+        attempts = [("Claude Pro (OAuth)", oauth_env)]
+
+    output = ""
     try:
-        result = subprocess.run(
-            ([cfg.claude_code_bin, "--dangerously-skip-permissions", "--print", prompt]
-            if os.getuid() != 0 else
-            [cfg.claude_code_bin, "--print", prompt]),
-            capture_output=True, text=True, timeout=SUBPROCESS_TIMEOUT, env=env,
-        )
-    except subprocess.TimeoutExpired:
-        logger.error("Claude Code timed out for %s", event.fingerprint)
-        return "error", None, ""
+        for label, env in attempts:
+            if env is None:
+                continue
+            logger.info("fix_engine: trying %s for %s", label, event.fingerprint)
+            output, timed_out = _run_claude_attempt(cfg.claude_code_bin, prompt, env)
+            if timed_out:
+                logger.error("Claude Code timed out for %s", event.fingerprint)
+                return "error", None, ""
+            if not _is_auth_error(output):
+                break
+            logger.warning("fix_engine: %s auth error for %s — trying next method", label, event.fingerprint)
+        else:
+            # All attempts failed with auth errors
+            msg = (
+                ":warning: *Sentinel — Fix Engine auth failure*\n"
+                f"Both API key and Claude Pro (OAuth) failed authentication for `{event.fingerprint}`.\n"
+                "• Check that `ANTHROPIC_API_KEY` is valid, or run `claude login` to refresh the OAuth session."
+            )
+            logger.error("fix_engine: all auth methods failed for %s", event.fingerprint)
+            slack_alert(cfg.slack_bot_token, cfg.slack_channel, msg)
+            return "error", None, ""
     except FileNotFoundError:
         msg = (
             f":warning: *Sentinel — Claude CLI not found*\n"
@@ -145,9 +212,7 @@ def generate_fix(
         slack_alert(cfg.slack_bot_token, cfg.slack_channel, msg)
         return "error", None, ""
 
-    output = (result.stdout or "") + (result.stderr or "")
-
-    # Alert Slack immediately on rate-limit / auth failure — never stay silent
+    # Alert Slack immediately on rate-limit — never stay silent
     alert_if_rate_limited(
         cfg.slack_bot_token,
         cfg.slack_channel,

package/python/sentinel/main.py

@@ -549,13 +549,14 @@ def _log_auth_status(cfg: SentinelConfig) -> None:
 
     if has_api_key and pro_for_tasks:
         logger.info(
-            "Claude auth: API key ✓ (Boss) + Claude Pro preferred for Fix Engine/Ask Codebase. "
+            "Claude auth: API key ✓ + Claude Pro (OAuth) ✓ — "
+            "Fix Engine will try Claude Pro first, falls back to API key on auth error. "
             "Run `claude login` if not already authenticated."
         )
     elif has_api_key and not pro_for_tasks:
         logger.info(
-            "Claude auth: API key ✓ (Boss + Fix Engine). "
-            "CLAUDE_PRO_FOR_TASKS=false — all tasks billed to API quota."
+            "Claude auth: API key ✓ — Boss + Fix Engine use API key. "
+            "CLAUDE_PRO_FOR_TASKS=false; falls back to Claude Pro (OAuth) if key auth fails."
         )
     elif not has_api_key and has_claude_bin:
         logger.warning(

package/python/sentinel/notify.py

@@ -7,6 +7,7 @@ Calls never raise — failures are logged and silently dropped.
 
 import logging
 import re
+import time
 
 import requests
 
@@ -28,8 +29,88 @@ def is_rate_limited(text: str) -> bool:
     return bool(_RATE_LIMIT_RE.search(text))
 
 
+# ── Circuit breaker ────────────────────────────────────────────────────────────
+#
+# Prevents alert storms when Claude is persistently rate-limited.
+# Each `source` string gets its own independent circuit:
+#   CLOSED → normal; alerts pass through immediately
+#   OPEN   → suppressed; one re-alert every CIRCUIT_COOLDOWN_SECONDS
+#
+# On recovery (first non-rate-limited output after OPEN):
+#   → post "resolved" to Slack, close the circuit
+
+CIRCUIT_COOLDOWN_SECONDS = 3600   # 1 h between repeat alerts while open
+
+# source → {opened_at, last_alerted_at, count}
+_circuits: dict[str, dict] = {}
+
+
+def get_circuit_status() -> dict:
+    """
+    Return a snapshot of all open circuits.
+    Used by the `check_auth_status` Boss tool.
+
+    Returns:
+        { source: { state, opened_at, open_for_seconds, alert_count } }
+        Only open circuits are included; an empty dict means everything is healthy.
+    """
+    now = time.time()
+    return {
+        src: {
+            "state": "open",
+            "opened_at": c["opened_at"],
+            "open_for_seconds": int(now - c["opened_at"]),
+            "alert_count": c["count"],
+        }
+        for src, c in _circuits.items()
+    }
+
+
+def _open_or_repeat(bot_token: str, channel: str, source: str, output: str) -> None:
+    """Open circuit on first hit; re-alert after cooldown if still failing."""
+    now = time.time()
+    circuit = _circuits.get(source)
+
+    if circuit is None:
+        # First occurrence — open and alert immediately
+        _circuits[source] = {"opened_at": now, "last_alerted_at": now, "count": 1}
+        logger.error("Circuit opened for %s: %s", source, output[:200])
+        slack_alert(bot_token, channel, rate_limit_message(source, output))
+        return
+
+    circuit["count"] += 1
+    elapsed = now - circuit["last_alerted_at"]
+    if elapsed >= CIRCUIT_COOLDOWN_SECONDS:
+        # Still failing after cooldown — remind admins once per hour
+        circuit["last_alerted_at"] = now
+        open_mins = int((now - circuit["opened_at"]) / 60)
+        msg = (
+            f":warning: *Sentinel — Claude usage/auth problem still active ({source})*\n"
+            f"Still failing after {open_mins} minutes. Total occurrences: {circuit['count']}.\n"
+            f"Last error:\n```{output.strip()[:300]}```\n"
+            f"Run `check_auth_status` in Slack to see the full picture."
+        )
+        logger.error("Circuit still open for %s (count=%d)", source, circuit["count"])
+        slack_alert(bot_token, channel, msg)
+    # else: within cooldown window — suppress
+
+
+def _close_if_open(bot_token: str, channel: str, source: str) -> None:
+    """If circuit was open, close it and post a recovery alert."""
+    circuit = _circuits.pop(source, None)
+    if circuit is None:
+        return
+    duration_mins = int((time.time() - circuit["opened_at"]) / 60)
+    msg = (
+        f":white_check_mark: *Sentinel — Claude auth restored ({source})*\n"
+        f"Fixed after {duration_mins} min. Total failures during outage: {circuit['count']}."
+    )
+    logger.info("Circuit closed for %s after %d min, %d failures", source, duration_mins, circuit["count"])
+    slack_alert(bot_token, channel, msg)
+
+
 def rate_limit_message(source: str, raw: str) -> str:
-    """Produce a human-readable Slack alert for a rate-limit event."""
+    """Produce a human-readable Slack alert for a rate-limit / auth event (first occurrence)."""
     snippet = raw.strip()[:300].replace("\n", " ")
     return (
         f":warning: *Sentinel — Claude usage/auth problem ({source})*\n"
@@ -37,9 +118,10 @@ def rate_limit_message(source: str, raw: str) -> str:
         f"```{snippet}```\n"
         f"*What to check:*\n"
         f"• API key: verify `ANTHROPIC_API_KEY` in `sentinel.properties` is valid and has credit\n"
-        f"• Claude Pro: run `claude login` on the server to refresh OAuth\n"
-        f"• Both: at least one auth method must be working\n"
-        f"Sentinel will retry on the next poll cycle."
+        f"• Claude Pro: run `claude login` on the server to refresh the OAuth session\n"
+        f"• Both: Sentinel tries both methods — at least one must be working\n"
+        f"Repeat alerts will be suppressed for 1 hour. "
+        f"Run `check_auth_status` in Slack to see current state."
     )
 
 
@@ -77,12 +159,15 @@ def alert_if_rate_limited(
     output: str,
 ) -> bool:
     """
-    Check output for rate-limit / auth signals.
-    If found, post a Slack alert and return True.
+    Check output for rate-limit / auth signals and manage the circuit breaker.
+
+    - Rate limited  → open/keep-open circuit, alert (with cooldown suppression)
+    - Not limited   → close circuit if it was open (recovery alert), return False
+
+    Returns True if a rate-limit signal was found.
     """
     if not is_rate_limited(output):
+        _close_if_open(bot_token, channel, source)
         return False
-    msg = rate_limit_message(source, output)
-    logger.error("Claude rate-limit/auth failure in %s: %s", source, output[:200])
-    slack_alert(bot_token, channel, msg)
+    _open_or_repeat(bot_token, channel, source, output)
     return True

package/python/sentinel/sentinel_boss.py

@@ -137,6 +137,7 @@ reply with a short summary grouped by category:
 *Fix management*
 • `get_fix_details` — full details of a specific fix — "show fix abc123"
 • `list_pending_prs` — all open Sentinel PRs awaiting review — "list open PRs"
+• `check_auth_status` — Claude auth health, rate-limit circuit state, fix engine 24 h stats — "is Claude working?", "any rate limits?", "auth issues?"
 
 *Project & task delivery*
 • `list_projects` — all projects and repos Sentinel manages — "what projects do you manage?"
@@ -275,6 +276,16 @@ _TOOLS = [
         "description": "List all open Sentinel PRs awaiting admin review.",
         "input_schema": {"type": "object", "properties": {}},
     },
+    {
+        "name": "check_auth_status",
+        "description": (
+            "Check Claude authentication health, current rate-limit / usage-limit circuit state, "
+            "and fix engine stats for the last 24 h. "
+            "Use when someone asks: 'is Claude working?', 'any rate limits?', 'why aren't fixes running?', "
+            "'is the API key OK?', 'auth issues?', 'fix engine status'."
+        ),
+        "input_schema": {"type": "object", "properties": {}},
+    },
     {
         "name": "pause_sentinel",
         "description": (
@@ -798,6 +809,67 @@ async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=Non
             "sentinel_paused": Path("SENTINEL_PAUSE").exists(),
         })
 
+    if name == "check_auth_status":
+        import subprocess as _sp
+        from .notify import get_circuit_status
+        cfg = cfg_loader.sentinel
+
+        # Auth configuration
+        has_key      = bool(cfg.anthropic_api_key)
+        pro_for_tasks = cfg.claude_pro_for_tasks
+        if pro_for_tasks and has_key:
+            primary, fallback = "claude_pro_oauth", "api_key"
+        elif pro_for_tasks:
+            primary, fallback = "claude_pro_oauth", None
+        else:
+            primary, fallback = "api_key", "claude_pro_oauth" if not has_key else "claude_pro_oauth"
+
+        # Claude CLI liveness check
+        cli_ok, cli_version = False, ""
+        try:
+            r = _sp.run(
+                [cfg.claude_code_bin, "--version"],
+                capture_output=True, text=True, timeout=10,
+            )
+            if r.returncode == 0:
+                cli_ok     = True
+                cli_version = r.stdout.strip() or r.stderr.strip()
+        except Exception:
+            pass
+
+        # Circuit breaker snapshot — only open (unhealthy) circuits appear here
+        circuits = get_circuit_status()
+
+        # Fix engine stats (last 24 h)
+        recent = store.get_recent_fixes(hours=24)
+        counts = {"applied": 0, "failed": 0, "skipped": 0, "pending": 0}
+        last_success = None
+        for f in recent:
+            s = f.get("status", "")
+            if s in counts:
+                counts[s] += 1
+            if s == "applied" and not last_success:
+                last_success = f.get("timestamp", "")
+
+        overall = "healthy"
+        if circuits:
+            overall = "degraded — rate/auth limit active on: " + ", ".join(circuits)
+        elif not cli_ok:
+            overall = "warning — claude CLI not reachable"
+
+        return json.dumps({
+            "overall": overall,
+            "auth": {
+                "api_key_configured": has_key,
+                "claude_pro_for_tasks": pro_for_tasks,
+                "primary_method": primary,
+                "fallback_method": fallback,
+            },
+            "claude_cli": {"available": cli_ok, "version": cli_version},
+            "rate_limit_circuits": circuits,
+            "fix_engine_24h": {**counts, "last_successful_fix": last_success},
+        })
+
     if name == "create_issue":
         description = inputs["description"]
         target_repo = inputs.get("target_repo", "")