@misterhuydo/sentinel 1.0.4 → 1.0.6

package/python/sentinel/main.py

@@ -1,223 +1,355 @@
-"""
-main.py — Sentinel entry point and watch loop.
-
-Usage:
-  python -m sentinel.main           # run watch loop
-  python -m sentinel.main --init    # first-time setup
-"""
-
-import argparse
-import asyncio
-import logging
-import signal
-import subprocess
-import sys
-from datetime import datetime, timezone
-from pathlib import Path
-
-from .cairn_client import ensure_installed as cairn_installed, index_repo
-from .config_loader import ConfigLoader
-from .fix_engine import generate_fix
-from .git_manager import apply_and_commit, publish
-from .cicd_trigger import trigger as cicd_trigger
-from .log_fetcher import fetch_all
-from .log_parser import parse_all, ErrorEvent
-from .repo_router import route
-from .reporter import build_and_send
-from .state_store import StateStore
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s %(levelname)-7s %(name)s — %(message)s",
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler("logs/sentinel.log", encoding="utf-8"),
-    ],
-)
-logger = logging.getLogger("sentinel")
-
-_report_requested = False
-
-
-def _on_sigusr1(*_):
-    global _report_requested
-    _report_requested = True
-    logger.info("SIGUSR1 received — health report queued")
-
-
-def _register_signals():
-    try:
-        signal.signal(signal.SIGUSR1, _on_sigusr1)
-    except (OSError, AttributeError):
-        pass
-
-
-# ── Fix pipeline ──────────────────────────────────────────────────────────────
-
-async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: StateStore):
-    sentinel = cfg_loader.sentinel
-
-    repo = route(event, cfg_loader.repos)
-    if not repo:
-        return
-
-    if Path("SENTINEL_PAUSE").exists():
-        logger.info("SENTINEL_PAUSE present — fix activity halted")
-        return
-
-    if event.is_infra_issue:
-        logger.info("Infra issue for %s — log only", event.fingerprint)
-        store.record_fix(event.fingerprint, "skipped", repo_name=repo.repo_name)
-        return
-
-    if event.severity == "CRITICAL" and repo.auto_publish:
-        logger.warning("CRITICAL in auto-publish repo '%s' — flagging for human review", repo.repo_name)
-        store.record_fix(event.fingerprint, "skipped", repo_name=repo.repo_name)
-        build_and_send(sentinel, store)
-        return
-
-    if store.fix_attempted_recently(event.fingerprint, hours=24):
-        logger.debug("Fix already attempted recently for %s", event.fingerprint)
-        return
-
-    patches_dir = Path(sentinel.workspace_dir) / "patches"
-    status, patch_path = generate_fix(event, repo, sentinel, patches_dir)
-
-    if status != "patch" or patch_path is None:
-        store.record_fix(event.fingerprint, "skipped" if status == "skip" else "failed", repo_name=repo.repo_name)
-        return
-
-    commit_status, commit_hash = apply_and_commit(event, patch_path, repo, sentinel)
-    if commit_status != "committed":
-        store.record_fix(event.fingerprint, "failed", repo_name=repo.repo_name)
-        return
-
-    branch, pr_url = publish(event, repo, sentinel, commit_hash)
-    store.record_fix(
-        event.fingerprint,
-        "applied" if repo.auto_publish else "pending",
-        patch_path=str(patch_path),
-        commit_hash=commit_hash,
-        branch=branch,
-        pr_url=pr_url,
-        repo_name=repo.repo_name,
-    )
-
-    if repo.auto_publish:
-        cicd_trigger(repo, store, event.fingerprint)
-
-
-# ── Poll cycle ────────────────────────────────────────────────────────────────
-
-async def poll_cycle(cfg_loader: ConfigLoader, store: StateStore):
-    global _report_requested
-
-    sources = list(cfg_loader.log_sources.values())
-    if not sources:
-        logger.warning("No log-configs found")
-        return
-
-    logger.info("Fetching logs from %d source(s)...", len(sources))
-    fetched = await fetch_all(sources, cfg_loader.sentinel)
-
-    events = parse_all(fetched, cfg_loader.log_sources)
-    logger.info("Parsed %d error/warn events", len(events))
-
-    new_events = []
-    for event in events:
-        store.record_error(event.fingerprint, event.source, event.message)
-        if not store.fix_attempted_recently(event.fingerprint):
-            new_events.append(event)
-
-    logger.info("%d new event(s) to process", len(new_events))
-    await asyncio.gather(*[_handle_error(e, cfg_loader, store) for e in new_events], return_exceptions=True)
-
-    if _report_requested or _report_due(cfg_loader, store):
-        _report_requested = False
-        logger.info("Sending health report...")
-        build_and_send(cfg_loader.sentinel, store)
-
-
-def _report_due(cfg_loader: ConfigLoader, store: StateStore) -> bool:
-    last = store.last_report_time()
-    if last is None:
-        return True
-    elapsed = (datetime.now(timezone.utc) - last).total_seconds()
-    return elapsed >= cfg_loader.sentinel.report_interval_hours * 3600
-
-
-# ── Init ──────────────────────────────────────────────────────────────────────
-
-def run_init(cfg_loader: ConfigLoader):
-    sentinel = cfg_loader.sentinel
-    logger.info("=== Sentinel --init ===")
-
-    if not cairn_installed():
-        logger.error("Cairn not installed. Run: npm install -g @misterhuydo/cairn-mcp")
-
-    for name, repo in cfg_loader.repos.items():
-        local = Path(repo.local_path)
-        if not local.exists():
-            logger.info("Cloning %s → %s", repo.repo_url, repo.local_path)
-            r = subprocess.run(["git", "clone", repo.repo_url, str(local)], capture_output=True, text=True)
-            if r.returncode != 0:
-                logger.error("Clone failed for %s: %s", name, r.stderr)
-                continue
-        index_repo(repo)
-
-    for src_name, src in cfg_loader.log_sources.items():
-        if src.source_type == "ssh" and src.hosts:
-            host = src.hosts[0]
-            logger.info("Testing SSH to %s (%s)...", src_name, host)
-            r = subprocess.run(
-                ["ssh", "-i", src.key, "-o", "StrictHostKeyChecking=no",
-                 "-o", "ConnectTimeout=5", f"ec2-user@{host}", "echo ok"],
-                capture_output=True, text=True, timeout=15,
-            )
-            logger.info("  SSH %s: %s", host, "OK" if r.returncode == 0 else f"FAILED — {r.stderr.strip()}")
-
-    logger.info("Sending test email...")
-    try:
-        build_and_send(sentinel, StateStore(sentinel.state_db))
-    except Exception as e:
-        logger.error("Test email failed: %s", e)
-
-    logger.info("=== Init complete ===")
-
-
-# ── Entry point ───────────────────────────────────────────────────────────────
-
-async def run_loop(cfg_loader: ConfigLoader, store: StateStore):
-    interval = cfg_loader.sentinel.poll_interval_seconds
-    logger.info("Sentinel starting — poll interval: %ds, repos: %s", interval, list(cfg_loader.repos.keys()))
-    while True:
-        try:
-            await poll_cycle(cfg_loader, store)
-        except Exception as e:
-            logger.exception("Unhandled error in poll cycle: %s", e)
-        await asyncio.sleep(interval)
-
-
-def main():
-    Path("logs").mkdir(exist_ok=True)
-    Path("workspace/fetched").mkdir(parents=True, exist_ok=True)
-    Path("workspace/patches").mkdir(parents=True, exist_ok=True)
-
-    parser = argparse.ArgumentParser(description="Sentinel — Autonomous DevOps Agent")
-    parser.add_argument("--init", action="store_true", help="First-time setup")
-    parser.add_argument("--config", default="./config", help="Config directory path")
-    args = parser.parse_args()
-
-    cfg_loader = ConfigLoader(config_dir=args.config)
-    store = StateStore(cfg_loader.sentinel.state_db)
-    _register_signals()
-
-    if args.init:
-        run_init(cfg_loader)
-        return
-
-    asyncio.run(run_loop(cfg_loader, store))
-
-
-if __name__ == "__main__":
-    main()
+"""
+main.py — Sentinel entry point and watch loop.
+
+Usage:
+  python -m sentinel.main           # run watch loop
+  python -m sentinel.main --init    # first-time setup
+"""
+
+import argparse
+import asyncio
+import logging
+import signal
+import subprocess
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .cairn_client import ensure_installed as cairn_installed, index_repo
+from .config_loader import ConfigLoader
+from .fix_engine import generate_fix
+from .git_manager import apply_and_commit, publish
+from .cicd_trigger import trigger as cicd_trigger
+from .log_fetcher import fetch_all
+from .log_parser import parse_all, ErrorEvent
+from .issue_watcher import scan_issues, mark_done, IssueEvent
+from .repo_router import route
+from .reporter import build_and_send, send_fix_notification, send_failure_notification
+from .state_store import StateStore
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)-7s %(name)s — %(message)s",
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler("logs/sentinel.log", encoding="utf-8"),
+    ],
+)
+logger = logging.getLogger("sentinel")
+
+_report_requested = False
+
+
+def _on_sigusr1(*_):
+    global _report_requested
+    _report_requested = True
+    logger.info("SIGUSR1 received — health report queued")
+
+
+def _register_signals():
+    try:
+        signal.signal(signal.SIGUSR1, _on_sigusr1)
+    except (OSError, AttributeError):
+        pass
+
+
+# ── Fix pipeline ──────────────────────────────────────────────────────────────
+
+async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: StateStore):
+    sentinel = cfg_loader.sentinel
+
+    repo = route(event, cfg_loader.repos)
+    if not repo:
+        return
+
+    if Path("SENTINEL_PAUSE").exists():
+        logger.info("SENTINEL_PAUSE present — fix activity halted")
+        return
+
+    if event.is_infra_issue:
+        logger.info("Infra issue for %s — log only", event.fingerprint)
+        store.record_fix(event.fingerprint, "skipped", repo_name=repo.repo_name)
+        return
+
+    if event.severity == "CRITICAL" and repo.auto_publish:
+        logger.warning("CRITICAL in auto-publish repo '%s' — flagging for human review", repo.repo_name)
+        store.record_fix(event.fingerprint, "skipped", repo_name=repo.repo_name)
+        return
+
+    if store.fix_attempted_recently(event.fingerprint, hours=24):
+        logger.debug("Fix already attempted recently for %s", event.fingerprint)
+        return
+
+    patches_dir = Path(sentinel.workspace_dir) / "patches"
+    status, patch_path = generate_fix(event, repo, sentinel, patches_dir)
+
+    if status != "patch" or patch_path is None:
+        outcome = "skipped" if status == "skip" else "failed"
+        store.record_fix(event.fingerprint, outcome, repo_name=repo.repo_name)
+        send_failure_notification(sentinel, {
+            "source":    event.source,
+            "message":   event.message,
+            "repo_name": repo.repo_name,
+            "reason":    f"Claude Code returned {status.upper()}",
+            "body":      event.full_text()[:500],
+        })
+        return
+
+    commit_status, commit_hash = apply_and_commit(event, patch_path, repo, sentinel)
+    if commit_status != "committed":
+        store.record_fix(event.fingerprint, "failed", repo_name=repo.repo_name)
+        send_failure_notification(sentinel, {
+            "source":    event.source,
+            "message":   event.message,
+            "repo_name": repo.repo_name,
+            "reason":    "patch generated but commit/tests failed",
+            "body":      event.full_text()[:500],
+        })
+        return
+
+    branch, pr_url = publish(event, repo, sentinel, commit_hash)
+    store.record_fix(
+        event.fingerprint,
+        "applied" if repo.auto_publish else "pending",
+        patch_path=str(patch_path),
+        commit_hash=commit_hash,
+        branch=branch,
+        pr_url=pr_url,
+        repo_name=repo.repo_name,
+    )
+
+    send_fix_notification(sentinel, {
+        "source":       event.source,
+        "severity":     event.severity,
+        "fingerprint":  event.fingerprint,
+        "first_seen":   str(event.timestamp),
+        "message":      event.message,
+        "stack_trace":  getattr(event, "stack_trace", ""),
+        "repo_name":    repo.repo_name,
+        "commit_hash":  commit_hash,
+        "branch":       branch,
+        "pr_url":       pr_url,
+        "auto_publish": repo.auto_publish,
+        "files_changed": [],
+    })
+
+    if repo.auto_publish:
+        cicd_trigger(repo, store, event.fingerprint)
+
+
+# ── Poll cycle ────────────────────────────────────────────────────────────────
+
+
+# ── Issue pipeline ────────────────────────────────────────────────────────────
+
+async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: StateStore):
+    """Process a single issue file from the issues/ directory."""
+    sentinel = cfg_loader.sentinel
+
+    if Path("SENTINEL_PAUSE").exists():
+        logger.info("SENTINEL_PAUSE present -- fix activity halted")
+        return
+
+    if store.fix_attempted_recently(event.fingerprint, hours=24):
+        logger.debug("Issue already processed recently: %s", event.source)
+        mark_done(event.issue_file)
+        return
+
+    # Route: explicit TARGET_REPO in file > single-repo shortcut > warn and leave
+    if event.target_repo:
+        repo = cfg_loader.repos.get(event.target_repo)
+        if not repo:
+            logger.warning("TARGET_REPO %r not found in config -- leaving %s for admin",
+                           event.target_repo, event.source)
+            return
+    elif len(cfg_loader.repos) == 1:
+        repo = next(iter(cfg_loader.repos.values()))
+    else:
+        logger.warning(
+            "Cannot auto-route %s -- add 'TARGET_REPO: <repo>' as first line in the file",
+            event.source,
+        )
+        return  # Leave the file so admin can add the header
+
+    patches_dir = Path(sentinel.workspace_dir) / "patches"
+    status, patch_path = generate_fix(event, repo, sentinel, patches_dir)
+
+    if status != "patch" or patch_path is None:
+        store.record_fix(event.fingerprint, "skipped" if status == "skip" else "failed",
+                         repo_name=repo.repo_name)
+        send_failure_notification(sentinel, {
+            "source":    event.source,
+            "message":   event.message,
+            "repo_name": repo.repo_name,
+            "reason":    f"Claude Code returned {status.upper()}",
+            "body":      event.body[:500],
+        })
+        mark_done(event.issue_file)
+        return
+
+    commit_status, commit_hash = apply_and_commit(event, patch_path, repo, sentinel)
+    if commit_status != "committed":
+        store.record_fix(event.fingerprint, "failed", repo_name=repo.repo_name)
+        send_failure_notification(sentinel, {
+            "source":    event.source,
+            "message":   event.message,
+            "repo_name": repo.repo_name,
+            "reason":    "patch generated but commit/tests failed",
+            "body":      event.body[:500],
+        })
+        mark_done(event.issue_file)
+        return
+
+    branch, pr_url = publish(event, repo, sentinel, commit_hash)
+    store.record_fix(
+        event.fingerprint,
+        "applied" if repo.auto_publish else "pending",
+        patch_path=str(patch_path),
+        commit_hash=commit_hash,
+        branch=branch,
+        pr_url=pr_url,
+        repo_name=repo.repo_name,
+    )
+    send_fix_notification(sentinel, {
+        "source":       event.source,
+        "severity":     "ERROR",
+        "fingerprint":  event.fingerprint,
+        "first_seen":   event.timestamp,
+        "message":      event.message,
+        "stack_trace":  event.body,
+        "repo_name":    repo.repo_name,
+        "commit_hash":  commit_hash,
+        "branch":       branch,
+        "pr_url":       pr_url,
+        "auto_publish": repo.auto_publish,
+        "files_changed": [],
+    })
+    mark_done(event.issue_file)
+
+    if repo.auto_publish:
+        cicd_trigger(repo, store, event.fingerprint)
+
+
+async def poll_cycle(cfg_loader: ConfigLoader, store: StateStore):
+    global _report_requested
+
+    # ── Log sources (optional) ────────────────────────────────────────────────
+    sources = list(cfg_loader.log_sources.values())
+    if sources:
+        logger.info("Fetching logs from %d source(s)...", len(sources))
+        fetched = await fetch_all(sources, cfg_loader.sentinel)
+        events = parse_all(fetched, cfg_loader.log_sources)
+        logger.info("Parsed %d error/warn events", len(events))
+
+        new_events = []
+        for event in events:
+            store.record_error(event.fingerprint, event.source, event.message)
+            if not store.fix_attempted_recently(event.fingerprint):
+                new_events.append(event)
+
+        if new_events:
+            logger.info("%d new log event(s) to process", len(new_events))
+            await asyncio.gather(
+                *[_handle_error(e, cfg_loader, store) for e in new_events],
+                return_exceptions=True,
+            )
+
+    # ── Issues directory (always checked) ────────────────────────────────────
+    issues = scan_issues(Path("."))
+    if issues:
+        logger.info("%d issue file(s) found in issues/", len(issues))
+        await asyncio.gather(
+            *[_handle_issue(e, cfg_loader, store) for e in issues],
+            return_exceptions=True,
+        )
+
+    if cfg_loader.sentinel.send_health and (_report_requested or _report_due(cfg_loader, store)):
+        _report_requested = False
+        logger.info("Sending health digest...")
+        build_and_send(cfg_loader.sentinel, store)
+
+
+def _report_due(cfg_loader: ConfigLoader, store: StateStore) -> bool:
+    last = store.last_report_time()
+    if last is None:
+        return True
+    elapsed = (datetime.now(timezone.utc) - last).total_seconds()
+    return elapsed >= cfg_loader.sentinel.report_interval_hours * 3600
+
+
+# ── Init ──────────────────────────────────────────────────────────────────────
+
+def run_init(cfg_loader: ConfigLoader):
+    sentinel = cfg_loader.sentinel
+    logger.info("=== Sentinel --init ===")
+
+    if not cairn_installed():
+        logger.error("Cairn not installed. Run: npm install -g @misterhuydo/cairn-mcp")
+
+    for name, repo in cfg_loader.repos.items():
+        local = Path(repo.local_path)
+        if not local.exists():
+            logger.info("Cloning %s → %s", repo.repo_url, repo.local_path)
+            r = subprocess.run(["git", "clone", repo.repo_url, str(local)], capture_output=True, text=True)
+            if r.returncode != 0:
+                logger.error("Clone failed for %s: %s", name, r.stderr)
+                continue
+        index_repo(repo)
+
+    for src_name, src in cfg_loader.log_sources.items():
+        if src.source_type == "ssh" and src.hosts:
+            host = src.hosts[0]
+            logger.info("Testing SSH to %s (%s)...", src_name, host)
+            r = subprocess.run(
+                ["ssh", "-i", src.key, "-o", "StrictHostKeyChecking=no",
+                 "-o", "ConnectTimeout=5", f"ec2-user@{host}", "echo ok"],
+                capture_output=True, text=True, timeout=15,
+            )
+            logger.info("  SSH %s: %s", host, "OK" if r.returncode == 0 else f"FAILED — {r.stderr.strip()}")
+
+    logger.info("Sending test email...")
+    try:
+        build_and_send(sentinel, StateStore(sentinel.state_db))
+    except Exception as e:
+        logger.error("Test email failed: %s", e)
+
+    logger.info("=== Init complete ===")
+
+
+# ── Entry point ───────────────────────────────────────────────────────────────
+
+async def run_loop(cfg_loader: ConfigLoader, store: StateStore):
+    interval = cfg_loader.sentinel.poll_interval_seconds
+    logger.info("Sentinel starting — poll interval: %ds, repos: %s", interval, list(cfg_loader.repos.keys()))
+    while True:
+        try:
+            await poll_cycle(cfg_loader, store)
+        except Exception as e:
+            logger.exception("Unhandled error in poll cycle: %s", e)
+        await asyncio.sleep(interval)
+
+
+def main():
+    Path("logs").mkdir(exist_ok=True)
+    Path("workspace/fetched").mkdir(parents=True, exist_ok=True)
+    Path("workspace/patches").mkdir(parents=True, exist_ok=True)
+    Path("issues").mkdir(exist_ok=True)
+
+    parser = argparse.ArgumentParser(description="Sentinel — Autonomous DevOps Agent")
+    parser.add_argument("--init", action="store_true", help="First-time setup")
+    parser.add_argument("--config", default="./config", help="Config directory path")
+    args = parser.parse_args()
+
+    cfg_loader = ConfigLoader(config_dir=args.config)
+    store = StateStore(cfg_loader.sentinel.state_db)
+    _register_signals()
+
+    if args.init:
+        run_init(cfg_loader)
+        return
+
+    asyncio.run(run_loop(cfg_loader, store))
+
+
+if __name__ == "__main__":
+    main()