@misterhuydo/sentinel 1.0.6 → 1.0.10

package/python/sentinel/log_parser.py

@@ -1,149 +1,175 @@
-"""
-log_parser.py — Parse fetched log files into ErrorEvent objects.
-
-Handles Java-style logs (Spring Boot / Logback format):
-  2024-01-15 12:34:56.789 ERROR [thread] class.ClassName - Message
-  followed by optional stack trace lines (^\tat ...)
-"""
-
-import hashlib
-import re
-import logging
-from dataclasses import dataclass, field
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-_LOG_HEADER = re.compile(
-    r"^(?P<ts>\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}:\d{2}[.,\d]*)\s+"
-    r"(?P<level>CRITICAL|ERROR|WARN(?:ING)?|INFO|DEBUG)\s+"
-    r"(?:\[(?P<thread>[^\]]*)\]\s+)?"
-    r"(?P<logger>\S+)\s+-\s+"
-    r"(?P<message>.+)$"
-)
-
-_STACK_LINE = re.compile(r"^\s+at |\s+\.\.\. \d+ more|^Caused by:")
-
-SEVERITY_MAP = {
-    "CRITICAL": "CRITICAL",
-    "ERROR": "ERROR",
-    "WARN": "WARN",
-    "WARNING": "WARN",
-    "INFO": "INFO",
-    "DEBUG": "DEBUG",
-}
-
-_CRITICAL_PATTERNS = re.compile(
-    r"OutOfMemoryError|StackOverflowError|OOMKilled", re.IGNORECASE
-)
-_INFRA_PATTERNS = re.compile(
-    r"ConnectException|TimeoutException|ConnectionRefused|SocketTimeout",
-    re.IGNORECASE,
-)
-
-
-@dataclass
-class ErrorEvent:
-    source: str           # log-source name (e.g. "SSOLWA")
-    log_file: str
-    timestamp: str
-    level: str            # CRITICAL / ERROR / WARN
-    thread: str
-    logger_name: str
-    message: str
-    stack_trace: list[str] = field(default_factory=list)
-    fingerprint: str = ""
-
-    def __post_init__(self):
-        if not self.fingerprint:
-            self.fingerprint = _fingerprint(self.message, self.stack_trace)
-
-    @property
-    def severity(self) -> str:
-        if _CRITICAL_PATTERNS.search(self.message) or _CRITICAL_PATTERNS.search(
-            "\n".join(self.stack_trace)
-        ):
-            return "CRITICAL"
-        return self.level
-
-    @property
-    def is_infra_issue(self) -> bool:
-        return bool(_INFRA_PATTERNS.search(self.message))
-
-    def short_summary(self) -> str:
-        return self.message[:120]
-
-    def full_text(self) -> str:
-        lines = [f"{self.timestamp} {self.level} [{self.thread}] {self.logger_name} - {self.message}"]
-        lines.extend(self.stack_trace)
-        return "\n".join(lines)
-
-
-def _normalize_message(msg: str) -> str:
-    msg = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", msg)
-    msg = re.sub(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", "UUID", msg)
-    msg = re.sub(r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}[.,\d]*\b", "TIMESTAMP", msg)
-    msg = re.sub(r"\b\d+\b", "N", msg)
-    return msg.strip()
-
-
-def _fingerprint(message: str, stack_trace: list[str]) -> str:
-    top_frames = [l for l in stack_trace if l.strip().startswith("at ")][:3]
-    raw = _normalize_message(message) + "\n" + "\n".join(top_frames)
-    return hashlib.sha1(raw.encode()).hexdigest()[:16]
-
-
-def parse_log_file(path: Path, source_name: str) -> list[ErrorEvent]:
-    """Parse a single log file and return all ERROR/WARN events."""
-    events: list[ErrorEvent] = []
-    current_header: re.Match | None = None
-    current_stack: list[str] = []
-
-    def flush():
-        if current_header is None:
-            return
-        level = SEVERITY_MAP.get(current_header.group("level").upper(), "WARN")
-        if level not in ("ERROR", "WARN", "CRITICAL"):
-            return
-        event = ErrorEvent(
-            source=source_name,
-            log_file=str(path),
-            timestamp=current_header.group("ts"),
-            level=level,
-            thread=current_header.group("thread") or "",
-            logger_name=current_header.group("logger"),
-            message=current_header.group("message"),
-            stack_trace=list(current_stack),
-        )
-        events.append(event)
-
-    try:
-        text = path.read_text(encoding="utf-8", errors="replace")
-    except OSError as e:
-        logger.error("Cannot read %s: %s", path, e)
-        return []
-
-    for line in text.splitlines():
-        m = _LOG_HEADER.match(line)
-        if m:
-            flush()
-            current_header = m
-            current_stack = []
-        elif current_header and _STACK_LINE.match(line):
-            current_stack.append(line)
-
-    flush()
-    logger.debug("Parsed %s: %d error/warn events", path.name, len(events))
-    return events
-
-
-def parse_all(
-    fetched_files: dict[str, list[Path]],
-    log_sources,  # dict[str, LogSourceConfig]
-) -> list[ErrorEvent]:
-    """Parse all fetched log files across all sources."""
-    all_events: list[ErrorEvent] = []
-    for source_name, files in fetched_files.items():
-        for f in files:
-            all_events.extend(parse_log_file(f, source_name))
-    return all_events
+"""
+log_parser.py — Parse fetched log files into ErrorEvent objects.
+
+Handles Java-style logs (Spring Boot / Logback format):
+  2024-01-15 12:34:56.789 ERROR [thread] class.ClassName - Message
+  followed by optional stack trace lines (^\tat ...)
+"""
+
+import hashlib
+import re
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+_LOG_HEADER = re.compile(
+    r"^(?P<ts>\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}:\d{2}[.,\d]*)\s+"
+    r"(?P<level>CRITICAL|ERROR|WARN(?:ING)?|INFO|DEBUG)\s+"
+    r"(?:\[(?P<thread>[^\]]*)\]\s+)?"
+    r"(?P<logger>\S+)\s+-\s+"
+    r"(?P<message>.+)$"
+)
+
+_STACK_LINE = re.compile(r"^\s+at |\s+\.\.\. \d+ more|^Caused by:")
+
+SEVERITY_MAP = {
+    "CRITICAL": "CRITICAL",
+    "ERROR": "ERROR",
+    "WARN": "WARN",
+    "WARNING": "WARN",
+    "INFO": "INFO",
+    "DEBUG": "DEBUG",
+}
+
+_CRITICAL_PATTERNS = re.compile(
+    r"OutOfMemoryError|StackOverflowError|OOMKilled", re.IGNORECASE
+)
+_INFRA_PATTERNS = re.compile(
+    r"ConnectException|TimeoutException|ConnectionRefused|SocketTimeout",
+    re.IGNORECASE,
+)
+
+
+@dataclass
+class ErrorEvent:
+    source: str           # log-source name (e.g. "SSOLWA")
+    log_file: str
+    timestamp: str
+    level: str            # CRITICAL / ERROR / WARN
+    thread: str
+    logger_name: str
+    message: str
+    stack_trace: list[str] = field(default_factory=list)
+    fingerprint: str = ""
+
+    def __post_init__(self):
+        if not self.fingerprint:
+            self.fingerprint = _fingerprint(self.message, self.stack_trace)
+
+    @property
+    def severity(self) -> str:
+        if _CRITICAL_PATTERNS.search(self.message) or _CRITICAL_PATTERNS.search(
+            "\n".join(self.stack_trace)
+        ):
+            return "CRITICAL"
+        return self.level
+
+    @property
+    def is_infra_issue(self) -> bool:
+        return bool(_INFRA_PATTERNS.search(self.message))
+
+    def short_summary(self) -> str:
+        return self.message[:120]
+
+    def full_text(self) -> str:
+        lines = [f"{self.timestamp} {self.level} [{self.thread}] {self.logger_name} - {self.message}"]
+        lines.extend(self.stack_trace)
+        return "\n".join(lines)
+
+
+def _normalize_message(msg: str) -> str:
+    msg = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", msg)
+    msg = re.sub(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", "UUID", msg)
+    msg = re.sub(r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}[.,\d]*\b", "TIMESTAMP", msg)
+    msg = re.sub(r"\b\d+\b", "N", msg)
+    return msg.strip()
+
+
+def _fingerprint(message: str, stack_trace: list[str]) -> str:
+    top_frames = [l for l in stack_trace if l.strip().startswith("at ")][:3]
+    raw = _normalize_message(message) + "\n" + "\n".join(top_frames)
+    return hashlib.sha1(raw.encode()).hexdigest()[:16]
+
+
+def parse_log_file(path: Path, source_name: str) -> list[ErrorEvent]:
+    """Parse a single log file and return all ERROR/WARN events."""
+    events: list[ErrorEvent] = []
+    current_header: re.Match | None = None
+    current_stack: list[str] = []
+
+    def flush():
+        if current_header is None:
+            return
+        level = SEVERITY_MAP.get(current_header.group("level").upper(), "WARN")
+        if level not in ("ERROR", "WARN", "CRITICAL"):
+            return
+        event = ErrorEvent(
+            source=source_name,
+            log_file=str(path),
+            timestamp=current_header.group("ts"),
+            level=level,
+            thread=current_header.group("thread") or "",
+            logger_name=current_header.group("logger"),
+            message=current_header.group("message"),
+            stack_trace=list(current_stack),
+        )
+        events.append(event)
+
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except OSError as e:
+        logger.error("Cannot read %s: %s", path, e)
+        return []
+
+    for line in text.splitlines():
+        m = _LOG_HEADER.match(line)
+        if m:
+            flush()
+            current_header = m
+            current_stack = []
+        elif current_header and _STACK_LINE.match(line):
+            current_stack.append(line)
+
+    flush()
+    logger.debug("Parsed %s: %d error/warn events", path.name, len(events))
+    return events
+
+
+def parse_all(
+    fetched_files: dict[str, list[Path]],
+    log_sources,  # dict[str, LogSourceConfig]
+) -> list[ErrorEvent]:
+    """Parse all fetched log files across all sources."""
+    all_events: list[ErrorEvent] = []
+    for source_name, files in fetched_files.items():
+        for f in files:
+            all_events.extend(parse_log_file(f, source_name))
+    return all_events
+
+
+# -- Sentinel marker detection -------------------------------------------------
+
+_SENTINEL_MARKER_RE = re.compile(r'SENTINEL:#([0-9a-f]{16})')
+
+
+def scan_for_markers(path: Path) -> list[str]:
+    """
+    Scan a single log file for SENTINEL:#<fingerprint> markers injected by fix_engine.
+    Returns a list of full marker strings (e.g. ['SENTINEL:#abc123de45678901']).
+    """
+    try:
+        text = path.read_text(encoding='utf-8', errors='replace')
+    except OSError:
+        return []
+    return [f'SENTINEL:#{m}' for m in _SENTINEL_MARKER_RE.findall(text)]
+
+
+def scan_all_for_markers(fetched_files: dict[str, list[Path]]) -> list[str]:
+    """Scan all fetched log files and return every SENTINEL marker found."""
+    markers: list[str] = []
+    for files in fetched_files.values():
+        for f in files:
+            markers.extend(scan_for_markers(f))
+    return markers

package/python/sentinel/main.py

@@ -21,10 +21,10 @@ from .fix_engine import generate_fix
 from .git_manager import apply_and_commit, publish
 from .cicd_trigger import trigger as cicd_trigger
 from .log_fetcher import fetch_all
-from .log_parser import parse_all, ErrorEvent
+from .log_parser import parse_all, scan_all_for_markers, ErrorEvent
 from .issue_watcher import scan_issues, mark_done, IssueEvent
 from .repo_router import route
-from .reporter import build_and_send, send_fix_notification, send_failure_notification
+from .reporter import build_and_send, send_fix_notification, send_failure_notification, send_confirmed_notification, send_regression_notification, send_startup_notification
 from .state_store import StateStore
 
 logging.basicConfig(
@@ -81,7 +81,7 @@ async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: Stat
         return
 
     patches_dir = Path(sentinel.workspace_dir) / "patches"
-    status, patch_path = generate_fix(event, repo, sentinel, patches_dir)
+    status, patch_path, marker = generate_fix(event, repo, sentinel, patches_dir, store)
 
     if status != "patch" or patch_path is None:
         outcome = "skipped" if status == "skip" else "failed"
@@ -116,6 +116,7 @@ async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: Stat
         branch=branch,
         pr_url=pr_url,
         repo_name=repo.repo_name,
+        sentinel_marker=marker,
     )
 
     send_fix_notification(sentinel, {
@@ -172,7 +173,7 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
         return  # Leave the file so admin can add the header
 
     patches_dir = Path(sentinel.workspace_dir) / "patches"
-    status, patch_path = generate_fix(event, repo, sentinel, patches_dir)
+    status, patch_path, marker = generate_fix(event, repo, sentinel, patches_dir, store)
 
     if status != "patch" or patch_path is None:
         store.record_fix(event.fingerprint, "skipped" if status == "skip" else "failed",
@@ -209,6 +210,7 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
         branch=branch,
         pr_url=pr_url,
         repo_name=repo.repo_name,
+        sentinel_marker=marker,
     )
     send_fix_notification(sentinel, {
         "source":       event.source,
@@ -232,6 +234,8 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
 
 async def poll_cycle(cfg_loader: ConfigLoader, store: StateStore):
     global _report_requested
+    events: list = []
+    fetched: dict = {}
 
     # ── Log sources (optional) ────────────────────────────────────────────────
     sources = list(cfg_loader.log_sources.values())
@@ -254,6 +258,36 @@ async def poll_cycle(cfg_loader: ConfigLoader, store: StateStore):
                 return_exceptions=True,
             )
 
+    # ── SENTINEL marker scanning (phase 1: record first seen in prod logs) ────
+    if sources and fetched:
+        for marker in set(scan_all_for_markers(fetched)):
+            fix = store.mark_marker_seen(marker)
+            if fix:
+                logger.info("Marker seen in production: %s repo=%s — quiet period started",
+                            marker, fix.get("repo_name"))
+
+    # ── Regression detection (error recurred before quiet period elapsed) ──────
+    if sources:
+        for event in events:
+            pending = store.get_marker_seen_fix(event.fingerprint)
+            if pending:
+                logger.warning("Regression: %s recurred after marker seen", event.fingerprint)
+                store.mark_regressed(event.fingerprint)
+                send_regression_notification(cfg_loader.sentinel, pending, {
+                    "source":  event.source,
+                    "message": event.message,
+                    "body":    event.full_text()[:500],
+                })
+
+    # ── Phase 2: confirm fixes whose quiet period has elapsed ────────────────
+    quiet_hours = cfg_loader.sentinel.marker_confirm_hours
+    for fix in store.get_fixes_pending_confirmation(quiet_hours):
+        confirmed = store.confirm_fix(fix["fingerprint"])
+        if confirmed:
+            logger.info("Fix confirmed after %dh quiet period: %s repo=%s",
+                        quiet_hours, fix["fingerprint"], fix.get("repo_name"))
+            send_confirmed_notification(cfg_loader.sentinel, confirmed)
+
     # ── Issues directory (always checked) ────────────────────────────────────
     issues = scan_issues(Path("."))
     if issues:
@@ -277,50 +311,99 @@ def _report_due(cfg_loader: ConfigLoader, store: StateStore) -> bool:
     return elapsed >= cfg_loader.sentinel.report_interval_hours * 3600
 
 
-# ── Init ──────────────────────────────────────────────────────────────────────
+# ── Startup checks (runs automatically on every start) ───────────────────────────
 
-def run_init(cfg_loader: ConfigLoader):
-    sentinel = cfg_loader.sentinel
-    logger.info("=== Sentinel --init ===")
+async def _startup_checks(cfg_loader: ConfigLoader) -> dict:
+    """
+    Clone missing repos, index with Cairn, test SSH sources.
+    Returns a results dict passed to the startup email.
+    """
+    results = {
+        "repos":    [],   # {name, status, message}
+        "cairn":    [],   # {name, status, message}
+        "ssh":      [],   # {name, host, status, message}
+        "warnings": [],
+    }
 
     if not cairn_installed():
-        logger.error("Cairn not installed. Run: npm install -g @misterhuydo/cairn-mcp")
+        results["warnings"].append("Cairn not found — run: npm install -g @misterhuydo/cairn-mcp")
 
     for name, repo in cfg_loader.repos.items():
         local = Path(repo.local_path)
         if not local.exists():
             logger.info("Cloning %s → %s", repo.repo_url, repo.local_path)
-            r = subprocess.run(["git", "clone", repo.repo_url, str(local)], capture_output=True, text=True)
+            r = subprocess.run(
+                ["git", "clone", repo.repo_url, str(local)],
+                capture_output=True, text=True,
+            )
             if r.returncode != 0:
-                logger.error("Clone failed for %s: %s", name, r.stderr)
+                msg = r.stderr.strip()
+                logger.error("Clone failed for %s: %s", name, msg)
+                results["repos"].append({"name": name, "status": "error", "message": msg})
                 continue
-        index_repo(repo)
+            results["repos"].append({"name": name, "status": "cloned", "message": repo.repo_url})
+        else:
+            results["repos"].append({"name": name, "status": "exists", "message": str(local)})
+
+        ok = index_repo(repo)
+        results["cairn"].append({
+            "name": name,
+            "status": "ok" if ok else "error",
+            "message": "indexed" if ok else "cairn index failed — check logs",
+        })
 
     for src_name, src in cfg_loader.log_sources.items():
         if src.source_type == "ssh" and src.hosts:
             host = src.hosts[0]
-            logger.info("Testing SSH to %s (%s)...", src_name, host)
-            r = subprocess.run(
-                ["ssh", "-i", src.key, "-o", "StrictHostKeyChecking=no",
-                 "-o", "ConnectTimeout=5", f"ec2-user@{host}", "echo ok"],
-                capture_output=True, text=True, timeout=15,
-            )
-            logger.info("  SSH %s: %s", host, "OK" if r.returncode == 0 else f"FAILED — {r.stderr.strip()}")
-
-    logger.info("Sending test email...")
+            try:
+                r = subprocess.run(
+                    ["ssh", "-i", src.key, "-o", "StrictHostKeyChecking=no",
+                     "-o", "ConnectTimeout=5", f"ec2-user@{host}", "echo ok"],
+                    capture_output=True, text=True, timeout=15,
+                )
+                ok = r.returncode == 0
+                results["ssh"].append({
+                    "name": src_name, "host": host,
+                    "status": "ok" if ok else "error",
+                    "message": "" if ok else r.stderr.strip(),
+                })
+            except Exception as e:
+                results["ssh"].append({"name": src_name, "host": host,
+                                       "status": "error", "message": str(e)})
+
+    return results
+
+
+async def _send_startup_email_delayed(cfg, results: dict, delay: int = 300):
+    """Wait delay seconds then send startup summary email."""
+    await asyncio.sleep(delay)
     try:
-        build_and_send(sentinel, StateStore(sentinel.state_db))
+        send_startup_notification(cfg, results)
     except Exception as e:
-        logger.error("Test email failed: %s", e)
-
-    logger.info("=== Init complete ===")
+        logger.error("Failed to send startup notification: %s", e)
 
 
-# ── Entry point ───────────────────────────────────────────────────────────────
+# ── Entry point ──────────────────────────────────────────────────────────────────────────────────
 
 async def run_loop(cfg_loader: ConfigLoader, store: StateStore):
     interval = cfg_loader.sentinel.poll_interval_seconds
-    logger.info("Sentinel starting — poll interval: %ds, repos: %s", interval, list(cfg_loader.repos.keys()))
+    logger.info("Sentinel starting — poll interval: %ds, repos: %s",
+                interval, list(cfg_loader.repos.keys()))
+
+    results = await _startup_checks(cfg_loader)
+
+    has_errors = any(
+        item["status"] == "error"
+        for key in ("repos", "cairn", "ssh")
+        for item in results[key]
+    )
+    if has_errors:
+        logger.warning("Startup completed with errors — check config and logs")
+    else:
+        logger.info("Startup checks passed — startup email in 5 minutes")
+
+    asyncio.ensure_future(_send_startup_email_delayed(cfg_loader.sentinel, results))
+
     while True:
         try:
             await poll_cycle(cfg_loader, store)
@@ -336,7 +419,6 @@ def main():
     Path("issues").mkdir(exist_ok=True)
 
     parser = argparse.ArgumentParser(description="Sentinel — Autonomous DevOps Agent")
-    parser.add_argument("--init", action="store_true", help="First-time setup")
     parser.add_argument("--config", default="./config", help="Config directory path")
     args = parser.parse_args()
 
@@ -344,10 +426,6 @@ def main():
     store = StateStore(cfg_loader.sentinel.state_db)
     _register_signals()
 
-    if args.init:
-        run_init(cfg_loader)
-        return
-
     asyncio.run(run_loop(cfg_loader, store))