@misterhuydo/sentinel 1.0.6 → 1.0.9

package/python/sentinel/log_parser.py

@@ -1,149 +1,175 @@
-"""
-log_parser.py — Parse fetched log files into ErrorEvent objects.
-
-Handles Java-style logs (Spring Boot / Logback format):
-  2024-01-15 12:34:56.789 ERROR [thread] class.ClassName - Message
-  followed by optional stack trace lines (^\tat ...)
-"""
-
-import hashlib
-import re
-import logging
-from dataclasses import dataclass, field
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-_LOG_HEADER = re.compile(
-    r"^(?P<ts>\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}:\d{2}[.,\d]*)\s+"
-    r"(?P<level>CRITICAL|ERROR|WARN(?:ING)?|INFO|DEBUG)\s+"
-    r"(?:\[(?P<thread>[^\]]*)\]\s+)?"
-    r"(?P<logger>\S+)\s+-\s+"
-    r"(?P<message>.+)$"
-)
-
-_STACK_LINE = re.compile(r"^\s+at |\s+\.\.\. \d+ more|^Caused by:")
-
-SEVERITY_MAP = {
-    "CRITICAL": "CRITICAL",
-    "ERROR": "ERROR",
-    "WARN": "WARN",
-    "WARNING": "WARN",
-    "INFO": "INFO",
-    "DEBUG": "DEBUG",
-}
-
-_CRITICAL_PATTERNS = re.compile(
-    r"OutOfMemoryError|StackOverflowError|OOMKilled", re.IGNORECASE
-)
-_INFRA_PATTERNS = re.compile(
-    r"ConnectException|TimeoutException|ConnectionRefused|SocketTimeout",
-    re.IGNORECASE,
-)
-
-
-@dataclass
-class ErrorEvent:
-    source: str           # log-source name (e.g. "SSOLWA")
-    log_file: str
-    timestamp: str
-    level: str            # CRITICAL / ERROR / WARN
-    thread: str
-    logger_name: str
-    message: str
-    stack_trace: list[str] = field(default_factory=list)
-    fingerprint: str = ""
-
-    def __post_init__(self):
-        if not self.fingerprint:
-            self.fingerprint = _fingerprint(self.message, self.stack_trace)
-
-    @property
-    def severity(self) -> str:
-        if _CRITICAL_PATTERNS.search(self.message) or _CRITICAL_PATTERNS.search(
-            "\n".join(self.stack_trace)
-        ):
-            return "CRITICAL"
-        return self.level
-
-    @property
-    def is_infra_issue(self) -> bool:
-        return bool(_INFRA_PATTERNS.search(self.message))
-
-    def short_summary(self) -> str:
-        return self.message[:120]
-
-    def full_text(self) -> str:
-        lines = [f"{self.timestamp} {self.level} [{self.thread}] {self.logger_name} - {self.message}"]
-        lines.extend(self.stack_trace)
-        return "\n".join(lines)
-
-
-def _normalize_message(msg: str) -> str:
-    msg = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", msg)
-    msg = re.sub(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", "UUID", msg)
-    msg = re.sub(r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}[.,\d]*\b", "TIMESTAMP", msg)
-    msg = re.sub(r"\b\d+\b", "N", msg)
-    return msg.strip()
-
-
-def _fingerprint(message: str, stack_trace: list[str]) -> str:
-    top_frames = [l for l in stack_trace if l.strip().startswith("at ")][:3]
-    raw = _normalize_message(message) + "\n" + "\n".join(top_frames)
-    return hashlib.sha1(raw.encode()).hexdigest()[:16]
-
-
-def parse_log_file(path: Path, source_name: str) -> list[ErrorEvent]:
-    """Parse a single log file and return all ERROR/WARN events."""
-    events: list[ErrorEvent] = []
-    current_header: re.Match | None = None
-    current_stack: list[str] = []
-
-    def flush():
-        if current_header is None:
-            return
-        level = SEVERITY_MAP.get(current_header.group("level").upper(), "WARN")
-        if level not in ("ERROR", "WARN", "CRITICAL"):
-            return
-        event = ErrorEvent(
-            source=source_name,
-            log_file=str(path),
-            timestamp=current_header.group("ts"),
-            level=level,
-            thread=current_header.group("thread") or "",
-            logger_name=current_header.group("logger"),
-            message=current_header.group("message"),
-            stack_trace=list(current_stack),
-        )
-        events.append(event)
-
-    try:
-        text = path.read_text(encoding="utf-8", errors="replace")
-    except OSError as e:
-        logger.error("Cannot read %s: %s", path, e)
-        return []
-
-    for line in text.splitlines():
-        m = _LOG_HEADER.match(line)
-        if m:
-            flush()
-            current_header = m
-            current_stack = []
-        elif current_header and _STACK_LINE.match(line):
-            current_stack.append(line)
-
-    flush()
-    logger.debug("Parsed %s: %d error/warn events", path.name, len(events))
-    return events
-
-
-def parse_all(
-    fetched_files: dict[str, list[Path]],
-    log_sources,  # dict[str, LogSourceConfig]
-) -> list[ErrorEvent]:
-    """Parse all fetched log files across all sources."""
-    all_events: list[ErrorEvent] = []
-    for source_name, files in fetched_files.items():
-        for f in files:
-            all_events.extend(parse_log_file(f, source_name))
-    return all_events
+"""
+log_parser.py — Parse fetched log files into ErrorEvent objects.
+
+Handles Java-style logs (Spring Boot / Logback format):
+  2024-01-15 12:34:56.789 ERROR [thread] class.ClassName - Message
+  followed by optional stack trace lines (^\tat ...)
+"""
+
+import hashlib
+import re
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+_LOG_HEADER = re.compile(
+    r"^(?P<ts>\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}:\d{2}[.,\d]*)\s+"
+    r"(?P<level>CRITICAL|ERROR|WARN(?:ING)?|INFO|DEBUG)\s+"
+    r"(?:\[(?P<thread>[^\]]*)\]\s+)?"
+    r"(?P<logger>\S+)\s+-\s+"
+    r"(?P<message>.+)$"
+)
+
+_STACK_LINE = re.compile(r"^\s+at |\s+\.\.\. \d+ more|^Caused by:")
+
+SEVERITY_MAP = {
+    "CRITICAL": "CRITICAL",
+    "ERROR": "ERROR",
+    "WARN": "WARN",
+    "WARNING": "WARN",
+    "INFO": "INFO",
+    "DEBUG": "DEBUG",
+}
+
+_CRITICAL_PATTERNS = re.compile(
+    r"OutOfMemoryError|StackOverflowError|OOMKilled", re.IGNORECASE
+)
+_INFRA_PATTERNS = re.compile(
+    r"ConnectException|TimeoutException|ConnectionRefused|SocketTimeout",
+    re.IGNORECASE,
+)
+
+
+@dataclass
+class ErrorEvent:
+    source: str           # log-source name (e.g. "SSOLWA")
+    log_file: str
+    timestamp: str
+    level: str            # CRITICAL / ERROR / WARN
+    thread: str
+    logger_name: str
+    message: str
+    stack_trace: list[str] = field(default_factory=list)
+    fingerprint: str = ""
+
+    def __post_init__(self):
+        if not self.fingerprint:
+            self.fingerprint = _fingerprint(self.message, self.stack_trace)
+
+    @property
+    def severity(self) -> str:
+        if _CRITICAL_PATTERNS.search(self.message) or _CRITICAL_PATTERNS.search(
+            "\n".join(self.stack_trace)
+        ):
+            return "CRITICAL"
+        return self.level
+
+    @property
+    def is_infra_issue(self) -> bool:
+        return bool(_INFRA_PATTERNS.search(self.message))
+
+    def short_summary(self) -> str:
+        return self.message[:120]
+
+    def full_text(self) -> str:
+        lines = [f"{self.timestamp} {self.level} [{self.thread}] {self.logger_name} - {self.message}"]
+        lines.extend(self.stack_trace)
+        return "\n".join(lines)
+
+
+def _normalize_message(msg: str) -> str:
+    msg = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", msg)
+    msg = re.sub(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", "UUID", msg)
+    msg = re.sub(r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}[.,\d]*\b", "TIMESTAMP", msg)
+    msg = re.sub(r"\b\d+\b", "N", msg)
+    return msg.strip()
+
+
+def _fingerprint(message: str, stack_trace: list[str]) -> str:
+    top_frames = [l for l in stack_trace if l.strip().startswith("at ")][:3]
+    raw = _normalize_message(message) + "\n" + "\n".join(top_frames)
+    return hashlib.sha1(raw.encode()).hexdigest()[:16]
+
+
+def parse_log_file(path: Path, source_name: str) -> list[ErrorEvent]:
+    """Parse a single log file and return all ERROR/WARN events."""
+    events: list[ErrorEvent] = []
+    current_header: re.Match | None = None
+    current_stack: list[str] = []
+
+    def flush():
+        if current_header is None:
+            return
+        level = SEVERITY_MAP.get(current_header.group("level").upper(), "WARN")
+        if level not in ("ERROR", "WARN", "CRITICAL"):
+            return
+        event = ErrorEvent(
+            source=source_name,
+            log_file=str(path),
+            timestamp=current_header.group("ts"),
+            level=level,
+            thread=current_header.group("thread") or "",
+            logger_name=current_header.group("logger"),
+            message=current_header.group("message"),
+            stack_trace=list(current_stack),
+        )
+        events.append(event)
+
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except OSError as e:
+        logger.error("Cannot read %s: %s", path, e)
+        return []
+
+    for line in text.splitlines():
+        m = _LOG_HEADER.match(line)
+        if m:
+            flush()
+            current_header = m
+            current_stack = []
+        elif current_header and _STACK_LINE.match(line):
+            current_stack.append(line)
+
+    flush()
+    logger.debug("Parsed %s: %d error/warn events", path.name, len(events))
+    return events
+
+
+def parse_all(
+    fetched_files: dict[str, list[Path]],
+    log_sources,  # dict[str, LogSourceConfig]
+) -> list[ErrorEvent]:
+    """Parse all fetched log files across all sources."""
+    all_events: list[ErrorEvent] = []
+    for source_name, files in fetched_files.items():
+        for f in files:
+            all_events.extend(parse_log_file(f, source_name))
+    return all_events
+
+
+# -- Sentinel marker detection -------------------------------------------------
+
+_SENTINEL_MARKER_RE = re.compile(r'SENTINEL:#([0-9a-f]{16})')
+
+
+def scan_for_markers(path: Path) -> list[str]:
+    """
+    Scan a single log file for SENTINEL:#<fingerprint> markers injected by fix_engine.
+    Returns a list of full marker strings (e.g. ['SENTINEL:#abc123de45678901']).
+    """
+    try:
+        text = path.read_text(encoding='utf-8', errors='replace')
+    except OSError:
+        return []
+    return [f'SENTINEL:#{m}' for m in _SENTINEL_MARKER_RE.findall(text)]
+
+
+def scan_all_for_markers(fetched_files: dict[str, list[Path]]) -> list[str]:
+    """Scan all fetched log files and return every SENTINEL marker found."""
+    markers: list[str] = []
+    for files in fetched_files.values():
+        for f in files:
+            markers.extend(scan_for_markers(f))
+    return markers

package/python/sentinel/main.py

@@ -21,10 +21,10 @@ from .fix_engine import generate_fix
 from .git_manager import apply_and_commit, publish
 from .cicd_trigger import trigger as cicd_trigger
 from .log_fetcher import fetch_all
-from .log_parser import parse_all, ErrorEvent
+from .log_parser import parse_all, scan_all_for_markers, ErrorEvent
 from .issue_watcher import scan_issues, mark_done, IssueEvent
 from .repo_router import route
-from .reporter import build_and_send, send_fix_notification, send_failure_notification
+from .reporter import build_and_send, send_fix_notification, send_failure_notification, send_confirmed_notification, send_regression_notification
 from .state_store import StateStore
 
 logging.basicConfig(
@@ -81,7 +81,7 @@ async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: Stat
         return
 
     patches_dir = Path(sentinel.workspace_dir) / "patches"
-    status, patch_path = generate_fix(event, repo, sentinel, patches_dir)
+    status, patch_path, marker = generate_fix(event, repo, sentinel, patches_dir, store)
 
     if status != "patch" or patch_path is None:
         outcome = "skipped" if status == "skip" else "failed"
@@ -116,6 +116,7 @@ async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: Stat
         branch=branch,
         pr_url=pr_url,
         repo_name=repo.repo_name,
+        sentinel_marker=marker,
     )
 
     send_fix_notification(sentinel, {
@@ -172,7 +173,7 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
         return  # Leave the file so admin can add the header
 
     patches_dir = Path(sentinel.workspace_dir) / "patches"
-    status, patch_path = generate_fix(event, repo, sentinel, patches_dir)
+    status, patch_path, marker = generate_fix(event, repo, sentinel, patches_dir, store)
 
     if status != "patch" or patch_path is None:
         store.record_fix(event.fingerprint, "skipped" if status == "skip" else "failed",
@@ -209,6 +210,7 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
         branch=branch,
         pr_url=pr_url,
         repo_name=repo.repo_name,
+        sentinel_marker=marker,
     )
     send_fix_notification(sentinel, {
         "source":       event.source,
@@ -232,6 +234,8 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
 
 async def poll_cycle(cfg_loader: ConfigLoader, store: StateStore):
     global _report_requested
+    events: list = []
+    fetched: dict = {}
 
     # ── Log sources (optional) ────────────────────────────────────────────────
     sources = list(cfg_loader.log_sources.values())
@@ -254,6 +258,36 @@ async def poll_cycle(cfg_loader: ConfigLoader, store: StateStore):
                 return_exceptions=True,
             )
 
+    # ── SENTINEL marker scanning (phase 1: record first seen in prod logs) ────
+    if sources and fetched:
+        for marker in set(scan_all_for_markers(fetched)):
+            fix = store.mark_marker_seen(marker)
+            if fix:
+                logger.info("Marker seen in production: %s repo=%s — quiet period started",
+                            marker, fix.get("repo_name"))
+
+    # ── Regression detection (error recurred before quiet period elapsed) ──────
+    if sources:
+        for event in events:
+            pending = store.get_marker_seen_fix(event.fingerprint)
+            if pending:
+                logger.warning("Regression: %s recurred after marker seen", event.fingerprint)
+                store.mark_regressed(event.fingerprint)
+                send_regression_notification(cfg_loader.sentinel, pending, {
+                    "source":  event.source,
+                    "message": event.message,
+                    "body":    event.full_text()[:500],
+                })
+
+    # ── Phase 2: confirm fixes whose quiet period has elapsed ────────────────
+    quiet_hours = cfg_loader.sentinel.marker_confirm_hours
+    for fix in store.get_fixes_pending_confirmation(quiet_hours):
+        confirmed = store.confirm_fix(fix["fingerprint"])
+        if confirmed:
+            logger.info("Fix confirmed after %dh quiet period: %s repo=%s",
+                        quiet_hours, fix["fingerprint"], fix.get("repo_name"))
+            send_confirmed_notification(cfg_loader.sentinel, confirmed)
+
     # ── Issues directory (always checked) ────────────────────────────────────
     issues = scan_issues(Path("."))
     if issues:

package/python/sentinel/reporter.py

@@ -186,3 +186,89 @@ def send_failure_notification(cfg: SentinelConfig, details: dict):
     _send_email(cfg, subject, html)
     logger.info('Failure notification sent for %s', source)
 
+
+
+# ---- Confirmed fix notification ----------------------------------------------
+
+def send_confirmed_notification(cfg: SentinelConfig, fix: dict):
+    """Notify admins that a fix has been confirmed running in production."""
+    if not cfg.mails:
+        return
+    repo_name = fix.get('repo_name', 'unknown')
+    fingerprint = fix.get('fingerprint', '')
+    marker = fix.get('sentinel_marker', '')
+    ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')
+    subject = f'[Sentinel] ✅ Fix confirmed in production: {repo_name} ({fingerprint[:8]})'
+    html = (
+        '<!DOCTYPE html><html><head><meta charset="utf-8">'
+        '<style>'
+        'body{font-family:Arial,sans-serif;font-size:14px;color:#222}'
+        'h2{color:#2e7d32}'
+        'table{border-collapse:collapse;width:100%;margin-bottom:16px}'
+        'th{background:#f1f3f4;text-align:left;padding:6px 10px}'
+        'td{padding:5px 10px;border-bottom:1px solid #eee;vertical-align:top}'
+        '.label{font-weight:bold;width:160px}'
+        '.mono{font-family:monospace;font-size:12px}'
+        '</style></head><body>'
+        '<h2>✅ Fix confirmed running in production</h2>'
+        f'<p><strong>{repo_name}</strong> &middot; {ts}</p>'
+        '<table>'
+        f'<tr><td class="label">Fingerprint</td><td class="mono">{fingerprint}</td></tr>'
+        f'<tr><td class="label">Sentinel marker</td><td class="mono">{marker}</td></tr>'
+        f'<tr><td class="label">Commit</td><td class="mono">{fix.get("commit_hash", "")}</td></tr>'
+        f'<tr><td class="label">Branch</td><td class="mono">{fix.get("branch", "")}</td></tr>'
+        f'<tr><td class="label">Confirmed at</td><td>{fix.get("confirmed_at", ts)}</td></tr>'
+        '</table>'
+        '<p>The marker log line was detected in production logs, confirming the fix is live and the fixed code path executed.</p>'
+        '<hr><small>Sentinel &mdash; Autonomous DevOps Agent</small>'
+        '</body></html>'
+    )
+    _send_email(cfg, subject, html)
+    logger.info('Confirmed notification sent for %s', fingerprint)
+
+
+# ---- Regression notification ------------------------------------------------
+
+def send_regression_notification(cfg: SentinelConfig, fix: dict, event: dict):
+    """Notify admins that a confirmed fix did not resolve the issue."""
+    if not cfg.mails:
+        return
+    repo_name = fix.get('repo_name', 'unknown')
+    fingerprint = fix.get('fingerprint', '')
+    ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')
+    subject = f'[Sentinel] ⚠ Regression: fix did not resolve issue in {repo_name}'
+    html = (
+        '<!DOCTYPE html><html><head><meta charset="utf-8">'
+        '<style>'
+        'body{font-family:Arial,sans-serif;font-size:14px;color:#222}'
+        'h2{color:#c62828}'
+        'h3{color:#444;border-bottom:1px solid #ddd;padding-bottom:4px}'
+        'table{border-collapse:collapse;width:100%;margin-bottom:16px}'
+        'th{background:#f1f3f4;text-align:left;padding:6px 10px}'
+        'td{padding:5px 10px;border-bottom:1px solid #eee;vertical-align:top}'
+        '.label{font-weight:bold;width:160px}'
+        '.mono{font-family:monospace;font-size:12px}'
+        'pre{background:#f8f8f8;border:1px solid #ddd;padding:10px;font-size:12px;white-space:pre-wrap}'
+        '</style></head><body>'
+        '<h2>⚠ Regression detected &mdash; fix did not resolve the issue</h2>'
+        f'<p><strong>{repo_name}</strong> &middot; {ts}</p>'
+        '<p>The original error recurred in production logs after the Sentinel fix was confirmed deployed.</p>'
+        '<h3>Fix Details</h3>'
+        '<table>'
+        f'<tr><td class="label">Fingerprint</td><td class="mono">{fingerprint}</td></tr>'
+        f'<tr><td class="label">Commit</td><td class="mono">{fix.get("commit_hash", "")}</td></tr>'
+        f'<tr><td class="label">Branch</td><td class="mono">{fix.get("branch", "")}</td></tr>'
+        f'<tr><td class="label">Confirmed at</td><td>{fix.get("confirmed_at", "")}</td></tr>'
+        '</table>'
+        '<h3>Recurring Error</h3>'
+        '<table>'
+        f'<tr><td class="label">Source</td><td class="mono">{event.get("source", "")}</td></tr>'
+        f'<tr><td class="label">Message</td><td class="mono">{event.get("message", "")}</td></tr>'
+        '</table>'
+        f'<pre>{event.get("body", "")}</pre>'
+        '<p>Sentinel will not attempt another automatic fix. Please investigate manually.</p>'
+        '<hr><small>Sentinel &mdash; Autonomous DevOps Agent</small>'
+        '</body></html>'
+    )
+    _send_email(cfg, subject, html)
+    logger.info('Regression notification sent for %s', fingerprint)