@misterhuydo/sentinel 1.0.5 → 1.0.9

package/python/sentinel/main.py

@@ -21,9 +21,10 @@ from .fix_engine import generate_fix
 from .git_manager import apply_and_commit, publish
 from .cicd_trigger import trigger as cicd_trigger
 from .log_fetcher import fetch_all
-from .log_parser import parse_all, ErrorEvent
+from .log_parser import parse_all, scan_all_for_markers, ErrorEvent
+from .issue_watcher import scan_issues, mark_done, IssueEvent
 from .repo_router import route
-from .reporter import build_and_send, send_fix_notification
+from .reporter import build_and_send, send_fix_notification, send_failure_notification, send_confirmed_notification, send_regression_notification
 from .state_store import StateStore
 
 logging.basicConfig(
@@ -80,15 +81,30 @@ async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: Stat
         return
 
     patches_dir = Path(sentinel.workspace_dir) / "patches"
-    status, patch_path = generate_fix(event, repo, sentinel, patches_dir)
+    status, patch_path, marker = generate_fix(event, repo, sentinel, patches_dir, store)
 
     if status != "patch" or patch_path is None:
-        store.record_fix(event.fingerprint, "skipped" if status == "skip" else "failed", repo_name=repo.repo_name)
+        outcome = "skipped" if status == "skip" else "failed"
+        store.record_fix(event.fingerprint, outcome, repo_name=repo.repo_name)
+        send_failure_notification(sentinel, {
+            "source":    event.source,
+            "message":   event.message,
+            "repo_name": repo.repo_name,
+            "reason":    f"Claude Code returned {status.upper()}",
+            "body":      event.full_text()[:500],
+        })
         return
 
     commit_status, commit_hash = apply_and_commit(event, patch_path, repo, sentinel)
     if commit_status != "committed":
         store.record_fix(event.fingerprint, "failed", repo_name=repo.repo_name)
+        send_failure_notification(sentinel, {
+            "source":    event.source,
+            "message":   event.message,
+            "repo_name": repo.repo_name,
+            "reason":    "patch generated but commit/tests failed",
+            "body":      event.full_text()[:500],
+        })
         return
 
     branch, pr_url = publish(event, repo, sentinel, commit_hash)
@@ -100,6 +116,7 @@ async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: Stat
         branch=branch,
         pr_url=pr_url,
         repo_name=repo.repo_name,
+        sentinel_marker=marker,
     )
 
     send_fix_notification(sentinel, {
@@ -123,28 +140,162 @@ async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: Stat
 
 # ── Poll cycle ────────────────────────────────────────────────────────────────
 
-async def poll_cycle(cfg_loader: ConfigLoader, store: StateStore):
-    global _report_requested
 
-    sources = list(cfg_loader.log_sources.values())
-    if not sources:
-        logger.warning("No log-configs found")
+# ── Issue pipeline ────────────────────────────────────────────────────────────
+
+async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: StateStore):
+    """Process a single issue file from the issues/ directory."""
+    sentinel = cfg_loader.sentinel
+
+    if Path("SENTINEL_PAUSE").exists():
+        logger.info("SENTINEL_PAUSE present -- fix activity halted")
         return
 
-    logger.info("Fetching logs from %d source(s)...", len(sources))
-    fetched = await fetch_all(sources, cfg_loader.sentinel)
+    if store.fix_attempted_recently(event.fingerprint, hours=24):
+        logger.debug("Issue already processed recently: %s", event.source)
+        mark_done(event.issue_file)
+        return
+
+    # Route: explicit TARGET_REPO in file > single-repo shortcut > warn and leave
+    if event.target_repo:
+        repo = cfg_loader.repos.get(event.target_repo)
+        if not repo:
+            logger.warning("TARGET_REPO %r not found in config -- leaving %s for admin",
+                           event.target_repo, event.source)
+            return
+    elif len(cfg_loader.repos) == 1:
+        repo = next(iter(cfg_loader.repos.values()))
+    else:
+        logger.warning(
+            "Cannot auto-route %s -- add 'TARGET_REPO: <repo>' as first line in the file",
+            event.source,
+        )
+        return  # Leave the file so admin can add the header
+
+    patches_dir = Path(sentinel.workspace_dir) / "patches"
+    status, patch_path, marker = generate_fix(event, repo, sentinel, patches_dir, store)
+
+    if status != "patch" or patch_path is None:
+        store.record_fix(event.fingerprint, "skipped" if status == "skip" else "failed",
+                         repo_name=repo.repo_name)
+        send_failure_notification(sentinel, {
+            "source":    event.source,
+            "message":   event.message,
+            "repo_name": repo.repo_name,
+            "reason":    f"Claude Code returned {status.upper()}",
+            "body":      event.body[:500],
+        })
+        mark_done(event.issue_file)
+        return
+
+    commit_status, commit_hash = apply_and_commit(event, patch_path, repo, sentinel)
+    if commit_status != "committed":
+        store.record_fix(event.fingerprint, "failed", repo_name=repo.repo_name)
+        send_failure_notification(sentinel, {
+            "source":    event.source,
+            "message":   event.message,
+            "repo_name": repo.repo_name,
+            "reason":    "patch generated but commit/tests failed",
+            "body":      event.body[:500],
+        })
+        mark_done(event.issue_file)
+        return
+
+    branch, pr_url = publish(event, repo, sentinel, commit_hash)
+    store.record_fix(
+        event.fingerprint,
+        "applied" if repo.auto_publish else "pending",
+        patch_path=str(patch_path),
+        commit_hash=commit_hash,
+        branch=branch,
+        pr_url=pr_url,
+        repo_name=repo.repo_name,
+        sentinel_marker=marker,
+    )
+    send_fix_notification(sentinel, {
+        "source":       event.source,
+        "severity":     "ERROR",
+        "fingerprint":  event.fingerprint,
+        "first_seen":   event.timestamp,
+        "message":      event.message,
+        "stack_trace":  event.body,
+        "repo_name":    repo.repo_name,
+        "commit_hash":  commit_hash,
+        "branch":       branch,
+        "pr_url":       pr_url,
+        "auto_publish": repo.auto_publish,
+        "files_changed": [],
+    })
+    mark_done(event.issue_file)
+
+    if repo.auto_publish:
+        cicd_trigger(repo, store, event.fingerprint)
+
 
-    events = parse_all(fetched, cfg_loader.log_sources)
-    logger.info("Parsed %d error/warn events", len(events))
+async def poll_cycle(cfg_loader: ConfigLoader, store: StateStore):
+    global _report_requested
+    events: list = []
+    fetched: dict = {}
 
-    new_events = []
-    for event in events:
-        store.record_error(event.fingerprint, event.source, event.message)
-        if not store.fix_attempted_recently(event.fingerprint):
-            new_events.append(event)
+    # ── Log sources (optional) ────────────────────────────────────────────────
+    sources = list(cfg_loader.log_sources.values())
+    if sources:
+        logger.info("Fetching logs from %d source(s)...", len(sources))
+        fetched = await fetch_all(sources, cfg_loader.sentinel)
+        events = parse_all(fetched, cfg_loader.log_sources)
+        logger.info("Parsed %d error/warn events", len(events))
+
+        new_events = []
+        for event in events:
+            store.record_error(event.fingerprint, event.source, event.message)
+            if not store.fix_attempted_recently(event.fingerprint):
+                new_events.append(event)
+
+        if new_events:
+            logger.info("%d new log event(s) to process", len(new_events))
+            await asyncio.gather(
+                *[_handle_error(e, cfg_loader, store) for e in new_events],
+                return_exceptions=True,
+            )
 
-    logger.info("%d new event(s) to process", len(new_events))
-    await asyncio.gather(*[_handle_error(e, cfg_loader, store) for e in new_events], return_exceptions=True)
+    # ── SENTINEL marker scanning (phase 1: record first seen in prod logs) ────
+    if sources and fetched:
+        for marker in set(scan_all_for_markers(fetched)):
+            fix = store.mark_marker_seen(marker)
+            if fix:
+                logger.info("Marker seen in production: %s repo=%s — quiet period started",
+                            marker, fix.get("repo_name"))
+
+    # ── Regression detection (error recurred before quiet period elapsed) ──────
+    if sources:
+        for event in events:
+            pending = store.get_marker_seen_fix(event.fingerprint)
+            if pending:
+                logger.warning("Regression: %s recurred after marker seen", event.fingerprint)
+                store.mark_regressed(event.fingerprint)
+                send_regression_notification(cfg_loader.sentinel, pending, {
+                    "source":  event.source,
+                    "message": event.message,
+                    "body":    event.full_text()[:500],
+                })
+
+    # ── Phase 2: confirm fixes whose quiet period has elapsed ────────────────
+    quiet_hours = cfg_loader.sentinel.marker_confirm_hours
+    for fix in store.get_fixes_pending_confirmation(quiet_hours):
+        confirmed = store.confirm_fix(fix["fingerprint"])
+        if confirmed:
+            logger.info("Fix confirmed after %dh quiet period: %s repo=%s",
+                        quiet_hours, fix["fingerprint"], fix.get("repo_name"))
+            send_confirmed_notification(cfg_loader.sentinel, confirmed)
+
+    # ── Issues directory (always checked) ────────────────────────────────────
+    issues = scan_issues(Path("."))
+    if issues:
+        logger.info("%d issue file(s) found in issues/", len(issues))
+        await asyncio.gather(
+            *[_handle_issue(e, cfg_loader, store) for e in issues],
+            return_exceptions=True,
+        )
 
     if cfg_loader.sentinel.send_health and (_report_requested or _report_due(cfg_loader, store)):
         _report_requested = False
@@ -216,6 +367,7 @@ def main():
     Path("logs").mkdir(exist_ok=True)
     Path("workspace/fetched").mkdir(parents=True, exist_ok=True)
     Path("workspace/patches").mkdir(parents=True, exist_ok=True)
+    Path("issues").mkdir(exist_ok=True)
 
     parser = argparse.ArgumentParser(description="Sentinel — Autonomous DevOps Agent")
     parser.add_argument("--init", action="store_true", help="First-time setup")

package/python/sentinel/reporter.py

@@ -136,3 +136,139 @@ def _age(ts_str: str) -> str:
         return f"{int(delta.total_seconds() // 60)}m" if hours < 1 else f"{hours}h"
     except Exception:
         return "?"
+
+def send_failure_notification(cfg: SentinelConfig, details: dict):
+    """
+    Notify admins when Claude Code cannot fix a problem (from logs or issues/).
+
+    details dict keys: source, message, repo_name, reason, body
+    """
+    if not cfg.mails:
+        return
+
+    source    = details.get('source', 'unknown')
+    repo_name = details.get('repo_name', 'unknown')
+    reason    = details.get('reason', 'unknown')
+    message   = details.get('message', '')
+    body      = details.get('body', '')[:1000]
+    ts        = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')
+
+    subject = f'[Sentinel] UNRESOLVED ({source}): {message[:80]}'
+
+    ctx_html = f'<h3>Context</h3><pre>{body}</pre>' if body else ''
+    html = (
+        '<!DOCTYPE html><html><head><meta charset="utf-8">'
+        '<style>'
+        'body{font-family:Arial,sans-serif;font-size:14px;color:#222}'
+        'h2{color:#c62828}'
+        'h3{color:#444;border-bottom:1px solid #ddd;padding-bottom:4px}'
+        'table{border-collapse:collapse;width:100%;margin-bottom:16px}'
+        'th{background:#f1f3f4;text-align:left;padding:6px 10px}'
+        'td{padding:5px 10px;border-bottom:1px solid #eee;vertical-align:top}'
+        '.label{font-weight:bold;width:160px}'
+        '.mono{font-family:monospace;font-size:12px}'
+        'pre{background:#f8f8f8;border:1px solid #ddd;padding:10px;font-size:12px;white-space:pre-wrap}'
+        '</style></head><body>'
+        '<h2>&#x26A0; Sentinel could not fix this issue</h2>'
+        f'<p><strong>{repo_name}</strong> &middot; {ts}</p>'
+        '<h3>Details</h3>'
+        '<table>'
+        f'<tr><td class="label">Source</td><td class="mono">{source}</td></tr>'
+        f'<tr><td class="label">Repository</td><td class="mono">{repo_name}</td></tr>'
+        f'<tr><td class="label">Message</td><td class="mono">{message}</td></tr>'
+        f'<tr><td class="label">Reason</td><td>{reason}</td></tr>'
+        '</table>'
+        + ctx_html +
+        '<hr><small>Sentinel &mdash; Autonomous DevOps Agent</small>'
+        '</body></html>'
+    )
+
+    _send_email(cfg, subject, html)
+    logger.info('Failure notification sent for %s', source)
+
+
+
+# ---- Confirmed fix notification ----------------------------------------------
+
+def send_confirmed_notification(cfg: SentinelConfig, fix: dict):
+    """Notify admins that a fix has been confirmed running in production."""
+    if not cfg.mails:
+        return
+    repo_name = fix.get('repo_name', 'unknown')
+    fingerprint = fix.get('fingerprint', '')
+    marker = fix.get('sentinel_marker', '')
+    ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')
+    subject = f'[Sentinel] ✅ Fix confirmed in production: {repo_name} ({fingerprint[:8]})'
+    html = (
+        '<!DOCTYPE html><html><head><meta charset="utf-8">'
+        '<style>'
+        'body{font-family:Arial,sans-serif;font-size:14px;color:#222}'
+        'h2{color:#2e7d32}'
+        'table{border-collapse:collapse;width:100%;margin-bottom:16px}'
+        'th{background:#f1f3f4;text-align:left;padding:6px 10px}'
+        'td{padding:5px 10px;border-bottom:1px solid #eee;vertical-align:top}'
+        '.label{font-weight:bold;width:160px}'
+        '.mono{font-family:monospace;font-size:12px}'
+        '</style></head><body>'
+        '<h2>✅ Fix confirmed running in production</h2>'
+        f'<p><strong>{repo_name}</strong> &middot; {ts}</p>'
+        '<table>'
+        f'<tr><td class="label">Fingerprint</td><td class="mono">{fingerprint}</td></tr>'
+        f'<tr><td class="label">Sentinel marker</td><td class="mono">{marker}</td></tr>'
+        f'<tr><td class="label">Commit</td><td class="mono">{fix.get("commit_hash", "")}</td></tr>'
+        f'<tr><td class="label">Branch</td><td class="mono">{fix.get("branch", "")}</td></tr>'
+        f'<tr><td class="label">Confirmed at</td><td>{fix.get("confirmed_at", ts)}</td></tr>'
+        '</table>'
+        '<p>The marker log line was detected in production logs, confirming the fix is live and the fixed code path executed.</p>'
+        '<hr><small>Sentinel &mdash; Autonomous DevOps Agent</small>'
+        '</body></html>'
+    )
+    _send_email(cfg, subject, html)
+    logger.info('Confirmed notification sent for %s', fingerprint)
+
+
+# ---- Regression notification ------------------------------------------------
+
+def send_regression_notification(cfg: SentinelConfig, fix: dict, event: dict):
+    """Notify admins that a confirmed fix did not resolve the issue."""
+    if not cfg.mails:
+        return
+    repo_name = fix.get('repo_name', 'unknown')
+    fingerprint = fix.get('fingerprint', '')
+    ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')
+    subject = f'[Sentinel] ⚠ Regression: fix did not resolve issue in {repo_name}'
+    html = (
+        '<!DOCTYPE html><html><head><meta charset="utf-8">'
+        '<style>'
+        'body{font-family:Arial,sans-serif;font-size:14px;color:#222}'
+        'h2{color:#c62828}'
+        'h3{color:#444;border-bottom:1px solid #ddd;padding-bottom:4px}'
+        'table{border-collapse:collapse;width:100%;margin-bottom:16px}'
+        'th{background:#f1f3f4;text-align:left;padding:6px 10px}'
+        'td{padding:5px 10px;border-bottom:1px solid #eee;vertical-align:top}'
+        '.label{font-weight:bold;width:160px}'
+        '.mono{font-family:monospace;font-size:12px}'
+        'pre{background:#f8f8f8;border:1px solid #ddd;padding:10px;font-size:12px;white-space:pre-wrap}'
+        '</style></head><body>'
+        '<h2>⚠ Regression detected &mdash; fix did not resolve the issue</h2>'
+        f'<p><strong>{repo_name}</strong> &middot; {ts}</p>'
+        '<p>The original error recurred in production logs after the Sentinel fix was confirmed deployed.</p>'
+        '<h3>Fix Details</h3>'
+        '<table>'
+        f'<tr><td class="label">Fingerprint</td><td class="mono">{fingerprint}</td></tr>'
+        f'<tr><td class="label">Commit</td><td class="mono">{fix.get("commit_hash", "")}</td></tr>'
+        f'<tr><td class="label">Branch</td><td class="mono">{fix.get("branch", "")}</td></tr>'
+        f'<tr><td class="label">Confirmed at</td><td>{fix.get("confirmed_at", "")}</td></tr>'
+        '</table>'
+        '<h3>Recurring Error</h3>'
+        '<table>'
+        f'<tr><td class="label">Source</td><td class="mono">{event.get("source", "")}</td></tr>'
+        f'<tr><td class="label">Message</td><td class="mono">{event.get("message", "")}</td></tr>'
+        '</table>'
+        f'<pre>{event.get("body", "")}</pre>'
+        '<p>Sentinel will not attempt another automatic fix. Please investigate manually.</p>'
+        '<hr><small>Sentinel &mdash; Autonomous DevOps Agent</small>'
+        '</body></html>'
+    )
+    _send_email(cfg, subject, html)
+    logger.info('Regression notification sent for %s', fingerprint)