@misterhuydo/sentinel 1.0.0

package/python/sentinel/log_fetcher.py

@@ -0,0 +1,200 @@
+"""
+log_fetcher.py — Fetch logs from SSH servers and Cloudflare workers.
+
+Each source maintains a single rolling log file:
+  workspace/fetched/<source-name>.log
+
+New content is appended each poll cycle; lines older than
+LOG_RETENTION_HOURS are pruned so Claude always has a meaningful
+window of history without unbounded disk growth.
+"""
+
+import asyncio
+import logging
+import os
+import re
+import subprocess
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import requests
+
+from .config_loader import LogSourceConfig, SentinelConfig
+
+logger = logging.getLogger(__name__)
+
+SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
+FETCH_LOG_SH = SCRIPTS_DIR / "fetch_log.sh"
+
+# Matches common Java/Spring log timestamps at the start of a line:
+#   2026-03-21 01:36:56,173   or   2026-03-21T01:36:56.173Z
+_TS_RE = re.compile(r"^(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2})")
+
+
+def _parse_line_ts(line):
+    m = _TS_RE.match(line)
+    if not m:
+        return None
+    try:
+        return datetime.strptime(m.group(1), "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc)
+    except ValueError:
+        pass
+    try:
+        return datetime.strptime(m.group(1), "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
+    except ValueError:
+        return None
+
+
+def _prune_lines(lines, cutoff):
+    """
+    Drop lines (and their continuation lines) whose timestamp is before cutoff.
+    Lines with no parseable timestamp are continuations; they inherit the
+    keep/drop decision of the preceding timestamped line.
+    """
+    kept = []
+    keep = True
+    for line in lines:
+        ts = _parse_line_ts(line)
+        if ts is not None:
+            keep = ts >= cutoff
+        if keep:
+            kept.append(line)
+    return kept
+
+
+def _rolling_update(rolling_path, new_content, retention_hours):
+    """Append new_content to rolling_path then prune lines older than retention_hours."""
+    existing = ""
+    if rolling_path.exists():
+        existing = rolling_path.read_text(encoding="utf-8", errors="replace")
+
+    combined = (existing.rstrip("\n") + "\n" + new_content.strip()) if existing else new_content.strip()
+    lines = combined.splitlines()
+
+    cutoff = datetime.now(timezone.utc) - timedelta(hours=retention_hours)
+    kept = _prune_lines(lines, cutoff)
+
+    rolling_path.write_text("\n".join(kept) + "\n", encoding="utf-8")
+    logger.debug(
+        "Rolling log %s: %d -> %d lines (kept %dh window)",
+        rolling_path.name, len(lines), len(kept), retention_hours,
+    )
+
+
+# -- Public API ----------------------------------------------------------------
+
+async def fetch_all(sources, sentinel_cfg):
+    tasks = [_fetch_one(src, sentinel_cfg) for src in sources]
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+    output = {}
+    for src, result in zip(sources, results):
+        if isinstance(result, Exception):
+            logger.error("Fetch failed for %s: %s", src.name, result)
+            output[src.name] = []
+        else:
+            output[src.name] = result
+    return output
+
+
+async def _fetch_one(src, cfg):
+    loop = asyncio.get_running_loop()
+    if src.source_type == "ssh":
+        return await loop.run_in_executor(None, _fetch_ssh, src, cfg)
+    elif src.source_type == "cloudflare":
+        return await loop.run_in_executor(None, _fetch_cloudflare, src, cfg)
+    else:
+        raise ValueError(f"Unknown SOURCE_TYPE '{src.source_type}' for {src.name}")
+
+
+# -- SSH -----------------------------------------------------------------------
+
+def _find_props_file(src):
+    candidates = [
+        Path("config/log-configs") / f"{src.name}.properties",
+        Path(__file__).parent.parent / "config" / "log-configs" / f"{src.name}.properties",
+    ]
+    for p in candidates:
+        if p.exists():
+            return p
+    return None
+
+
+def _fetch_ssh(src, cfg):
+    props_file = _find_props_file(src)
+    if not props_file:
+        raise FileNotFoundError(f"Properties file not found for {src.name}")
+
+    workspace = Path(cfg.workspace_dir) / "fetched"
+    workspace.mkdir(parents=True, exist_ok=True)
+
+    tmp_dir = workspace / f"_tmp_{src.name}"
+    tmp_dir.mkdir(exist_ok=True)
+
+    result = subprocess.run(
+        ["bash", str(FETCH_LOG_SH), str(props_file)],
+        env={**os.environ.copy(), "OUTPUT_DIR": str(tmp_dir)},
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    if result.returncode != 0:
+        logger.warning("fetch_log.sh stderr for %s:\n%s", src.name, result.stderr)
+
+    new_lines = []
+    for host_file in sorted(tmp_dir.rglob("*.log")):
+        new_lines.extend(host_file.read_text(encoding="utf-8", errors="replace").splitlines())
+        host_file.unlink()
+    for d in sorted(tmp_dir.rglob("*"), reverse=True):
+        if d.is_dir():
+            try:
+                d.rmdir()
+            except OSError:
+                pass
+    try:
+        tmp_dir.rmdir()
+    except OSError:
+        pass
+
+    rolling_path = workspace / f"{src.name}.log"
+    if not new_lines:
+        logger.info("SSH fetch %s: no lines fetched", src.name)
+        return [rolling_path] if rolling_path.exists() else []
+
+    _rolling_update(rolling_path, "\n".join(new_lines), cfg.log_retention_hours)
+    logger.info("SSH fetch %s: %d new lines -> %s", src.name, len(new_lines), rolling_path)
+    return [rolling_path]
+
+
+# -- Cloudflare ----------------------------------------------------------------
+
+def _fetch_cloudflare(src, cfg):
+    if not src.cf_url or not src.cf_token:
+        raise ValueError(f"CF_URL or CF_TOKEN missing for {src.name}")
+
+    workspace = Path(cfg.workspace_dir) / "fetched"
+    workspace.mkdir(parents=True, exist_ok=True)
+
+    headers = {"Authorization": f"Bearer {src.cf_token}"}
+    lines = []
+    cursor = None
+
+    while True:
+        params = {"cursor": cursor} if cursor else {}
+        resp = requests.get(src.cf_url, headers=headers, params=params, timeout=30)
+        resp.raise_for_status()
+        ct = resp.headers.get("content-type", "")
+        if ct.startswith("application/json"):
+            data = resp.json()
+            batch = data.get("lines") or data.get("logs") or []
+            lines.extend(str(l) for l in batch)
+            cursor = data.get("next_cursor") or data.get("cursor")
+        else:
+            lines.extend(resp.text.splitlines())
+            cursor = None
+        if not cursor:
+            break
+
+    rolling_path = workspace / f"{src.name}.log"
+    _rolling_update(rolling_path, "\n".join(lines), cfg.log_retention_hours)
+    logger.info("Cloudflare fetch %s: %d new lines -> %s", src.name, len(lines), rolling_path)
+    return [rolling_path]

package/python/sentinel/log_parser.py

@@ -0,0 +1,149 @@
+"""
+log_parser.py — Parse fetched log files into ErrorEvent objects.
+
+Handles Java-style logs (Spring Boot / Logback format):
+  2024-01-15 12:34:56.789 ERROR [thread] class.ClassName - Message
+  followed by optional stack trace lines (^\tat ...)
+"""
+
+import hashlib
+import re
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+_LOG_HEADER = re.compile(
+    r"^(?P<ts>\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}:\d{2}[.,\d]*)\s+"
+    r"(?P<level>CRITICAL|ERROR|WARN(?:ING)?|INFO|DEBUG)\s+"
+    r"(?:\[(?P<thread>[^\]]*)\]\s+)?"
+    r"(?P<logger>\S+)\s+-\s+"
+    r"(?P<message>.+)$"
+)
+
+_STACK_LINE = re.compile(r"^\s+at |\s+\.\.\. \d+ more|^Caused by:")
+
+SEVERITY_MAP = {
+    "CRITICAL": "CRITICAL",
+    "ERROR": "ERROR",
+    "WARN": "WARN",
+    "WARNING": "WARN",
+    "INFO": "INFO",
+    "DEBUG": "DEBUG",
+}
+
+_CRITICAL_PATTERNS = re.compile(
+    r"OutOfMemoryError|StackOverflowError|OOMKilled", re.IGNORECASE
+)
+_INFRA_PATTERNS = re.compile(
+    r"ConnectException|TimeoutException|ConnectionRefused|SocketTimeout",
+    re.IGNORECASE,
+)
+
+
+@dataclass
+class ErrorEvent:
+    source: str           # log-source name (e.g. "SSOLWA")
+    log_file: str
+    timestamp: str
+    level: str            # CRITICAL / ERROR / WARN
+    thread: str
+    logger_name: str
+    message: str
+    stack_trace: list[str] = field(default_factory=list)
+    fingerprint: str = ""
+
+    def __post_init__(self):
+        if not self.fingerprint:
+            self.fingerprint = _fingerprint(self.message, self.stack_trace)
+
+    @property
+    def severity(self) -> str:
+        if _CRITICAL_PATTERNS.search(self.message) or _CRITICAL_PATTERNS.search(
+            "\n".join(self.stack_trace)
+        ):
+            return "CRITICAL"
+        return self.level
+
+    @property
+    def is_infra_issue(self) -> bool:
+        return bool(_INFRA_PATTERNS.search(self.message))
+
+    def short_summary(self) -> str:
+        return self.message[:120]
+
+    def full_text(self) -> str:
+        lines = [f"{self.timestamp} {self.level} [{self.thread}] {self.logger_name} - {self.message}"]
+        lines.extend(self.stack_trace)
+        return "\n".join(lines)
+
+
+def _normalize_message(msg: str) -> str:
+    msg = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", msg)
+    msg = re.sub(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", "UUID", msg)
+    msg = re.sub(r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}[.,\d]*\b", "TIMESTAMP", msg)
+    msg = re.sub(r"\b\d+\b", "N", msg)
+    return msg.strip()
+
+
+def _fingerprint(message: str, stack_trace: list[str]) -> str:
+    top_frames = [l for l in stack_trace if l.strip().startswith("at ")][:3]
+    raw = _normalize_message(message) + "\n" + "\n".join(top_frames)
+    return hashlib.sha1(raw.encode()).hexdigest()[:16]
+
+
+def parse_log_file(path: Path, source_name: str) -> list[ErrorEvent]:
+    """Parse a single log file and return all ERROR/WARN events."""
+    events: list[ErrorEvent] = []
+    current_header: re.Match | None = None
+    current_stack: list[str] = []
+
+    def flush():
+        if current_header is None:
+            return
+        level = SEVERITY_MAP.get(current_header.group("level").upper(), "WARN")
+        if level not in ("ERROR", "WARN", "CRITICAL"):
+            return
+        event = ErrorEvent(
+            source=source_name,
+            log_file=str(path),
+            timestamp=current_header.group("ts"),
+            level=level,
+            thread=current_header.group("thread") or "",
+            logger_name=current_header.group("logger"),
+            message=current_header.group("message"),
+            stack_trace=list(current_stack),
+        )
+        events.append(event)
+
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except OSError as e:
+        logger.error("Cannot read %s: %s", path, e)
+        return []
+
+    for line in text.splitlines():
+        m = _LOG_HEADER.match(line)
+        if m:
+            flush()
+            current_header = m
+            current_stack = []
+        elif current_header and _STACK_LINE.match(line):
+            current_stack.append(line)
+
+    flush()
+    logger.debug("Parsed %s: %d error/warn events", path.name, len(events))
+    return events
+
+
+def parse_all(
+    fetched_files: dict[str, list[Path]],
+    log_sources,  # dict[str, LogSourceConfig]
+) -> list[ErrorEvent]:
+    """Parse all fetched log files across all sources."""
+    all_events: list[ErrorEvent] = []
+    for source_name, files in fetched_files.items():
+        for f in files:
+            all_events.extend(parse_log_file(f, source_name))
+    return all_events

package/python/sentinel/main.py

@@ -0,0 +1,223 @@
+"""
+main.py — Sentinel entry point and watch loop.
+
+Usage:
+  python -m sentinel.main           # run watch loop
+  python -m sentinel.main --init    # first-time setup
+"""
+
+import argparse
+import asyncio
+import logging
+import signal
+import subprocess
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .cairn_client import ensure_installed as cairn_installed, index_repo
+from .config_loader import ConfigLoader
+from .fix_engine import generate_fix
+from .git_manager import apply_and_commit, publish
+from .cicd_trigger import trigger as cicd_trigger
+from .log_fetcher import fetch_all
+from .log_parser import parse_all, ErrorEvent
+from .repo_router import route
+from .reporter import build_and_send
+from .state_store import StateStore
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)-7s %(name)s — %(message)s",
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler("logs/sentinel.log", encoding="utf-8"),
+    ],
+)
+logger = logging.getLogger("sentinel")
+
+_report_requested = False
+
+
+def _on_sigusr1(*_):
+    global _report_requested
+    _report_requested = True
+    logger.info("SIGUSR1 received — health report queued")
+
+
+def _register_signals():
+    try:
+        signal.signal(signal.SIGUSR1, _on_sigusr1)
+    except (OSError, AttributeError):
+        pass
+
+
+# ── Fix pipeline ──────────────────────────────────────────────────────────────
+
+async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: StateStore):
+    sentinel = cfg_loader.sentinel
+
+    repo = route(event, cfg_loader.repos)
+    if not repo:
+        return
+
+    if Path("SENTINEL_PAUSE").exists():
+        logger.info("SENTINEL_PAUSE present — fix activity halted")
+        return
+
+    if event.is_infra_issue:
+        logger.info("Infra issue for %s — log only", event.fingerprint)
+        store.record_fix(event.fingerprint, "skipped", repo_name=repo.repo_name)
+        return
+
+    if event.severity == "CRITICAL" and repo.auto_publish:
+        logger.warning("CRITICAL in auto-publish repo '%s' — flagging for human review", repo.repo_name)
+        store.record_fix(event.fingerprint, "skipped", repo_name=repo.repo_name)
+        build_and_send(sentinel, store)
+        return
+
+    if store.fix_attempted_recently(event.fingerprint, hours=24):
+        logger.debug("Fix already attempted recently for %s", event.fingerprint)
+        return
+
+    patches_dir = Path(sentinel.workspace_dir) / "patches"
+    status, patch_path = generate_fix(event, repo, sentinel, patches_dir)
+
+    if status != "patch" or patch_path is None:
+        store.record_fix(event.fingerprint, "skipped" if status == "skip" else "failed", repo_name=repo.repo_name)
+        return
+
+    commit_status, commit_hash = apply_and_commit(event, patch_path, repo, sentinel)
+    if commit_status != "committed":
+        store.record_fix(event.fingerprint, "failed", repo_name=repo.repo_name)
+        return
+
+    branch, pr_url = publish(event, repo, sentinel, commit_hash)
+    store.record_fix(
+        event.fingerprint,
+        "applied" if repo.auto_publish else "pending",
+        patch_path=str(patch_path),
+        commit_hash=commit_hash,
+        branch=branch,
+        pr_url=pr_url,
+        repo_name=repo.repo_name,
+    )
+
+    if repo.auto_publish:
+        cicd_trigger(repo, store, event.fingerprint)
+
+
+# ── Poll cycle ────────────────────────────────────────────────────────────────
+
+async def poll_cycle(cfg_loader: ConfigLoader, store: StateStore):
+    global _report_requested
+
+    sources = list(cfg_loader.log_sources.values())
+    if not sources:
+        logger.warning("No log-configs found")
+        return
+
+    logger.info("Fetching logs from %d source(s)...", len(sources))
+    fetched = await fetch_all(sources, cfg_loader.sentinel)
+
+    events = parse_all(fetched, cfg_loader.log_sources)
+    logger.info("Parsed %d error/warn events", len(events))
+
+    new_events = []
+    for event in events:
+        store.record_error(event.fingerprint, event.source, event.message)
+        if not store.fix_attempted_recently(event.fingerprint):
+            new_events.append(event)
+
+    logger.info("%d new event(s) to process", len(new_events))
+    await asyncio.gather(*[_handle_error(e, cfg_loader, store) for e in new_events], return_exceptions=True)
+
+    if _report_requested or _report_due(cfg_loader, store):
+        _report_requested = False
+        logger.info("Sending health report...")
+        build_and_send(cfg_loader.sentinel, store)
+
+
+def _report_due(cfg_loader: ConfigLoader, store: StateStore) -> bool:
+    last = store.last_report_time()
+    if last is None:
+        return True
+    elapsed = (datetime.now(timezone.utc) - last).total_seconds()
+    return elapsed >= cfg_loader.sentinel.report_interval_hours * 3600
+
+
+# ── Init ──────────────────────────────────────────────────────────────────────
+
+def run_init(cfg_loader: ConfigLoader):
+    sentinel = cfg_loader.sentinel
+    logger.info("=== Sentinel --init ===")
+
+    if not cairn_installed():
+        logger.error("Cairn not installed. Run: npm install -g @misterhuydo/cairn-mcp")
+
+    for name, repo in cfg_loader.repos.items():
+        local = Path(repo.local_path)
+        if not local.exists():
+            logger.info("Cloning %s → %s", repo.repo_url, repo.local_path)
+            r = subprocess.run(["git", "clone", repo.repo_url, str(local)], capture_output=True, text=True)
+            if r.returncode != 0:
+                logger.error("Clone failed for %s: %s", name, r.stderr)
+                continue
+        index_repo(repo)
+
+    for src_name, src in cfg_loader.log_sources.items():
+        if src.source_type == "ssh" and src.hosts:
+            host = src.hosts[0]
+            logger.info("Testing SSH to %s (%s)...", src_name, host)
+            r = subprocess.run(
+                ["ssh", "-i", src.key, "-o", "StrictHostKeyChecking=no",
+                 "-o", "ConnectTimeout=5", f"ec2-user@{host}", "echo ok"],
+                capture_output=True, text=True, timeout=15,
+            )
+            logger.info("  SSH %s: %s", host, "OK" if r.returncode == 0 else f"FAILED — {r.stderr.strip()}")
+
+    logger.info("Sending test email...")
+    try:
+        build_and_send(sentinel, StateStore(sentinel.state_db))
+    except Exception as e:
+        logger.error("Test email failed: %s", e)
+
+    logger.info("=== Init complete ===")
+
+
+# ── Entry point ───────────────────────────────────────────────────────────────
+
+async def run_loop(cfg_loader: ConfigLoader, store: StateStore):
+    interval = cfg_loader.sentinel.poll_interval_seconds
+    logger.info("Sentinel starting — poll interval: %ds, repos: %s", interval, list(cfg_loader.repos.keys()))
+    while True:
+        try:
+            await poll_cycle(cfg_loader, store)
+        except Exception as e:
+            logger.exception("Unhandled error in poll cycle: %s", e)
+        await asyncio.sleep(interval)
+
+
+def main():
+    Path("logs").mkdir(exist_ok=True)
+    Path("workspace/fetched").mkdir(parents=True, exist_ok=True)
+    Path("workspace/patches").mkdir(parents=True, exist_ok=True)
+
+    parser = argparse.ArgumentParser(description="Sentinel — Autonomous DevOps Agent")
+    parser.add_argument("--init", action="store_true", help="First-time setup")
+    parser.add_argument("--config", default="./config", help="Config directory path")
+    args = parser.parse_args()
+
+    cfg_loader = ConfigLoader(config_dir=args.config)
+    store = StateStore(cfg_loader.sentinel.state_db)
+    _register_signals()
+
+    if args.init:
+        run_init(cfg_loader)
+        return
+
+    asyncio.run(run_loop(cfg_loader, store))
+
+
+if __name__ == "__main__":
+    main()

package/python/sentinel/repo_router.py

@@ -0,0 +1,24 @@
+"""
+repo_router.py — Map an ErrorEvent to its RepoConfig by filename stem.
+
+Convention: log-configs/<name>.properties links to repo-configs/<name>.properties.
+If no repo-config exists for a given log-config stem, the error is unroutable
+and the fix is skipped.
+"""
+
+import logging
+from .log_parser import ErrorEvent
+from .config_loader import RepoConfig
+
+logger = logging.getLogger(__name__)
+
+
+def route(event: ErrorEvent, repos: dict[str, RepoConfig]) -> RepoConfig | None:
+    """Return the RepoConfig whose stem matches the log-source name, or None."""
+    repo = repos.get(event.source)
+    if repo is None:
+        logger.warning(
+            "No repo-config found for log-source '%s' (fingerprint: %s) — skipping fix",
+            event.source, event.fingerprint,
+        )
+    return repo