@misterhuydo/sentinel 1.0.5 → 1.0.9

package/python/sentinel/config_loader.py

@@ -51,6 +51,7 @@ class SentinelConfig:
     fix_confidence_threshold: float = 0.7
     log_retention_hours: int = 48
     anthropic_api_key: str = ""
+    marker_confirm_hours: int = 24  # quiet period before confirming a fix
 
 
 @dataclass
@@ -104,11 +105,21 @@ class ConfigLoader:
         )
 
     def _load_sentinel(self):
+        # Load workspace-level config first (~/sentinel/sentinel.properties),
+        # then overlay per-project config so project values win.
+        d: dict[str, str] = {}
+        workspace_props = self.config_dir.parent.parent / "sentinel.properties"
+        if workspace_props.exists():
+            d.update(_parse_properties(str(workspace_props)))
+            logger.debug("Loaded workspace config from %s", workspace_props)
+
         path = self.config_dir / "sentinel.properties"
         if not path.exists():
-            logger.warning("sentinel.properties not found at %s", path)
-            return
-        d = _parse_properties(str(path))
+            if not d:
+                logger.warning("sentinel.properties not found at %s", path)
+        else:
+            d.update(_parse_properties(str(path)))
+
         c = SentinelConfig()
         c.poll_interval_seconds = int(d.get("POLL_INTERVAL_SECONDS", 120))
         c.smtp_host = d.get("SMTP_HOST", "")
@@ -125,6 +136,7 @@ class ConfigLoader:
         c.fix_confidence_threshold = float(d.get("FIX_CONFIDENCE_THRESHOLD", 0.7))
         c.log_retention_hours = int(d.get("LOG_RETENTION_HOURS", 48))
         c.anthropic_api_key = d.get("ANTHROPIC_API_KEY", "")
+        c.marker_confirm_hours = int(d.get("MARKER_CONFIRM_HOURS", 24))
         self.sentinel = c
 
     def _load_log_sources(self):

package/python/sentinel/fix_engine.py

@@ -26,36 +26,74 @@ _DIFF_BLOCK = re.compile(r"```(?:diff|patch)?\n(.*?)```", re.DOTALL)
 _DIFF_HEADER = re.compile(r"^diff --git|^---\s+\S+|^\+\+\+\s+\S+", re.MULTILINE)
 
 
-def _build_prompt(event: ErrorEvent, repo: RepoConfig, log_file: Path) -> str:
-    return textwrap.dedent(f"""\
-        You are fixing a production bug in the repository at {repo.local_path}.
-        Repository: {repo.repo_name}
-
-        LOG FILE: {log_file}
-        Read this file first. It contains the last 48h of logs from {event.source} —
-        use it to understand the frequency, surrounding context, and any warnings
-        that preceded this error.
-
-        ERROR fingerprint to fix (from {event.source}):
-        {event.full_text()}
-
-        Task:
-        1. Read the log file above to understand what led up to this error.
-        2. Use your available tools to explore the codebase and identify the root cause.
-        3. Output ONLY a unified diff patch (git diff format) fixing the issue.
-        4. Do not explain. Output only the patch.
-        5. If you cannot determine a safe fix, output: SKIP: <reason>
-    """)
-
+def _build_prompt(event, repo: RepoConfig, log_file, marker: str, stale_markers: list[str] = None) -> str:
+    if log_file and log_file.exists():
+        ctx = (
+            "LOG FILE: " + str(log_file) + "
+"
+            "Read this file first -- it contains the last 48h of logs from "
+            + event.source + ".
+"
+            "Use it to understand frequency, context, and preceding warnings."
+        )
+        step1 = "Read the log file above to understand what led up to this error."
+    else:
+        ctx = (
+            "SOURCE: " + event.source + "
+"
+            "No rolling log file available. The full issue description is below."
+        )
+        step1 = "Use the issue description above as your primary context."
+
+    marker_label = marker + " sentinel-auto-fix [safe to remove after verification]"
+    marker_instruction = "
+".join([
+        "For EVERY method and constructor you modify, add this as the FIRST executable line:",
+        f"  Java/Kotlin : log.info("{marker_label}");",
+        f"  Python      : logger.info("{marker_label}")",
+        f"  Node.js     : logger.info("{marker_label}")",
+        "Use the logger already present in the file. Do not add new imports.",
+        "This applies to ALL modified methods and constructors without exception.",
+    ])
 
-def _extract_patch(output: str) -> str | None:
-    m = _DIFF_BLOCK.search(output)
-    if m:
-        return m.group(1).strip()
-    if _DIFF_HEADER.search(output):
-        return output.strip()
-    return None
+    cleanup = ""
+    if stale_markers:
+        marker_list = "
+".join(f"  - {m}" for m in stale_markers)
+        cleanup = (
+            "CLEANUP (do this first, before the fix):
+"
+            "Remove any log lines containing these stale Sentinel markers from the codebase:
+"
+            + marker_list + "
+"
+            "Commit the cleanup separately with message: 'chore(sentinel): remove stale markers'
+"
+        )
 
+    lines_out = [
+        f"You are fixing a production bug in the repository at {repo.local_path}.",
+        f"Repository: {repo.repo_name}",
+        "",
+    ]
+    if cleanup:
+        lines_out += [cleanup, ""]
+    lines_out += [
+        ctx,
+        "",
+        f"ISSUE TO FIX (from {event.source}):",
+        event.full_text(),
+        "",
+        "Task:",
+        f"1. {step1}",
+        "2. Use your available tools to explore the codebase and identify the root cause.",
+        f"3. {marker_instruction}",
+        "4. Output ONLY a unified diff patch (git diff format) fixing the issue.",
+        "5. Do not explain. Output only the patch.",
+        "6. If you cannot determine a safe fix, output: SKIP: <reason>",
+    ]
+    return "
+".join(lines_out)
 
 def _validate_patch(patch: str) -> tuple[bool, str]:
     files_changed = len(re.findall(r"^diff --git", patch, re.MULTILINE))
@@ -83,7 +121,10 @@ def generate_fix(
         (status, patch_path)
         status: "patch" | "skip" | "error"
     """
+    # Issues have source like "issues/filename" — no rolling log file exists
     log_file = Path(cfg.workspace_dir) / "fetched" / f"{event.source}.log"
+    if not log_file.exists():
+        log_file = None
     prompt = _build_prompt(event, repo, log_file)
 
     logger.info("Invoking Claude Code for %s (fp=%s)", event.source, event.fingerprint)
@@ -98,30 +139,30 @@ def generate_fix(
         )
     except subprocess.TimeoutExpired:
         logger.error("Claude Code timed out for %s", event.fingerprint)
-        return "error", None
+        return "error", None, ""
     except FileNotFoundError:
         logger.error("Claude Code binary not found at '%s'", cfg.claude_code_bin)
-        return "error", None
+        return "error", None, ""
 
     output = (result.stdout or "") + (result.stderr or "")
 
     if output.strip().upper().startswith("SKIP:"):
         reason = output.strip()[5:].strip()
         logger.info("Claude skipped fix for %s: %s", event.fingerprint, reason)
-        return "skip", None
+        return "skip", None, ""
 
     patch = _extract_patch(output)
     if not patch:
         logger.warning("No patch found in Claude output for %s", event.fingerprint)
-        return "error", None
+        return "error", None, ""
 
     ok, reason = _validate_patch(patch)
     if not ok:
         logger.warning("Patch rejected for %s: %s", event.fingerprint, reason)
-        return "skip", None
+        return "skip", None, ""
 
     patches_dir.mkdir(parents=True, exist_ok=True)
     patch_path = patches_dir / f"{event.fingerprint}.diff"
     patch_path.write_text(patch, encoding="utf-8")
     logger.info("Patch written to %s", patch_path)
-    return "patch", patch_path
+    return "patch", patch_path, marker

package/python/sentinel/issue_watcher.py

@@ -0,0 +1,146 @@
+"""
+issue_watcher.py — Scan the issues/ directory for manually-submitted bug reports.
+
+Admins drop plain-text or markdown files into <project>/issues/.
+Each file is treated as a fix request. Processed files are archived to issues/.done/.
+
+File format (TARGET_REPO header is optional):
+
+    TARGET_REPO: my-repo-name
+
+    Short summary of the problem (becomes the email subject line)
+
+    Any details: customer feedback, stack traces, screenshots text, etc.
+    If TARGET_REPO is omitted and only one repo is configured, it is used automatically.
+"""
+
+import hashlib
+import logging
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+_TARGET_REPO_PREFIX = "TARGET_REPO:"
+
+
+@dataclass
+class IssueEvent:
+    """
+    A fix request sourced from the issues/ directory.
+    Implements the same interface as ErrorEvent so it can flow through
+    the same fix pipeline (_handle_error / generate_fix / git_manager).
+    """
+    source: str        # "issues/<filename>" — shown in emails and logs
+    issue_file: Path   # full path, used for archiving after processing
+    message: str       # first non-blank body line — used as subject summary
+    body: str          # full file content (the issue description)
+    target_repo: str   # explicit TARGET_REPO value, or "" for auto-select
+    fingerprint: str = ""
+    severity: str = "ERROR"
+    timestamp: str = ""
+
+    # Compatibility fields matching ErrorEvent interface
+    level: str = "ERROR"
+    thread: str = ""
+    logger_name: str = ""
+    stack_trace: list[str] = field(default_factory=list)
+    log_file: str = ""
+
+    def __post_init__(self):
+        if not self.fingerprint:
+            raw = f"issue:{self.source}:{self.message[:200]}"
+            self.fingerprint = hashlib.sha1(raw.encode()).hexdigest()[:16]
+        if not self.timestamp:
+            self.timestamp = datetime.now(timezone.utc).isoformat()
+        if not self.stack_trace:
+            self.stack_trace = self.body.splitlines()
+
+    @property
+    def is_infra_issue(self) -> bool:
+        return False
+
+    def short_summary(self) -> str:
+        return self.message[:120]
+
+    def full_text(self) -> str:
+        return self.body
+
+
+# Binary extensions Sentinel will never try to process
+_BINARY_EXTENSIONS = {
+    ".zip", ".tar", ".gz", ".bz2", ".xz", ".7z",
+    ".jar", ".war", ".ear", ".class",
+    ".exe", ".dll", ".so", ".bin", ".pyc",
+    ".pdf", ".doc", ".docx", ".xls", ".xlsx",
+    ".mp3", ".mp4", ".avi", ".mov",
+}
+
+
+def scan_issues(project_dir: Path) -> list[IssueEvent]:
+    """
+    Return all pending issue files from <project_dir>/issues/.
+
+    Accepts text, markdown, logs, images, JSON — anything Claude can read.
+    Skips dotfiles, archives, and compiled binaries.
+    """
+    issues_dir = project_dir / "issues"
+    if not issues_dir.exists():
+        return []
+
+    events = []
+    for f in sorted(issues_dir.iterdir()):
+        if not f.is_file() or f.name.startswith("."):
+            continue
+        if f.suffix.lower() in _BINARY_EXTENSIONS:
+            logger.debug("Skipping binary issue file: %s", f.name)
+            continue
+
+        try:
+            content = f.read_text(encoding="utf-8", errors="replace").strip()
+        except OSError as e:
+            logger.error("Cannot read issue file %s: %s", f, e)
+            continue
+
+        if not content:
+            continue
+
+        lines = content.splitlines()
+        target_repo = ""
+        body_start = 0
+
+        # Parse optional TARGET_REPO: header (must be the first non-blank line)
+        for i, line in enumerate(lines):
+            stripped = line.strip()
+            if stripped.upper().startswith(_TARGET_REPO_PREFIX):
+                target_repo = stripped[len(_TARGET_REPO_PREFIX):].strip()
+                body_start = i + 1
+            elif stripped:
+                break
+
+        body = "\n".join(lines[body_start:]).strip() or content
+        message = next((l.strip() for l in lines[body_start:] if l.strip()), f.name)
+
+        events.append(IssueEvent(
+            source=f"issues/{f.name}",
+            issue_file=f,
+            message=message,
+            body=body,
+            target_repo=target_repo,
+        ))
+        logger.info("Found issue: %s (target_repo=%r)", f.name, target_repo or "auto")
+
+    return events
+
+
+def mark_done(issue_file: Path) -> None:
+    """Archive a processed issue to issues/.done/ regardless of outcome."""
+    done_dir = issue_file.parent / ".done"
+    done_dir.mkdir(exist_ok=True)
+    dest = done_dir / issue_file.name
+    if dest.exists():
+        dest = done_dir / f"{issue_file.stem}-{int(time.time())}{issue_file.suffix}"
+    issue_file.rename(dest)
+    logger.info("Issue archived: %s -> .done/%s", issue_file.name, dest.name)

package/python/sentinel/log_parser.py

@@ -1,149 +1,175 @@
-"""
-log_parser.py — Parse fetched log files into ErrorEvent objects.
-
-Handles Java-style logs (Spring Boot / Logback format):
-  2024-01-15 12:34:56.789 ERROR [thread] class.ClassName - Message
-  followed by optional stack trace lines (^\tat ...)
-"""
-
-import hashlib
-import re
-import logging
-from dataclasses import dataclass, field
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-_LOG_HEADER = re.compile(
-    r"^(?P<ts>\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}:\d{2}[.,\d]*)\s+"
-    r"(?P<level>CRITICAL|ERROR|WARN(?:ING)?|INFO|DEBUG)\s+"
-    r"(?:\[(?P<thread>[^\]]*)\]\s+)?"
-    r"(?P<logger>\S+)\s+-\s+"
-    r"(?P<message>.+)$"
-)
-
-_STACK_LINE = re.compile(r"^\s+at |\s+\.\.\. \d+ more|^Caused by:")
-
-SEVERITY_MAP = {
-    "CRITICAL": "CRITICAL",
-    "ERROR": "ERROR",
-    "WARN": "WARN",
-    "WARNING": "WARN",
-    "INFO": "INFO",
-    "DEBUG": "DEBUG",
-}
-
-_CRITICAL_PATTERNS = re.compile(
-    r"OutOfMemoryError|StackOverflowError|OOMKilled", re.IGNORECASE
-)
-_INFRA_PATTERNS = re.compile(
-    r"ConnectException|TimeoutException|ConnectionRefused|SocketTimeout",
-    re.IGNORECASE,
-)
-
-
-@dataclass
-class ErrorEvent:
-    source: str           # log-source name (e.g. "SSOLWA")
-    log_file: str
-    timestamp: str
-    level: str            # CRITICAL / ERROR / WARN
-    thread: str
-    logger_name: str
-    message: str
-    stack_trace: list[str] = field(default_factory=list)
-    fingerprint: str = ""
-
-    def __post_init__(self):
-        if not self.fingerprint:
-            self.fingerprint = _fingerprint(self.message, self.stack_trace)
-
-    @property
-    def severity(self) -> str:
-        if _CRITICAL_PATTERNS.search(self.message) or _CRITICAL_PATTERNS.search(
-            "\n".join(self.stack_trace)
-        ):
-            return "CRITICAL"
-        return self.level
-
-    @property
-    def is_infra_issue(self) -> bool:
-        return bool(_INFRA_PATTERNS.search(self.message))
-
-    def short_summary(self) -> str:
-        return self.message[:120]
-
-    def full_text(self) -> str:
-        lines = [f"{self.timestamp} {self.level} [{self.thread}] {self.logger_name} - {self.message}"]
-        lines.extend(self.stack_trace)
-        return "\n".join(lines)
-
-
-def _normalize_message(msg: str) -> str:
-    msg = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", msg)
-    msg = re.sub(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", "UUID", msg)
-    msg = re.sub(r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}[.,\d]*\b", "TIMESTAMP", msg)
-    msg = re.sub(r"\b\d+\b", "N", msg)
-    return msg.strip()
-
-
-def _fingerprint(message: str, stack_trace: list[str]) -> str:
-    top_frames = [l for l in stack_trace if l.strip().startswith("at ")][:3]
-    raw = _normalize_message(message) + "\n" + "\n".join(top_frames)
-    return hashlib.sha1(raw.encode()).hexdigest()[:16]
-
-
-def parse_log_file(path: Path, source_name: str) -> list[ErrorEvent]:
-    """Parse a single log file and return all ERROR/WARN events."""
-    events: list[ErrorEvent] = []
-    current_header: re.Match | None = None
-    current_stack: list[str] = []
-
-    def flush():
-        if current_header is None:
-            return
-        level = SEVERITY_MAP.get(current_header.group("level").upper(), "WARN")
-        if level not in ("ERROR", "WARN", "CRITICAL"):
-            return
-        event = ErrorEvent(
-            source=source_name,
-            log_file=str(path),
-            timestamp=current_header.group("ts"),
-            level=level,
-            thread=current_header.group("thread") or "",
-            logger_name=current_header.group("logger"),
-            message=current_header.group("message"),
-            stack_trace=list(current_stack),
-        )
-        events.append(event)
-
-    try:
-        text = path.read_text(encoding="utf-8", errors="replace")
-    except OSError as e:
-        logger.error("Cannot read %s: %s", path, e)
-        return []
-
-    for line in text.splitlines():
-        m = _LOG_HEADER.match(line)
-        if m:
-            flush()
-            current_header = m
-            current_stack = []
-        elif current_header and _STACK_LINE.match(line):
-            current_stack.append(line)
-
-    flush()
-    logger.debug("Parsed %s: %d error/warn events", path.name, len(events))
-    return events
-
-
-def parse_all(
-    fetched_files: dict[str, list[Path]],
-    log_sources,  # dict[str, LogSourceConfig]
-) -> list[ErrorEvent]:
-    """Parse all fetched log files across all sources."""
-    all_events: list[ErrorEvent] = []
-    for source_name, files in fetched_files.items():
-        for f in files:
-            all_events.extend(parse_log_file(f, source_name))
-    return all_events
+"""
+log_parser.py — Parse fetched log files into ErrorEvent objects.
+
+Handles Java-style logs (Spring Boot / Logback format):
+  2024-01-15 12:34:56.789 ERROR [thread] class.ClassName - Message
+  followed by optional stack trace lines (^\tat ...)
+"""
+
+import hashlib
+import re
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+_LOG_HEADER = re.compile(
+    r"^(?P<ts>\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}:\d{2}[.,\d]*)\s+"
+    r"(?P<level>CRITICAL|ERROR|WARN(?:ING)?|INFO|DEBUG)\s+"
+    r"(?:\[(?P<thread>[^\]]*)\]\s+)?"
+    r"(?P<logger>\S+)\s+-\s+"
+    r"(?P<message>.+)$"
+)
+
+_STACK_LINE = re.compile(r"^\s+at |\s+\.\.\. \d+ more|^Caused by:")
+
+SEVERITY_MAP = {
+    "CRITICAL": "CRITICAL",
+    "ERROR": "ERROR",
+    "WARN": "WARN",
+    "WARNING": "WARN",
+    "INFO": "INFO",
+    "DEBUG": "DEBUG",
+}
+
+_CRITICAL_PATTERNS = re.compile(
+    r"OutOfMemoryError|StackOverflowError|OOMKilled", re.IGNORECASE
+)
+_INFRA_PATTERNS = re.compile(
+    r"ConnectException|TimeoutException|ConnectionRefused|SocketTimeout",
+    re.IGNORECASE,
+)
+
+
+@dataclass
+class ErrorEvent:
+    source: str           # log-source name (e.g. "SSOLWA")
+    log_file: str
+    timestamp: str
+    level: str            # CRITICAL / ERROR / WARN
+    thread: str
+    logger_name: str
+    message: str
+    stack_trace: list[str] = field(default_factory=list)
+    fingerprint: str = ""
+
+    def __post_init__(self):
+        if not self.fingerprint:
+            self.fingerprint = _fingerprint(self.message, self.stack_trace)
+
+    @property
+    def severity(self) -> str:
+        if _CRITICAL_PATTERNS.search(self.message) or _CRITICAL_PATTERNS.search(
+            "\n".join(self.stack_trace)
+        ):
+            return "CRITICAL"
+        return self.level
+
+    @property
+    def is_infra_issue(self) -> bool:
+        return bool(_INFRA_PATTERNS.search(self.message))
+
+    def short_summary(self) -> str:
+        return self.message[:120]
+
+    def full_text(self) -> str:
+        lines = [f"{self.timestamp} {self.level} [{self.thread}] {self.logger_name} - {self.message}"]
+        lines.extend(self.stack_trace)
+        return "\n".join(lines)
+
+
+def _normalize_message(msg: str) -> str:
+    msg = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", msg)
+    msg = re.sub(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", "UUID", msg)
+    msg = re.sub(r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}[.,\d]*\b", "TIMESTAMP", msg)
+    msg = re.sub(r"\b\d+\b", "N", msg)
+    return msg.strip()
+
+
+def _fingerprint(message: str, stack_trace: list[str]) -> str:
+    top_frames = [l for l in stack_trace if l.strip().startswith("at ")][:3]
+    raw = _normalize_message(message) + "\n" + "\n".join(top_frames)
+    return hashlib.sha1(raw.encode()).hexdigest()[:16]
+
+
+def parse_log_file(path: Path, source_name: str) -> list[ErrorEvent]:
+    """Parse a single log file and return all ERROR/WARN events."""
+    events: list[ErrorEvent] = []
+    current_header: re.Match | None = None
+    current_stack: list[str] = []
+
+    def flush():
+        if current_header is None:
+            return
+        level = SEVERITY_MAP.get(current_header.group("level").upper(), "WARN")
+        if level not in ("ERROR", "WARN", "CRITICAL"):
+            return
+        event = ErrorEvent(
+            source=source_name,
+            log_file=str(path),
+            timestamp=current_header.group("ts"),
+            level=level,
+            thread=current_header.group("thread") or "",
+            logger_name=current_header.group("logger"),
+            message=current_header.group("message"),
+            stack_trace=list(current_stack),
+        )
+        events.append(event)
+
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except OSError as e:
+        logger.error("Cannot read %s: %s", path, e)
+        return []
+
+    for line in text.splitlines():
+        m = _LOG_HEADER.match(line)
+        if m:
+            flush()
+            current_header = m
+            current_stack = []
+        elif current_header and _STACK_LINE.match(line):
+            current_stack.append(line)
+
+    flush()
+    logger.debug("Parsed %s: %d error/warn events", path.name, len(events))
+    return events
+
+
+def parse_all(
+    fetched_files: dict[str, list[Path]],
+    log_sources,  # dict[str, LogSourceConfig]
+) -> list[ErrorEvent]:
+    """Parse all fetched log files across all sources."""
+    all_events: list[ErrorEvent] = []
+    for source_name, files in fetched_files.items():
+        for f in files:
+            all_events.extend(parse_log_file(f, source_name))
+    return all_events
+
+
+# -- Sentinel marker detection -------------------------------------------------
+
+_SENTINEL_MARKER_RE = re.compile(r'SENTINEL:#([0-9a-f]{16})')
+
+
+def scan_for_markers(path: Path) -> list[str]:
+    """
+    Scan a single log file for SENTINEL:#<fingerprint> markers injected by fix_engine.
+    Returns a list of full marker strings (e.g. ['SENTINEL:#abc123de45678901']).
+    """
+    try:
+        text = path.read_text(encoding='utf-8', errors='replace')
+    except OSError:
+        return []
+    return [f'SENTINEL:#{m}' for m in _SENTINEL_MARKER_RE.findall(text)]
+
+
+def scan_all_for_markers(fetched_files: dict[str, list[Path]]) -> list[str]:
+    """Scan all fetched log files and return every SENTINEL marker found."""
+    markers: list[str] = []
+    for files in fetched_files.values():
+        for f in files:
+            markers.extend(scan_for_markers(f))
+    return markers