@misterhuydo/sentinel 1.6.15 → 1.6.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@misterhuydo/sentinel",
-  "version": "1.6.15",
+  "version": "1.6.17",
   "description": "Sentinel — Autonomous DevOps Agent installer and manager",
   "bin": {
     "sentinel": "./bin/sentinel.js"

package/python/sentinel/__init__.py

@@ -1 +1 @@
-__version__ = "1.6.15"
+__version__ = "1.6.17"

package/python/sentinel/main.py

@@ -61,6 +61,42 @@ def _project_lock(project_name: str) -> "asyncio.Lock":
     return _project_locks[project_name]
 
 
+def _is_paused(store: "StateStore", project_name: str, repo_name: str = "") -> tuple[bool, str]:
+    """Return (paused, reason_text). Checks project-level then optional repo-level pause."""
+    if project_name and store.is_project_paused(project_name):
+        row = store.get_pause("project", project_name) or {}
+        rsn = row.get("reason") or "(no reason given)"
+        by = row.get("paused_by") or "admin"
+        return True, f"project `{project_name}` is paused by <@{by}> — {rsn}"
+    if repo_name and store.is_repo_paused(repo_name):
+        row = store.get_pause("repo", repo_name) or {}
+        rsn = row.get("reason") or "(no reason given)"
+        by = row.get("paused_by") or "admin"
+        return True, f"repo `{repo_name}` is paused by <@{by}> — {rsn}"
+    return False, ""
+
+
+def _notify_skip(sentinel: SentinelConfig, event: "IssueEvent", reason: str) -> None:
+    """Tell the submitter (and channel) when an issue is skipped via dedupe.
+
+    Without this, Boss's "I'll @-mention you when it completes" promise breaks —
+    the user is left waiting indefinitely after a silent skip.
+    """
+    from .notify import slack_alert as _alert
+    submitter = getattr(event, "submitter_user_id", "")
+    channel = getattr(event, "origin_channel", "") or sentinel.slack_channel
+    if not channel or not sentinel.slack_bot_token:
+        return
+    mention = f"<@{submitter}> " if submitter else ""
+    _alert(
+        sentinel.slack_bot_token, channel,
+        f":fast_forward: {mention}*Issue skipped* — fingerprint `{event.fingerprint[:8]}`. "
+        f"{reason}. To force a new attempt, use `retry_issue` (it clears prior `failed` rows). "
+        f"If a successful fix already shipped for this fingerprint, the retry will still skip "
+        f"unless an admin clears the `applied` row.",
+    )
+
+
 def _restart_via_execv() -> None:
     """Re-exec the current process, preserving the original `python -m sentinel.main` invocation.
 
@@ -213,6 +249,11 @@ async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: Stat
         logger.debug("Error %s is dismissed — skipping", event.fingerprint)
         return
 
+    # ── Project-level pause — silent skip; we'll re-detect on resume ────────────
+    if store.is_project_paused(sentinel.project_name or ""):
+        logger.debug("Error %s skipped — project paused", event.fingerprint[:8])
+        return
+
     repo = route(event, cfg_loader.repos)
     if not repo:
         from .notify import slack_alert as _slack_alert
@@ -239,6 +280,11 @@ async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: Stat
         logger.info("SENTINEL_PAUSE present — fix activity halted")
         return
 
+    # ── Repo-level pause — silent skip; admin must resume to act on this ────────
+    if store.is_repo_paused(repo.repo_name):
+        logger.debug("Error %s skipped — repo %s paused", event.fingerprint[:8], repo.repo_name)
+        return
+
     # ── Suppress repeat notifications for same error (notified within 4h) ───────
     if store.was_notified_recently(event.fingerprint):
         logger.debug("Error %s already notified recently — suppressing repeat", event.fingerprint[:8])
@@ -576,12 +622,21 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
         logger.info("SENTINEL_PAUSE present -- fix activity halted")
         return
 
+    # Project-level pause check (admin pause via Boss)
+    paused, prsn = _is_paused(store, sentinel.project_name or "")
+    if paused:
+        logger.info("Issue %s skipped — %s", event.source, prsn)
+        _notify_skip(sentinel, event, prsn)
+        mark_done(event.issue_file)
+        return
+
     if store.fix_attempted_recently(event.fingerprint, hours=24):
         logger.info(
             "Issue %s skipped — fingerprint %s attempted in last 24h "
             "(use Boss `retry_issue` to clear the prior row and re-attempt)",
             event.source, event.fingerprint,
         )
+        _notify_skip(sentinel, event, "An attempt was already made in the last 24h")
         mark_done(event.issue_file)
         return
 
@@ -624,6 +679,14 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
             mark_done(event.issue_file)  # archive so it doesn't re-prompt every poll
             return
 
+    # Repo-level pause check now that we know the target repo
+    paused, rrsn = _is_paused(store, "", repo.repo_name)
+    if paused:
+        logger.info("Issue %s skipped — %s", event.source, rrsn)
+        _notify_skip(sentinel, event, rrsn)
+        mark_done(event.issue_file)
+        return
+
     # Per-project lock — serialise issue processing within a project so two
     # concurrent claude sessions never race on the working tree of any repo.
     async with _project_lock(sentinel.project_name or "_default"):
@@ -643,6 +706,7 @@ async def _handle_issue_locked(event, repo, cfg_loader, store):
             "Issue %s skipped (post-lock recheck) — fingerprint %s already attempted",
             event.source, event.fingerprint,
         )
+        _notify_skip(sentinel, event, "Another worker already attempted this in the last 24h")
         mark_done(event.issue_file)
         return None
 
@@ -1103,15 +1167,32 @@ async def poll_cycle(cfg_loader: ConfigLoader, store: StateStore):
                 cfg_loader.sentinel.workspace_dir, store=store,
             )
         )
+        # If the project is paused, skip ALL health-checker actions silently —
+        # we'll re-detect anything still wrong on resume.
+        _project_paused_now = store.is_project_paused(cfg_loader.sentinel.project_name or "")
         for hr in health_results:
+            if _project_paused_now:
+                continue
+            # Repo-level pause: don't auto-fix or alert_once for paused repos
+            # (recovered notifications still fire — they're just informational).
+            if hr["action"] in ("fix", "alert_once") and store.is_repo_paused(hr["repo_name"]):
+                logger.debug(
+                    "health_checker: %s action=%s skipped — repo paused",
+                    hr["repo_name"], hr["action"],
+                )
+                continue
             if hr["action"] == "fix":
                 fp = f"health-{hr['repo_name']}"
                 store.record_error(fp, f"health_checker/{hr['repo_name']}", hr["message"])
                 if not store.fix_attempted_recently(fp, hours=6):
                     from .log_parser import ErrorEvent as _EE
                     from datetime import datetime, timezone as _tz
+                    # Use the repo name as the source so repo_router.route()
+                    # can map directly to the affected repo. The original
+                    # "health_checker/<repo>" provenance is preserved in
+                    # thread/logger_name and in the errors table audit row above.
                     synth = _EE(
-                        source=f"health_checker/{hr['repo_name']}",
+                        source=hr["repo_name"],
                         log_file="",
                         timestamp=datetime.now(_tz.utc).isoformat(),
                         level="ERROR",

package/python/sentinel/sentinel_boss.py

@@ -1998,6 +1998,84 @@ _TOOLS = [
             "required": ["repo_name"],
         },
     },
+    {
+        "name": "pause_project",
+        "description": (
+            "Halt ALL Sentinel fix activity for the current project until resume_project is called. "
+            "Stops auto-fixes from logs, health-check auto-fixes, and Boss-submitted issues. Recovery "
+            "notifications still fire. Admin-only. "
+            "Use when: 'pause sentinel for elprint', 'stop working on this project', 'we're doing manual deploys today'."
+        ),
+        "input_schema": {
+            "type": "object",
+            "properties": {
+                "reason": {
+                    "type": "string",
+                    "description": "Optional reason — shown to anyone whose issue gets skipped while paused.",
+                },
+            },
+            "required": [],
+        },
+    },
+    {
+        "name": "resume_project",
+        "description": (
+            "Lift a project pause set by pause_project. Sentinel resumes auto-fixing on the next poll. "
+            "Admin-only. Use when: 'resume sentinel', 'unpause this project', 'we're back online'."
+        ),
+        "input_schema": {
+            "type": "object",
+            "properties": {},
+            "required": [],
+        },
+    },
+    {
+        "name": "pause_repo",
+        "description": (
+            "Halt Sentinel fix activity for ONE repo. Other repos in the project keep working normally. "
+            "Useful when one service is being manually rebuilt or its build is broken in CI. Admin-only. "
+            "Use when: 'pause elprint-reporting-service', 'stop working on repo X for now'."
+        ),
+        "input_schema": {
+            "type": "object",
+            "properties": {
+                "repo_name": {
+                    "type": "string",
+                    "description": "Repo name as configured (from repo-configs/*.properties).",
+                },
+                "reason": {
+                    "type": "string",
+                    "description": "Optional reason — shown to anyone whose issue gets skipped while paused.",
+                },
+            },
+            "required": ["repo_name"],
+        },
+    },
+    {
+        "name": "resume_repo",
+        "description": (
+            "Lift a repo pause set by pause_repo. Admin-only. "
+            "Use when: 'resume elprint-reporting-service', 'unpause repo X'."
+        ),
+        "input_schema": {
+            "type": "object",
+            "properties": {
+                "repo_name": {
+                    "type": "string",
+                    "description": "Repo name to resume.",
+                },
+            },
+            "required": ["repo_name"],
+        },
+    },
+    {
+        "name": "list_paused",
+        "description": (
+            "List all current pauses (project-level and per-repo) with who paused them and why. "
+            "Anyone in the allowed list can run this."
+        ),
+        "input_schema": {"type": "object", "properties": {}, "required": []},
+    },
     {
         "name": "chain_release",
         "description": (
@@ -2285,6 +2363,28 @@ def _format_duration(seconds: int) -> str:
 # ── Tool execution ────────────────────────────────────────────────────────────
 
 async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=None, user_id: str = "", channel: str = "", is_admin: bool = False) -> str:
+    if name == "list_paused":
+        pauses = store.list_pauses()
+        if not pauses:
+            return json.dumps({
+                "total": 0,
+                "pauses": [],
+                "message": "No active pauses. Sentinel is fully operational.",
+            })
+        return json.dumps({
+            "total": len(pauses),
+            "pauses": [
+                {
+                    "scope":     p["scope_type"],   # 'project' or 'repo'
+                    "name":      p["scope_value"],
+                    "paused_by": p.get("paused_by") or "",
+                    "paused_at": (p.get("paused_at") or "")[:19],
+                    "reason":    p.get("reason") or "",
+                }
+                for p in pauses
+            ],
+        })
+
     if name == "get_status":
         hours = int(inputs.get("hours", 24))
         errors = store.get_recent_errors(hours)
@@ -2603,25 +2703,37 @@ async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=Non
         if store:
             try:
                 with store._conn() as _c:
-                    _row = _c.execute(
-                        "SELECT status, pr_url, commit_hash FROM fixes "
-                        "WHERE fingerprint=? ORDER BY timestamp DESC LIMIT 1",
+                    # Check for a successful fix in the last 24h FIRST. We can't
+                    # use ORDER BY timestamp DESC LIMIT 1 because a spurious
+                    # 'failed' row inserted by a post-success exception (see
+                    # the catch-all in main.py) sorts ahead of the real
+                    # 'applied' row by milliseconds, hiding the success.
+                    _applied = _c.execute(
+                        "SELECT status, commit_hash FROM fixes "
+                        "WHERE fingerprint=? AND status IN ('applied', 'merged') "
+                        "AND timestamp >= datetime('now', '-24 hours') "
+                        "ORDER BY timestamp DESC LIMIT 1",
                         (_fp,),
                     ).fetchone()
-                if _row:
-                    _status = _row["status"]
-                    if _status in ("merged", "applied"):
-                        _commit = _row["commit_hash"] or ""
+                    if _applied:
+                        _commit = _applied["commit_hash"] or ""
                         return json.dumps({
                             "error": (
                                 f"Already fixed — this issue was resolved "
                                 + (f"in commit `{_commit[:8]}`" if _commit else "successfully")
-                                + f". Status: `{_status}`. "
+                                + f". Status: `{_applied['status']}`. "
                                 f"If the problem recurred, describe it as a new issue."
                             )
                         })
-                    if _status == "pending":
-                        _pr = _row["pr_url"] or ""
+                    _pending = _c.execute(
+                        "SELECT pr_url FROM fixes "
+                        "WHERE fingerprint=? AND status='pending' "
+                        "AND timestamp >= datetime('now', '-24 hours') "
+                        "ORDER BY timestamp DESC LIMIT 1",
+                        (_fp,),
+                    ).fetchone()
+                    if _pending:
+                        _pr = _pending["pr_url"] or ""
                         return json.dumps({
                             "error": (
                                 f"There is already an open PR for this issue"
@@ -4130,7 +4242,7 @@ async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=Non
         return json.dumps({"error": "cannot determine user — not clearing"})
 
     # ── Admin-only tools ──────────────────────────────────────────────────────
-    _ADMIN_TOOLS = {"list_all_users", "clear_user_history", "reset_fingerprint", "list_all_errors", "export_db", "merge_pr", "install_tool", "manage_release", "chain_release"}
+    _ADMIN_TOOLS = {"list_all_users", "clear_user_history", "reset_fingerprint", "list_all_errors", "export_db", "merge_pr", "install_tool", "manage_release", "chain_release", "pause_project", "resume_project", "pause_repo", "resume_repo"}
     if name in _ADMIN_TOOLS:
         if not is_admin:
             return json.dumps({"error": "This operation is admin-only (SLACK_ADMIN_USERS). Contact a Sentinel admin if you need access."})
@@ -4225,6 +4337,71 @@ async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=Non
                 ),
             })
 
+        if name == "pause_project":
+            project_name = cfg_loader.sentinel.project_name or "_default"
+            reason = (inputs.get("reason") or "").strip()
+            store.pause_project(project_name, paused_by=user_id, reason=reason)
+            logger.info("Boss: project %s paused by %s (reason: %s)", project_name, user_id, reason or "none")
+            return json.dumps({
+                "status": "paused",
+                "project": project_name,
+                "reason": reason or None,
+                "message": (
+                    f"Project `{project_name}` is paused. Sentinel will skip all auto-fixes and Boss-submitted "
+                    f"issues until `resume_project` is called. Recovery notifications still fire."
+                ),
+            })
+
+        if name == "resume_project":
+            project_name = cfg_loader.sentinel.project_name or "_default"
+            existed = store.resume_project(project_name)
+            logger.info("Boss: project %s resume by %s (was_paused=%s)", project_name, user_id, existed)
+            return json.dumps({
+                "status": "resumed" if existed else "not_paused",
+                "project": project_name,
+                "message": (
+                    f"Project `{project_name}` is back online — Sentinel will resume on the next poll."
+                    if existed else
+                    f"Project `{project_name}` was not paused — nothing to do."
+                ),
+            })
+
+        if name == "pause_repo":
+            repo_name = (inputs.get("repo_name") or "").strip()
+            reason = (inputs.get("reason") or "").strip()
+            if not repo_name:
+                return json.dumps({"error": "repo_name is required"})
+            if repo_name not in cfg_loader.repos:
+                return json.dumps({"error": f"Unknown repo `{repo_name}`. Configured repos: "
+                                            f"{', '.join(sorted(cfg_loader.repos.keys()))}"})
+            store.pause_repo(repo_name, paused_by=user_id, reason=reason)
+            logger.info("Boss: repo %s paused by %s (reason: %s)", repo_name, user_id, reason or "none")
+            return json.dumps({
+                "status": "paused",
+                "repo": repo_name,
+                "reason": reason or None,
+                "message": (
+                    f"Repo `{repo_name}` is paused. Other repos keep working normally. "
+                    f"Use `resume_repo` to lift the pause."
+                ),
+            })
+
+        if name == "resume_repo":
+            repo_name = (inputs.get("repo_name") or "").strip()
+            if not repo_name:
+                return json.dumps({"error": "repo_name is required"})
+            existed = store.resume_repo(repo_name)
+            logger.info("Boss: repo %s resume by %s (was_paused=%s)", repo_name, user_id, existed)
+            return json.dumps({
+                "status": "resumed" if existed else "not_paused",
+                "repo": repo_name,
+                "message": (
+                    f"Repo `{repo_name}` is back online."
+                    if existed else
+                    f"Repo `{repo_name}` was not paused — nothing to do."
+                ),
+            })
+
         if name == "reset_fingerprint":
             fp = inputs.get("fingerprint", "").strip()
             if not fp:

package/python/sentinel/state_store.py

@@ -111,6 +111,18 @@ class StateStore:
                     total_cost_usd  REAL NOT NULL DEFAULT 0,
                     turn_count      INTEGER NOT NULL DEFAULT 0
                 );
+
+                -- Admin-controlled pauses. scope_type is 'project' or 'repo';
+                -- scope_value is the project name or the repo name. While a row
+                -- exists, fix activity for that scope is halted.
+                CREATE TABLE IF NOT EXISTS pauses (
+                    scope_type   TEXT NOT NULL CHECK(scope_type IN ('project','repo')),
+                    scope_value  TEXT NOT NULL,
+                    paused_by    TEXT,
+                    paused_at    TEXT NOT NULL,
+                    reason       TEXT,
+                    PRIMARY KEY (scope_type, scope_value)
+                );
             """)
         self._migrate()
         logger.debug("StateStore initialised at %s", self.db_path)
@@ -942,6 +954,75 @@ class StateStore:
             ).fetchall()
             return [dict(r) for r in rows]
 
+    # -- Admin pauses (project-wide and per-repo) ----------------------------------
+
+    def _set_pause(self, scope_type: str, scope_value: str, paused_by: str, reason: str) -> None:
+        if scope_type not in ("project", "repo"):
+            raise ValueError(f"scope_type must be 'project' or 'repo', got {scope_type!r}")
+        with self._conn() as conn:
+            conn.execute(
+                "INSERT OR REPLACE INTO pauses (scope_type, scope_value, paused_by, paused_at, reason) "
+                "VALUES (?, ?, ?, ?, ?)",
+                (scope_type, scope_value, paused_by, _now(), reason),
+            )
+
+    def _clear_pause(self, scope_type: str, scope_value: str) -> bool:
+        with self._conn() as conn:
+            cur = conn.execute(
+                "DELETE FROM pauses WHERE scope_type=? AND scope_value=?",
+                (scope_type, scope_value),
+            )
+            return cur.rowcount > 0
+
+    def pause_project(self, project_name: str, paused_by: str = "", reason: str = "") -> None:
+        """Halt all fix activity for the named project until resume_project is called."""
+        self._set_pause("project", project_name, paused_by, reason)
+
+    def resume_project(self, project_name: str) -> bool:
+        """Lift project pause. Returns True if a pause existed, False if no-op."""
+        return self._clear_pause("project", project_name)
+
+    def is_project_paused(self, project_name: str) -> bool:
+        with self._conn() as conn:
+            row = conn.execute(
+                "SELECT 1 FROM pauses WHERE scope_type='project' AND scope_value=?",
+                (project_name,),
+            ).fetchone()
+            return row is not None
+
+    def pause_repo(self, repo_name: str, paused_by: str = "", reason: str = "") -> None:
+        """Halt fix activity for one repo. Other repos in the project keep working."""
+        self._set_pause("repo", repo_name, paused_by, reason)
+
+    def resume_repo(self, repo_name: str) -> bool:
+        return self._clear_pause("repo", repo_name)
+
+    def is_repo_paused(self, repo_name: str) -> bool:
+        with self._conn() as conn:
+            row = conn.execute(
+                "SELECT 1 FROM pauses WHERE scope_type='repo' AND scope_value=?",
+                (repo_name,),
+            ).fetchone()
+            return row is not None
+
+    def get_pause(self, scope_type: str, scope_value: str) -> dict | None:
+        """Return the pause row (paused_by, paused_at, reason) or None."""
+        with self._conn() as conn:
+            row = conn.execute(
+                "SELECT * FROM pauses WHERE scope_type=? AND scope_value=?",
+                (scope_type, scope_value),
+            ).fetchone()
+            return dict(row) if row else None
+
+    def list_pauses(self) -> list[dict]:
+        """Return all active pauses (project + repo) ordered by paused_at DESC."""
+        with self._conn() as conn:
+            rows = conn.execute(
+                "SELECT scope_type, scope_value, paused_by, paused_at, reason "
+                "FROM pauses ORDER BY paused_at DESC"
+            ).fetchall()
+            return [dict(r) for r in rows]
+
     def get_all_user_stats(self) -> list[dict]:
         """Return activity summary for every known Slack user."""
         users = self.get_all_users()  # {user_id: display_name}

package/python/tests/test_pauses.py

@@ -0,0 +1,81 @@
+"""Unit tests for project + repo pause API on StateStore."""
+import pytest
+from sentinel.state_store import StateStore
+
+
+@pytest.fixture
+def store(tmp_path):
+    return StateStore(str(tmp_path / "pauses.db"))
+
+
+# ── project pause ────────────────────────────────────────────────────────────
+
+def test_project_pause_default_unpaused(store):
+    assert store.is_project_paused("elprint") is False
+
+
+def test_pause_project_then_check(store):
+    store.pause_project("elprint", paused_by="U123", reason="manual deploy")
+    assert store.is_project_paused("elprint") is True
+    row = store.get_pause("project", "elprint")
+    assert row["paused_by"] == "U123"
+    assert row["reason"] == "manual deploy"
+    assert row["paused_at"]
+
+
+def test_resume_project_clears_pause(store):
+    store.pause_project("elprint", paused_by="U1", reason="x")
+    assert store.resume_project("elprint") is True
+    assert store.is_project_paused("elprint") is False
+
+
+def test_resume_project_returns_false_when_not_paused(store):
+    assert store.resume_project("elprint") is False
+
+
+def test_pause_project_is_idempotent(store):
+    store.pause_project("elprint", paused_by="U1", reason="first")
+    store.pause_project("elprint", paused_by="U2", reason="second")
+    row = store.get_pause("project", "elprint")
+    assert row["paused_by"] == "U2"
+    assert row["reason"] == "second"
+
+
+# ── repo pause ───────────────────────────────────────────────────────────────
+
+def test_repo_pause_independent_of_project(store):
+    store.pause_repo("elprint-reporting-service", paused_by="U1", reason="broken build")
+    assert store.is_repo_paused("elprint-reporting-service") is True
+    assert store.is_project_paused("elprint") is False
+
+
+def test_pause_one_repo_doesnt_affect_others(store):
+    store.pause_repo("elprint-reporting-service", paused_by="U1", reason="x")
+    assert store.is_repo_paused("elprint-reporting-service") is True
+    assert store.is_repo_paused("elprint-component-service") is False
+
+
+def test_resume_repo_clears_pause(store):
+    store.pause_repo("foo", paused_by="U1", reason="x")
+    assert store.resume_repo("foo") is True
+    assert store.is_repo_paused("foo") is False
+
+
+# ── list_pauses ──────────────────────────────────────────────────────────────
+
+def test_list_pauses_empty(store):
+    assert store.list_pauses() == []
+
+
+def test_list_pauses_combined(store):
+    store.pause_project("elprint", paused_by="U1", reason="r1")
+    store.pause_repo("elprint-reporting-service", paused_by="U2", reason="r2")
+    pauses = store.list_pauses()
+    assert len(pauses) == 2
+    scopes = {p["scope_type"] for p in pauses}
+    assert scopes == {"project", "repo"}
+
+
+def test_invalid_scope_type_rejected(store):
+    with pytest.raises(ValueError):
+        store._set_pause("BOGUS", "x", "U1", "r")