@smilintux/skcapstone 0.3.1 → 0.3.2

package/src/skcapstone/scheduled_tasks.py

@@ -374,6 +374,50 @@ def make_dreaming_task(
     return _run
 
 
+def make_itil_auto_close_task(home: Path) -> Callable[[], None]:
+    """Return a callback that auto-closes resolved incidents after 24h stable.
+
+    Args:
+        home: Shared root directory.
+    """
+
+    def _run() -> None:
+        from .itil import ITILManager
+
+        mgr = ITILManager(home)
+        closed = mgr.auto_close_resolved(stable_hours=24)
+        if closed:
+            logger.info("ITIL auto-close: %d incident(s) closed: %s", len(closed), closed)
+        else:
+            logger.debug("ITIL auto-close: no incidents to close")
+
+    return _run
+
+
+def make_itil_escalation_task(home: Path) -> Callable[[], None]:
+    """Return a callback that checks SLA breaches on open incidents.
+
+    Args:
+        home: Shared root directory.
+    """
+
+    def _run() -> None:
+        from .itil import ITILManager
+
+        mgr = ITILManager(home)
+        breaches = mgr.check_sla_breaches()
+        if breaches:
+            for b in breaches:
+                logger.warning(
+                    "ITIL SLA breach: %s (%s) unacknowledged for %d min (limit: %d min)",
+                    b["id"], b["severity"], b["elapsed_minutes"], b["sla_minutes"],
+                )
+        else:
+            logger.debug("ITIL escalation check: no SLA breaches")
+
+    return _run
+
+
 # ---------------------------------------------------------------------------
 # Convenience builder
 # ---------------------------------------------------------------------------
@@ -481,4 +525,22 @@ def build_scheduler(
     except ImportError:
         logger.debug("service_health not available — service_health_check task skipped")
 
+    # ITIL escalation check — SLA breach detection every 5 minutes
+    try:
+        from . import SHARED_ROOT
+
+        shared = Path(SHARED_ROOT).expanduser()
+        scheduler.register(
+            name="itil_escalation_check",
+            interval_seconds=300,  # 5 minutes
+            callback=make_itil_escalation_task(shared),
+        )
+        scheduler.register(
+            name="itil_auto_close",
+            interval_seconds=1800,  # 30 minutes
+            callback=make_itil_auto_close_task(shared),
+        )
+    except Exception:
+        logger.debug("ITIL scheduled tasks not available — skipped")
+
     return scheduler

package/src/skcapstone/service_health.py

@@ -197,16 +197,90 @@ def check_all_services() -> list[dict[str, Any]]:
 # ---------------------------------------------------------------------------
 
 
+def _create_incident_for_down_service(service_result: dict[str, Any]) -> None:
+    """Auto-create an ITIL incident for a down service (with dedup).
+
+    Only creates a new incident if there is no existing open incident
+    for the same service. Uses best-effort: failures are logged but
+    never block the health check.
+    """
+    try:
+        from . import SHARED_ROOT
+        from .itil import ITILManager
+
+        svc_name = service_result["name"]
+        mgr = ITILManager(os.path.expanduser(SHARED_ROOT))
+
+        # Dedup: skip if there's already an open incident for this service
+        existing = mgr.find_open_incident_for_service(svc_name)
+        if existing:
+            logger.debug(
+                "Skipping incident creation for %s — open incident %s exists",
+                svc_name, existing.id,
+            )
+            return
+
+        error_info = service_result.get("error") or "unreachable"
+        mgr.create_incident(
+            title=f"{svc_name} down",
+            severity="sev3",
+            source="service_health",
+            affected_services=[svc_name],
+            impact=f"Service unreachable: {error_info}",
+            managed_by="lumina",
+            created_by="service_health",
+            tags=["auto-detected", "service-health"],
+        )
+        logger.info("Auto-created incident for down service: %s", svc_name)
+    except Exception as exc:
+        logger.debug("Failed to create incident for %s: %s", service_result.get("name"), exc)
+
+
+def _auto_resolve_recovered_service(service_result: dict[str, Any]) -> None:
+    """Auto-resolve sev4 incidents when a service recovers."""
+    try:
+        from . import SHARED_ROOT
+        from .itil import ITILManager
+
+        svc_name = service_result["name"]
+        mgr = ITILManager(os.path.expanduser(SHARED_ROOT))
+        existing = mgr.find_open_incident_for_service(svc_name)
+        if existing is None:
+            return
+
+        if existing.severity.value == "sev4":
+            mgr.update_incident(
+                existing.id, "service_health",
+                new_status="resolved",
+                note=f"Service {svc_name} recovered automatically",
+                resolution_summary="Auto-resolved: service came back up",
+            )
+            logger.info("Auto-resolved sev4 incident %s for recovered service %s",
+                        existing.id, svc_name)
+        else:
+            mgr.update_incident(
+                existing.id, "service_health",
+                note=f"Service {svc_name} appears to be back up",
+            )
+    except Exception as exc:
+        logger.debug("Failed to auto-resolve incident for %s: %s",
+                      service_result.get("name"), exc)
+
+
 def make_service_health_task() -> callable:
     """Return a zero-arg callback suitable for TaskScheduler.register().
 
     Runs check_all_services() and logs results.  Down services are logged
-    at WARNING level; all-up is logged at DEBUG level.
+    at WARNING level; all-up is logged at DEBUG level.  Auto-creates ITIL
+    incidents for down services and auto-resolves sev4 incidents for
+    recovered services.
     """
 
     def _run() -> None:
         results = check_all_services()
         down = [r for r in results if r["status"] == "down"]
+        up = [r for r in results if r["status"] == "up"]
+
         if down:
             names = ", ".join(r["name"] for r in down)
             logger.warning(
@@ -216,8 +290,9 @@ def make_service_health_task() -> callable:
                 logger.warning(
                     "  %s (%s): %s", r["name"], r["url"], r["error"] or "unreachable"
                 )
+                _create_incident_for_down_service(r)
         else:
-            up_count = sum(1 for r in results if r["status"] == "up")
+            up_count = len(up)
             logger.debug(
                 "Service health: %d/%d up, %d unknown",
                 up_count,
@@ -225,4 +300,8 @@ def make_service_health_task() -> callable:
                 len(results) - up_count,
             )
 
+        # Check for recovered services
+        for r in up:
+            _auto_resolve_recovered_service(r)
+
     return _run