@smilintux/skcapstone 0.9.0 → 0.12.5

package/src/skcapstone/scheduled_tasks.py

@@ -20,12 +20,21 @@ Usage:
 from __future__ import annotations
 
 import logging
+import os
+import random
+import shutil
+import subprocess
 import threading
+import time
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Callable, Optional
 
+from .scheduler_jobs import JobSpec, is_due, job_runs_here
+from .scheduler_runner import JobRunner
+from .scheduler_state import SchedulerState
+
 logger = logging.getLogger("skcapstone.scheduled_tasks")
 
 
@@ -55,16 +64,25 @@ class ScheduledTask:
     last_error: Optional[str] = None
     run_count: int = 0
     error_count: int = 0
+    delay_first_run: float = 0.0
 
     def is_due(self, now: Optional[datetime] = None) -> bool:
         """Return True if the task interval has elapsed since last_run.
 
-        A task with no prior run is always considered due.
+        A task with no prior run is always considered due, unless
+        ``delay_first_run`` is set — in that case the first run is
+        deferred by that many seconds from process start.
 
         Args:
             now: Reference time for the check (defaults to UTC now).
         """
         if self.last_run is None:
+            if self.delay_first_run > 0:
+                if not hasattr(self, "_created_at"):
+                    object.__setattr__(self, "_created_at", datetime.now(timezone.utc))
+                reference = now or datetime.now(timezone.utc)
+                elapsed = (reference - self._created_at).total_seconds()
+                return elapsed >= self.delay_first_run
             return True
         reference = now or datetime.now(timezone.utc)
         elapsed = (reference - self.last_run).total_seconds()
@@ -122,6 +140,10 @@ class TaskScheduler:
         self._tasks: list[ScheduledTask] = []
         self._lock = threading.Lock()
         self._thread: Optional[threading.Thread] = None
+        self._config_jobs: list[JobSpec] = []
+        self._host_aliases: set[str] = set()
+        self._state: Optional[SchedulerState] = None
+        self._job_runner: Optional[JobRunner] = None
 
     # ------------------------------------------------------------------
     # Public API
@@ -132,6 +154,7 @@ class TaskScheduler:
         name: str,
         interval_seconds: float,
         callback: Callable[[], None],
+        delay_first_run: float = 0.0,
     ) -> ScheduledTask:
         """Register a recurring task.
 
@@ -139,11 +162,12 @@ class TaskScheduler:
             name: Unique task name (used in logs and status output).
             interval_seconds: Minimum seconds between executions.
             callback: Zero-argument callable to invoke.
+            delay_first_run: Seconds to wait before first execution (default 0 = immediate).
 
         Returns:
             The created ScheduledTask (caller may inspect it at runtime).
         """
-        task = ScheduledTask(name=name, interval_seconds=interval_seconds, callback=callback)
+        task = ScheduledTask(name=name, interval_seconds=interval_seconds, callback=callback, delay_first_run=delay_first_run)
         with self._lock:
             self._tasks.append(task)
         logger.debug("Registered scheduled task '%s' every %.0fs", name, interval_seconds)
@@ -188,6 +212,181 @@ class TaskScheduler:
                 for t in self._tasks
             ]
 
+    def load_config_jobs(
+        self,
+        jobs: list[JobSpec],
+        hostname: str,
+        host_aliases: set[str],
+        state_root: Path,
+    ) -> None:
+        """Load config-driven jobs and initialise per-host execution state.
+
+        Filters *jobs* to only those that are enabled and whose node affinity
+        matches *host_aliases*.  Initialises a :class:`SchedulerState` for
+        tracking run history and a :class:`JobRunner` for dispatching jobs.
+
+        **Call before** :meth:`start`.  The attributes ``_config_jobs``,
+        ``_state``, and ``_job_runner`` are not lock-protected against
+        concurrent mutation while the scheduler thread is running.
+        ``build_scheduler`` already calls this before ``start()``, so
+        documenting this constraint is sufficient for v1.
+
+        Args:
+            jobs: Full list of :class:`~skcapstone.scheduler_jobs.JobSpec`
+                instances as returned by
+                :func:`~skcapstone.scheduler_jobs.load_jobs`.
+            hostname: The current host's primary identifier (typically
+                ``socket.gethostname()``), used as the state sub-directory.
+            host_aliases: Full set of aliases for the current host, used for
+                node-affinity matching via
+                :func:`~skcapstone.scheduler_jobs.job_runs_here`.
+            state_root: Root directory under which per-host scheduler state
+                (``scheduler/<hostname>/state.json``) and log files are stored.
+        """
+        self._host_aliases = host_aliases
+        self._state = SchedulerState(root=state_root, hostname=hostname)
+        self._job_runner = JobRunner(
+            log_dir=state_root / "scheduler" / hostname / "logs"
+        )
+        self._config_jobs = [
+            j for j in jobs if j.enabled and job_runs_here(j, host_aliases)
+        ]
+        logger.info(
+            "Loaded %d config job(s) for host %s",
+            len(self._config_jobs),
+            hostname,
+        )
+
+    def tick_config_jobs(self, now: Optional[datetime] = None) -> None:
+        """Fire any config-driven jobs that are due at *now*.
+
+        Skips silently when no config jobs are loaded or state/runner are not
+        initialised (i.e. :meth:`load_config_jobs` has not been called).
+
+        Each due job is dispatched to its own short-lived daemon thread so
+        the tick returns immediately.  Long-running jobs (e.g. ``agent``
+        type, timeout up to 900 s) therefore never block the scheduler daemon
+        thread — which also drives heartbeats and all built-in tasks.
+
+        The due-check is intentionally kept in the tick thread (it is cheap).
+        The overlap lock is acquired *inside* the worker thread so it spans
+        the actual run; :meth:`_run_config_job` handles lock + run + state.
+
+        Note: because ``record_run`` is called asynchronously inside the
+        worker, the next tick may evaluate the same job as "due" before
+        ``record_run`` completes.  The per-job overlap lock prevents a second
+        concurrent execution in that window — the second worker acquires
+        ``got=False`` and returns immediately.  :class:`SchedulerState` uses
+        a ``threading.Lock`` so concurrent ``record_run`` calls are safe.
+
+        Args:
+            now: Reference UTC timestamp for due-checks.  Defaults to
+                ``datetime.now(timezone.utc)`` when not provided.
+        """
+        if not self._config_jobs or self._state is None or self._job_runner is None:
+            return
+        now = now or datetime.now(timezone.utc)
+        for job in self._config_jobs:
+            if not is_due(job, self._state.last_run(job.name), now):
+                continue
+            threading.Thread(
+                target=self._run_config_job,
+                args=(job, now),
+                name=f"skjob-{job.name}",
+                daemon=True,
+            ).start()
+
+    def _run_config_job(self, job: JobSpec, fire_time: datetime) -> None:
+        """Run a single config job in its own thread: lock, execute, record.
+
+        This method is the body of the per-job daemon thread spawned by
+        :meth:`tick_config_jobs`.  It acquires the per-job overlap lock,
+        runs the job via the configured :class:`~skcapstone.scheduler_runner.JobRunner`,
+        then records the result via
+        :class:`~skcapstone.scheduler_state.SchedulerState`.
+
+        If the lock cannot be obtained the method returns immediately without
+        running or recording — this is the safe path when the previous run is
+        still in progress (which can happen if a job's execution time exceeds
+        one tick interval).
+
+        Args:
+            job: The :class:`~skcapstone.scheduler_jobs.JobSpec` to execute.
+            fire_time: The UTC timestamp at which this job was determined to be
+                due (propagated to :meth:`~skcapstone.scheduler_state.SchedulerState.record_run`
+                so state timestamps reflect the scheduled fire time rather than
+                the wall-clock time of completion).
+        """
+        with self._job_runner.lock(job) as got:
+            if not got:
+                logger.debug("job '%s' still running — skip", job.name)
+                return
+            # Jitter: random splay before dispatch so fleet nodes sharing a cron
+            # slot don't stampede a shared resource (LLM endpoint, registry, etc).
+            if getattr(job, "jitter", 0.0) > 0:
+                time.sleep(random.uniform(0.0, float(job.jitter)))
+            # Run with retries + linear backoff for transient infra failures.
+            attempts = max(1, int(getattr(job, "retries", 0)) + 1)
+            result = None
+            for i in range(attempts):
+                result = self._job_runner.run(job)
+                if result.ok:
+                    break
+                if i < attempts - 1:
+                    logger.warning(
+                        "job '%s' attempt %d/%d failed: %s — retrying",
+                        job.name, i + 1, attempts, result.error,
+                    )
+                    backoff = float(getattr(job, "retry_backoff", 0.0))
+                    if backoff > 0:
+                        time.sleep(backoff)
+            self._state.record_run(
+                job.name, now=fire_time, ok=result.ok, error=result.error
+            )
+            if not result.ok:
+                logger.warning(
+                    "job '%s' failed after %d attempt(s): %s",
+                    job.name, attempts, result.error,
+                )
+            self._maybe_notify(job, result, attempts)
+
+    @staticmethod
+    def _maybe_notify(job: JobSpec, result, attempts: int) -> None:
+        """Fire an sk-alert per the job's ``notify`` policy.
+
+        Policy values: ``off`` (default), ``on_failure``, ``on_success``,
+        ``always``.  Sends the job name, status, attempt count, and a tail of
+        the captured output to Chef's Telegram via the ``sk-alert`` primitive.
+        Never raises — notification failure must not break the scheduler.
+
+        Args:
+            job: The job that ran.
+            result: The :class:`~skcapstone.scheduler_runner.JobResult`.
+            attempts: Number of attempts made (for the message).
+        """
+        mode = getattr(job, "notify", "off")
+        if mode == "off":
+            return
+        want = (
+            mode == "always"
+            or (mode == "on_failure" and not result.ok)
+            or (mode == "on_success" and result.ok)
+        )
+        if not want:
+            return
+        status = "✅ ok" if result.ok else "❌ FAILED"
+        suffix = f" (after {attempts} attempts)" if attempts > 1 else ""
+        tail = "\n".join((result.output or result.error or "").strip().splitlines()[-12:])
+        msg = f"🗓️ skscheduler · {job.name} · {status}{suffix}"
+        if tail:
+            msg += "\n" + tail
+        level = "info" if result.ok else getattr(job, "notify_level", "warn")
+        alert = shutil.which("sk-alert") or os.path.expanduser("~/.skenv/bin/sk-alert")
+        try:
+            subprocess.run([alert, "-l", level, msg], timeout=30, check=False)
+        except Exception as exc:  # noqa: BLE001 — notify must never break the loop
+            logger.warning("notify failed for job '%s': %s", job.name, exc)
+
     # ------------------------------------------------------------------
     # Internal
     # ------------------------------------------------------------------
@@ -203,6 +402,8 @@ class TaskScheduler:
                 if task.is_due(now):
                     task.run()
 
+            self.tick_config_jobs(now)
+
             self._stop_event.wait(timeout=self._tick_interval)
 
 
@@ -214,29 +415,50 @@ class TaskScheduler:
 def make_memory_promotion_task(home: Path) -> Callable[[], None]:
     """Return a callback that runs an hourly memory promotion sweep.
 
-    Instantiates PromotionEngine lazily (so import errors are deferred until
-    first run, matching the graceful-import pattern used elsewhere in the daemon).
+    The sweep runs in a dedicated background thread so it never blocks the
+    scheduler (and therefore never blocks watchdog pings or other scheduled
+    tasks).  A ``threading.Event`` gate prevents overlapping sweeps.
+
+    The sweep is rate-limited to 50 promotions per run to bound I/O time.
 
     Args:
         home: Agent home directory containing the ``memory/`` subtree.
     """
+    _running = threading.Event()
 
-    def _run() -> None:
-        from .memory_promoter import PromotionEngine
+    def _sweep() -> None:
+        try:
+            from .memory_promoter import PromotionEngine
+
+            engine = PromotionEngine(home)
+            result = engine.sweep(limit=50)
+            if result.promoted:
+                logger.info(
+                    "Memory promotion sweep: %d promoted of %d scanned",
+                    len(result.promoted),
+                    result.scanned,
+                )
+            else:
+                logger.debug(
+                    "Memory promotion sweep: %d scanned, 0 promoted",
+                    result.scanned,
+                )
+        except Exception as exc:
+            logger.error("Memory promotion sweep error: %s", exc)
+        finally:
+            _running.clear()
 
-        engine = PromotionEngine(home)
-        result = engine.sweep()
-        if result.promoted:
-            logger.info(
-                "Memory promotion sweep: %d promoted of %d scanned",
-                len(result.promoted),
-                result.scanned,
-            )
-        else:
-            logger.debug(
-                "Memory promotion sweep: %d scanned, 0 promoted",
-                result.scanned,
-            )
+    def _run() -> None:
+        if _running.is_set():
+            logger.debug("Memory promotion sweep already running — skipping")
+            return
+        _running.set()
+        t = threading.Thread(
+            target=_sweep,
+            name="memory-promotion-sweep",
+            daemon=True,
+        )
+        t.start()
 
     return _run
 
@@ -498,6 +720,7 @@ def build_scheduler(
         name="memory_promotion_sweep",
         interval_seconds=3600,  # 1 hour
         callback=make_memory_promotion_task(home),
+        delay_first_run=120,  # let daemon stabilize before first sweep
     )
 
     scheduler.register(
@@ -543,4 +766,16 @@ def build_scheduler(
     except Exception:
         logger.debug("ITIL scheduled tasks not available — skipped")
 
+    from .scheduler_jobs import load_jobs_with_dropins, current_host_aliases
+    import socket
+    jobs_path = Path(home) / "config" / "jobs.yaml"
+    jobs = load_jobs_with_dropins(jobs_path)
+    if jobs:
+        scheduler.load_config_jobs(
+            jobs=jobs,
+            hostname=socket.gethostname(),
+            host_aliases=current_host_aliases(),
+            state_root=Path(home),
+        )
+
     return scheduler