@maudecode/talaria-web 1.18.0 → 1.19.0

package/sidecar/talaria_sidecar/methods/cron.py

@@ -14,9 +14,13 @@ import importlib
 import logging
 import multiprocessing
 import queue
+import os
 import re
+import signal
 import threading
 import time
+from datetime import datetime
+from functools import wraps
 from pathlib import Path
 
 from ..errors import InvalidParams, RpcError
@@ -31,6 +35,18 @@ _PASSTHROUGH_FIELDS = ("script", "no_agent", "context_from", "reasoning_effort",
 _RUNNING: dict[str, float] = {}
 _RUNNING_LOCK = threading.Lock()
 _SNAPSHOT_LOCK = threading.Lock()
+# The Agent's fire claim lives FIRE_CLAIM_TTL_SECONDS (300); a run refreshes it well inside that, re-checks a missed
+# refresh once before treating the claim as lost (as the Agent's own runs do), and gives up once no refresh has landed
+# for most of the lease.
+_CLAIM_HEARTBEAT_SECONDS = 60.0
+_CLAIM_MISS_CONFIRM_SECONDS = 1.0
+_CLAIM_HEARTBEAT_GRACE_SECONDS = 240.0
+_CLAIM_LOST = "Another process took over this task's run claim, so this run stopped before saving or delivering anything"
+
+
+class _ClaimLost(RuntimeError):
+    def __init__(self) -> None:
+        super().__init__(_CLAIM_LOST)
 
 
 def _main_model_pin() -> tuple[str | None, str | None]:
@@ -97,6 +113,25 @@ def _job_ref(params: dict, key: str = "job_id") -> str:
     return ref
 
 
+def _settled_mutation(method):
+    """Fence schedule edits against manual execution, including its retained recovery obligation."""
+    @wraps(method)
+    def guarded(ctx, params):
+        from .cron_execution import ManualExecution, Unverified
+        home, jobs = profile_home_param(params), _jobs()
+        with _store(home):
+            job = getattr(jobs, "resolve_job_ref", jobs.get_job)(_job_ref(params))
+        if job is None:
+            return method(ctx, params)
+        job_id = _job_id({"job_id": job["id"]})
+        execution = ManualExecution(home, home, {"id": job_id}, None)
+        with execution.acquire():
+            if execution.file.exists():
+                raise Unverified("Schedule edits and deletion are blocked until the manual execution is reconciled.")
+            return method(ctx, dict(params, job_id=job_id))
+    return guarded
+
+
 # ── job payload shaping ───────────────────────────────────────────────────
 
 def _monitor_storage(monitor) -> dict:
@@ -144,6 +179,87 @@ def job_for_api(job: dict) -> dict:
     return payload
 
 
+# ── store health and foreign runs (TAL-810) ─────────────────────────────
+
+def _store_health(cron_dir: Path) -> dict | None:
+    """The Agent's cross-process write probe: ``None`` when the store accepts writes or the Agent predates it."""
+    try:
+        from cron.store_health import probe_report
+    except ImportError:
+        return None
+    return probe_report(cron_dir)
+
+
+def _live_executions(home: Path, job_ids: list[str], *, strict: bool = False) -> dict[str, dict]:
+    """Jobs whose latest attempt is claimed or running under a live owner: the gateway scheduler or another process.
+
+    The executions ledger lives under ``get_hermes_home()``, not the cron store override, so it is read in the
+    profile's scope. A store without a ledger never ran anything, and is not created by this read. An unreadable
+    ledger reads idle for a listing; ``strict`` (run admission) refuses instead, since it cannot prove the job idle."""
+    try:
+        from cron.executions import live_inflight_execution
+    except ImportError:
+        return {}
+    if not job_ids or not (Path(home).expanduser() / "cron" / "executions.db").is_file():
+        return {}
+    try:
+        with scoped_home(home):
+            return {job_id: record for job_id in job_ids if job_id and (record := live_inflight_execution(job_id))}
+    except Exception as exc:  # noqa: BLE001 - an unreadable ledger cannot prove a run; the list still loads
+        if strict:
+            raise RpcError("Cannot tell whether this task is already running: its executions ledger is unreadable", condition="cron_ledger_unreadable") from exc
+        log.warning("Cron executions ledger unreadable in %s", home, exc_info=True)
+        return {}
+
+
+def _claim_fire(store_home: Path, job: dict) -> dict | bool | None:
+    """The Agent's own run-now lease (``claim_job_for_fire(manual=True)``), which a gateway tick must also win.
+
+    The exact record the claim took (the one to run), ``False`` when another process holds the job's claim, or
+    ``None`` for a job the Agent does not claim (paused, disabled, or terminal), which no tick fires. ``mark_job_run``
+    releases a claim when the run ends; the run heartbeats it meanwhile."""
+    jobs = _jobs()
+    with _store(store_home):
+        if not jobs.is_job_runnable(job):
+            return None
+        claimed = jobs.claim_job_for_fire(job["id"], manual=True, return_job=True)
+        if not isinstance(claimed, dict):
+            # The claim only loses to a live owner's claim; any other refusal is a record the Agent never claims.
+            current = jobs.get_job(job["id"]) or {}
+            return False if isinstance(current.get("fire_claim"), dict) else None
+    return {key: value for key, value in claimed.items() if key != "_scheduled_instant"}
+
+
+@contextlib.contextmanager
+def _owner_fence(jobs, job_id: str, owner: str | None, *, protect_unclaimed=False):
+    """Hold the Agent fire fence across side effects; canonical unclaimed runs also exclude a successor claim."""
+    if owner is None and protect_unclaimed:
+        with jobs._fire_job_lock(job_id) as locked:
+            current = jobs.get_job(job_id) if locked else None
+            yield bool(current) and not current.get("fire_claim")
+    else:
+        with jobs.fire_claim_fence(job_id, expected_owner=owner) if owner else contextlib.nullcontext(True) as owns:
+            yield owns
+
+
+def _mark_run(jobs, job_id: str, success: bool, error, owner: str | None, delivery_error=None) -> None:
+    """``mark_job_run`` fenced by the run's fire claim: a run that lost its claim records nothing over the new owner's."""
+    fence = {"expected_fire_owner": owner} if owner else {}
+    try:
+        jobs.mark_job_run(job_id, success, error, delivery_error=delivery_error, **fence)
+    except TypeError:
+        if owner:
+            raise
+        jobs.mark_job_run(job_id, success, error)
+
+
+def _elapsed(record: dict) -> float:
+    try:
+        return round(max(0.0, time.time() - datetime.fromisoformat(str(record.get("claimed_at"))).timestamp()), 1)
+    except ValueError:
+        return 0.0
+
+
 # ── output files ──────────────────────────────────────────────────────────
 
 def _response_marker_index(text: str) -> int:
@@ -228,12 +344,20 @@ def _snippet(text: str, limit: int = 600) -> str:
 
 # ── manual runs ───────────────────────────────────────────────────────────
 
-def _child_main(job, execution_home, result_queue):
+def _child_main(job, execution_home, result_queue, cancel_event=None, execution_id=None, recovering=False):
     try:
         from cron.scheduler import run_job
 
+        def execute():
+            if recovering:
+                from .cron_execution import recover
+                return recover(job, execution_id, cancel_event)
+            if execution_id is not None:
+                return run_job(job, cancel_event=cancel_event, execution_id=execution_id)
+            return run_job(job)
+
         if execution_home is None:
-            result = run_job(job)
+            result = execute()
         else:
             from talaria_sidecar.home import scoped_home as _scoped
 
@@ -241,12 +365,16 @@ def _child_main(job, execution_home, result_queue):
 
             os.environ["HERMES_HOME"] = str(execution_home)
             with _scoped(Path(execution_home)):
-                result = run_job(job)
-        result_queue.put(("ok", result))
+                result = execute()
+        result_queue.put(("ok", result, {key: job[key] for key in ("_model_unreachable", "_quota_hold_seconds") if key in job}))
     except BaseException as exc:  # noqa: BLE001 - surfaced in the parent
         import traceback
+        from .cron_execution import Unverified
 
-        result_queue.put(("error", f"{type(exc).__name__}: {exc}", traceback.format_exc()))
+        if isinstance(exc, Unverified):
+            result_queue.put(("unverified", exc.detail))
+        else:
+            result_queue.put(("error", f"{type(exc).__name__}: {exc}", traceback.format_exc()))
 
 
 def _result_timeout(job) -> float:
@@ -263,44 +391,137 @@ def _result_timeout(job) -> float:
     return 6 * 60 * 60.0
 
 
-def _run_in_child(job, execution_home, ctx: CallContext):
+def _worker(main, *args) -> None:
+    """A run's worker leads its own process group, so stopping the run also reaches every process it started."""
+    if hasattr(os, "setsid"):
+        os.setsid()
+    main(*args)
+
+
+def _signal_group(process, sig) -> None:
+    with contextlib.suppress(ProcessLookupError, PermissionError):
+        os.killpg(process.pid, sig)
+
+
+def _stop(process) -> None:
+    """End a run's worker and everything it started, returning only once the worker has exited: SIGTERM to the
+    worker's process group, then SIGKILL to whatever is left, so nothing keeps running after the run gives up its
+    claim or its running guard. Without process groups (Windows) only the worker itself is stopped."""
+    if not hasattr(os, "killpg"):
+        process.terminate()
+        process.join(timeout=5)
+        if process.is_alive():
+            process.kill()
+            process.join()
+        return
+    _signal_group(process, signal.SIGTERM)
+    process.join(timeout=5)
+    _signal_group(process, signal.SIGKILL)
+    process.join()
+
+
+def _run_in_child(job, execution_home, ctx: CallContext, heartbeat=None, execution=None):
+    from .cron_execution import SETTLEMENT_SECONDS, Unverified
+
     ctx_mp = multiprocessing.get_context("spawn")
     result_queue = ctx_mp.Queue(maxsize=1)
-    process = ctx_mp.Process(target=_child_main, args=(job, str(execution_home) if execution_home else None, result_queue))
-    process.start()
+    cancel_event = ctx_mp.Event() if execution is not None else None
+
+    def start(recovering=False):
+        args = (job, str(execution_home) if execution_home else None, result_queue)
+        if execution is not None:
+            args += (cancel_event, execution.execution_id, recovering)
+        child = ctx_mp.Process(target=_worker, args=(_child_main, *args))
+        child.start()
+        return child
+
+    process = start(execution.recovering if execution else False)
     deadline = time.monotonic() + _result_timeout(job)
+    next_beat = time.monotonic() + _CLAIM_HEARTBEAT_SECONDS
     status, payload = "error", ["cron run subprocess failed before producing a result", ""]
+    settling = None
+    recovery_started = bool(execution and execution.recovering)
+
+    def cancel(reason):
+        nonlocal settling
+        if execution is None:
+            return
+        execution.cancel(reason)
+        cancel_event.set()
+        if settling is None:
+            settling = time.monotonic() + SETTLEMENT_SECONDS
+
+    def recover_child(detail):
+        nonlocal process, recovery_started, result_queue
+        process.join(timeout=5)
+        if process.is_alive():
+            _stop(process)
+        if recovery_started:
+            raise Unverified(detail)
+        recovery_started = True
+        cancel("submitter interrupted")
+        # A killed queue writer can leave a partial frame. Recovery gets a fresh pipe.
+        result_queue.close()
+        result_queue.join_thread()
+        result_queue = ctx_mp.Queue(maxsize=1)
+        process = start(True)
+
+    if execution and execution.record["cancel_reason"]:
+        cancel(execution.record["cancel_reason"])
     try:
         while True:
             try:
                 status, *payload = result_queue.get(timeout=1.0)
-                break
+                if execution is not None and status == "unverified":
+                    raise Unverified(payload[0])
+                if execution is None or status == "ok":
+                    break
+                # A transport failure is not a failed task. Recover the original identity once.
+                recover_child(payload[0])
             except queue.Empty:
+                if heartbeat is not None and time.monotonic() >= next_beat:
+                    next_beat = time.monotonic() + _CLAIM_HEARTBEAT_SECONDS
+                    if not heartbeat():
+                        if execution is None:
+                            _stop(process)
+                            raise _ClaimLost()
+                        cancel("claim_lost")
+                        heartbeat = None
                 if ctx.cancelled:
-                    process.terminate()
-                    process.join(timeout=5)
-                    status, payload = "cancelled", ["cancelled", ""]
-                    break
+                    if execution is None:
+                        _stop(process)
+                        status, payload = "cancelled", ["cancelled", ""]
+                        break
+                    cancel("cancelled")
                 if not process.is_alive():
                     # The child may have put its result just before exiting; otherwise it died (OOM, signal) without one.
                     try:
                         status, *payload = result_queue.get_nowait()
                     except queue.Empty:
                         payload = [f"cron run subprocess exited with code {process.exitcode} without producing a result", ""]
+                    if execution is not None and status != "ok":
+                        recover_child(payload[0])
+                        continue
                     break
                 if time.monotonic() >= deadline:
-                    process.terminate()
-                    process.join(timeout=5)
-                    payload = [f"cron run subprocess produced no result within {_result_timeout(job):g}s and was terminated", ""]
-                    break
+                    if execution is None:
+                        _stop(process)
+                        payload = [f"cron run subprocess produced no result within {_result_timeout(job):g}s and was terminated", ""]
+                        break
+                    cancel("Manual task timed out")
+                if settling is not None and time.monotonic() >= settling:
+                    raise Unverified("The owner has not provided a terminal receipt.")
         process.join(timeout=5)
         if process.is_alive():
-            process.terminate()
-            process.join(timeout=5)
+            _stop(process)
     finally:
+        if process.is_alive():
+            _stop(process)
         result_queue.close()
         result_queue.join_thread()
     if status == "ok":
+        if execution is not None and len(payload) > 1:
+            execution.job.update(payload[1])
         return payload[0]
     if status == "cancelled":
         raise RpcError("cancelled", condition="cancelled")
@@ -331,7 +552,32 @@ def _delivery_home(scheduler, home: Path):
                 pass
 
 
-def run_tracked(job: dict, store_home: Path, execution_home: Path | None, ctx: CallContext) -> dict:
+def _heartbeat(store_home, job_id, claim_owner):
+    jobs = _jobs()
+    last_beat = time.monotonic()
+
+    def heartbeat() -> bool:
+        """Refresh only this fire's claim, giving transient store errors less than a lease to recover."""
+        nonlocal last_beat
+        try:
+            with _store(store_home):
+                owned = jobs.heartbeat_fire_claim(job_id, expected_owner=claim_owner) is not False
+                if not owned:
+                    time.sleep(_CLAIM_MISS_CONFIRM_SECONDS)
+                    owned = jobs.heartbeat_fire_claim(job_id, expected_owner=claim_owner) is not False
+        except Exception:  # noqa: BLE001
+            log.warning("Fire claim heartbeat failed for job %s", job_id, exc_info=True)
+            return time.monotonic() - last_beat < _CLAIM_HEARTBEAT_GRACE_SECONDS
+        if owned:
+            last_beat = time.monotonic()
+        return owned
+    return heartbeat
+
+
+def run_tracked(job: dict, store_home: Path, execution_home: Path | None, ctx: CallContext, claim_owner: str | None = None, *, recovery_only=False) -> dict:
+    from . import cron_execution
+    if recovery_only or cron_execution.path(store_home, job["id"]).exists() or cron_execution.supported(job):
+        return cron_execution.run(job, store_home, execution_home, ctx, claim_owner, recovery_only=recovery_only)
     jobs = _jobs()
     scheduler = importlib.import_module("cron.scheduler")
     silent_marker = getattr(scheduler, "SILENT_MARKER", "[SILENT]")
@@ -340,8 +586,12 @@ def run_tracked(job: dict, store_home: Path, execution_home: Path | None, ctx: C
     execution_home = execution_home or store_home
     outcome: dict = {"job_id": job_id, "status": "completed"}
     try:
-        success, output, final_response, error = _run_in_child(job, execution_home, ctx)
-        with _store(store_home), _delivery_home(scheduler, store_home):
+        heartbeat = _heartbeat(store_home, job_id, claim_owner)
+        success, output, final_response, error = _run_in_child(job, execution_home, ctx, heartbeat if claim_owner else None)
+        with _store(store_home), _owner_fence(jobs, job_id, claim_owner) as owns, contextlib.ExitStack() as delivery:
+            if not owns:
+                raise _ClaimLost()
+            delivery.enter_context(_delivery_home(scheduler, store_home))
             jobs.save_job_output(job_id, output)
             content = final_response if success else f"⚠️ Cron job '{job.get('name', job_id)}' failed:\n{error}"
             should_deliver = bool(content) and not (success and silent_marker in content.strip().upper())
@@ -353,15 +603,12 @@ def run_tracked(job: dict, store_home: Path, execution_home: Path | None, ctx: C
                     delivery_error = str(exc)
             if success and not final_response:
                 success, error = False, "Agent completed but produced empty response (model error, timeout, or misconfiguration)"
-            try:
-                jobs.mark_job_run(job_id, success, error, delivery_error=delivery_error)
-            except TypeError:
-                jobs.mark_job_run(job_id, success, error)
+            _mark_run(jobs, job_id, success, error, claim_owner, delivery_error)
         outcome.update(success=bool(success), error=error, delivery_error=delivery_error)
     except RpcError:
         with _store(store_home):
             try:
-                jobs.mark_job_run(job_id, False, "cancelled")
+                _mark_run(jobs, job_id, False, "cancelled", claim_owner)
             except Exception:  # noqa: BLE001
                 pass
         raise
@@ -369,7 +616,7 @@ def run_tracked(job: dict, store_home: Path, execution_home: Path | None, ctx: C
         log.exception("Manual cron run failed for job %s", job_id)
         with _store(store_home):
             try:
-                jobs.mark_job_run(job_id, False, str(exc))
+                _mark_run(jobs, job_id, False, str(exc), claim_owner)
             except Exception:  # noqa: BLE001
                 pass
         outcome.update(status="failed", success=False, error=str(exc))
@@ -382,17 +629,79 @@ def run_tracked(job: dict, store_home: Path, execution_home: Path | None, ctx: C
 # ── methods ───────────────────────────────────────────────────────────────
 
 def register(registry) -> None:
+    @registry.method("cron.blueprints")
+    def blueprints(ctx: CallContext, params: dict) -> dict:
+        try:
+            catalog = importlib.import_module("cron.blueprint_catalog")
+        except ModuleNotFoundError as exc:
+            if exc.name != "cron.blueprint_catalog":
+                raise
+            return {"blueprints": []}
+        with scoped_home(profile_home_param(params)):
+            rows = []
+            for blueprint in catalog.list_blueprints():
+                entry = catalog.blueprint_catalog_entry(blueprint)
+                schema = catalog.blueprint_form_schema(blueprint)
+                slots = [{**slot, "default": str(slot["default"]) if slot.get("default") is not None else None,
+                          "options": [str(value) for value in slot.get("options", [])]} for slot in schema["fields"]]
+                rows.append({key: entry[key] for key in ("key", "title", "description", "category", "source", "plugin")})
+                rows[-1].update(schedule_label=entry["scheduleHuman"], slots=slots)
+            return {"blueprints": rows}
+
+    @registry.method("cron.blueprint_fill")
+    def blueprint_fill(ctx: CallContext, params: dict) -> dict:
+        key, values = params.get("key"), params.get("values")
+        if not isinstance(key, str) or not key.strip() or not isinstance(values, dict) or not all(isinstance(v, str) for v in values.values()):
+            raise InvalidParams("key and string slot values are required")
+        try:
+            catalog = importlib.import_module("cron.blueprint_catalog")
+        except ModuleNotFoundError as exc:
+            if exc.name != "cron.blueprint_catalog":
+                raise
+            raise InvalidParams("Automation blueprints are unavailable") from exc
+        with scoped_home(profile_home_param(params)):
+            blueprint = catalog.get_blueprint(key)
+            if blueprint is None:
+                raise InvalidParams("Unknown automation blueprint")
+            try:
+                draft = catalog.fill_blueprint(blueprint, values)
+            except catalog.BlueprintFillError as exc:
+                raise InvalidParams(str(exc)) from exc
+            draft.setdefault("skills", [])
+            return {"draft": draft}
+
     @registry.method("cron.list")
     def list_(ctx: CallContext, params: dict) -> dict:
-        """Jobs of one store, shaped for the API; the server merges profiles."""
+        """Jobs of one store, shaped for the API; the server merges profiles. ``probe_store`` adds the store's write health."""
         jobs = _jobs()
-        with _store(profile_home_param(params)):
-            return {"jobs": [job_for_api(job) for job in jobs.list_jobs(include_disabled=True)]}
+        home = profile_home_param(params)
+        with _store(home):
+            from .cron_execution import pending_jobs
+            pending = {job["id"]: job for job in pending_jobs(home)}
+            stored = jobs.list_jobs(include_disabled=True)
+            ids = {job["id"] for job in stored}
+            stored.extend(job for jid, job in pending.items() if jid not in ids)
+            listed = [job_for_api(job) for job in stored]
+            result: dict = {"jobs": listed}
+            if params.get("probe_store"):
+                result["store_health"] = _store_health(jobs.get_cron_output_dir().parent)
+        live = _live_executions(home, [str(job.get("id") or "") for job in listed])
+        with _RUNNING_LOCK:
+            active = set(_RUNNING)
+        for index, job in enumerate(listed):
+            jid = str(job.get("id") or "")
+            if jid in pending and jid not in live and jid not in active:
+                # Web preflight must use the accepted fire's profile, even after external job edits.
+                job = listed[index] = job_for_api(pending[jid])
+            job["running_elsewhere"] = jid in live
+        return result
 
     @registry.method("cron.get")
     def get(ctx: CallContext, params: dict) -> dict:
-        with _store(profile_home_param(params)):
-            job = _jobs().get_job(_job_id(params))
+        from .cron_execution import pending_job
+        home, job_id = profile_home_param(params), _job_id(params)
+        with _store(home):
+            job = pending_job(home, job_id) or _jobs().get_job(job_id)
         return {"job": job_for_api(job) if job else None}
 
     @registry.method("cron.create")
@@ -444,6 +753,7 @@ def register(registry) -> None:
         return {"job": job_for_api(job)}
 
     @registry.method("cron.update")
+    @_settled_mutation
     def update(ctx: CallContext, params: dict) -> dict:
         jobs = _jobs()
         job_id = _job_ref(params)
@@ -472,6 +782,7 @@ def register(registry) -> None:
         return {"job": job_for_api(job)}
 
     @registry.method("cron.delete")
+    @_settled_mutation
     def delete(ctx: CallContext, params: dict) -> dict:
         job_id = _job_ref(params)
         with _store(profile_home_param(params)):
@@ -489,6 +800,7 @@ def register(registry) -> None:
         return {"job": dict(job)}
 
     @registry.method("cron.resume")
+    @_settled_mutation
     def resume(ctx: CallContext, params: dict) -> dict:
         with _store(profile_home_param(params)):
             try:
@@ -505,17 +817,37 @@ def register(registry) -> None:
         store_home = profile_home_param(params)
         execution_home = Path(params["execution_home"]).expanduser() if params.get("execution_home") else None
         job_id = _job_id(params)
+        from .cron_execution import path
+        if path(store_home, job_id).exists():
+            return run_tracked({"id": job_id}, store_home, execution_home, ctx, recovery_only=True)
         with _store(store_home):
             job = _jobs().get_job(job_id)
         if not job:
             raise RpcError("Job not found", condition="not_found")
+        foreign = _live_executions(store_home, [job_id], strict=True).get(job_id)
+        if foreign is not None:
+            return {"job_id": job_id, "status": "already_running", "elapsed": _elapsed(foreign)}
         with _RUNNING_LOCK:
             started = _RUNNING.get(job_id)
             if started is not None:
                 return {"job_id": job_id, "status": "already_running", "elapsed": round(time.time() - started, 1)}
             _RUNNING[job_id] = time.time()
+        try:
+            claimed = _claim_fire(store_home, job)
+        except BaseException:
+            with _RUNNING_LOCK:
+                _RUNNING.pop(job_id, None)
+            raise
+        if claimed is False:
+            with _RUNNING_LOCK:
+                _RUNNING.pop(job_id, None)
+            return {"job_id": job_id, "status": "already_running", "elapsed": 0.0}
+        owner = None
+        if claimed:
+            job = claimed
+            owner = str((claimed.get("fire_claim") or {}).get("by") or "") or None
         ctx.emit("started", {"job_id": job_id})
-        return run_tracked(job, store_home, execution_home, ctx)
+        return run_tracked(job, store_home, execution_home, ctx, owner)
 
     @registry.method("cron.status", requires_agent=False)
     def status(ctx: CallContext, params: dict) -> dict: