@maudecode/talaria-web 1.17.2 → 1.19.0

package/sidecar/talaria_sidecar/methods/cron_execution.py

@@ -0,0 +1,284 @@
+"""Exact manual-fire recovery. The Agent owns admission, delivery and bookkeeping formats.
+
+Its admission journal lacks the Web caller's cancellation intent and tail ownership.
+This additive record retains those two obligations across a lost submitter or sidecar.
+"""
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import importlib
+import json
+import os
+import time
+from pathlib import Path
+
+from ..dependency_sync import _lock
+from ..errors import RpcError
+from ..home import scoped_home
+
+SETTLEMENT_SECONDS = 30.0
+
+
+class Unverified(RpcError):
+    def __init__(self, detail):
+        self.detail = str(detail)
+        super().__init__(f"Task outcome is unverified. {detail}",
+                         condition="cron_execution_unverified")
+
+
+def path(home, job_id):
+    return Path(home) / "cron" / "manual_runs" / (job_id + ".json")
+
+
+def _read_record(home, job_id):
+    try:
+        record = json.loads(path(home, job_id).read_text())
+    except FileNotFoundError:
+        return None
+    except (OSError, ValueError) as exc:
+        raise Unverified("The retained task record could not be read.") from exc
+    job = record.get("job") if isinstance(record, dict) else None
+    if not isinstance(job, dict) or job.get("id") != job_id:
+        raise Unverified("Recovery identity does not match this task.")
+    return record
+
+
+def pending_job(home, job_id):
+    """The frozen job keeps an externally deleted execution reachable through Web's list/get preflight."""
+    record = _read_record(home, job_id)
+    if record is None:
+        return None
+    job = record["job"]
+    return dict(job, recovery_only=True, state="paused", enabled=False, last_run_at=None, last_status="error",
+                last_error="Task outcome is unverified. Use Run now to check it.")
+
+
+def pending_jobs(home):
+    from .cron import _job_id
+    result = []
+    for file in sorted((Path(home) / "cron" / "manual_runs").glob("*.json")):
+        job = pending_job(home, _job_id({"job_id": file.stem}))
+        if job is not None:
+            result.append(job)
+    return result
+
+
+def supported(job, *, recovery=False):
+    if job.get("no_agent"):
+        return False
+    try:
+        authority = importlib.import_module("cron.scheduler_authority")
+        if recovery:
+            surfaces = ((authority, ("journal_path", "_lift_pause")),
+                        (importlib.import_module("cron.scheduler_bookkeeping"), ("fail_empty_response", "finish_completed_run")),
+                        (importlib.import_module("cron.executions"), ("give_up_execution", "recover_receipted_execution")),
+                        (importlib.import_module("hermes_cli.gateway_client"), ("connect_gateway",)),
+                        (importlib.import_module("hermes_cli.gateway_runtime"), ("discover_gateway_endpoint",)))
+            return all(callable(getattr(module, name, None)) for module, names in surfaces for name in names)
+    except ImportError as exc:
+        if recovery or (isinstance(exc, ModuleNotFoundError) and exc.name in {"cron.scheduler_authority", "cron"}):
+            return False
+        raise
+    return True
+
+
+def recover(job, execution_id, cancel_event):
+    """Observe only. A lost submission reply never authorizes another submit."""
+    from hermes_cli.gateway_client import connect_gateway
+
+    async def observe():
+        endpoint = None
+        if not os.environ.get("HERMES_TUI_GATEWAY_URL", "").strip():
+            from hermes_cli.gateway_runtime import discover_gateway_endpoint
+            from hermes_constants import get_hermes_home
+            discovery = await asyncio.to_thread(discover_gateway_endpoint, get_hermes_home())
+            if discovery.state != "ready" or discovery.endpoint is None:
+                raise Unverified("The owning gateway is unavailable.")
+            endpoint = discovery.endpoint
+        async with connect_gateway(endpoint=endpoint) as client:
+            state = await client.rpc("cron.recover", job_id=job["id"], request_id=execution_id, extra_prompt=None)
+            if state["status"] == "missing":
+                return False, "", "", "Cron admission missing: the owner never admitted this fire"
+            if state["status"] == "unknown":
+                raise Unverified("The owner cannot prove this admission's outcome.")
+            receipt = {key: state[key] for key in ("session_id", "admission_id")}
+            while True:
+                if state["status"] == "terminal":
+                    job.update(state.get("job_flags") or {})
+                    return tuple(state["result"])
+                if state["status"] not in {"queued", "started"}:
+                    raise Unverified(f"Owner status: {state['status']}")
+                if cancel_event.is_set():
+                    await client.rpc("cron.cancel", **receipt)
+                await asyncio.sleep(.2)
+                state = await client.rpc("cron.status", **receipt)
+    return asyncio.run(observe())
+
+
+class ManualExecution:
+    def __init__(self, home, execution_home, job, owner):
+        self.home = Path(home)
+        self.file = path(home, job["id"])
+        self.execution_home = Path(execution_home or home)
+        self.job, self.owner = dict(job), owner
+        self.record = None
+        self.recovering = False
+        self.settled = False
+
+    def save(self):
+        from utils import atomic_json_write
+        atomic_json_write(self.file, self.record, mode=0o600, fsync_dir=True)
+
+    @contextlib.contextmanager
+    def acquire(self):
+        self.file.parent.mkdir(parents=True, exist_ok=True)
+        with self.file.with_suffix(".lock").open("a+b") as lease:
+            try:
+                _lock(lease)
+            except OSError as exc:
+                raise RpcError("This manual task is still being reconciled", condition="cron_execution_unverified") from exc
+            yield
+
+    def prepare(self, *, recovery_only=False):
+        from cron import executions, jobs
+        self.record = _read_record(self.home, self.job["id"])
+        if self.record is not None:
+            self.job, self.owner = self.record["job"], self.record["owner"]
+            self.execution_home = Path(self.record["execution_home"])
+            self.recovering = True
+        else:
+            if recovery_only:
+                raise Unverified("The retained task record is no longer available. Refresh its status.")
+            with scoped_home(self.execution_home):
+                row = executions.create_execution(self.job["id"], source="web-manual")
+                executions.mark_execution_running(row["id"])
+            self.record = {"job": self.job, "owner": self.owner, "execution_home": str(self.execution_home),
+                           "execution_id": row["id"], "cancel_reason": None, "tail_started": False}
+            self.execution_id = row["id"]
+            self.save()
+        self.execution_id = self.record["execution_id"]
+        if self.recovering:
+            self.cancel("submitter interrupted")
+        # Pause before dispatch: a dead sidecar must not let claim expiry re-fire an accepted task.
+        # Only this exact pause is lifted. Existing operator pauses are never changed.
+        with jobs.use_cron_store(self.home):
+            from .cron import _owner_fence
+            with _owner_fence(jobs, self.job["id"], self.owner, protect_unclaimed=True) as owns:
+                current = jobs.get_job(self.job["id"])
+                if not owns and not self.recovering:
+                    self.cancel("claim_lost")
+                    self.record["result"] = [False, "", "", "Fire claim lost before submission"]
+                    self.save()
+                if owns and current and jobs.is_job_runnable(current) and not self.recovering:
+                    current = jobs.pause_job(self.job["id"], reason=f"Manual execution pending: {self.execution_id}")
+                # The job write may have committed before the submitter died saving its pause receipt.
+                if (current and current.get("state") == "paused" and not self.record.get("pause")
+                        and current.get("paused_reason") == f"Manual execution pending: {self.execution_id}"):
+                    self.record["pause"] = {"paused_at": current["paused_at"], "reason": current["paused_reason"]}
+                    self.save()
+
+    def cancel(self, reason):
+        if self.record["cancel_reason"] is None:
+            self.record["cancel_reason"] = reason
+            self.save()
+
+    def finish(self, result):
+        from cron import executions, jobs, scheduler
+        from cron.scheduler_authority import journal_path
+        from .cron import _delivery_home, _owner_fence, _ClaimLost
+
+        success, output, answer, error = result
+        with scoped_home(self.execution_home), jobs.use_cron_store(self.home), _owner_fence(jobs, self.job["id"], self.owner, protect_unclaimed=True) as owns:
+            current = jobs.get_job(self.job["id"]) or {}
+            row = executions.get_execution(self.execution_id)
+            marked = self.execution_id in current.get("canonical_completions", [])
+            booked = marked or (row and row["status"] in {"completed", "failed"})
+            # Our own mark releases the claim before pause cleanup. A successor's claim never qualifies.
+            may_finish = bool(current) and (owns or (marked and not current.get("fire_claim")))
+            if not booked and self.record["tail_started"]:
+                raise Unverified("Bookkeeping was interrupted; its side effects will not be repeated.")
+            # Stop the Agent's orphan journal reader from replaying OUR tail after a crash.
+            # Our record now owns recovery, including the at-most-once tail barrier.
+            self.record["result"] = list(result)
+            self.save()
+            journal_path(self.job["id"], self.execution_id).unlink(missing_ok=True)
+            if not may_finish:
+                self._finish_ledger(False, "Fire claim ownership lost before terminal completion.")
+            elif booked:
+                self._finish_ledger(success, error)
+            else:
+                self.record["tail_started"] = True
+                self.save()
+                self.job["execution_id"] = self.execution_id
+                with _delivery_home(scheduler, self.home):
+                    from cron.scheduler_bookkeeping import fail_empty_response, finish_completed_run
+                    success, error, declared = scheduler._apply_agent_failure_marker(self.job, success, error, answer)
+                    delivery = scheduler._RunDelivery(self.job, success, error, agent_declared=declared)
+                    scheduler._save_compose_deliver(delivery, scheduler._FireOwnership(self.job), answer, output,
+                                                 adapters=None, loop=None, verbose=False, execution_token=None)
+                    fail_empty_response(delivery, answer)
+                    if not finish_completed_run(delivery, self.owner, self.execution_id, recovered=self.recovering):
+                        raise Unverified("A prior submitter still owns this execution's bookkeeping.")
+                    success, error = delivery.success, delivery.error
+            pause = self.record.get("pause")
+            if may_finish and pause:
+                from cron.scheduler_authority import _lift_pause
+                _lift_pause(self.job["id"], pause)
+            self.file.unlink()
+            self.settled = True
+        reason = self.record["cancel_reason"]
+        if not may_finish or reason == "claim_lost":
+            return {"job_id": self.job["id"], "status": "failed", "success": False, "error": str(_ClaimLost())}
+        if reason == "cancelled" and not success:
+            raise RpcError("cancelled", condition="cancelled")
+        if reason and not success:
+            return {"job_id": self.job["id"], "status": "failed", "success": False, "error": reason}
+        return {"job_id": self.job["id"], "status": "completed", "success": bool(success), "error": error}
+
+    def _finish_ledger(self, success, error):
+        from cron import executions
+        finish = executions.recover_receipted_execution if self.recovering else executions.finish_execution
+        kwargs = {"job_id": self.job["id"]} if self.recovering else {}
+        finish(self.execution_id, success=success, error=error, **kwargs)
+
+
+def run(job, home, execution_home, ctx, owner, *, recovery_only=False):
+    from . import cron
+    # Refuse before prepare/cancel can rewrite retained evidence or cleanup can enter a removed owner API.
+    if (recovery_only or path(home, job["id"]).exists()) and not supported({}, recovery=True):
+        raise Unverified("This Agent has an unsupported recovery protocol. Reconcile the retained manual task "
+                         "with a capable Agent before retrying. No task was resubmitted or settled.")
+    execution = ManualExecution(home, execution_home, job, owner)
+    with execution.acquire():
+        with cron._RUNNING_LOCK:
+            cron._RUNNING.setdefault(job["id"], time.time())
+        try:
+            execution.prepare(recovery_only=recovery_only)
+            if recovery_only:
+                ctx.emit("started", {"job_id": job["id"]})
+            heartbeat = cron._heartbeat(home, execution.job["id"], execution.owner) if execution.owner else None
+            if execution.record.get("result") is not None:
+                result = execution.record["result"]
+            else:
+                result = cron._run_in_child(execution.job, execution.execution_home, ctx, heartbeat, execution)
+            return execution.finish(result)
+        except Exception as exc:
+            if execution.settled:
+                raise
+            if execution.record is not None and hasattr(execution, "execution_id"):
+                execution.cancel(execution.record["cancel_reason"] or "submitter interrupted")
+                # The Web record, not the Agent's generic orphan replay, now owns cancellation.
+                # Give up only our own audit row; this never releases or stamps a fire claim.
+                with scoped_home(execution.execution_home):
+                    from cron.scheduler_authority import journal_path
+                    from cron.executions import give_up_execution
+                    journal_path(job["id"], execution.execution_id).unlink(missing_ok=True)
+                    give_up_execution(execution.execution_id, reason=str(exc))
+            if isinstance(exc, Unverified):
+                raise
+            raise Unverified(str(exc)) from exc
+        finally:
+            # The durable record guards an unverified owner; this map guards only an active call.
+            with cron._RUNNING_LOCK:
+                cron._RUNNING.pop(job["id"], None)

package/sidecar/talaria_sidecar/methods/gateway.py

@@ -1,9 +1,10 @@
-"""``gateway.*``: Hermes gateway lifecycle through the ``hermes`` CLI (ported from api/gateway_restart.py)."""
+"""``gateway.*``: Hermes gateway lifecycle (start, stop, restart) through the ``hermes`` CLI."""
 
 from __future__ import annotations
 
 import logging
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -12,11 +13,22 @@ import time
 from pathlib import Path
 
 from ..errors import InvalidParams
-from ..home import profile_home_param
+from ..home import _is_named_profile, profile_home_param
 from ..rpc import CallContext
+from .profiles import _bounded, gateway_mode
 
 log = logging.getLogger("talaria_sidecar.gateway")
-_RESTART_LOCK = threading.Lock()
+# One lifecycle action at a time across every action: overlapping `hermes gateway` subprocesses race each other.
+_ACTION_LOCK = threading.Lock()
+ACTIONS = ("start", "stop", "restart")
+_DONE = {"start": "started", "stop": "stopped", "restart": "restarted"}
+# EX_CONFIG: Agent v0.21.6 refuses a named profile's own start/restart with it, since the host gateway serves every profile.
+_REFUSED = 78
+_SERVED_WAIT_SECONDS = 15.0
+# A restart waits out the CLI's own exit budget plus this headroom; 240 s on an Agent without the budget helper.
+_RESTART_HEADROOM_SECONDS = 30.0
+_RESTART_FALLBACK_SECONDS = 240.0
+_ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
 
 
 def _hermes_command() -> str:
@@ -39,68 +51,213 @@ def _drain(stream, sink: list[str]) -> None:
             pass
 
 
-def restart(home: Path, cli_profile: str | None, ctx: CallContext, *, quick_timeout: float, background_wait: float) -> dict:
-    if not _RESTART_LOCK.acquire(blocking=False):
-        return {"status": "busy", "message": "Restart already in progress. Please wait a moment and try again."}
+def _reason(stdout: str, stderr: str) -> str | None:
+    """The Agent's first error line: its first ``✗`` line, else the first line it wrote; one bounded printable line without colour codes."""
+    def lines(text: str) -> list[str]:
+        return [line.strip() for line in _ANSI.sub("", text).splitlines() if line.strip()]
+
+    first = next((line for line in lines(f"{stdout}\n{stderr}") if line.startswith("✗")), None) or next(iter(lines(stderr or stdout)), "")
+    return _bounded(first.lstrip("✗ "), 300) or None
+
+
+def _host_root(home: Path) -> Path | None:
+    """The Hermes root of a ``<root>/profiles/<name>`` home, as the Agent's ``profile_name_for_home`` reads a named profile."""
+    return home.parent.parent if home.parent.name == "profiles" and not home.name.startswith(".") and home.name != "default" else None
+
+
+def _host_serves(home: Path, budget_deadline: float) -> bool:
+    """True once the host gateway serves the named profile at ``home`` (TAL-805 ``served``); the host may still be coming up."""
+    deadline = min(time.monotonic() + _SERVED_WAIT_SECONDS, budget_deadline)
+    while True:
+        try:
+            if gateway_mode(home, home.name, False) == "served":
+                return True
+        except Exception:  # noqa: BLE001
+            log.debug("gateway mode probe failed for %s", home, exc_info=True)
+            return False
+        if time.monotonic() >= deadline:
+            return False
+        time.sleep(0.5)
+
+
+def control(action: str, home: Path, cli_profile: str | None, ctx: CallContext, *, quick_timeout: float, background_wait: float) -> dict:
+    if action not in ACTIONS:
+        raise InvalidParams("unsupported gateway action")
+    if not _ACTION_LOCK.acquire(blocking=False):
+        return {"status": "busy", "message": "Another gateway action is already in progress. Please wait a moment and try again."}
     try:
+        # One budget for the whole action, host fallback included: each step gets only what is left of it.
+        deadline = time.monotonic() + quick_timeout + background_wait
+
+        def run(action: str, home: Path, cli_profile: str | None) -> dict:
+            left = deadline - time.monotonic()
+            if left <= 0:
+                return {"status": "failed", "message": f"Gateway {action} ran out of time before it could start", "timed_out": True}
+            quick = min(quick_timeout, left)
+            return _run(action, home, cli_profile, ctx, quick_timeout=quick, background_wait=left - quick)
+
+        result = run(action, home, cli_profile)
+        root = _host_root(home)
+        # TAL-884: Agent v0.21.6 gives a named profile no gateway of its own. Start the host gateway from the root profile
+        # (`--profile default`, so a sticky active profile cannot retarget it), then rerun the profile's action, which
+        # serves or unparks it there. Never `--force`: that grows the per-profile fleet member the Agent refuses.
+        if action == "stop" or root is None or result.get("returncode") != _REFUSED:
+            return result
+        log.info("Gateway %s refused for %s; starting the host gateway from %s", action, home, root)
+        host = run("start", root, "default")
+        if host["status"] != "completed":
+            return host
+        result = run(action, home, cli_profile)
+        if result.get("returncode") == _REFUSED and _host_serves(home, deadline):
+            return {"status": "completed", "message": f"Gateway service {_DONE[action]} successfully", "detail": result.get("detail", "")}
+        return result
+    finally:
+        _ACTION_LOCK.release()
+
+
+def _profile_selector(home: Path) -> str | None:
+    """The profile the Agent names ``home`` (``default`` for the root; None for no profile home): a selector-less verb would
+    follow the sticky ``active_profile``. Raises ImportError on an Agent without ``profile_name_for_home``."""
+    from hermes_constants import profile_name_for_home
+
+    return profile_name_for_home(home)
+
+
+def _child_env(home: Path, profile: str | None, action: str) -> dict:
+    """The CLI's environment. The sidecar's ``os.environ`` carries its launch profile's ``.env`` once chat ran, so only the
+    launch home's own root-profile action keeps it; any other profile starts scrubbed of it and loads its own ``.env``."""
+    if profile is None or (profile == "default" and not _is_named_profile(home)):
         env = dict(os.environ)
         env["HERMES_HOME"] = str(home)
-        env.pop("HERMES_EXEC_ASK", None)  # the sidecar's own approval posture, not the gateway's
-        cmd = [_hermes_command()]
-        if cli_profile:
-            cmd.extend(["--profile", cli_profile])
-        cmd.extend(["gateway", "restart"])
-        log.info("Restarting gateway via %s (HERMES_HOME=%s)", " ".join(cmd), home)
+    else:
+        from hermes_cli.web_server_gateway import _profile_action_environment
+
+        env = _profile_action_environment(["--profile", profile, "gateway", action])
+    env.pop("HERMES_EXEC_ASK", None)  # the sidecar's own approval posture, not the gateway's
+    return env
+
+
+# The CLI's own restart budget, read as the CLI reads it: in the child's environment once its home's .env has loaded.
+_BUDGET_SCRIPT = (
+    "import os, sys; from pathlib import Path; sys.path.insert(0, sys.argv[1]); "
+    "from hermes_cli.env_loader import load_hermes_dotenv; "
+    "load_hermes_dotenv(hermes_home=os.environ['HERMES_HOME'], project_env=Path(sys.argv[1]) / '.env'); "
+    "from hermes_cli.gateway import _get_restart_exit_wait_budget; print(_get_restart_exit_wait_budget())"
+)
+
+
+def _agent_restart_budget(env: dict) -> float:
+    import hermes_cli
+
+    agent_dir = str(Path(hermes_cli.__file__).resolve().parent.parent)
+    out = subprocess.run([sys.executable, "-c", _BUDGET_SCRIPT, agent_dir], env=env, capture_output=True, text=True, timeout=60, check=True).stdout
+    return float(out.split()[-1])
+
+
+def _restart_wait(home: Path, cli_profile: str | None = None) -> float:
+    """The Agent CLI's own restart wait for ``home`` plus headroom, so Web never kills a restart the CLI still waits on."""
+    try:
+        profile = cli_profile or _profile_selector(home)
+        return _agent_restart_budget(_child_env(home, profile, "restart")) + _RESTART_HEADROOM_SECONDS
+    except Exception:  # noqa: BLE001 - an Agent without the helper, or an unreadable config, gets the historical wait
+        log.warning("Could not read the Agent's restart budget for %s", home, exc_info=True)
+        return _RESTART_FALLBACK_SECONDS
+
+
+def _run(action: str, home: Path, cli_profile: str | None, ctx: CallContext, *, quick_timeout: float, background_wait: float) -> dict:
+    """One ``hermes --profile <name> gateway <action>`` subprocess for ``home``; the caller holds ``_ACTION_LOCK``."""
+    try:
+        profile = cli_profile or _profile_selector(home)
+    except ImportError:
+        # Selector-less, the CLI would follow the sticky profile, and a named profile's would inherit the launch profile's credentials.
+        return {"status": "failed", "message": f"Gateway {action} refused: the installed Hermes Agent cannot name the profile at {home}"}
+    try:
+        env = _child_env(home, profile, action)
+    except Exception as exc:  # noqa: BLE001 - never start a named profile's gateway with another profile's credentials
+        log.warning("Could not build the %s profile's gateway environment", profile, exc_info=True)
+        return {"status": "failed", "message": f"Could not isolate the {profile} profile's environment: {getattr(exc, 'detail', None) or exc}"}
+    cmd = [_hermes_command()]
+    if profile:
+        cmd.extend(["--profile", profile])
+    cmd.extend(["gateway", action])
+    log.info("Running gateway %s via %s (HERMES_HOME=%s)", action, " ".join(cmd), home)
+    try:
         proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env)
-        # Drain both pipes from the start: a chatty restart must never block on a full pipe and be mistaken for a hang.
-        out: list[str] = []
-        err: list[str] = []
-        drainers = [threading.Thread(target=_drain, args=(proc.stdout, out), daemon=True), threading.Thread(target=_drain, args=(proc.stderr, err), daemon=True)]
+    except FileNotFoundError:
+        return {"status": "failed", "message": "Hermes CLI not found"}
+    # Drain both pipes from the start: a chatty restart must never block on a full pipe and be mistaken for a hang.
+    out: list[str] = []
+    err: list[str] = []
+    drainers = [threading.Thread(target=_drain, args=(proc.stdout, out), daemon=True), threading.Thread(target=_drain, args=(proc.stderr, err), daemon=True)]
+    for t in drainers:
+        t.start()
+
+    def collected() -> tuple[str, str]:
         for t in drainers:
-            t.start()
-
-        def collected() -> tuple[str, str]:
-            for t in drainers:
-                t.join(timeout=5)
-            return "".join(out).strip(), "".join(err).strip()
-
-        def outcome() -> dict:
-            stdout, stderr = collected()
-            if proc.returncode == 0:
-                return {"status": "completed", "message": "Gateway service restarted successfully", "detail": stdout or stderr}
-            return {"status": "failed", "message": f"Restart failed: {stderr or stdout}", "detail": stdout or stderr, "returncode": proc.returncode}
-
-        ctx.emit("progress", {"phase": "started"})
-        deadline = time.monotonic() + quick_timeout
-        while proc.poll() is None and time.monotonic() < deadline:
-            time.sleep(0.1)
-        if proc.poll() is not None:
-            return outcome()
-        ctx.emit("progress", {"phase": "draining"})
-        deadline = time.monotonic() + background_wait
-        while proc.poll() is None and time.monotonic() < deadline:
-            if ctx.cancelled:
-                break
-            time.sleep(0.5)
-        if proc.poll() is None:
-            proc.terminate()
-            try:
-                proc.wait(timeout=5)
-            except subprocess.TimeoutExpired:
-                proc.kill()
-                proc.wait(timeout=5)
-            collected()
-            return {"status": "failed", "message": f"Gateway restart process timed out after {background_wait:.0f}s and was terminated"}
+            t.join(timeout=5)
+        return "".join(out).strip(), "".join(err).strip()
+
+    def outcome() -> dict:
+        stdout, stderr = collected()
+        if proc.returncode == 0:
+            return {"status": "completed", "message": f"Gateway service {_DONE[action]} successfully", "detail": stdout or stderr}
+        failed = {"status": "failed", "message": f"{action.capitalize()} failed: {stderr or stdout}", "detail": stdout or stderr, "returncode": proc.returncode}
+        reason = _reason(stdout, stderr)
+        return {**failed, "reason": reason} if reason else failed
+
+    ctx.emit("progress", {"phase": "started"})
+    deadline = time.monotonic() + quick_timeout
+    while proc.poll() is None and time.monotonic() < deadline:
+        time.sleep(0.1)
+    if proc.poll() is not None:
         return outcome()
-    finally:
-        _RESTART_LOCK.release()
+    ctx.emit("progress", {"phase": "draining"})
+    deadline = time.monotonic() + background_wait
+    while proc.poll() is None and time.monotonic() < deadline:
+        if ctx.cancelled:
+            break
+        time.sleep(0.5)
+    if proc.poll() is None:
+        proc.terminate()
+        try:
+            proc.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait(timeout=5)
+        collected()
+        if ctx.cancelled:
+            return {"status": "failed", "message": f"Gateway {action} was cancelled and its process terminated", "timed_out": True}
+        return {"status": "failed", "message": f"Gateway {action} process timed out after {background_wait:.0f}s and was terminated", "timed_out": True}
+    return outcome()
 
 
 def register(registry) -> None:
-    @registry.method("gateway.restart", requires_agent=False)
-    def restart_(ctx: CallContext, params: dict) -> dict:
+    @registry.method("gateway.session")
+    def session_(ctx: CallContext, params: dict) -> dict:
+        from .gateway_sessions import session
+
+        return session(ctx, params)
+
+    def run(ctx: CallContext, params: dict, action: str) -> dict:
         home = profile_home_param(params)
         cli_profile = params.get("cli_profile")
         if cli_profile is not None and not isinstance(cli_profile, str):
             raise InvalidParams("cli_profile must be a string")
-        return restart(home, cli_profile or None, ctx, quick_timeout=float(params.get("quick_timeout_seconds") or 2.0), background_wait=float(params.get("background_wait_seconds") or 240.0))
+        # A restart drains the old gateway first, for as long as the CLI's own budget allows (infinite: no deadline).
+        wait = params.get("background_wait_seconds") or (_restart_wait(home, cli_profile or None) if action == "restart" else 60.0)
+        return control(action, home, cli_profile or None, ctx, quick_timeout=float(params.get("quick_timeout_seconds") or 2.0), background_wait=float(wait))
+
+    @registry.method("gateway.control", requires_agent=False)
+    def control_(ctx: CallContext, params: dict) -> dict:
+        return run(ctx, params, str(params.get("action") or ""))
+
+    @registry.method("gateway.restart", requires_agent=False)
+    def restart_(ctx: CallContext, params: dict) -> dict:
+        return run(ctx, params, "restart")
+
+    @registry.method("gateway.mode")
+    def mode_(ctx: CallContext, params: dict) -> dict:
+        name = params.get("name")
+        if not isinstance(name, str) or not name.strip():
+            raise InvalidParams("name is required")
+        return {"gateway_mode": gateway_mode(profile_home_param(params), name, params.get("is_default") is True)}