@maudecode/talaria-web 1.19.0 → 1.20.0

package/sidecar/talaria_sidecar/agent_mutation.py

@@ -29,6 +29,8 @@ _SESSIONS = {}
 _SESSIONS_LOCK = threading.Lock()
 _CONSENTS = {}
 CONSENT_SECONDS = 300
+# One lease exchange, including the worker's unwind after a refused drain.
+LEASE_SECONDS = 330
 
 
 def _close(session):
@@ -119,7 +121,59 @@ def _native_source_repair(root):
         raise
 
 
-def recovery_receipts(runtime, *, watch=False):
+def _receipt_candidates(path, pattern, cache):
+    """Cache discovery, never receipt presence; missing paths watch their nearest existing ancestor."""
+    key = (path, pattern)
+    if cache is not None:
+        ancestor = path
+        while True:
+            try:
+                stat = ancestor.stat()
+                break
+            except FileNotFoundError:
+                if ancestor == ancestor.parent:
+                    raise
+                ancestor = ancestor.parent
+        stamp = (ancestor, stat.st_dev, stat.st_ino, stat.st_mode, stat.st_mtime_ns, stat.st_ctime_ns, stat.st_size)
+        now = time.monotonic()
+        previous = cache.get(key)
+        if previous is not None and previous[0] == stamp and 0 <= now - previous[1] < 1:
+            return previous[2]
+    if pattern is not None:
+        try:
+            # glob suppresses enumeration errors, which cannot authorize Agent admission.
+            with os.scandir(path) as children:
+                found = tuple(path / child.name for child in children if Path(child.name).match(pattern))
+        except FileNotFoundError:
+            found = ()
+    else:
+        try:
+            found = (path,) if any(' refs/talaria/autostash/pending/' in line for line in path.read_text().splitlines()) else ()
+        except FileNotFoundError:
+            found = ()
+    if cache is not None:
+        if len(cache) >= 16:
+            cache.clear()  # Bound retained paths if an installer repeatedly redirects Git metadata.
+        cache[key] = (stamp, now, found)
+    return found
+
+
+def _process_presence(pid):
+    """A stdlib rearm trigger only; native identity verification alone decides recovery."""
+    if os.name == 'nt' or pid <= 0:
+        return 'unknown'
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return 'exited'
+    except PermissionError:
+        pass
+    except (OSError, OverflowError):
+        return 'unknown'
+    return 'present'
+
+
+def recovery_receipts(runtime, *, watch=False, discovery_cache=None):
     """Read only the configured checkout's receipts without importing its possibly torn code."""
     root, home = runtime.agent_dir, runtime.hermes_home
     if root is None:
@@ -133,19 +187,15 @@ def recovery_receipts(runtime, *, watch=False):
     paths = [services.recovery_path(root, home), marker, native,
              root / '.update-incomplete', root / '.lazy-refresh-incomplete',
              gitdir / 'hermes-update-pull', root / '.hermes-update-zip-swap']
-    paths.extend(home.glob(native.name + '.*.claim'))
+    paths.extend(_receipt_candidates(home, native.name + '.*.claim', discovery_cache))
     common = gitdir
     try:
         common = (gitdir / (gitdir / 'commondir').read_text().strip()).resolve()
     except FileNotFoundError:
         pass
-    paths.extend((common / 'refs/talaria/autostash/pending').glob('*'))
+    paths.extend(_receipt_candidates(common / 'refs/talaria/autostash/pending', '*', discovery_cache))
     packed = common / 'packed-refs'
-    try:
-        if any(' refs/talaria/autostash/pending/' in line for line in packed.read_text().splitlines()):
-            paths.append(packed)
-    except FileNotFoundError:
-        pass
+    paths.extend(_receipt_candidates(packed, None, discovery_cache))
     def stamps(candidates):
         result = []
         for path in sorted(set(candidates)):
@@ -169,15 +219,23 @@ def recovery_receipts(runtime, *, watch=False):
                     watched.append(common / ref)
         except (OSError, UnicodeError):
             pass
+        lifecycle = []
         try:
             body = json.loads(services.recovery_path(root, home).read_text())
             for baseline in body.get('tree', {}).get('baselines', []):
                 changed = [*baseline.get('dirty_at_pause', []), *baseline.get('move_paths', [])]
                 changed.extend(path for paths in baseline.get('landed', {}).values() for path in paths)
                 watched.extend(root / path for path in changed if isinstance(path, str) and not Path(path).is_absolute() and '..' not in Path(path).parts)
+            # A retained gateway's exit or successor rearms exhausted recovery without changing any receipt.
+            for entry in body.get('entries', []):
+                if entry.get('purpose') == 'gateway' and isinstance(entry.get('home'), str):
+                    watched.append(Path(entry['home']) / 'gateway.pid')
+                    pids = {**entry.get('retired_processes', {}), **entry.get('processes', {})}
+                    lifecycle.extend(('process', int(pid), _process_presence(int(pid))) for pid in sorted(pids, key=int))
         except (OSError, ValueError, TypeError, AttributeError):
             pass  # The receipt's own stamp still rearms corrected evidence; the worker reports its exact refusal.
         found.extend(stamps(watched))
+        found.extend(lifecycle)
     return found, marker
 
 
@@ -203,6 +261,8 @@ def recover_retained(runtime, ctx, *, probe=False, complete_revision=None):
     env.update(HERMES_HOME=str(runtime.hermes_home), HERMES_DISABLE_LAZY_INSTALLS='1',
                PYTHONPATH=str(Path(__file__).parents[1]) + os.pathsep + str(root))
     try:
+        if complete_revision is None:
+            runtime.retire()
         with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr:
             child = subprocess.Popen([sys.executable, '-m', 'talaria_sidecar.agent_mutation', '--recover-current', str(root), complete_revision or ''],
                                      cwd=root, env=env, stdin=subprocess.PIPE, stdout=stdout, stderr=stderr)
@@ -237,12 +297,22 @@ def recover_retained(runtime, ctx, *, probe=False, complete_revision=None):
         _ACTION_LOCK.release()
 
 
+def _drain_progress(result):
+    progress = result.get("progress") if isinstance(result, dict) and result.keys() == {"progress"} else None
+    if (not isinstance(progress, dict) or progress.get("phase") != "gateway_drain"
+            or not isinstance(progress.get("deadline_seconds"), (int, float)) or isinstance(progress["deadline_seconds"], bool)
+            or not 0 <= progress["deadline_seconds"] <= services.GATEWAY_DRAIN_CEILING
+            or not isinstance(progress.get("services"), list) or not all(isinstance(name, str) for name in progress["services"])):
+        return None
+    return {"phase": "gateway_drain", "deadline_seconds": progress["deadline_seconds"], "services": progress["services"]}
+
+
 def _exchange(session, request, ctx):
     with session["lock"]:
         if request is not None:
             session["proc"].stdin.write(json.dumps(request) + "\n")
             session["proc"].stdin.flush()
-        deadline = time.monotonic() + 330
+        deadline = time.monotonic() + LEASE_SECONDS
         while True:
             ctx.check_cancelled()
             if time.monotonic() >= deadline:
@@ -254,6 +324,14 @@ def _exchange(session, request, ctx):
             if line is None:
                 raise RpcError("Agent writer lease exited; its pause journal retains recovery.", condition="agent_update_unsafe")
             result = json.loads(line)
+            if isinstance(result, dict) and "progress" in result:
+                progress = _drain_progress(result)
+                if progress is None:
+                    raise RpcError("Invalid Agent writer lease response.", condition="agent_update_unsafe")
+                # The worker's own drain deadline is bounded; unwind keeps the ordinary lease margin.
+                deadline = max(deadline, time.monotonic() + progress["deadline_seconds"] + LEASE_SECONDS)
+                ctx.emit("progress", progress)
+                continue
             if not isinstance(result, dict) or not isinstance(result.get("ok"), bool):
                 raise RpcError("Invalid Agent writer lease response.", condition="agent_update_unsafe")
             return {"ok": result["ok"], "skipped": result.get("skipped") is True, "detail": str(result.get("detail") or ""),
@@ -358,6 +436,7 @@ def call(runtime, ctx, params):
                 _CONSENTS[nonce].update(binding=confirmation['binding'], expires=time.monotonic() + CONSENT_SECONDS)
             result['refresh_confirmation'] = {'token': nonce, 'services': confirmation['services']}
         if action == "begin" and result["ok"]:
+            runtime.retire()
             return {**result, "token": token}
         if action != "finish" and result["ok"]:
             return result
@@ -583,7 +662,11 @@ def _worker(root, revision, homes, requests, stopped, send, refresh_binding=None
         records.mark_stop_requested(token, sorted({pid for e in gateway_entries for pid in (e['pid'], e.get('service_pid', e['pid']))}))
         for entry in backend_entries:
             recovery.stop(entry, stopped)
-        gateway_deadline = recovery.gateway_deadline()
+        # Every profile shares the longest native drain; the lease reports it so its RPC keeps custody.
+        gateway_deadline = recovery.gateway_deadline(homes)
+        if gateway_entries:
+            send({'progress': {'phase': 'gateway_drain', 'deadline_seconds': round(max(gateway_deadline - time.monotonic(), 0), 3),
+                               'services': sorted({services._name(e) for e in gateway_entries})}})
         for entry in gateway_entries:
             if stopped.is_set() or time.monotonic() >= gateway_deadline:
                 raise RuntimeError('Gateway drain has not completed; its durable resume obligation is retained.')
@@ -708,6 +791,8 @@ def _worker(root, revision, homes, requests, stopped, send, refresh_binding=None
         error = str(exc)
     finally:
         try:
+            # Cancellation or a lost lease pipe ends the drain wait; the retained debt keeps the gateway owed.
+            unwind_deadline = time.monotonic() if stopped.is_set() else gateway_deadline
             if token:
                 live = any(records.identity_is_live({"pid": int(pid), "ct": ct}) for pid, ct in token.get("identities", {}).items())
                 found = _command(modules["update_custody"].git_argv(["git"], ["rev-parse", "HEAD"]), root, 10, threading.Event(), modules["update_custody"])
@@ -720,7 +805,7 @@ def _worker(root, revision, homes, requests, stopped, send, refresh_binding=None
                 if live and not changed and not sync_failed and records.tree_is_whole(token, root)[0]:
                     # An untouched gateway must not strand a backend stopped earlier in a partial pause.
                     recovery.save(token)
-                    recovery.resume(_writer_ledger, gateway_deadline=gateway_deadline)
+                    recovery.resume(_writer_ledger, gateway_deadline=unwind_deadline)
                 if live or sync_failed or (changed and needs_sync and not locals().get("response", {}).get("ok")):
                     raise RuntimeError("Gateway resume remains owed until writer drain and dependency repair complete.")
                 whole, why = records.tree_is_whole(token, root)
@@ -742,7 +827,7 @@ def _worker(root, revision, homes, requests, stopped, send, refresh_binding=None
                 if token.get("resume_needed") or token.get("resume_deferred"):
                     raise RuntimeError("Gateway resume could not be verified; its obligation is retained.")
                 recovery.save(token)
-                recovery.resume(_writer_ledger, gateway_deadline=gateway_deadline)
+                recovery.resume(_writer_ledger, gateway_deadline=unwind_deadline)
             if token and not handed_off:
                 records.discharge(token)
                 from .runtime import AgentRuntime
@@ -763,16 +848,19 @@ def _worker(root, revision, homes, requests, stopped, send, refresh_binding=None
 
 
 def _resume_current(root, custody, stopped=None, complete_revision=None):
+    # Resolve target PM code in a fresh stdlib process, after any source/dependency change.
     try:
-        from hermes_cli._launchers import resolve_store_python
-        python = resolve_store_python(root)
+        launchers = importlib.import_module('hermes_cli._launchers')
+        if not Path(launchers.__file__).resolve().is_relative_to(root):
+            raise RuntimeError('Recovery loaded another Agent bootstrap selector.')
+        bootstrap = launchers.resolve_store_python(root)
     except ModuleNotFoundError as exc:
-        if exc.name != "hermes_cli._launchers":
+        if exc.name != 'hermes_cli._launchers':
             raise
-        python = None
-    if python is None:
-        python = next((path for path in (root / ".venv/bin/python", root / "venv/bin/python", root / "venv/Scripts/python.exe") if path.is_file()), Path(sys.executable))
-    result = _command([str(python), "-m", "talaria_sidecar.agent_mutation", "--resume-leased", str(root), complete_revision or ""], root, 660, stopped or threading.Event(), custody, cooperative=True)
+        bootstrap = None
+    # A copied macOS venv interpreter can be its own _base_executable and later be retired.
+    bootstrap = bootstrap or getattr(sys, '_base_executable', None) or sys.executable
+    result = _command([str(bootstrap), "-m", "talaria_sidecar.agent_mutation", "--select-resume", str(root), complete_revision or "", sys.executable], root, 660, stopped or threading.Event(), custody, cooperative=True)
     try:
         reply = json.loads(result["detail"]) if result["ok"] else {}
         return {**reply, "ok": result["ok"] and reply.get("ok") is True, "detail": str(reply.get("detail") or result["detail"])}
@@ -780,6 +868,46 @@ def _resume_current(root, custody, stopped=None, complete_revision=None):
         return {"ok": False, "detail": "The current Agent could not verify its retained gateway resume obligation."}
 
 
+def _recovery_environment(root):
+    """Reuse PM's selector and packaged launch policy without copying its facts/path rules."""
+    sys.path.insert(0, str(root))
+    environments = importlib.import_module('pm.environments')
+    if any(not Path(sys.modules[name].__file__).resolve().is_relative_to(root)
+           for name in ('pm', 'pm.environments', 'hermes_constants')):
+        raise RuntimeError('Recovery loaded another Agent environment selector.')
+    selected = environments.committed_venv(root)
+    if selected is None:
+        if environments.install_state_dir(root).exists():
+            raise RuntimeError('No dependency environment is committed for this Agent install; recovery remains owed.')
+        selected = environments.selected_venv(root)
+    return environments, selected
+
+
+def _select_resume(root, complete_revision, legacy_python):
+    if (root / 'pm').exists():
+        environments, selected = _recovery_environment(root)
+        command = environments.venv_command(root, selected)
+        environment = str(selected)
+    else:
+        python = next((path for venv in ('.venv', 'venv')
+                       for path in (root / venv / 'bin/python', root / venv / 'Scripts/python.exe') if path.is_file()), Path(legacy_python))
+        command, environment = [str(python)], ''
+    # Keep bin/python intact: realpath would turn a POSIX venv into its bare tool.
+    argv = [*command, '-m', 'talaria_sidecar.agent_mutation', '--resume-leased', str(root), complete_revision or '', environment]
+    os.execv(argv[0], argv)
+
+
+def _verify_recovery_environment(root, expected):
+    from .runtime import _check_site_packages_python
+    environments, selected = _recovery_environment(root)
+    if str(selected) != expected or environments.venv_python_version(selected) != sys.version_info[:2]:
+        raise RuntimeError('Agent recovery dependency environment changed or has an incompatible Python ABI.')
+    site = environments.site_packages(selected)
+    if not site.is_dir() or site.resolve() not in {Path(entry).resolve() for entry in sys.path if entry}:
+        raise RuntimeError('Agent recovery did not enter its selected dependency environment.')
+    _check_site_packages_python()
+
+
 def _resume_worker(root, stopped=None, complete_revision=None):
     sys.path.insert(0, str(root))
     from hermes_cli import update_lock, update_pause_record as records, process_identity, update_cmd_posix_pause
@@ -833,9 +961,10 @@ def _resume_worker(root, stopped=None, complete_revision=None):
                 if response.get('ok') is not True:
                     return {'ok': False, 'detail': 'Recorded dependency repair failed: ' + str(response.get('detail') or synced['detail'])}
                 recovery.state['sync_failed'] = False
-                if transaction:
-                    transaction['dependencies_verified'] = True
+                transaction = recovery.state.setdefault('transaction', {'revision': revision, 'homes': [str(runtime.hermes_home)]})
+                transaction['dependencies_verified'] = True
                 recovery.save(tree)
+                return _resume_current(root, custody, stopped, complete_revision)
             elif recovery.state.get('sync_failed'):
                 return {'ok': False, 'detail': 'The recorded dependency repair module is missing.'}
         whole, why = records.tree_is_whole(tree, root)
@@ -850,7 +979,10 @@ def _resume_worker(root, stopped=None, complete_revision=None):
         if not Path(main_module.__file__).resolve().is_relative_to(root):
             raise RuntimeError("Gateway recovery loaded another Agent runtime.")
         records.recover(argv=[])
-        recovery.resume(_writer_ledger)
+        try:
+            recovery.resume(_writer_ledger)
+        except services.GatewayDraining as exc:
+            return {'ok': False, 'draining': True, 'detail': str(exc)}
         owed = records.record_path().exists() or bool(records._claims(records.record_path())) or bool(recovery.state['entries'])
         runtime_pending = bool((marker and marker.exists()) or (recovery.state.get('transaction') or {}).get('runtime_revision'))
         if not owed and runtime_pending:
@@ -926,9 +1058,11 @@ def _command(argv, root, timeout, stopped, custody, *, cooperative=False):
 def main():
     for key in ("GIT_DIR", "GIT_WORK_TREE", "GIT_COMMON_DIR", "GIT_INDEX_FILE"):
         os.environ.pop(key, None)
-    if len(sys.argv) > 2 and sys.argv[1] in {'--recover-current', '--resume', '--resume-leased'}:
+    if len(sys.argv) > 2 and sys.argv[1] in {'--recover-current', '--resume', '--resume-leased', '--select-resume'}:
         try:
             _native_source_repair(Path(sys.argv[2]).resolve())
+            if sys.argv[1] == '--select-resume':
+                _select_resume(Path(sys.argv[2]).resolve(), sys.argv[3], sys.argv[4])
         except BaseException as exc:
             print(json.dumps({'ok': False, 'detail': str(exc) if str(exc) != '1' else 'Native recorded source repair could not complete; its receipt remains owed.'}), flush=True)
             return
@@ -962,6 +1096,8 @@ def main():
             return
         if sys.argv[1] in {"--resume", "--resume-leased"}:
             with contextlib.redirect_stdout(sys.stderr):
+                if len(sys.argv) > 4 and sys.argv[4]:
+                    _verify_recovery_environment(Path(sys.argv[2]).resolve(), sys.argv[4])
                 reply = _resume_worker(Path(sys.argv[2]).resolve(), stopped if sys.argv[1] == "--resume-leased" else None, sys.argv[3] if len(sys.argv) > 3 and sys.argv[3] else None)
             send(reply)
             return

package/sidecar/talaria_sidecar/agent_services.py

@@ -11,6 +11,7 @@ import hashlib
 import http.client
 import ipaddress
 import json
+import math
 import os
 from pathlib import Path
 import plistlib
@@ -27,6 +28,42 @@ import time
 STOP_SECONDS = 30.0
 READY_SECONDS = 30.0
 SNAPSHOT_ATTEMPTS = 4
+# Bounds a native drain configured to wait indefinitely; the retained debt outlives it.
+GATEWAY_DRAIN_CEILING = 4 * 3600.0
+# One orphan-recovery attempt must fit its 660-second recovery RPC; its debt drives the next attempt.
+RECOVERY_DRAIN_SECONDS = 300.0
+
+
+@contextlib.contextmanager
+def _native_home(home):
+    """Resolve native per-profile configuration for one home in this single-purpose worker."""
+    previous = os.environ.get('HERMES_HOME')
+    os.environ['HERMES_HOME'] = str(home)
+    try:
+        from hermes_constants import reset_hermes_home_override, set_hermes_home_override
+    except ImportError:  # Older Agents resolve the home from the environment alone.
+        set_hermes_home_override = None
+    token = set_hermes_home_override(home) if set_hermes_home_override else None
+    try:
+        yield
+    finally:
+        if token is not None:
+            reset_hermes_home_override(token)
+        if previous is None:
+            os.environ.pop('HERMES_HOME', None)
+        else:
+            os.environ['HERMES_HOME'] = previous
+
+
+def drain_budget(pause, homes, ceiling=GATEWAY_DRAIN_CEILING):
+    """One shared drain budget: the longest native after-turn + stop-drain budget among the homes, bounded."""
+    budget = 0.0
+    for home in sorted({str(home) for home in homes}):
+        with _native_home(home):
+            native = float(pause._drain_budget())
+        # A native wait-indefinitely budget (inf or NaN) takes the ceiling before any comparison.
+        budget = max(budget, native if math.isfinite(native) else ceiling)
+    return min(budget, ceiling)
 
 
 @contextlib.contextmanager
@@ -640,6 +677,10 @@ def _start(entry, pause):
         raise RuntimeError(f'{_name(entry)} supervisor failed to start its backend.')
 
 
+class GatewayDraining(RuntimeError):
+    """An acknowledged native drain is still within its native budget: progress, not a recovery failure."""
+
+
 class ServiceRecovery:
     """Companion to the native pause token; all calls hold its existing checkout lock."""
     def __init__(self, root, home, records, identity, pause):
@@ -825,7 +866,7 @@ class ServiceRecovery:
         owed = next(e for e in self.state['entries'] if _service_key(e) == _service_key(entry))
         owed['phase'] = 'retiring'
         if acknowledged:
-            owed['drain_acknowledged'] = True
+            owed.update(drain_acknowledged=True, drain_acknowledged_at=time.time())
         self.save()
 
     def _gateway_candidate(self, entry):
@@ -864,14 +905,27 @@ class ServiceRecovery:
                    for pid, p in processes.items()
                    if int(pid) != entry['service_pid'] or int(pid) == entry['pid'])
 
-    def gateway_deadline(self):
-        return time.monotonic() + min(self.pause._drain_budget(), 300)
+    def gateway_deadline(self, homes=None, ceiling=GATEWAY_DRAIN_CEILING):
+        if homes is None:
+            homes = [e['home'] for e in self.state['entries'] if e['purpose'] == 'gateway']
+        return time.monotonic() + drain_budget(self.pause, homes, ceiling)
+
+    def _drain_progressing(self, entry):
+        """Live recorded writers still finishing an acknowledged drain within the native budget."""
+        if not entry.get('drain_acknowledged') or not self._gateway_writers_live(entry):
+            return False
+        now = time.time()
+        if not isinstance(entry.get('drain_acknowledged_at'), (int, float)):
+            entry['drain_acknowledged_at'] = now  # Older custody has no time; its native budget starts when first observed.
+            self.save()
+        return 0 <= now - entry['drain_acknowledged_at'] < drain_budget(self.pause, [entry['home']])
 
     def settle_gateway(self, entry, stopped, *, deadline=None):
         owed = next(e for e in self.state['entries'] if _service_key(e) == _service_key(entry))
         self._settle_gateway(owed, stopped, deadline=deadline)
 
-    def _settle_gateway(self, entry, stopped=None, *, deadline=None):
+    def _settle_gateway(self, entry, stopped=None, *, deadline=None, ready=None):
+        """Stop the recorded gateway; True when ``ready`` adopts a verified successor after the drain instead."""
         if not self._gateway_writers_live(entry) and not any(_service_pid(entry)):
             if _config_stamp(entry) != entry['config_stamp']:
                 raise RuntimeError(f'{_name(entry)} service configuration changed; recovery remains owed.')
@@ -880,9 +934,11 @@ class ServiceRecovery:
             return
         stopped = stopped or threading.Event()
         if deadline is None:
-            deadline = self.gateway_deadline()
+            deadline = self.gateway_deadline([entry['home']], RECOVERY_DRAIN_SECONDS)
         while True:
             if stopped.is_set() or time.monotonic() >= deadline:
+                if not stopped.is_set() and self._drain_progressing(entry):
+                    raise GatewayDraining(f'{_name(entry)} gateway drain remains owed while its acknowledged native drain finishes; recovery will retry.')
                 raise RuntimeError(f'{_name(entry)} gateway drain remains owed; retry Agent update recovery.')
             if _config_stamp(entry) != entry['config_stamp']:
                 raise RuntimeError(f'{_name(entry)} service configuration changed; recovery remains owed.')
@@ -891,6 +947,8 @@ class ServiceRecovery:
                     raise RuntimeError(f'{_name(entry)} gateway drain remains owed; retry Agent update recovery.')
                 stopped.wait(.05)
                 continue
+            if ready is not None and ready():
+                return True  # The original drain finished under its supervisor; never drain or stop a ready successor.
             entry['runtime_fence_requested'] = True
             self.save()
             with gateway_fence(entry) as fence:
@@ -929,14 +987,14 @@ class ServiceRecovery:
                     raise RuntimeError(f'{_name(entry)} repeatedly respawned during drain; recovery remains owed.')
                 retired = {**entry.get('retired_processes', {}), **entry['processes']}
                 entry.update(candidate, retired_processes=retired, phase='retiring',
-                             drain_acknowledged=False, settle_attempts=attempts)
+                             drain_acknowledged=False, drain_acknowledged_at=None, settle_attempts=attempts)
                 self.save()  # Successor custody precedes its cooperative lifecycle request.
                 verify_processes(entry, self.records)
                 if _config_stamp(entry) != entry['config_stamp']:
                     raise RuntimeError(f'{_name(entry)} service configuration changed; recovery remains owed.')
                 if not self.pause._ask_to_drain(entry):
                     raise RuntimeError(f'{_name(entry)} successor did not acknowledge cooperative drain; recovery remains owed.')
-                entry['drain_acknowledged'] = True
+                entry.update(drain_acknowledged=True, drain_acknowledged_at=time.time())
                 self.save()
             stopped.wait(.05)
 
@@ -946,7 +1004,7 @@ class ServiceRecovery:
         whole, why = self.records.tree_is_whole(self.state['tree'], self.root)
         if not whole or self.state.get('sync_failed'):
             raise RuntimeError('Backend recovery remains owed until source/dependency repair completes: ' + why)
-        failures = []
+        failures, draining = [], True
         for entry in list(self.state['entries']):
             try:
                 if self._ready(entry, ledger):
@@ -963,8 +1021,11 @@ class ServiceRecovery:
                     continue
                 if entry['purpose'] == 'gateway':
                     if gateway_deadline is None:
-                        gateway_deadline = self.gateway_deadline()
-                    self._settle_gateway(entry, deadline=gateway_deadline)
+                        gateway_deadline = self.gateway_deadline(ceiling=RECOVERY_DRAIN_SECONDS)
+                    if self._settle_gateway(entry, deadline=gateway_deadline, ready=lambda entry=entry: self._ready(entry, ledger)):
+                        self.state['entries'].remove(entry)
+                        self.save()
+                        continue
                     if not _service_pid(entry)[1]:
                         _start(entry, self.pause)
                     self._wait_ready(entry, ledger)
@@ -984,9 +1045,11 @@ class ServiceRecovery:
                     _start(entry, self.pause)
                 self._wait_ready(entry, ledger)
             except Exception as exc:
+                draining = draining and isinstance(exc, GatewayDraining)
                 failures.append(str(exc))
         if failures:
-            raise RuntimeError('; '.join(failures))
+            # Sibling obligations recover independently; only an all-draining remainder is progress.
+            raise (GatewayDraining if draining else RuntimeError)('; '.join(failures))
 
     def _wait_ready(self, entry, ledger):
         deadline = time.monotonic() + READY_SECONDS

package/sidecar/talaria_sidecar/context_detail.py

@@ -0,0 +1,59 @@
+"""The shared allowlist for native/resident context reads. Never includes prompt bodies or paths."""
+from __future__ import annotations
+
+import math
+import re
+
+CATEGORIES = {'system_prompt', 'tool_definitions', 'rules', 'skills', 'mcp', 'subagent_definitions', 'memory', 'conversation'}
+STATUSES = {'loaded', 'truncated', 'flagged', 'shadowed', 'blocked', 'empty', 'unreadable', 'suppressed'}
+FILE_NAMES = {'AGENTS.md', 'SOUL.md', '.hermes.md', 'CLAUDE.md', 'GEMINI.md', 'CONTEXT.md'}
+
+
+def number(value):
+    return value if type(value) in (int, float) and 0 <= value <= 2**53 - 1 and math.isfinite(value) and int(value) == value else None
+
+
+def unavailable(reason='unavailable') -> dict:
+    return {'available': False, 'reason': reason, 'used_tokens': None, 'window_tokens': None,
+            'source': None, 'estimated': None, 'estimated_total': None, 'categories': [], 'files': [],
+            'truncated': False, 'current_turn_output_tokens': None, 'last_turn_output_tokens': None}
+
+
+def project(payload, *, usage=None, running=False) -> dict:
+    if not isinstance(payload, dict):
+        return unavailable('unsupported')
+    rows = payload.get('categories')
+    if not isinstance(rows, list) or not rows:
+        return unavailable('not_built')
+    maximum = number(payload.get('context_max')) or None
+    categories, seen = [], set()
+    for row in rows[:64]:
+        if not isinstance(row, dict) or not isinstance(row.get('id'), str) or row.get('id') not in CATEGORIES or row['id'] in seen:
+            continue
+        tokens = number(row.get('tokens'))
+        if tokens is None:
+            continue
+        seen.add(row['id'])
+        categories.append({'id': row['id'], 'tokens': int(tokens), 'percent': min(100, tokens / maximum * 100) if maximum else None, 'estimated': True})
+    if not categories:
+        return unavailable('not_built')
+    source = payload.get('context_source')
+    source = source if isinstance(source, str) and source in {'provider_usage', 'local_estimate', 'provider_usage_plus_estimate'} else None
+    files = payload.get('context_files')
+    files = files if isinstance(files, list) else []
+    safe_files = []
+    for row in files[:64]:
+        if not isinstance(row, dict):
+            continue
+        # Labels from owners can include the entire private directory. Keep known context basenames only.
+        name = re.split(r'[/\\]', str(row.get('path') or ''))[-1]
+        label = name if name in FILE_NAMES else row.get('label') if isinstance(row.get('label'), str) and row.get('label') in FILE_NAMES else 'Context file'
+        status = row.get('status') if isinstance(row.get('status'), str) and row.get('status') in STATUSES else 'unknown'
+        safe_files.append({'label': label, 'characters': number(row.get('chars')), 'tokens': number(row.get('est_tokens')),
+                           'status': status, 'loaded': status in {'loaded', 'truncated', 'flagged'} if status != 'unknown' else None})
+    output = number(usage.get('last_turn_tokens')) if isinstance(usage, dict) else None
+    return {'available': True, 'reason': None, 'used_tokens': number(payload.get('context_used')), 'window_tokens': maximum,
+            'source': source, 'estimated': source != 'provider_usage' if source else None,
+            'estimated_total': number(payload.get('estimated_total')), 'categories': categories, 'files': safe_files,
+            'truncated': len(rows) > 64 or len(files) > 64,
+            'current_turn_output_tokens': output if running else None, 'last_turn_output_tokens': output if not running else None}