@maudecode/talaria-web 1.19.0 → 1.21.0

package/sidecar/talaria_sidecar/agent_services.py

@@ -11,6 +11,7 @@ import hashlib
 import http.client
 import ipaddress
 import json
+import math
 import os
 from pathlib import Path
 import plistlib
@@ -27,6 +28,42 @@ import time
 STOP_SECONDS = 30.0
 READY_SECONDS = 30.0
 SNAPSHOT_ATTEMPTS = 4
+# Bounds a native drain configured to wait indefinitely; the retained debt outlives it.
+GATEWAY_DRAIN_CEILING = 4 * 3600.0
+# One orphan-recovery attempt must fit its 660-second recovery RPC; its debt drives the next attempt.
+RECOVERY_DRAIN_SECONDS = 300.0
+
+
+@contextlib.contextmanager
+def _native_home(home):
+    """Resolve native per-profile configuration for one home in this single-purpose worker."""
+    previous = os.environ.get('HERMES_HOME')
+    os.environ['HERMES_HOME'] = str(home)
+    try:
+        from hermes_constants import reset_hermes_home_override, set_hermes_home_override
+    except ImportError:  # Older Agents resolve the home from the environment alone.
+        set_hermes_home_override = None
+    token = set_hermes_home_override(home) if set_hermes_home_override else None
+    try:
+        yield
+    finally:
+        if token is not None:
+            reset_hermes_home_override(token)
+        if previous is None:
+            os.environ.pop('HERMES_HOME', None)
+        else:
+            os.environ['HERMES_HOME'] = previous
+
+
+def drain_budget(pause, homes, ceiling=GATEWAY_DRAIN_CEILING):
+    """One shared drain budget: the longest native after-turn + stop-drain budget among the homes, bounded."""
+    budget = 0.0
+    for home in sorted({str(home) for home in homes}):
+        with _native_home(home):
+            native = float(pause._drain_budget())
+        # A native wait-indefinitely budget (inf or NaN) takes the ceiling before any comparison.
+        budget = max(budget, native if math.isfinite(native) else ceiling)
+    return min(budget, ceiling)
 
 
 @contextlib.contextmanager
@@ -127,12 +164,15 @@ def snapshot(entry, *, records=None, expected=None):
     from gateway.status import get_process_start_time, _pid_exists
     service, pid = entry.get('service_pid', entry['pid']), entry['pid']
     anchors = {service, pid}
-    observed = dict(expected or {})
+    observed = {**entry.get('departed_processes', {}), **(expected or {})}
     seen_parents = {int(p): proof['parent'] for p, proof in observed.items() if 'parent' in proof}
-    departed = set()
+    departed = set(map(int, entry.get('departed_processes', {})))
 
     def exited(p):
-        if _pid_exists(p) is False:
+        live = _pid_exists(p)
+        if live is None:
+            raise RuntimeError(f'{_name(entry)} process {p} liveness is unreadable; source was not changed.')
+        if live is False:
             if p in anchors:
                 raise RuntimeError(f'{_name(entry)} anchor process {p} disappeared during discovery; source was not changed.')
             departed.add(p)
@@ -142,7 +182,7 @@ def snapshot(entry, *, records=None, expected=None):
     def inventory():
         parents = _process_table()
         if service not in _ancestors(pid, parents):
-            raise RuntimeError(f'{_name(entry)} does not own backend/gateway PID {pid}.')
+            raise RuntimeError(f'{_name(entry)} identity/ancestry changed: does not own backend/gateway PID {pid}.')
         if service in _ancestors(os.getpid(), parents):
             raise RuntimeError(f'{_name(entry)} owns the updater or an ancestor; source was not changed.')
         for p, parent in seen_parents.items():
@@ -151,6 +191,10 @@ def snapshot(entry, *, records=None, expected=None):
                     continue
                 raise RuntimeError(f'{_name(entry)} process {p} identity/ancestry changed during discovery; source was not changed.')
         descendants = _descendants(service, parents)
+        # Successor retries cannot discard a live writer from a previously observed tree.
+        for p in seen_parents.keys() - descendants - departed:
+            if not exited(p):
+                raise RuntimeError(f'{_name(entry)} process {p} remains live outside the attested tree; recovery remains owed.')
         seen_parents.update({p: parents[p] for p in descendants})
         # Zombies may remain in ps. Recheck departed PIDs at every inventory so
         # neither a recycled PID nor unknown liveness can be silently excluded.
@@ -168,6 +212,8 @@ def snapshot(entry, *, records=None, expected=None):
             if exited(p):
                 return None
             raise RuntimeError(f'{_name(entry)} process {p} birth identity is live or unreadable; source was not changed.')
+        if exited(p):
+            return None
         if any(proof.get(key) != value for key, value in observed.get(str(p), {}).items()):
             raise RuntimeError(f'{_name(entry)} process {p} identity/ancestry changed during discovery; source was not changed.')
         observed[str(p)] = proof
@@ -182,6 +228,8 @@ def snapshot(entry, *, records=None, expected=None):
         _, final = inventory()
         if live & departed or remaining != live or final != live:
             continue
+        if departed:
+            entry['departed_processes'] = {str(p): observed.get(str(p), {'parent': seen_parents[p]}) for p in departed}
         return proofs
     raise RuntimeError(f'{_name(entry)} process inventory kept changing after {SNAPSHOT_ATTEMPTS} samples; retry when descendants settle. Source was not changed and recovery remains owed.')
 
@@ -366,8 +414,23 @@ def _service_pid(entry):
 
 def verify_processes(entry, records, *, stopping=False, fenced=False):
     from gateway.status import get_process_start_time, _pid_exists
+    if not stopping:
+        # Reuse discovery's bounded proof without granting permission to lose an anchor.
+        candidate = dict(entry)
+        for boundary in range(2):
+            _, service_pid = _service_pid(entry)
+            if service_pid != entry['service_pid']:
+                raise RuntimeError(f'{_name(entry)} supervisor respawned during quiescence; source was not changed.')
+            if 'config_stamp' in entry and _config_stamp(entry) != entry['config_stamp']:
+                raise RuntimeError(f'{_name(entry)} service configuration changed; recovery remains owed.')
+            if boundary == 0:
+                candidate['processes'] = snapshot(candidate, records=records, expected=entry['processes'])
+        entry.update(processes=candidate['processes'])
+        if candidate.get('departed_processes'):
+            entry['departed_processes'] = candidate['departed_processes']
+        return True
     parents = _process_table()
-    departed = set()
+    departed = set(map(int, entry.get('departed_processes', {})))
     for pid, expected in entry['processes'].items():
         pid = int(pid)
         if _pid_exists(pid) is False:
@@ -405,7 +468,6 @@ def verify_processes(entry, records, *, stopping=False, fenced=False):
 
 def bind_processes(entry, records):
     if entry.get('processes'):
-        entry['processes'] = snapshot(entry, records=records, expected=entry['processes'])
         verify_processes(entry, records)
 
 
@@ -640,6 +702,10 @@ def _start(entry, pause):
         raise RuntimeError(f'{_name(entry)} supervisor failed to start its backend.')
 
 
+class GatewayDraining(RuntimeError):
+    """An acknowledged native drain is still within its native budget: progress, not a recovery failure."""
+
+
 class ServiceRecovery:
     """Companion to the native pause token; all calls hold its existing checkout lock."""
     def __init__(self, root, home, records, identity, pause):
@@ -672,8 +738,20 @@ class ServiceRecovery:
                 or ('runtime_revision' in transaction and not re.fullmatch(r'[a-f0-9]{40}', str(transaction['runtime_revision'])))):
             raise RuntimeError('Service recovery transaction has an unknown identity or format; its evidence is retained.')
 
+        operation = self.state.get('update_operation')
+        if operation is not None and (
+                not isinstance(operation, dict)
+                or not re.fullmatch(r'[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}', str(operation.get('operation_id', '')))
+                or any(not re.fullmatch(r'[a-f0-9]{40}', str(operation.get(key, ''))) for key in ('baseline_revision', 'target_revision'))
+                or ('attempt_error' in operation and (not isinstance(operation['attempt_error'], str) or len(operation['attempt_error']) > 2000))
+                or ('source_moved' in operation and not isinstance(operation['source_moved'], bool))
+                or ('failed_stage' in operation and operation['failed_stage'] not in {'install', 'gateway_restart', 'agent_ready'})
+                or operation.get('stage') not in {'install', 'gateway_restart', 'agent_ready'}
+                or ('recovered_revision' in operation and not re.fullmatch(r'[a-f0-9]{40}', str(operation['recovered_revision'])))):
+            raise RuntimeError('Update operation correlation is malformed; its evidence is retained.')
+
     def save(self, token=None):
-        if not self.state['entries'] and not self.state.get('sync_failed') and not self.state.get('transaction'):
+        if not self.state['entries'] and not self.state.get('sync_failed') and not self.state.get('transaction') and not self.state.get('update_operation'):
             if self.path.exists():
                 self.path.unlink()
                 self._sync_directory()
@@ -693,9 +771,9 @@ class ServiceRecovery:
             with contextlib.suppress(FileNotFoundError):
                 os.unlink(temporary)
 
-    def _sync_directory(self):
+    def _sync_directory(self, directory=None):
         if os.name != 'nt':
-            fd = os.open(self.path.parent, os.O_RDONLY)
+            fd = os.open(directory if directory is not None else self.path.parent, os.O_RDONLY)
             try:
                 os.fsync(fd)
             finally:
@@ -726,10 +804,17 @@ class ServiceRecovery:
             self.records.write(token)
             self.save(token)
 
+    def verify_processes(self, entry, **permissions):
+        owed = next(e for e in self.state['entries'] if _service_key(e) == _service_key(entry))
+        result = verify_processes(owed, self.records, **permissions)
+        self.save()  # Updated custody must precede the next lifecycle effect.
+        return result
+
     def stop(self, entry, stopped):
         owed = next(e for e in self.state['entries'] if _service_key(e) == _service_key(entry))
-        verify_processes(owed, self.records)
+        self.verify_processes(owed)
         backend_request(owed)
+        self.verify_processes(owed)
         permit = backend_request(owed, 'prepare')
         if permit.get('ok') is not True or permit.get('idle') is not True or not isinstance(permit.get('token'), str):
             if owed['phase'] == 'recorded':
@@ -742,7 +827,7 @@ class ServiceRecovery:
         except BaseException:
             backend_request(owed, 'cancel', permit['token'])
             raise
-        verify_processes(owed, self.records)
+        self.verify_processes(owed)
         if backend_request(owed, 'commit', permit['token']).get('ok') is not True:
             raise RuntimeError(f'{_name(owed)} did not commit its backend retirement; recovery remains owed.')
         owed['backend_committed'] = True
@@ -753,14 +838,14 @@ class ServiceRecovery:
         # Disarm through the original manager before its KeepAlive can admit a successor.
         owed['backend_stop_requested'] = True
         self.save()
-        verify_processes(owed, self.records, stopping=stopping)
+        self.verify_processes(owed, stopping=stopping)
         if _config_stamp(owed) != owed['config_stamp']:
             raise RuntimeError(f'{_name(owed)} service configuration changed before backend stop; recovery remains owed.')
         if stopped.is_set():
             raise RuntimeError(f'{_name(owed)} backend stop was cancelled; recovery remains owed.')
         self.pause._disarm(service_entry(owed))
         wait_stopped(owed, self.records, stopped)
-        verify_processes(owed, self.records, stopping=True)
+        self.verify_processes(owed, stopping=True)
         owed['phase'] = 'stopped'
         self.save()
 
@@ -770,15 +855,15 @@ class ServiceRecovery:
             raise RuntimeError(f'{_name(entry)} legacy shutdown remains owed; no signal was replayed.')
         if _config_stamp(owed) != owed['config_stamp']:
             raise RuntimeError(f'{_name(entry)} configuration changed before legacy refresh.')
-        verify_processes(owed, self.records)
+        self.verify_processes(owed)
         verify_listener(owed)
         owed.update(legacy_refresh=True, phase='refreshing')
         self.save()  # Older readers reject this phase and retain debt; no reader replays its signal.
-        verify_processes(owed, self.records)
+        self.verify_processes(owed)
         verify_listener(owed)
         if _config_stamp(owed) != owed['config_stamp']:
             raise RuntimeError(f'{_name(owed)} configuration changed before legacy refresh.')
-        verify_processes(owed, self.records)
+        self.verify_processes(owed)
         try:
             if stopped.is_set():
                 raise InterruptedError('Legacy refresh was cancelled before interruption.')
@@ -825,7 +910,7 @@ class ServiceRecovery:
         owed = next(e for e in self.state['entries'] if _service_key(e) == _service_key(entry))
         owed['phase'] = 'retiring'
         if acknowledged:
-            owed['drain_acknowledged'] = True
+            owed.update(drain_acknowledged=True, drain_acknowledged_at=time.time())
         self.save()
 
     def _gateway_candidate(self, entry):
@@ -842,6 +927,8 @@ class ServiceRecovery:
         if (_config_stamp(candidate) != entry['config_stamp']
                 or candidate.get('profile') != entry.get('profile')):
             raise RuntimeError(f'{_name(entry)} successor configuration/profile changed; recovery remains owed.')
+        if entry.get('departed_processes'):
+            candidate['departed_processes'] = entry['departed_processes']
         bind_processes(candidate, self.records)
         parents = {int(pid): p['parent'] for pid, p in candidate['processes'].items()}
         chain, pid = set(), candidate['pid']
@@ -853,7 +940,8 @@ class ServiceRecovery:
         if chain | _descendants(candidate['pid'], parents) != set(parents):
             raise RuntimeError(f'{_name(entry)} successor has unrelated descendants; recovery remains owed.')
         for pid, process in candidate['processes'].items():
-            old = entry['processes'].get(pid) or entry.get('retired_processes', {}).get(pid)
+            old = (entry['processes'].get(pid) or entry.get('departed_processes', {}).get(pid)
+                   or entry.get('retired_processes', {}).get(pid))
             if old is not None and old != process:
                 raise RuntimeError(f'{_name(entry)} successor reused a recorded process identity; recovery remains owed.')
         return candidate
@@ -864,14 +952,27 @@ class ServiceRecovery:
                    for pid, p in processes.items()
                    if int(pid) != entry['service_pid'] or int(pid) == entry['pid'])
 
-    def gateway_deadline(self):
-        return time.monotonic() + min(self.pause._drain_budget(), 300)
+    def gateway_deadline(self, homes=None, ceiling=GATEWAY_DRAIN_CEILING):
+        if homes is None:
+            homes = [e['home'] for e in self.state['entries'] if e['purpose'] == 'gateway']
+        return time.monotonic() + drain_budget(self.pause, homes, ceiling)
+
+    def _drain_progressing(self, entry):
+        """Live recorded writers still finishing an acknowledged drain within the native budget."""
+        if not entry.get('drain_acknowledged') or not self._gateway_writers_live(entry):
+            return False
+        now = time.time()
+        if not isinstance(entry.get('drain_acknowledged_at'), (int, float)):
+            entry['drain_acknowledged_at'] = now  # Older custody has no time; its native budget starts when first observed.
+            self.save()
+        return 0 <= now - entry['drain_acknowledged_at'] < drain_budget(self.pause, [entry['home']])
 
     def settle_gateway(self, entry, stopped, *, deadline=None):
         owed = next(e for e in self.state['entries'] if _service_key(e) == _service_key(entry))
         self._settle_gateway(owed, stopped, deadline=deadline)
 
-    def _settle_gateway(self, entry, stopped=None, *, deadline=None):
+    def _settle_gateway(self, entry, stopped=None, *, deadline=None, ready=None):
+        """Stop the recorded gateway; True when ``ready`` adopts a verified successor after the drain instead."""
         if not self._gateway_writers_live(entry) and not any(_service_pid(entry)):
             if _config_stamp(entry) != entry['config_stamp']:
                 raise RuntimeError(f'{_name(entry)} service configuration changed; recovery remains owed.')
@@ -880,9 +981,11 @@ class ServiceRecovery:
             return
         stopped = stopped or threading.Event()
         if deadline is None:
-            deadline = self.gateway_deadline()
+            deadline = self.gateway_deadline([entry['home']], RECOVERY_DRAIN_SECONDS)
         while True:
             if stopped.is_set() or time.monotonic() >= deadline:
+                if not stopped.is_set() and self._drain_progressing(entry):
+                    raise GatewayDraining(f'{_name(entry)} gateway drain remains owed while its acknowledged native drain finishes; recovery will retry.')
                 raise RuntimeError(f'{_name(entry)} gateway drain remains owed; retry Agent update recovery.')
             if _config_stamp(entry) != entry['config_stamp']:
                 raise RuntimeError(f'{_name(entry)} service configuration changed; recovery remains owed.')
@@ -891,6 +994,8 @@ class ServiceRecovery:
                     raise RuntimeError(f'{_name(entry)} gateway drain remains owed; retry Agent update recovery.')
                 stopped.wait(.05)
                 continue
+            if ready is not None and ready():
+                return True  # The original drain finished under its supervisor; never drain or stop a ready successor.
             entry['runtime_fence_requested'] = True
             self.save()
             with gateway_fence(entry) as fence:
@@ -908,7 +1013,7 @@ class ServiceRecovery:
                         if pending:
                             stopped.wait(.05)
                             continue
-                        if not verify_processes(entry, self.records, stopping=True, fenced=True):
+                        if not self.verify_processes(entry, stopping=True, fenced=True):
                             stopped.wait(.05)
                             continue
                         if self._gateway_writers_live(entry) or _config_stamp(entry) != entry['config_stamp']:
@@ -929,14 +1034,14 @@ class ServiceRecovery:
                     raise RuntimeError(f'{_name(entry)} repeatedly respawned during drain; recovery remains owed.')
                 retired = {**entry.get('retired_processes', {}), **entry['processes']}
                 entry.update(candidate, retired_processes=retired, phase='retiring',
-                             drain_acknowledged=False, settle_attempts=attempts)
+                             drain_acknowledged=False, drain_acknowledged_at=None, settle_attempts=attempts)
                 self.save()  # Successor custody precedes its cooperative lifecycle request.
-                verify_processes(entry, self.records)
+                self.verify_processes(entry)
                 if _config_stamp(entry) != entry['config_stamp']:
                     raise RuntimeError(f'{_name(entry)} service configuration changed; recovery remains owed.')
                 if not self.pause._ask_to_drain(entry):
                     raise RuntimeError(f'{_name(entry)} successor did not acknowledge cooperative drain; recovery remains owed.')
-                entry['drain_acknowledged'] = True
+                entry.update(drain_acknowledged=True, drain_acknowledged_at=time.time())
                 self.save()
             stopped.wait(.05)
 
@@ -946,7 +1051,7 @@ class ServiceRecovery:
         whole, why = self.records.tree_is_whole(self.state['tree'], self.root)
         if not whole or self.state.get('sync_failed'):
             raise RuntimeError('Backend recovery remains owed until source/dependency repair completes: ' + why)
-        failures = []
+        failures, draining = [], True
         for entry in list(self.state['entries']):
             try:
                 if self._ready(entry, ledger):
@@ -963,8 +1068,11 @@ class ServiceRecovery:
                     continue
                 if entry['purpose'] == 'gateway':
                     if gateway_deadline is None:
-                        gateway_deadline = self.gateway_deadline()
-                    self._settle_gateway(entry, deadline=gateway_deadline)
+                        gateway_deadline = self.gateway_deadline(ceiling=RECOVERY_DRAIN_SECONDS)
+                    if self._settle_gateway(entry, deadline=gateway_deadline, ready=lambda entry=entry: self._ready(entry, ledger)):
+                        self.state['entries'].remove(entry)
+                        self.save()
+                        continue
                     if not _service_pid(entry)[1]:
                         _start(entry, self.pause)
                     self._wait_ready(entry, ledger)
@@ -975,7 +1083,7 @@ class ServiceRecovery:
                         raise RuntimeError(f'{_name(entry)} uncommitted retirement could not yet be cancelled; recovery remains owed.')
                     self.stop(entry, threading.Event())
                 elif any(self.records.identity_is_live({'pid': int(pid), 'ct': p['ct']}) for pid, p in entry['processes'].items()):
-                    verify_processes(entry, self.records, stopping=True)
+                    self.verify_processes(entry, stopping=True)
                     if any(self.records.identity_is_live({'pid': int(pid), 'ct': p['ct']}) for pid, p in entry['processes'].items() if int(pid) != entry['service_pid']):
                         raise RuntimeError(f'{_name(entry)} still has live writer descendants; recovery remains owed.')
                     self._stop_backend_supervisor(entry, threading.Event(), stopping=True)
@@ -984,9 +1092,11 @@ class ServiceRecovery:
                     _start(entry, self.pause)
                 self._wait_ready(entry, ledger)
             except Exception as exc:
+                draining = draining and isinstance(exc, GatewayDraining)
                 failures.append(str(exc))
         if failures:
-            raise RuntimeError('; '.join(failures))
+            # Sibling obligations recover independently; only an all-draining remainder is progress.
+            raise (GatewayDraining if draining else RuntimeError)('; '.join(failures))
 
     def _wait_ready(self, entry, ledger):
         deadline = time.monotonic() + READY_SECONDS
@@ -999,6 +1109,18 @@ class ServiceRecovery:
                 raise RuntimeError(f'{_name(entry)} readiness is unverified at its original endpoint; recovery remains owed.')
             time.sleep(.1)
 
+    def _verify_candidate(self, candidate, entry):
+        # Readiness retries must retain both live observations and proved departures.
+        candidate['config_stamp'] = entry['config_stamp']
+        candidate['processes'] = {**candidate['processes'], **entry.get('readiness_processes', {})}
+        candidate['departed_processes'] = {**entry.get('departed_processes', {}),
+                                            **candidate.get('departed_processes', {})}
+        bind_processes(candidate, self.records)
+        entry['readiness_processes'] = candidate['processes']
+        if candidate['departed_processes']:
+            entry['departed_processes'] = candidate['departed_processes']
+        self.save()
+
     def _ready(self, entry, ledger):
         if _config_stamp(entry) != entry['config_stamp']:
             raise RuntimeError(f'{_name(entry)} service configuration changed; backend recovery remains owed.')
@@ -1012,8 +1134,9 @@ class ServiceRecovery:
             candidate = self._gateway_candidate(entry)
             if candidate is None:
                 return False
+            self._verify_candidate(candidate, entry)
             ready = _gateway_home_matches(entry['home'], (candidate['pid'], candidate['ct']), self.pause)
-            verify_processes(candidate, self.records)
+            self._verify_candidate(candidate, entry)
             return ready
         if entry.get('legacy_refresh') and any(self.records.identity_is_live({'pid': int(pid), 'ct': p['ct']})
                 for pid, p in entry['processes'].items()
@@ -1032,7 +1155,7 @@ class ServiceRecovery:
         if (candidate['profile'] != entry['profile'] or _service_key(candidate) != _service_key(entry)
                 or _config_stamp(candidate) != entry['config_stamp']):
             return False
-        bind_processes(candidate, self.records)
+        self._verify_candidate(candidate, entry)
         try:
             if entry.get('legacy_refresh'):
                 verify_listener(candidate)
@@ -1043,7 +1166,7 @@ class ServiceRecovery:
                     return False
                 if backend_request(candidate, 'cancel', permit['token']).get('ok') is not True:
                     return False
-            verify_processes(candidate, self.records)
+            self._verify_candidate(candidate, entry)
             if self.identity._pid_alive_matches(row['pid'], row['create_time'], strict=True) is not True:
                 return False
         except BackendLifecycleUnsupported:

package/sidecar/talaria_sidecar/context_detail.py

@@ -0,0 +1,59 @@
+"""The shared allowlist for native/resident context reads. Never includes prompt bodies or paths."""
+from __future__ import annotations
+
+import math
+import re
+
+CATEGORIES = {'system_prompt', 'tool_definitions', 'rules', 'skills', 'mcp', 'subagent_definitions', 'memory', 'conversation'}
+STATUSES = {'loaded', 'truncated', 'flagged', 'shadowed', 'blocked', 'empty', 'unreadable', 'suppressed'}
+FILE_NAMES = {'AGENTS.md', 'SOUL.md', '.hermes.md', 'CLAUDE.md', 'GEMINI.md', 'CONTEXT.md'}
+
+
+def number(value):
+    return value if type(value) in (int, float) and 0 <= value <= 2**53 - 1 and math.isfinite(value) and int(value) == value else None
+
+
+def unavailable(reason='unavailable') -> dict:
+    return {'available': False, 'reason': reason, 'used_tokens': None, 'window_tokens': None,
+            'source': None, 'estimated': None, 'estimated_total': None, 'categories': [], 'files': [],
+            'truncated': False, 'current_turn_output_tokens': None, 'last_turn_output_tokens': None}
+
+
+def project(payload, *, usage=None, running=False) -> dict:
+    if not isinstance(payload, dict):
+        return unavailable('unsupported')
+    rows = payload.get('categories')
+    if not isinstance(rows, list) or not rows:
+        return unavailable('not_built')
+    maximum = number(payload.get('context_max')) or None
+    categories, seen = [], set()
+    for row in rows[:64]:
+        if not isinstance(row, dict) or not isinstance(row.get('id'), str) or row.get('id') not in CATEGORIES or row['id'] in seen:
+            continue
+        tokens = number(row.get('tokens'))
+        if tokens is None:
+            continue
+        seen.add(row['id'])
+        categories.append({'id': row['id'], 'tokens': int(tokens), 'percent': min(100, tokens / maximum * 100) if maximum else None, 'estimated': True})
+    if not categories:
+        return unavailable('not_built')
+    source = payload.get('context_source')
+    source = source if isinstance(source, str) and source in {'provider_usage', 'local_estimate', 'provider_usage_plus_estimate'} else None
+    files = payload.get('context_files')
+    files = files if isinstance(files, list) else []
+    safe_files = []
+    for row in files[:64]:
+        if not isinstance(row, dict):
+            continue
+        # Labels from owners can include the entire private directory. Keep known context basenames only.
+        name = re.split(r'[/\\]', str(row.get('path') or ''))[-1]
+        label = name if name in FILE_NAMES else row.get('label') if isinstance(row.get('label'), str) and row.get('label') in FILE_NAMES else 'Context file'
+        status = row.get('status') if isinstance(row.get('status'), str) and row.get('status') in STATUSES else 'unknown'
+        safe_files.append({'label': label, 'characters': number(row.get('chars')), 'tokens': number(row.get('est_tokens')),
+                           'status': status, 'loaded': status in {'loaded', 'truncated', 'flagged'} if status != 'unknown' else None})
+    output = number(usage.get('last_turn_tokens')) if isinstance(usage, dict) else None
+    return {'available': True, 'reason': None, 'used_tokens': number(payload.get('context_used')), 'window_tokens': maximum,
+            'source': source, 'estimated': source != 'provider_usage' if source else None,
+            'estimated_total': number(payload.get('estimated_total')), 'categories': categories, 'files': safe_files,
+            'truncated': len(rows) > 64 or len(files) > 64,
+            'current_turn_output_tokens': output if running else None, 'last_turn_output_tokens': output if not running else None}