@vibelord/lanes 1.0.0 → 1.0.2

package/payload/lanes/libexec/statusd.py

@@ -1,18 +1,22 @@
 #!/usr/bin/env python3
 """lanes status daemon: observes lanes, never touches them.
 
-Every few seconds: find LIVE tmux servers via `ps` (dead sockets are never probed), run one read-only
-`list-panes -a` per live server that the registry references, and publish ~/.local/state/lanes/status.json
+Every few seconds: find LIVE tmux servers in each workspace container's own cgroup (`guard`, plain /proc reads;
+dead sockets are never probed), run one read-only `list-panes -a` per live server that the registry references —
+at most PROBE_WORKERS at once, the whole round cut at PROBE_TIMEOUT + PROBE_GRACE seconds, the servers the last
+round cut started first — and publish ~/.local/state/lanes/status.json
 atomically, plus the one-line ~/.local/state/lanes/health. The sidebar, `lanes status`, `lanes doctor` and the
 future Throttle web harness all read those files.
 
-It never INVENTS a state. "No answer" (a tmux, ps, docker or zmx call that timed out or failed on an overloaded
+It never INVENTS a state. "No answer" (a tmux, cgroup, docker or zmx read that timed out or failed on an overloaded
 host) is not an observation: the lane keeps its last published state, marked stale, for at most STALE_MAX seconds
 and is 'unknown' after that. 'absent' / 'missing' are only published when a server answered without the session,
 or positively does not exist.
 """
+import concurrent.futures
 import json
 import os
+import subprocess
 import sys
 import time
 from pathlib import Path
@@ -20,12 +24,19 @@ from pathlib import Path
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 import lanes_lib as L  # noqa: E402
 import session_actions as A  # noqa: E402
-import legacy_sessions as Legacy  # noqa: E402
+import agent_state as AS  # noqa: E402
+import guard as G  # noqa: E402
 
 INTERVAL = float(os.environ.get('LANES_STATUS_INTERVAL', '3'))
 SLOWDOWN = {'ok': 1, 'busy': 2, 'critical': 4}   # a host under pressure gets FEWER spawns from us, not the same
 HEARTBEAT = 15          # status.json is rewritten at least this often, so its age proves that statusd is alive
 PROBE_TIMEOUT = 5
+# Plan Task 16: the servers of one cycle are probed concurrently, never more than PROBE_WORKERS tmux clients at once,
+# and the cycle stops waiting PROBE_GRACE seconds after one PROBE_TIMEOUT: run_bounded's post-kill wait is 2 s, so a
+# probe still running then is a child the kernel cannot reap (D state) or a fork that has not even started. Its socket
+# is published as a 'timeout' (keep(): last state, stale) and the others' answers are not held back by it.
+PROBE_WORKERS = 8
+PROBE_GRACE = 2
 CONTAINER_REFRESH = 300  # docker is asked again this often as a safety net; liveness itself comes from /proc/<pid>
 CONTAINER_RETRY = 30     # ... and at most this often while a container is down or docker did not answer
 ACTIVE_WINDOW = 20  # seconds since last output for a lane to count as active
@@ -35,7 +46,12 @@ FIELDS = ['session_name', 'pane_pid', 'pane_dead', 'window_activity', 'pane_curr
           'session_id', 'session_created',
           # Alignment contract A4a: the group a session is attached onto (a Throttle Studio browser view is a
           # grouped session), and which pane of which window represents the session's state.
-          'session_group', 'pane_active', 'window_active']
+          'session_group', 'pane_active', 'window_active',
+          # Task 5 (`agent{}`): the window bell, the ids a Claude session file names its pane by
+          # (`<session>:@<win>.%<pane>`), and the pane title Codex writes its status into. APPENDED, so every
+          # column above keeps its index; the title is the one free-text column and comes LAST, so a tab inside it
+          # stays in the title (the row is split at most len(FIELDS) - 1 times) instead of shifting the ids.
+          'window_bell_flag', 'window_id', 'pane_id', 'pane_title']
 FORMAT = '\t'.join('#{' + f + '}' for f in FIELDS)
 VIEW_PREFIX = 'view-'   # a Throttle Studio browser attach: a grouped session named view-<studio>-<ch>
 
@@ -160,21 +176,40 @@ def board_named_by(name, boards):
     return None
 
 
-def live_tmux_sockets():
-    """Socket names (-L) of running tmux servers, from the process table only.
-    None = ps did not answer: the caller must then NOT conclude that any server is gone."""
-    rc, out, _err = L.run_bounded(['ps', '-eo', 'comm=,args='], 10)
-    if rc != 0:
+def live_by_container(containers, errors=None):
+    """{container: socket names (-L) of the tmux servers running in it | None}, read from each container's own cgroup
+    (`guard.tmux_servers_by_container`: cgroup.procs + comm + cmdline, no subprocess). It replaced a host-wide
+    `ps -eo comm=,args=` per cycle — one fork that read every process of a 64-core host (plan Task 16).
+
+    This is statusd's own reading, PER CONTAINER: a container whose recorded init pid cannot be walked is None — only
+    ITS sockets are "nobody knows" (every socket file of that container is probed, none is concluded gone) — and the
+    other containers keep their real live sets (Task 16 review: one unwalkable pid used to collapse the WHOLE fleet's
+    reading to None). `errors` receives {container: why} for each None. A container without an init pid is absent
+    (it runs nothing; its lanes are 'missing' / kept by `probe_server`)."""
+    found = G.tmux_servers_by_container(G.Env(), containers or {}, errors)
+    return {name: (None if servers is None else set(servers)) for name, servers in found.items()}
+
+
+def live_tmux_sockets(containers):
+    """All live socket names of these containers as ONE set, or None when any container that has an init pid could
+    not be walked. All-or-nothing on purpose: `lanes invariant` (bin/lanes `all_panes`) must never save or compare a
+    PARTIAL pane list. statusd itself reads `live_by_container`, where one unwalkable container costs only its own."""
+    found = live_by_container(containers)
+    if any(servers is None for servers in found.values()):
         return None
-    names = set()
-    for line in out.splitlines():
-        if not line.startswith('tmux: server'):
-            continue
-        parts = line.split()
-        for index, part in enumerate(parts):
-            if part == '-L' and index + 1 < len(parts):
-                names.add(parts[index + 1])
-    return names
+    return {socket for servers in found.values() for socket in servers}
+
+
+def live_in(live, container):
+    """The live socket names of ONE container from statusd's `live`, or None = nobody knows (probe its socket files,
+    conclude nothing gone). `live` is `live_by_container`'s {container: set | None} (a container missing from it is
+    unknown, never empty), None (no reading at all), or a flat set, which applies to every container — the shape
+    `live_tmux_sockets` returns and the unit fixtures and `tests/cursor_lane.py` pass."""
+    if live is None:
+        return None
+    if isinstance(live, dict):
+        return live.get(container)
+    return live
 
 
 def keep(previous, lane_id, outcome):
@@ -191,15 +226,18 @@ def keep(previous, lane_id, outcome):
     # 'active' is the one state that is a claim about NOW (output in the last 20 s): never true of a cached entry,
     # and the one that readers which know nothing about 'stale' count and draw green. 'idle' is the seventh word
     # a pre-DEF-823 statusd published for a zmx lane: a status.json written by that build seeds `new_context()`
-    # across the upgrade, so it is normalised here rather than drawn as an unknown glyph by the sidebar.
+    # across the upgrade, so it is normalised here rather than drawn as an unknown glyph by the sidebar. (Plan
+    # Task 19 listed this line as dead weight; it stays because test_alignment's vocabulary gate pins it — removing
+    # it would mean weakening that assert.)
     state = 'quiet' if old['state'] in ('active', 'idle') else old['state']
     return dict(old, state=state, stale=True, observed=outcome, stale_since=since)
 
 
-def lane_entry(pane, now, activity=None, viewing=()):
+def lane_entry(pane, now, activity=None, viewing=(), agent=None):
     """One live session's published row. `activity` is the NEWEST `window_activity` across the session's panes
     (see `newest_activity`) and `viewing` the rows of the sessions that VIEW it (see `split_viewers`); both
-    default to "this pane alone", which is what a caller with a single row has."""
+    default to "this pane alone", which is what a caller with a single row has. `agent` is `agent_state.merge`'s
+    verdict for the session's pane: published as the additive `agent` key, absent when no signal names an agent."""
     if activity is None:
         try:
             activity = int(pane['window_activity'])
@@ -224,16 +262,110 @@ def lane_entry(pane, now, activity=None, viewing=()):
              'tty': pane['pane_tty']}
     if viewing:   # skipped when 0, so a lane nobody views publishes byte-identically to before A4a
         entry['viewers'] = len(viewing)
+    if agent:     # absent = no agent: a plain shell's row is byte-identical to before Task 5
+        entry['agent'] = agent
     return entry
 
 
+def agent_signals(containers, sessions_by_server):
+    """The per-container inputs `agent_state.merge` needs, read ONCE per container per cycle through the
+    container's root as the host sees it (`/proc/<init pid>/root`), so liveness is judged in the container's pid
+    namespace and nothing is spawned. -> {container: claude_by_pane}.
+
+    Also sets `pane_has_child` on the session rows that need it, in place: only a pane whose title Codex wrote
+    (`parse_codex_title`) over a bare-shell foreground can be a harness that ended into its shell, and only the
+    child read (`child_liveness`) tells that apart from a live harness launched under `bash -c` or a wrapper. A pane
+    a live Claude session file already claims is not probed (the file outranks the title). The key is set only for
+    a known answer: unknown liveness leaves it absent, and an absent key never demotes a title (DEF-1476). A
+    container whose files cannot be read costs only its own agents, never a lane its `state`."""
+    claude = {}
+    for container in sorted({container for container, _socket in sessions_by_server}):
+        info = containers.get(container) or {}
+        if not info.get('pid'):
+            continue
+        root = Path(L.proc_root()) / str(info['pid']) / 'root'
+        try:
+            claude[container] = AS.claude_sessions(root / L.CONTAINER_HOME.lstrip('/') / '.claude' / 'sessions',
+                                                   root / 'proc')
+        except Exception as error:
+            log(f'agents {container}', error)
+            claude[container] = {}
+        probe = [pane for (owner, _socket), sessions in sessions_by_server.items() if owner == container
+                 for pane in sessions.values()
+                 if str(pane.get('pane_pid') or '').isdigit() and AS.parse_codex_title(pane.get('pane_title'))
+                 and AS._shell_foreground(pane)
+                 and f"{pane.get('session_name')}:{pane.get('window_id')}.{pane.get('pane_id')}" not in claude[container]]
+        if not probe:
+            continue
+        try:
+            live = AS.child_liveness(root / 'proc', [pane['pane_pid'] for pane in probe])
+        except Exception as error:
+            log(f'liveness {container}', error)
+            continue
+        for pane in probe:
+            known = live.get(int(pane['pane_pid']))
+            if known is not None:
+                pane['pane_has_child'] = known
+    return claude
+
+
+SIDECAR_MAX_AGE = 3600   # a hook sidecar whose last event is older than this loses to the live signals
+# The hook sidecars' own directory under the container home. NOT `.lz/state` itself: lane-run writes each zmx lane's
+# `{phase, ...}` record there as `<zmx name>.json` and exports that name as LANE_ID, so a shared name would let a hook
+# overwrite the record `observe_zmx_container` and the container `lane ls` read (DEF-1436).
+SIDECAR_DIR = ('.lz', 'state', 'hooks')
+
+
+def sidecar_dir(info):
+    return Path(L.proc_root()) / str(info['pid']) / 'root' / L.CONTAINER_HOME.lstrip('/') / Path(*SIDECAR_DIR)
+
+
+def sidecar_for(info, lane_id, now=None):
+    """The `lane-signal` hook sidecar of one lane (Task 8): `~/.lz/state/hooks/<lane id>.json` in the container, read
+    through `/proc/<container init pid>/root` like the Claude session files, judged by `agent_state.read_sidecar_dict`
+    in the container's pid namespace (a dead or recycled agent pid, or a last event older than SIDECAR_MAX_AGE, is
+    None)."""
+    if not info.get('pid'):
+        return None
+    root = Path(L.proc_root()) / str(info['pid']) / 'root'
+    return AS.read_sidecar_dict(L.read_json(sidecar_dir(info) / f'{lane_id}.json'),
+                                time.time() if now is None else now, SIDECAR_MAX_AGE, proc_root=root / 'proc')
+
+
+def agent_sidecars(containers, wanted, now=None):
+    """{container: {lane id: sidecar}} for the containers in `wanted`: ONE listing of each container's
+    ~/.lz/state/hooks per cycle, and a read only for the `.json` names in it — lane-run's phase records in the parent
+    directory are never listed or read here. A container whose directory cannot be listed simply has no sidecars —
+    absent is "no hook installed", never an error a lane pays for. A zmx lane's hook sidecar is named by its zmx
+    session (lane-run's LANE_ID), not by its published `<ws>--z-<name>` id, so it joins no lane yet (DEF-1437)."""
+    out = {}
+    for container in sorted(wanted):
+        info = containers.get(container) or {}
+        if not info.get('pid'):
+            continue
+        state = sidecar_dir(info)
+        try:
+            names = os.listdir(state)
+        except OSError:
+            continue
+        found = {}
+        for name in names:
+            if name.endswith('.json') and not name.startswith('.'):
+                side = sidecar_for(info, name[:-5], now)
+                if side:
+                    found[name[:-5]] = side
+        out[container] = found
+    return out
+
+
 def probe_server(info, socket, live):
+    """`live` = the live socket names of THIS server's container (`live_in`), or None when nobody knows them."""
     if not info.get('pid'):
         # Only a docker inspect that ANSWERED "pid 0" says the container is down. No record at all, or one that
         # carries an error (docker did not answer) = nobody knows.
         return ('noserver' if info and not info.get('error') else 'error'), ''
     if live is not None and socket not in live:
-        return 'noserver', ''   # ps answered and lists no such server: a stale socket file is never probed
+        return 'noserver', ''   # the cgroups were read and run no such server: a stale socket file is never probed
     if (live is not None and info.get('running', True) and os.path.exists(f"{L.proc_root()}/{info['pid']}")
             and not os.path.exists(L.tmux_socket(info, socket))):
         # (container init alive: a container that went DOWN takes its sockets along and is 'missing', not this)
@@ -257,17 +389,78 @@ def discovered_sockets(containers, live):
         if not info.get('pid'):
             continue
         directory = Path(L.tmux_socket(info, 'discovery')).parent
+        known = live_in(live, container)
         try:
             for path in sorted(directory.iterdir()):
-                if L.IDENT.fullmatch(path.name) and path.is_socket() and (live is None or path.name in live):
+                if L.IDENT.fullmatch(path.name) and path.is_socket() and (known is None or path.name in known):
                     found.append((container, path.name))
         except OSError:
             pass
     return found
 
 
-def observe(registry, containers, live, previous=None):
-    """previous = the last published payload: what a lane keeps when its server gave no answer this time."""
+def probe_one(container, info, socket, live):
+    """One server's answer -> (outcome, sessions, activities). Runs on the probe pool, one call per socket: one server
+    (or one odd row) must not cost the other servers their cycle, so whatever goes wrong in here ends as "no answer
+    from THIS server" and never reaches another socket's result."""
+    rows_by_session = {}
+    try:
+        outcome, text = probe_server(info, socket, live_in(live, container))
+        for row in text.splitlines() if outcome == 'ok' else []:
+            values = dict(zip(FIELDS, row.split('\t', len(FIELDS) - 1)))
+            rows_by_session.setdefault(values['session_name'], []).append(values)
+    except Exception as error:
+        log(f'observe {container}/{socket}', error)
+        outcome, rows_by_session = 'error', {}
+    # A4a: every session is read through its ACTIVE pane and its NEWEST window activity, the same two
+    # readings the Host makes, so `dead`/`active` cannot mean one thing per surface on a multi-window lane.
+    return (outcome, {name: session_row(rows) for name, rows in rows_by_session.items()},
+            {name: newest_activity(rows) for name, rows in rows_by_session.items()})
+
+
+def probe_all(containers, keys, live, first=()):
+    """{(container, socket): probe_one's answer} for every key, at most PROBE_WORKERS probes at once, plus the keys
+    that were cut. A probe still running PROBE_TIMEOUT + PROBE_GRACE seconds after the round began answers 'timeout' —
+    keep() then republishes its lanes as last seen — and is abandoned, never waited for: `run_bounded` bounds it on
+    its own (PROBE_TIMEOUT + its 2 s post-kill wait), and `pause` lets the next round start only after that.
+    A probe that never started by then is cancelled. Both read at call time, so a test can shorten PROBE_TIMEOUT.
+
+    The keys in `first` — the ones the previous round's deadline cut — are submitted AHEAD of the others. The deadline
+    counts from the start of the round and the pool's queue is FIFO, so with more sockets than PROBE_WORKERS the
+    sockets at the end of the (deterministic, registry-ordered) key list start last: on a host where every probe
+    answers in, say, 0.4 x PROBE_TIMEOUT, the same tail was cut EVERY cycle and went 'unknown' after STALE_MAX while
+    the head stayed live (Task 16 review, MEASURED with 17 sockets at 1.5 s against a 4 s deadline). A socket cut once
+    now starts at t=0 next round, so no socket that answers inside PROBE_TIMEOUT is cut two rounds running unless more
+    sockets were cut than the pool can start at once."""
+    keys = list(keys)
+    first = set(first)
+    keys = [key for key in keys if key in first] + [key for key in keys if key not in first]
+    probed, cut = {}, []
+    if not keys:
+        return probed, cut
+    pool = concurrent.futures.ThreadPoolExecutor(max_workers=PROBE_WORKERS, thread_name_prefix='statusd-probe')
+    try:
+        futures = {pool.submit(probe_one, container, containers.get(container) or {}, socket, live): (container, socket)
+                   for container, socket in keys}
+        done, _pending = concurrent.futures.wait(futures, timeout=PROBE_TIMEOUT + PROBE_GRACE)
+        for future, key in futures.items():
+            if future in done:
+                probed[key] = future.result()   # probe_one never raises
+            else:
+                cut.append(key)
+                probed[key] = ('timeout', {}, {})
+    finally:
+        pool.shutdown(wait=False, cancel_futures=True)
+    if cut:
+        names = ' '.join(f'{container}/{socket}' for container, socket in cut[:8]) + (' …' if len(cut) > 8 else '')
+        log('observe', TimeoutError(f'{len(cut)} tmux probe(s) still running at the {PROBE_TIMEOUT + PROBE_GRACE} s '
+                                    f'cycle deadline (first next round): {names}'))
+    return probed, cut
+
+
+def observe(registry, containers, live, previous=None, report=None):
+    """previous = the last published payload: what a lane keeps when its server gave no answer this time.
+    report, when given, receives {'cut': n}: how many probes were still running at the cycle deadline."""
     previous = previous or {}
     now = time.time()
     lanes, servers, unregistered = {}, {}, {}
@@ -301,23 +494,30 @@ def observe(registry, containers, live, previous=None):
     # servers and servers with no registered lane. Never start a server to ask whether it exists.
     for key in discovered_sockets(containers, live):
         wanted.setdefault(key, [])
+    # The servers the previous round's deadline cut go first this round (`probe_all`). Read from the published
+    # `servers.*.cut` mark, so a statusd that restarts from status.json keeps the order too.
+    last_servers = previous.get('servers') or {}
+    first = {key for key in wanted if (last_servers.get(f'{key[0]}/{key[1]}') or {}).get('cut')}
+    probed, cut = probe_all(containers, wanted, live, first)
+    cut = set(cut)
+    if report is not None:
+        report['cut'] = len(cut)
+    # Task 5: the agent signals of every container, read once per cycle over all of its servers' sessions.
+    try:
+        claude = agent_signals(containers, {key: sessions for key, (_o, sessions, _a) in probed.items()})
+    except Exception as error:
+        log('agents', error)
+        claude = {}
+    # Task 8: the hook sidecars `lane-signal` writes, one directory listing per container per cycle. They outrank the
+    # session file and the title in `agent_state.merge`; a container without hooks costs one failed listdir.
+    try:
+        sidecars = agent_sidecars(containers, {container for container, _socket in probed}, now)
+    except Exception as error:
+        log('sidecars', error)
+        sidecars = {}
     for (container, socket), members in wanted.items():
         info = containers.get(container) or {}
-        rows_by_session = {}
-        # One server (or one odd row) must not cost the other servers their cycle: whatever goes wrong in here
-        # ends as "no answer from THIS server".
-        try:
-            outcome, text = probe_server(info, socket, live)
-            for row in text.splitlines() if outcome == 'ok' else []:
-                values = dict(zip(FIELDS, row.split('\t')))
-                rows_by_session.setdefault(values['session_name'], []).append(values)
-        except Exception as error:
-            log(f'observe {container}/{socket}', error)
-            outcome, rows_by_session = 'error', {}
-        # A4a: every session is read through its ACTIVE pane and its NEWEST window activity, the same two
-        # readings the Host makes, so `dead`/`active` cannot mean one thing per surface on a multi-window lane.
-        sessions = {name: session_row(rows) for name, rows in rows_by_session.items()}
-        activities = {name: newest_activity(rows) for name, rows in rows_by_session.items()}
+        outcome, sessions, activities = probed[(container, socket)]
         # ... and an UNDECLARED session grouped onto another live session is that session's viewer, not a lane.
         registered = {lane['tmux']['session'] for lane in members}
         own_sessions, viewers = split_viewers(sessions, registered)
@@ -326,11 +526,16 @@ def observe(registry, containers, live, previous=None):
         servers[f'{container}/{socket}'] = {'live': outcome == 'ok' if answered else last.get('live'),
                                             'sessions': len(own_sessions) if answered else last.get('sessions', 0),
                                             'outcome': outcome}
+        if (container, socket) in cut:
+            # Additive key, present only on a server the cycle DEADLINE abandoned (a probe that ran into its own
+            # PROBE_TIMEOUT is not marked): the next round starts it first. Every other row is byte-identical.
+            servers[f'{container}/{socket}']['cut'] = True
         if not answered:
             # The sessions nobody registered stay visible as well while their server does not answer.
             for workspace_id, entries in (previous.get('unregistered') or {}).items():
                 for entry in entries:
-                    if entry.get('container') == container and (entry.get('tmux') or {}).get('socket') == socket:
+                    if (entry.get('container') == container and (entry.get('tmux') or {}).get('socket') == socket
+                            and not any(A.session_key(entry) == A.session_key(lane) for lane in members)):
                         unregistered.setdefault(workspace_id, []).append(entry)
                         lanes[entry['id']] = keep(previous, entry['id'], outcome)
         if own_sessions:
@@ -380,7 +585,9 @@ def observe(registry, containers, live, previous=None):
                     # RUNBOOK nor the design doc may promise that it does.
                     if not L.TMUX_SESSION.match(name):
                         entry['note'] = 'session name outside the lane declaration grammar'
-                    found = lane_entry(pane, now, activities.get(name), viewers.get(name, []))
+                    found = lane_entry(pane, now, activities.get(name), viewers.get(name, []),
+                                       AS.merge(pane, claude.get(container, {}),
+                                                sidecars.get(container, {}).get(entry['id'])))
                 except Exception as error:
                     log(f'observe {container}/{socket}/{name}', error)
                     continue
@@ -398,7 +605,9 @@ def observe(registry, containers, live, previous=None):
                 lanes[lane['id']] = {'state': 'absent' if info.get('running') else 'missing'}
             else:
                 try:
-                    lanes[lane['id']] = lane_entry(pane, now, activities.get(session), viewers.get(session, []))
+                    lanes[lane['id']] = lane_entry(pane, now, activities.get(session), viewers.get(session, []),
+                                                   AS.merge(pane, claude.get(container, {}),
+                                                            sidecars.get(container, {}).get(lane['id'])))
                 except Exception as error:   # a torn row: the session is there, its details are not readable
                     log(f"observe {lane['id']}", error)
                     lanes[lane['id']] = keep(previous, lane['id'], 'error')
@@ -486,7 +695,8 @@ def reconcile_client_sizes(registry, containers, live):
     sockets.update(discovered_sockets(containers, live))
     for container, socket in sorted(sockets):
         info = containers.get(container) or {}
-        if socket not in live or not info.get('pid'):
+        known = live_in(live, container)   # cosmetics only on a server KNOWN to be live: unknown is never touched
+        if known is None or socket not in known or not info.get('pid'):
             continue
         output = L.tmux_query(info, socket, ['list-clients', '-F', CLIENT_FORMAT])
         rows = [row.split('|') for row in (output or '').splitlines() if row.count('|') == 4]
@@ -547,21 +757,28 @@ def observe_zmx_container(container):
         name = fields.get('name', '')
         if not L.IDENT.match(name):
             continue
-        meta = L.read_json(link / 'meta' / f'{name}.json', {}) or {}
-        agent = L.read_json(link / 'state' / f'{name}.json', {}) or {}
-        workspace = meta.get('ws') or fields.get('ws') or '-'
+        # The meta and state sidecars are written INSIDE the container (DEF-1461): each one is read as an object or
+        # nothing, each field as one printable line or nothing, and the workspace must be a lanes identifier — it
+        # names a host file below. A malformed sidecar costs its own row's labels, never the container's other rows.
+        # `lane new` writes exactly these shapes (`ws` checked against IDENT or empty, `zmx_version` a string).
+        meta, agent = (L.read_json(link / sub / f'{name}.json', {}) for sub in ('meta', 'state'))
+        meta, agent = (d if isinstance(d, dict) else {} for d in (meta, agent))
+        text = {key: meta[key] if isinstance(meta.get(key), str) and meta[key].isprintable() else ''
+                for key in ('ws', 'title', 'zmx_version', 'color', 'icon', 'cwd')}
+        workspace = next((w for w in (text['ws'], fields.get('ws') or '') if L.IDENT.match(w)), '-')
         lane_id = f'{workspace}--z-{name}'
-        version = str(meta.get('zmx_version') or binaries[-1].name.split('-', 1)[1])
+        version = text['zmx_version'] or binaries[-1].name.split('-', 1)[1]
+        title = text['title'] or name
         groups.setdefault(workspace, []).append({
-            'id': lane_id, 'name': meta.get('title') or name, 'holder': 'zmx', 'container': container,
-            'zmx': {'name': name, 'version': version}, 'color': L.ANSI_SLOTS.get(meta.get('color')),
-            'icon': L.LANE_ICONS.get(meta.get('icon'), '🧵')})
+            'id': lane_id, 'name': title, 'holder': 'zmx', 'container': container,
+            'zmx': {'name': name, 'version': version}, 'color': L.ANSI_SLOTS.get(text['color']),
+            'icon': L.LANE_ICONS.get(text['icon'], '🧵')})
         L.atomic_write(L.STATE / 'resolve.dynamic' / lane_id,
-                       '\n'.join(['zmx', container, name, version, meta.get('title') or name]) + '\n')
+                       '\n'.join(['zmx', container, name, version, title]) + '\n')
         state, evidence = zmx_state(agent)
         lanes[lane_id] = {'state': state, 'evidence': evidence,
                           'clients': int(fields.get('clients', 0) or 0), 'pid': int(fields.get('pid', 0) or 0),
-                          'agent': agent, 'cwd': meta.get('cwd', '')}
+                          'agent': agent, 'cwd': text['cwd']}
     return lanes, groups
 
 
@@ -591,7 +808,7 @@ def update_containers(ctx, registry, level):
     5-minute safety net on a host that is not busy — and at most every CONTAINER_RETRY seconds. NEVER because a
     probe failed: `docker inspect` (2 spawns, 20 s timeouts) every 3 s is a load generator on a host that is
     already too slow to answer it."""
-    names = {lane['container'] for w in registry['workspaces'] for lane in w['lanes'] + w['extras']}
+    names = {lane['container'] for w in registry['workspaces'] for lane in w['lanes'] if lane.get('container')}
     known = ctx['containers']
     age = time.time() - ctx['containers_at']
     unsettled = set(known) != names or any(not pid_alive(c.get('pid')) for c in known.values())
@@ -608,8 +825,11 @@ def update_containers(ctx, registry, level):
 
 def build_payload(ctx, level, reconcile=True):
     """One observation pass -> the payload of status.json (None = there is no registry to observe). Every phase
-    fails alone: what it could not observe is republished as last seen (stale), the other phases still run."""
+    fails alone: what it could not observe is republished as last seen (stale), the other phases still run.
+    ctx['cut'] = how many tmux probes the cycle deadline abandoned (0 when nothing was probed): `pause` reads it."""
     previous = ctx['payload']
+    report = {'cut': 0}
+    ctx['cut'] = 0
     try:
         stamp = os.stat(L.STATE / 'registry.json').st_mtime_ns
         if stamp != ctx['stamp']:
@@ -624,17 +844,32 @@ def build_payload(ctx, level, reconcile=True):
     except Exception as error:
         log('containers', error)
     containers = ctx['containers']
+    unwalkable = {}
     try:
-        live = live_tmux_sockets()
+        live = live_by_container(containers, unwalkable)
     except Exception as error:
-        log('ps', error)
+        log('live servers', error)
         live = None
+    # Name the container whose init pid broke the cgroup read, once when it starts (not every INTERVAL): only ITS
+    # sockets are probed blind, and the operator needs to know which one that is.
+    for name in sorted(set(unwalkable) - set(ctx.get('unwalkable') or ())):
+        log(f'live servers {name}', L.LanesError(f'{unwalkable[name]} — its sockets are probed as unknown, none concluded gone'))
+    ctx['unwalkable'] = sorted(unwalkable)
     try:
-        lanes, servers, unregistered = observe(registry, containers, live, previous)
+        lanes, servers, unregistered = observe(registry, containers, live, previous, report)
     except Exception as error:
         log('observe', error)
         lanes = {lane_id: keep(previous, lane_id, 'error') for lane_id in previous.get('lanes') or {}}
-        servers, unregistered = previous.get('servers') or {}, previous.get('unregistered') or {}
+        # DEF-1457: COPIES of what was published, never the previous payload itself (`previous` is the very object
+        # `ctx['published']` holds, so a kept list extended in place would grow the published payload), and only
+        # observe's own rows (holder 'tmux', the rows `observe` carries over too).
+        servers = dict(previous.get('servers') or {})
+        unregistered = {}
+        for workspace, entries in (previous.get('unregistered') or {}).items():
+            kept = [entry for entry in entries if entry.get('holder') == 'tmux']
+            if kept:
+                unregistered[workspace] = kept
+    ctx['cut'] = report.get('cut', 0)
     # Half of all our spawns, and only cosmetics (who dictates a window size): not on a host under pressure.
     if reconcile and level == 'ok':
         try:
@@ -647,10 +882,6 @@ def build_payload(ctx, level, reconcile=True):
         log('zmx', error)
         zmx_lanes, zmx_groups = kept_zmx(previous, 'error')
     lanes.update(zmx_lanes)
-    legacy_lanes, legacy_groups = Legacy.observe()
-    lanes.update(legacy_lanes)
-    for workspace, entries in legacy_groups.items():
-        unregistered.setdefault(workspace, []).extend(entries)
     return {'lanes': lanes, 'servers': servers, 'unregistered': unregistered, 'zmx': zmx_groups,
             'containers': {k: {'running': v.get('running', False)} for k, v in containers.items()}}
 
@@ -664,7 +895,7 @@ def new_context():
     for entry in (seed.get('lanes') or {}).values():
         entry.setdefault('stale_since', int(written))
     return {'registry': None, 'stamp': 0, 'containers': {}, 'containers_at': 0.0,
-            'payload': seed, 'published': None, 'status_at': written}
+            'payload': seed, 'published': None, 'status_at': written, 'cut': 0, 'unwalkable': []}
 
 
 def cycle(ctx, reconcile=True):
@@ -709,47 +940,78 @@ def cycle(ctx, reconcile=True):
     return level, payload
 
 
-def pause(level, elapsed):
-    return max(0.5, INTERVAL * SLOWDOWN.get(level, 1) - elapsed)
+def pause(level, elapsed, cut=0):
+    """Seconds to sleep before the next cycle. After a cycle that abandoned probes at its deadline (`cut`), at least
+    PROBE_TIMEOUT + PROBE_GRACE: run_bounded ends each of those within that time, so the next round never runs
+    beside them and at most PROBE_WORKERS tmux clients of ours are ever in flight on a host that is already slow."""
+    rest = max(0.5, INTERVAL * SLOWDOWN.get(level, 1) - elapsed)
+    return max(rest, PROBE_TIMEOUT + PROBE_GRACE) if cut else rest
+
 
+def registry_source_paths(registry, containers):
+    """(container path, HOST path) for every terminals.json the registry references. The daemon runs on the
+    host: a lane's `source` is spelled as the container sees it and only exists under the container's home
+    volume (`L.host_path`). A container with no record is skipped — nobody can stat its files anyway.
 
-def _registry_sources(registry):
-    """All terminals.json paths referenced by the current registry (unique, extant)."""
-    seen = set()
+    Deduplicated by the HOST path, never by `source` alone: the same spelling in two containers (one repository
+    cloned in two workspace boxes) is two distinct files, and each must be stat'ed on its own."""
+    seen, out = set(), []
     for w in (registry or {}).get('workspaces', []):
         for lane in w.get('lanes', []):
             src = lane.get('source')
-            if src and src not in seen:
-                seen.add(src)
-                yield src
+            if not src:
+                continue
+            info = (containers or {}).get(lane.get('container') or '') or {}
+            try:
+                path = L.host_path(info, src)
+            except L.LanesError:
+                continue
+            if path in seen:
+                continue
+            seen.add(path)
+            out.append((src, path))
+    return out
+
+
+def hot_sync(ctx, source_mtimes):
+    """One hot-reload check: start ONE `lanes sync` when a referenced terminals.json changed since the last
+    check (read through the host volume). Returns True when a sync was started. The new mtime is recorded
+    once the sync is started, so one edit costs one sync — not one every rate-limit window forever. Keyed by the
+    host path: a `source` spelling is shared by every container that clones the same repository."""
+    for _src, path in registry_source_paths(ctx['registry'], ctx.get('containers') or {}):
+        key = str(path)
+        try:
+            mtime = path.stat().st_mtime_ns
+        except OSError:
+            continue
+        if key in source_mtimes and mtime != source_mtimes[key]:
+            try:
+                proc = subprocess.Popen([str(L.REPO / 'bin' / 'lanes'), 'sync'],
+                                        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            except Exception as sync_err:
+                log('hot-sync', sync_err)
+                return False
+            ctx.setdefault('children', []).append(proc)
+            source_mtimes[key] = mtime
+            return True
+        source_mtimes[key] = mtime
+    return False
 
 
 def main():
     ctx = new_context()
     last_triggered_sync = 0.0
-    source_mtimes = {}   # path -> mtime_ns at last check
+    source_mtimes = {}   # host path of a terminals.json -> its mtime_ns at last check
     while True:
         started = time.time()
         level, _payload = cycle(ctx)
         # Hot-reload: check if any terminals.json changed since we last synced.
         # Rate-limited to at most one triggered sync per 10 s to avoid thrashing.
         if time.time() - last_triggered_sync > 10 and ctx['registry']:
-            for src in _registry_sources(ctx['registry']):
-                try:
-                    mtime = os.stat(src).st_mtime_ns
-                except OSError:
-                    continue
-                if src in source_mtimes and mtime != source_mtimes[src]:
-                    try:
-                        import subprocess as _sp
-                        lanes_bin = str(L.REPO / 'bin' / 'lanes')
-                        _sp.Popen([lanes_bin, 'sync'], stdout=_sp.DEVNULL, stderr=_sp.DEVNULL)
-                        last_triggered_sync = time.time()
-                    except Exception as sync_err:
-                        log('hot-sync', sync_err)
-                    break
-                source_mtimes[src] = mtime
-        time.sleep(pause(level, time.time() - started))
+            if hot_sync(ctx, source_mtimes):
+                last_triggered_sync = time.time()
+        ctx['children'] = [p for p in ctx.get('children', []) if p.poll() is None]   # reap finished syncs
+        time.sleep(pause(level, time.time() - started, ctx.get('cut', 0)))
 
 
 if __name__ == '__main__':