@vibelord/lanes 1.0.1 → 1.0.3

package/payload/lanes/libexec/statusd.py

@@ -1,16 +1,19 @@
 #!/usr/bin/env python3
 """lanes status daemon: observes lanes, never touches them.
 
-Every few seconds: find LIVE tmux servers via `ps` (dead sockets are never probed), run one read-only
-`list-panes -a` per live server that the registry references, and publish ~/.local/state/lanes/status.json
+Every few seconds: find LIVE tmux servers in each workspace container's own cgroup (`guard`, plain /proc reads;
+dead sockets are never probed), run one read-only `list-panes -a` per live server that the registry references —
+at most PROBE_WORKERS at once, the whole round cut at PROBE_TIMEOUT + PROBE_GRACE seconds, the servers the last
+round cut started first — and publish ~/.local/state/lanes/status.json
 atomically, plus the one-line ~/.local/state/lanes/health. The sidebar, `lanes status`, `lanes doctor` and the
 future Throttle web harness all read those files.
 
-It never INVENTS a state. "No answer" (a tmux, ps, docker or zmx call that timed out or failed on an overloaded
+It never INVENTS a state. "No answer" (a tmux, cgroup, docker or zmx read that timed out or failed on an overloaded
 host) is not an observation: the lane keeps its last published state, marked stale, for at most STALE_MAX seconds
 and is 'unknown' after that. 'absent' / 'missing' are only published when a server answered without the session,
 or positively does not exist.
 """
+import concurrent.futures
 import json
 import os
 import subprocess
@@ -22,12 +25,18 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 import lanes_lib as L  # noqa: E402
 import session_actions as A  # noqa: E402
 import agent_state as AS  # noqa: E402
-import legacy_sessions as Legacy  # noqa: E402
+import guard as G  # noqa: E402
 
 INTERVAL = float(os.environ.get('LANES_STATUS_INTERVAL', '3'))
 SLOWDOWN = {'ok': 1, 'busy': 2, 'critical': 4}   # a host under pressure gets FEWER spawns from us, not the same
 HEARTBEAT = 15          # status.json is rewritten at least this often, so its age proves that statusd is alive
 PROBE_TIMEOUT = 5
+# Plan Task 16: the servers of one cycle are probed concurrently, never more than PROBE_WORKERS tmux clients at once,
+# and the cycle stops waiting PROBE_GRACE seconds after one PROBE_TIMEOUT: run_bounded's post-kill wait is 2 s, so a
+# probe still running then is a child the kernel cannot reap (D state) or a fork that has not even started. Its socket
+# is published as a 'timeout' (keep(): last state, stale) and the others' answers are not held back by it.
+PROBE_WORKERS = 8
+PROBE_GRACE = 2
 CONTAINER_REFRESH = 300  # docker is asked again this often as a safety net; liveness itself comes from /proc/<pid>
 CONTAINER_RETRY = 30     # ... and at most this often while a container is down or docker did not answer
 ACTIVE_WINDOW = 20  # seconds since last output for a lane to count as active
@@ -167,21 +176,40 @@ def board_named_by(name, boards):
     return None
 
 
-def live_tmux_sockets():
-    """Socket names (-L) of running tmux servers, from the process table only.
-    None = ps did not answer: the caller must then NOT conclude that any server is gone."""
-    rc, out, _err = L.run_bounded(['ps', '-eo', 'comm=,args='], 10)
-    if rc != 0:
+def live_by_container(containers, errors=None):
+    """{container: socket names (-L) of the tmux servers running in it | None}, read from each container's own cgroup
+    (`guard.tmux_servers_by_container`: cgroup.procs + comm + cmdline, no subprocess). It replaced a host-wide
+    `ps -eo comm=,args=` per cycle — one fork that read every process of a 64-core host (plan Task 16).
+
+    This is statusd's own reading, PER CONTAINER: a container whose recorded init pid cannot be walked is None — only
+    ITS sockets are "nobody knows" (every socket file of that container is probed, none is concluded gone) — and the
+    other containers keep their real live sets (Task 16 review: one unwalkable pid used to collapse the WHOLE fleet's
+    reading to None). `errors` receives {container: why} for each None. A container without an init pid is absent
+    (it runs nothing; its lanes are 'missing' / kept by `probe_server`)."""
+    found = G.tmux_servers_by_container(G.Env(), containers or {}, errors)
+    return {name: (None if servers is None else set(servers)) for name, servers in found.items()}
+
+
+def live_tmux_sockets(containers):
+    """All live socket names of these containers as ONE set, or None when any container that has an init pid could
+    not be walked. All-or-nothing on purpose: `lanes invariant` (bin/lanes `all_panes`) must never save or compare a
+    PARTIAL pane list. statusd itself reads `live_by_container`, where one unwalkable container costs only its own."""
+    found = live_by_container(containers)
+    if any(servers is None for servers in found.values()):
         return None
-    names = set()
-    for line in out.splitlines():
-        if not line.startswith('tmux: server'):
-            continue
-        parts = line.split()
-        for index, part in enumerate(parts):
-            if part == '-L' and index + 1 < len(parts):
-                names.add(parts[index + 1])
-    return names
+    return {socket for servers in found.values() for socket in servers}
+
+
+def live_in(live, container):
+    """The live socket names of ONE container from statusd's `live`, or None = nobody knows (probe its socket files,
+    conclude nothing gone). `live` is `live_by_container`'s {container: set | None} (a container missing from it is
+    unknown, never empty), None (no reading at all), or a flat set, which applies to every container — the shape
+    `live_tmux_sockets` returns and the unit fixtures and `tests/cursor_lane.py` pass."""
+    if live is None:
+        return None
+    if isinstance(live, dict):
+        return live.get(container)
+    return live
 
 
 def keep(previous, lane_id, outcome):
@@ -198,7 +226,9 @@ def keep(previous, lane_id, outcome):
     # 'active' is the one state that is a claim about NOW (output in the last 20 s): never true of a cached entry,
     # and the one that readers which know nothing about 'stale' count and draw green. 'idle' is the seventh word
     # a pre-DEF-823 statusd published for a zmx lane: a status.json written by that build seeds `new_context()`
-    # across the upgrade, so it is normalised here rather than drawn as an unknown glyph by the sidebar.
+    # across the upgrade, so it is normalised here rather than drawn as an unknown glyph by the sidebar. (Plan
+    # Task 19 listed this line as dead weight; it stays because test_alignment's vocabulary gate pins it — removing
+    # it would mean weakening that assert.)
     state = 'quiet' if old['state'] in ('active', 'idle') else old['state']
     return dict(old, state=state, stale=True, observed=outcome, stale_since=since)
 
@@ -246,7 +276,7 @@ def agent_signals(containers, sessions_by_server):
     (`parse_codex_title`) over a bare-shell foreground can be a harness that ended into its shell, and only the
     child read (`child_liveness`) tells that apart from a live harness launched under `bash -c` or a wrapper. A pane
     a live Claude session file already claims is not probed (the file outranks the title). The key is set only for
-    a known answer: unknown liveness leaves it absent, and an absent key never demotes a title (DEF-1420). A
+    a known answer: unknown liveness leaves it absent, and an absent key never demotes a title (DEF-1557). A
     container whose files cannot be read costs only its own agents, never a lane its `state`."""
     claude = {}
     for container in sorted({container for container, _socket in sessions_by_server}):
@@ -279,13 +309,63 @@ def agent_signals(containers, sessions_by_server):
     return claude
 
 
+SIDECAR_MAX_AGE = 3600   # a hook sidecar whose last event is older than this loses to the live signals
+# The hook sidecars' own directory under the container home. NOT `.lz/state` itself: lane-run writes each zmx lane's
+# `{phase, ...}` record there as `<zmx name>.json` and exports that name as LANE_ID, so a shared name would let a hook
+# overwrite the record `observe_zmx_container` and the container `lane ls` read (DEF-1572).
+SIDECAR_DIR = ('.lz', 'state', 'hooks')
+
+
+def sidecar_dir(info):
+    return Path(L.proc_root()) / str(info['pid']) / 'root' / L.CONTAINER_HOME.lstrip('/') / Path(*SIDECAR_DIR)
+
+
+def sidecar_for(info, lane_id, now=None):
+    """The `lane-signal` hook sidecar of one lane (Task 8): `~/.lz/state/hooks/<lane id>.json` in the container, read
+    through `/proc/<container init pid>/root` like the Claude session files, judged by `agent_state.read_sidecar_dict`
+    in the container's pid namespace (a dead or recycled agent pid, or a last event older than SIDECAR_MAX_AGE, is
+    None)."""
+    if not info.get('pid'):
+        return None
+    root = Path(L.proc_root()) / str(info['pid']) / 'root'
+    return AS.read_sidecar_dict(L.read_json(sidecar_dir(info) / f'{lane_id}.json'),
+                                time.time() if now is None else now, SIDECAR_MAX_AGE, proc_root=root / 'proc')
+
+
+def agent_sidecars(containers, wanted, now=None):
+    """{container: {lane id: sidecar}} for the containers in `wanted`: ONE listing of each container's
+    ~/.lz/state/hooks per cycle, and a read only for the `.json` names in it — lane-run's phase records in the parent
+    directory are never listed or read here. A container whose directory cannot be listed simply has no sidecars —
+    absent is "no hook installed", never an error a lane pays for. A zmx lane's hook sidecar is named by its zmx
+    session (lane-run's LANE_ID), not by its published `<ws>--z-<name>` id, so it joins no lane yet (DEF-1573)."""
+    out = {}
+    for container in sorted(wanted):
+        info = containers.get(container) or {}
+        if not info.get('pid'):
+            continue
+        state = sidecar_dir(info)
+        try:
+            names = os.listdir(state)
+        except OSError:
+            continue
+        found = {}
+        for name in names:
+            if name.endswith('.json') and not name.startswith('.'):
+                side = sidecar_for(info, name[:-5], now)
+                if side:
+                    found[name[:-5]] = side
+        out[container] = found
+    return out
+
+
 def probe_server(info, socket, live):
+    """`live` = the live socket names of THIS server's container (`live_in`), or None when nobody knows them."""
     if not info.get('pid'):
         # Only a docker inspect that ANSWERED "pid 0" says the container is down. No record at all, or one that
         # carries an error (docker did not answer) = nobody knows.
         return ('noserver' if info and not info.get('error') else 'error'), ''
     if live is not None and socket not in live:
-        return 'noserver', ''   # ps answered and lists no such server: a stale socket file is never probed
+        return 'noserver', ''   # the cgroups were read and run no such server: a stale socket file is never probed
     if (live is not None and info.get('running', True) and os.path.exists(f"{L.proc_root()}/{info['pid']}")
             and not os.path.exists(L.tmux_socket(info, socket))):
         # (container init alive: a container that went DOWN takes its sockets along and is 'missing', not this)
@@ -309,17 +389,78 @@ def discovered_sockets(containers, live):
         if not info.get('pid'):
             continue
         directory = Path(L.tmux_socket(info, 'discovery')).parent
+        known = live_in(live, container)
         try:
             for path in sorted(directory.iterdir()):
-                if L.IDENT.fullmatch(path.name) and path.is_socket() and (live is None or path.name in live):
+                if L.IDENT.fullmatch(path.name) and path.is_socket() and (known is None or path.name in known):
                     found.append((container, path.name))
         except OSError:
             pass
     return found
 
 
-def observe(registry, containers, live, previous=None):
-    """previous = the last published payload: what a lane keeps when its server gave no answer this time."""
+def probe_one(container, info, socket, live):
+    """One server's answer -> (outcome, sessions, activities). Runs on the probe pool, one call per socket: one server
+    (or one odd row) must not cost the other servers their cycle, so whatever goes wrong in here ends as "no answer
+    from THIS server" and never reaches another socket's result."""
+    rows_by_session = {}
+    try:
+        outcome, text = probe_server(info, socket, live_in(live, container))
+        for row in text.splitlines() if outcome == 'ok' else []:
+            values = dict(zip(FIELDS, row.split('\t', len(FIELDS) - 1)))
+            rows_by_session.setdefault(values['session_name'], []).append(values)
+    except Exception as error:
+        log(f'observe {container}/{socket}', error)
+        outcome, rows_by_session = 'error', {}
+    # A4a: every session is read through its ACTIVE pane and its NEWEST window activity, the same two
+    # readings the Host makes, so `dead`/`active` cannot mean one thing per surface on a multi-window lane.
+    return (outcome, {name: session_row(rows) for name, rows in rows_by_session.items()},
+            {name: newest_activity(rows) for name, rows in rows_by_session.items()})
+
+
+def probe_all(containers, keys, live, first=()):
+    """{(container, socket): probe_one's answer} for every key, at most PROBE_WORKERS probes at once, plus the keys
+    that were cut. A probe still running PROBE_TIMEOUT + PROBE_GRACE seconds after the round began answers 'timeout' —
+    keep() then republishes its lanes as last seen — and is abandoned, never waited for: `run_bounded` bounds it on
+    its own (PROBE_TIMEOUT + its 2 s post-kill wait), and `pause` lets the next round start only after that.
+    A probe that never started by then is cancelled. Both read at call time, so a test can shorten PROBE_TIMEOUT.
+
+    The keys in `first` — the ones the previous round's deadline cut — are submitted AHEAD of the others. The deadline
+    counts from the start of the round and the pool's queue is FIFO, so with more sockets than PROBE_WORKERS the
+    sockets at the end of the (deterministic, registry-ordered) key list start last: on a host where every probe
+    answers in, say, 0.4 x PROBE_TIMEOUT, the same tail was cut EVERY cycle and went 'unknown' after STALE_MAX while
+    the head stayed live (Task 16 review, MEASURED with 17 sockets at 1.5 s against a 4 s deadline). A socket cut once
+    now starts at t=0 next round, so no socket that answers inside PROBE_TIMEOUT is cut two rounds running unless more
+    sockets were cut than the pool can start at once."""
+    keys = list(keys)
+    first = set(first)
+    keys = [key for key in keys if key in first] + [key for key in keys if key not in first]
+    probed, cut = {}, []
+    if not keys:
+        return probed, cut
+    pool = concurrent.futures.ThreadPoolExecutor(max_workers=PROBE_WORKERS, thread_name_prefix='statusd-probe')
+    try:
+        futures = {pool.submit(probe_one, container, containers.get(container) or {}, socket, live): (container, socket)
+                   for container, socket in keys}
+        done, _pending = concurrent.futures.wait(futures, timeout=PROBE_TIMEOUT + PROBE_GRACE)
+        for future, key in futures.items():
+            if future in done:
+                probed[key] = future.result()   # probe_one never raises
+            else:
+                cut.append(key)
+                probed[key] = ('timeout', {}, {})
+    finally:
+        pool.shutdown(wait=False, cancel_futures=True)
+    if cut:
+        names = ' '.join(f'{container}/{socket}' for container, socket in cut[:8]) + (' …' if len(cut) > 8 else '')
+        log('observe', TimeoutError(f'{len(cut)} tmux probe(s) still running at the {PROBE_TIMEOUT + PROBE_GRACE} s '
+                                    f'cycle deadline (first next round): {names}'))
+    return probed, cut
+
+
+def observe(registry, containers, live, previous=None, report=None):
+    """previous = the last published payload: what a lane keeps when its server gave no answer this time.
+    report, when given, receives {'cut': n}: how many probes were still running at the cycle deadline."""
     previous = previous or {}
     now = time.time()
     lanes, servers, unregistered = {}, {}, {}
@@ -353,30 +494,27 @@ def observe(registry, containers, live, previous=None):
     # servers and servers with no registered lane. Never start a server to ask whether it exists.
     for key in discovered_sockets(containers, live):
         wanted.setdefault(key, [])
-    probed = {}
-    for container, socket in wanted:
-        info = containers.get(container) or {}
-        rows_by_session = {}
-        # One server (or one odd row) must not cost the other servers their cycle: whatever goes wrong in here
-        # ends as "no answer from THIS server".
-        try:
-            outcome, text = probe_server(info, socket, live)
-            for row in text.splitlines() if outcome == 'ok' else []:
-                values = dict(zip(FIELDS, row.split('\t', len(FIELDS) - 1)))
-                rows_by_session.setdefault(values['session_name'], []).append(values)
-        except Exception as error:
-            log(f'observe {container}/{socket}', error)
-            outcome, rows_by_session = 'error', {}
-        # A4a: every session is read through its ACTIVE pane and its NEWEST window activity, the same two
-        # readings the Host makes, so `dead`/`active` cannot mean one thing per surface on a multi-window lane.
-        probed[(container, socket)] = (outcome, {name: session_row(rows) for name, rows in rows_by_session.items()},
-                                       {name: newest_activity(rows) for name, rows in rows_by_session.items()})
+    # The servers the previous round's deadline cut go first this round (`probe_all`). Read from the published
+    # `servers.*.cut` mark, so a statusd that restarts from status.json keeps the order too.
+    last_servers = previous.get('servers') or {}
+    first = {key for key in wanted if (last_servers.get(f'{key[0]}/{key[1]}') or {}).get('cut')}
+    probed, cut = probe_all(containers, wanted, live, first)
+    cut = set(cut)
+    if report is not None:
+        report['cut'] = len(cut)
     # Task 5: the agent signals of every container, read once per cycle over all of its servers' sessions.
     try:
         claude = agent_signals(containers, {key: sessions for key, (_o, sessions, _a) in probed.items()})
     except Exception as error:
         log('agents', error)
         claude = {}
+    # Task 8: the hook sidecars `lane-signal` writes, one directory listing per container per cycle. They outrank the
+    # session file and the title in `agent_state.merge`; a container without hooks costs one failed listdir.
+    try:
+        sidecars = agent_sidecars(containers, {container for container, _socket in probed}, now)
+    except Exception as error:
+        log('sidecars', error)
+        sidecars = {}
     for (container, socket), members in wanted.items():
         info = containers.get(container) or {}
         outcome, sessions, activities = probed[(container, socket)]
@@ -388,6 +526,10 @@ def observe(registry, containers, live, previous=None):
         servers[f'{container}/{socket}'] = {'live': outcome == 'ok' if answered else last.get('live'),
                                             'sessions': len(own_sessions) if answered else last.get('sessions', 0),
                                             'outcome': outcome}
+        if (container, socket) in cut:
+            # Additive key, present only on a server the cycle DEADLINE abandoned (a probe that ran into its own
+            # PROBE_TIMEOUT is not marked): the next round starts it first. Every other row is byte-identical.
+            servers[f'{container}/{socket}']['cut'] = True
         if not answered:
             # The sessions nobody registered stay visible as well while their server does not answer.
             for workspace_id, entries in (previous.get('unregistered') or {}).items():
@@ -444,7 +586,8 @@ def observe(registry, containers, live, previous=None):
                     if not L.TMUX_SESSION.match(name):
                         entry['note'] = 'session name outside the lane declaration grammar'
                     found = lane_entry(pane, now, activities.get(name), viewers.get(name, []),
-                                       AS.merge(pane, claude.get(container, {}), None))
+                                       AS.merge(pane, claude.get(container, {}),
+                                                sidecars.get(container, {}).get(entry['id'])))
                 except Exception as error:
                     log(f'observe {container}/{socket}/{name}', error)
                     continue
@@ -462,9 +605,9 @@ def observe(registry, containers, live, previous=None):
                 lanes[lane['id']] = {'state': 'absent' if info.get('running') else 'missing'}
             else:
                 try:
-                    # The hook sidecar (Task 8) is the third argument; until it lands nothing is passed.
                     lanes[lane['id']] = lane_entry(pane, now, activities.get(session), viewers.get(session, []),
-                                                   AS.merge(pane, claude.get(container, {}), None))
+                                                   AS.merge(pane, claude.get(container, {}),
+                                                            sidecars.get(container, {}).get(lane['id'])))
                 except Exception as error:   # a torn row: the session is there, its details are not readable
                     log(f"observe {lane['id']}", error)
                     lanes[lane['id']] = keep(previous, lane['id'], 'error')
@@ -552,7 +695,8 @@ def reconcile_client_sizes(registry, containers, live):
     sockets.update(discovered_sockets(containers, live))
     for container, socket in sorted(sockets):
         info = containers.get(container) or {}
-        if socket not in live or not info.get('pid'):
+        known = live_in(live, container)   # cosmetics only on a server KNOWN to be live: unknown is never touched
+        if known is None or socket not in known or not info.get('pid'):
             continue
         output = L.tmux_query(info, socket, ['list-clients', '-F', CLIENT_FORMAT])
         rows = [row.split('|') for row in (output or '').splitlines() if row.count('|') == 4]
@@ -613,30 +757,47 @@ def observe_zmx_container(container):
         name = fields.get('name', '')
         if not L.IDENT.match(name):
             continue
-        meta = L.read_json(link / 'meta' / f'{name}.json', {}) or {}
-        agent = L.read_json(link / 'state' / f'{name}.json', {}) or {}
-        workspace = meta.get('ws') or fields.get('ws') or '-'
+        # The meta and state sidecars are written INSIDE the container (DEF-1542): each one is read as an object or
+        # nothing, each field as one printable line or nothing, and the workspace must be a lanes identifier — it
+        # names a host file below. A malformed sidecar costs its own row's labels, never the container's other rows.
+        # `lane new` writes exactly these shapes (`ws` checked against IDENT or empty, `zmx_version` a string).
+        meta, agent = (L.read_json(link / sub / f'{name}.json', {}) for sub in ('meta', 'state'))
+        meta, agent = (d if isinstance(d, dict) else {} for d in (meta, agent))
+        text = {key: meta[key] if isinstance(meta.get(key), str) and meta[key].isprintable() else ''
+                for key in ('ws', 'title', 'zmx_version', 'color', 'icon', 'cwd')}
+        workspace = next((w for w in (text['ws'], fields.get('ws') or '') if L.IDENT.match(w)), '-')
         lane_id = f'{workspace}--z-{name}'
-        version = str(meta.get('zmx_version') or binaries[-1].name.split('-', 1)[1])
+        version = text['zmx_version'] or binaries[-1].name.split('-', 1)[1]
+        title = text['title'] or name
         groups.setdefault(workspace, []).append({
-            'id': lane_id, 'name': meta.get('title') or name, 'holder': 'zmx', 'container': container,
-            'zmx': {'name': name, 'version': version}, 'color': L.ANSI_SLOTS.get(meta.get('color')),
-            'icon': L.LANE_ICONS.get(meta.get('icon'), '🧵')})
+            'id': lane_id, 'name': title, 'holder': 'zmx', 'container': container,
+            'zmx': {'name': name, 'version': version}, 'color': L.ANSI_SLOTS.get(text['color']),
+            'icon': L.LANE_ICONS.get(text['icon'], '🧵')})
         L.atomic_write(L.STATE / 'resolve.dynamic' / lane_id,
-                       '\n'.join(['zmx', container, name, version, meta.get('title') or name]) + '\n')
+                       '\n'.join(['zmx', container, name, version, title]) + '\n')
         state, evidence = zmx_state(agent)
         lanes[lane_id] = {'state': state, 'evidence': evidence,
                           'clients': int(fields.get('clients', 0) or 0), 'pid': int(fields.get('pid', 0) or 0),
-                          'agent': agent, 'cwd': meta.get('cwd', '')}
+                          'agent': agent, 'cwd': text['cwd']}
     return lanes, groups
 
 
-def observe_zmx(containers, previous=None):
-    """zmx-held lanes describe themselves (labels + meta sidecar): no registry edit is needed to see one."""
+def observe_zmx(containers, previous=None, listed=None):
+    """zmx-held lanes describe themselves (labels + meta sidecar): no registry edit is needed to see one.
+
+    `listed`, when given, receives `{container: unix time the zmx list was ISSUED}` for every container whose listing
+    ANSWERED this cycle (status.json `zmx_listed_at`). A container that timed out or failed is absent from it: its
+    groups are the previous ones republished, which say nothing about a lane created since. The co-located Studio
+    Host prunes a lane it started only on such a listing (design §8.10 C1, review round 6 — it read absence from any
+    fresh status.json and lost stop authority over lanes a timed-out listing had never seen)."""
     lanes, groups = {}, {}
     for container in containers:
+        issued = time.time()
         try:
-            found = observe_zmx_container(container) or kept_zmx(previous or {}, 'timeout', container)
+            answered = observe_zmx_container(container)
+            found = answered or kept_zmx(previous or {}, 'timeout', container)
+            if answered is not None and listed is not None:
+                listed[container] = issued
         except Exception as error:
             log(f'zmx {container}', error)
             found = kept_zmx(previous or {}, 'error', container)
@@ -657,7 +818,7 @@ def update_containers(ctx, registry, level):
     5-minute safety net on a host that is not busy — and at most every CONTAINER_RETRY seconds. NEVER because a
     probe failed: `docker inspect` (2 spawns, 20 s timeouts) every 3 s is a load generator on a host that is
     already too slow to answer it."""
-    names = {lane['container'] for w in registry['workspaces'] for lane in w['lanes'] + w['extras'] if lane.get('container')}
+    names = {lane['container'] for w in registry['workspaces'] for lane in w['lanes'] if lane.get('container')}
     known = ctx['containers']
     age = time.time() - ctx['containers_at']
     unsettled = set(known) != names or any(not pid_alive(c.get('pid')) for c in known.values())
@@ -674,8 +835,11 @@ def update_containers(ctx, registry, level):
 
 def build_payload(ctx, level, reconcile=True):
     """One observation pass -> the payload of status.json (None = there is no registry to observe). Every phase
-    fails alone: what it could not observe is republished as last seen (stale), the other phases still run."""
+    fails alone: what it could not observe is republished as last seen (stale), the other phases still run.
+    ctx['cut'] = how many tmux probes the cycle deadline abandoned (0 when nothing was probed): `pause` reads it."""
     previous = ctx['payload']
+    report = {'cut': 0}
+    ctx['cut'] = 0
     try:
         stamp = os.stat(L.STATE / 'registry.json').st_mtime_ns
         if stamp != ctx['stamp']:
@@ -690,33 +854,48 @@ def build_payload(ctx, level, reconcile=True):
     except Exception as error:
         log('containers', error)
     containers = ctx['containers']
+    unwalkable = {}
     try:
-        live = live_tmux_sockets()
+        live = live_by_container(containers, unwalkable)
     except Exception as error:
-        log('ps', error)
+        log('live servers', error)
         live = None
+    # Name the container whose init pid broke the cgroup read, once when it starts (not every INTERVAL): only ITS
+    # sockets are probed blind, and the operator needs to know which one that is.
+    for name in sorted(set(unwalkable) - set(ctx.get('unwalkable') or ())):
+        log(f'live servers {name}', L.LanesError(f'{unwalkable[name]} — its sockets are probed as unknown, none concluded gone'))
+    ctx['unwalkable'] = sorted(unwalkable)
     try:
-        lanes, servers, unregistered = observe(registry, containers, live, previous)
+        lanes, servers, unregistered = observe(registry, containers, live, previous, report)
     except Exception as error:
         log('observe', error)
         lanes = {lane_id: keep(previous, lane_id, 'error') for lane_id in previous.get('lanes') or {}}
-        servers, unregistered = previous.get('servers') or {}, previous.get('unregistered') or {}
+        # DEF-1538: COPIES of what was published, never the previous payload itself (`previous` is the very object
+        # `ctx['published']` holds, so a kept list extended in place would grow the published payload), and only
+        # observe's own rows (holder 'tmux', the rows `observe` carries over too).
+        servers = dict(previous.get('servers') or {})
+        unregistered = {}
+        for workspace, entries in (previous.get('unregistered') or {}).items():
+            kept = [entry for entry in entries if entry.get('holder') == 'tmux']
+            if kept:
+                unregistered[workspace] = kept
+    ctx['cut'] = report.get('cut', 0)
     # Half of all our spawns, and only cosmetics (who dictates a window size): not on a host under pressure.
     if reconcile and level == 'ok':
         try:
             reconcile_client_sizes(registry, containers, live)
         except Exception as error:
             log('reconcile', error)
+    listed = {}
     try:
-        zmx_lanes, zmx_groups = observe_zmx(containers, previous)
+        zmx_lanes, zmx_groups = observe_zmx(containers, previous, listed)
     except Exception as error:
         log('zmx', error)
         zmx_lanes, zmx_groups = kept_zmx(previous, 'error')
+        listed = {}
+    # Outside the payload: it changes every cycle, and status.json is rewritten only on change (plus the heartbeat).
+    ctx['zmx_listed_at'] = listed
     lanes.update(zmx_lanes)
-    legacy_lanes, legacy_groups = Legacy.observe()
-    lanes.update(legacy_lanes)
-    for workspace, entries in legacy_groups.items():
-        unregistered.setdefault(workspace, []).extend(entries)
     return {'lanes': lanes, 'servers': servers, 'unregistered': unregistered, 'zmx': zmx_groups,
             'containers': {k: {'running': v.get('running', False)} for k, v in containers.items()}}
 
@@ -730,7 +909,7 @@ def new_context():
     for entry in (seed.get('lanes') or {}).values():
         entry.setdefault('stale_since', int(written))
     return {'registry': None, 'stamp': 0, 'containers': {}, 'containers_at': 0.0,
-            'payload': seed, 'published': None, 'status_at': written}
+            'payload': seed, 'published': None, 'status_at': written, 'cut': 0, 'unwalkable': []}
 
 
 def cycle(ctx, reconcile=True):
@@ -754,7 +933,8 @@ def cycle(ctx, reconcile=True):
         if payload != ctx['published'] or started - ctx['status_at'] > HEARTBEAT:
             try:
                 written = time.time()
-                L.atomic_write(L.STATE / 'status.json', json.dumps(dict(payload, health=health, updated_at=written)) + '\n')
+                L.atomic_write(L.STATE / 'status.json', json.dumps(dict(payload, health=health, updated_at=written,
+                                                                        zmx_listed_at=ctx.get('zmx_listed_at') or {})) + '\n')
                 ctx['published'], ctx['status_at'] = payload, written
             except Exception as error:
                 log('status.json', error)
@@ -775,8 +955,12 @@ def cycle(ctx, reconcile=True):
     return level, payload
 
 
-def pause(level, elapsed):
-    return max(0.5, INTERVAL * SLOWDOWN.get(level, 1) - elapsed)
+def pause(level, elapsed, cut=0):
+    """Seconds to sleep before the next cycle. After a cycle that abandoned probes at its deadline (`cut`), at least
+    PROBE_TIMEOUT + PROBE_GRACE: run_bounded ends each of those within that time, so the next round never runs
+    beside them and at most PROBE_WORKERS tmux clients of ours are ever in flight on a host that is already slow."""
+    rest = max(0.5, INTERVAL * SLOWDOWN.get(level, 1) - elapsed)
+    return max(rest, PROBE_TIMEOUT + PROBE_GRACE) if cut else rest
 
 
 def registry_source_paths(registry, containers):
@@ -842,7 +1026,7 @@ def main():
             if hot_sync(ctx, source_mtimes):
                 last_triggered_sync = time.time()
         ctx['children'] = [p for p in ctx.get('children', []) if p.poll() is None]   # reap finished syncs
-        time.sleep(pause(level, time.time() - started))
+        time.sleep(pause(level, time.time() - started, ctx.get('cut', 0)))
 
 
 if __name__ == '__main__':