@vibelord/lanes 1.3.0 → 1.5.0

package/payload/lanes/libexec/statusd.py

@@ -17,11 +17,19 @@ or positively does not exist.
 The docs cockpit (plan W10 W7): a lane row carries `docs: {n, at}` while docs/counts.json counts documents for it
 (one stat of that file per cycle), and the document `web_origin`, the https origin the docs page answers on, once
 main() has learnt it between two cycles (refresh_origin).
+
+Fleet facts (spec W11 §2.2): every FLEET_INTERVAL, unless the host is critical, `fleet` — the host's cpu, memory,
+pressure, inotify ceiling and the biggest workspace containers, from `lanes_lib.compute_health(sample=True)` and the
+previous sample kept in the context — and on every cycle `throttle`, the Throttle pulse's `~/.lz/state/throttle.json`
+in the daemon's container (one stat, read only when it moved). The one spawn either adds is `rearm_pulse`, between two
+cycles: one bounded `docker exec -d` at most every PULSE_REARM_EVERY seconds while the pulse file has gone quiet.
 """
 import concurrent.futures
 import json
+import math
 import os
 import re
+import stat
 import subprocess
 import sys
 import time
@@ -72,12 +80,39 @@ WEB_PORT = 8443         # bin/lanes SERVE_PORT: `tailscale serve --https=8443` -
 ORIGIN_TTL = 300        # enrold.TAILNET_TTL
 HOST_NAME = re.compile(r'(?=.{1,253}\Z)[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?'
                        r'(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*\Z', re.ASCII)
+# Fleet facts and the Throttle pulse (spec W11 §2.1/§2.2, DM-LANES-FLEET, DM-LANES-THROTTLE-PULSE).
+FLEET_INTERVAL = 10     # seconds between two fleet samples (cpu rates are over this); none taken while critical
+HOSTNAME_FILE = '/etc/hostname'   # `host.name`, read once per daemon start (new_context)
+THROTTLE_TOOL = 'tools--throttle-monitor'   # lanes_lib.build_registry's tool tab: its container hosts the daemon
+PULSE_DIR = ('home', 'coder', '.lz', 'state')   # under /proc/<daemon container init>/root, walked without a link
+PULSE_FILE = 'throttle.json'
+PULSE_MAX = 64 << 10    # a bigger file is not the pulse's: refused, never read
+# A healthy pulse's `at` is seen up to ~84 s old here (its 60 s HEARTBEAT is judged on an int-truncated `at`, one
+# 10 s loop cycle late at a second boundary, a slow cycle's five 3 s GET timeouts, then statusd's own 3 s cycle):
+# older than PULSE_STALE_S is published `stale: true` — the same 90 s after which `rearm_pulse` calls the loop stopped.
+PULSE_STALE_S = 90
+PULSE_REARM_S = 90      # a pulse file quiet for longer is a stopped loop ...
+PULSE_REARM_EVERY = 600  # ... restarted at most this often, never on a critical host
+PULSE_REARM_TIMEOUT = 10
+DAEMON_REFRESH = 300    # site.conf is re-read this often for LANES_THROTTLE_CONTAINER (CONTAINER_REFRESH)
+PULSE_BIN = ('home', 'coder', '.lz', 'bin')
+PULSE_PROGRAM = 'throttle-pulse'
+PULSE_POOLS = ('claude', 'codex', 'kimi')   # throttle-pulse POOLS (W12 adds kimi; its overflow is always null)
+PULSE_COUNTS = ('configured', 'usable', 'cooling', 'exhausted', 'benched', 'auth_failed')
+PULSE_OVERFLOW = ('spend_today_usd', 'budget_usd', 'fraction', 'alert_level_pct')
+PULSE_PHASE = re.compile(r'[a-z][a-z0-9_-]{0,31}\Z', re.ASCII)
+PULSE_ERROR_MAX = 160
 
 
 def log(where, error):
     print(time.strftime('%FT%T'), where, type(error).__name__, error, flush=True)
 
 
+def note(where, message):
+    """A journal line that is no exception (the pulse re-arm and its outcome, a pulse file that went or came back)."""
+    print(time.strftime('%FT%T'), where, message, flush=True)
+
+
 # ------------------------------------------------- the viewer rule (A4a)
 # A Throttle Studio browser attaches with `new-session -t <lane> -s view-<studio>-<ch>`: an independent
 # current window on the lane's windows. Such a session is a CLIENT of the lane, not a lane of its own, and
@@ -295,34 +330,92 @@ def settled(previous, lane_id, now, raw):
     return AS.settle(old.get('agent') if isinstance(old, dict) else None, raw, now)
 
 
-def agent_signals(containers, sessions_by_server):
+# W12 (spec §2): the harness each pane runs, read from the container's /proc and kept per pane pid between cycles —
+# a pane whose own children did not change is not walked again (`agent_state.HarnessCache`; reads only, no spawn).
+HARNESS_CACHE = AS.HarnessCache()
+# The Grok Build observer's title memory between cycles (`agent_state.grok_sessions(..., titles=)`), one dict per
+# container root ({pane key: the last title, held only while it carries `⚠ Action Required`}): unfocused, Grok blinks
+# that prefix away half of every second, so a busy title right after a prefixed one is still an approval. Kept like
+# HARNESS_CACHE; each cycle prunes it to the panes it passed and drops a container root it did not read.
+GROK_TITLES = {}
+# The harnesses whose panes go to the Grok observer: Grok Build itself; a bare shell (the observer reads its title: a
+# Grok title over a tree with no grok left is that harness ended, `grok` / `shell`); and `unknown` WITH a harness pid —
+# the walk's `grok` whose /proc/<pid>/exe could not be read, Grok Build only when active_sessions.json names its pid.
+GROK_OBSERVED = ('grok', 'shell')
+
+
+def agent_signals(containers, sessions_by_server, now=None):
     """The per-container inputs `agent_state.merge` needs, read ONCE per container per cycle through the
     container's root as the host sees it (`/proc/<init pid>/root`), so liveness is judged in the container's pid
-    namespace and nothing is spawned. -> {container: claude_by_pane}.
+    namespace and nothing is spawned. -> {container: {'claude': claude_by_pane, 'kimi': kimi_by_pane,
+    'grok': grok_by_pane}}.
+
+    Sets `harness` (and `harness_pid` when one was found) on every session row whose harness could be observed, in
+    place (W12, spec §2: `agent_state.HarnessCache` over `harness_walk` — the pane's process tree in the container's
+    /proc); the Kimi Code observer (`agent_state.kimi_sessions`) reads only for the panes found running `kimi`, and the
+    Grok Build observer (`agent_state.grok_sessions` -> `harness_grok.grok_sessions`) gets the ROWS of the panes found
+    running `grok`, ending in a bare shell, or running a grok whose exe could not be read (GROK_OBSERVED), with the
+    container's GROK_TITLES memory.
 
     Also sets `pane_has_child` on the session rows that need it, in place: only a pane whose title Codex wrote
-    (`parse_codex_title`) over a bare-shell foreground can be a harness that ended into its shell, and only the
-    child read (`child_liveness`) tells that apart from a live harness launched under `bash -c` or a wrapper. A pane
-    a live Claude session file already claims is not probed (the file outranks the title). The key is set only for
-    a known answer: unknown liveness leaves it absent, and an absent key never demotes a title (DEF-1557). A
-    container whose files cannot be read costs only its own agents, never a lane its `state`."""
-    claude = {}
+    (`parse_codex_title`, gated by the observed harness) over a bare-shell foreground can be a harness that ended into
+    its shell, and only the child read (`child_liveness`) tells that apart from a live harness launched under `bash -c`
+    or a wrapper. A pane a live Claude session file already claims is not probed (the file outranks the title), nor one
+    whose walk found a harness running (alive by that very read). The key is set only for a known answer: unknown
+    liveness leaves it absent, and an absent key never demotes a title (DEF-1557). A container whose files cannot be
+    read costs only its own agents, never a lane its `state`."""
+    signals = {}
+    walked = set()
+    titled = set()
+    home = tuple(part for part in L.CONTAINER_HOME.split('/') if part)
     for container in sorted({container for container, _socket in sessions_by_server}):
         info = containers.get(container) or {}
         if not info.get('pid'):
             continue
         root = Path(L.proc_root()) / str(info['pid']) / 'root'
         try:
-            claude[container] = AS.claude_sessions(root / L.CONTAINER_HOME.lstrip('/') / '.claude' / 'sessions',
-                                                   root / 'proc')
+            claude = AS.claude_sessions(root / L.CONTAINER_HOME.lstrip('/') / '.claude' / 'sessions', root / 'proc')
         except Exception as error:
             log(f'agents {container}', error)
-            claude[container] = {}
-        probe = [pane for (owner, _socket), sessions in sessions_by_server.items() if owner == container
-                 for pane in sessions.values()
-                 if str(pane.get('pane_pid') or '').isdigit() and AS.parse_codex_title(pane.get('pane_title'))
-                 and AS._shell_foreground(pane)
-                 and f"{pane.get('session_name')}:{pane.get('window_id')}.{pane.get('pane_id')}" not in claude[container]]
+            claude = {}
+        signals[container] = mine = {'claude': claude, 'kimi': {}, 'grok': {}}
+        panes = [pane for (owner, _socket), sessions in sessions_by_server.items() if owner == container
+                 for pane in sessions.values() if str(pane.get('pane_pid') or '').isdigit()]
+        observers = {'kimi': {}, 'grok': {}}
+        for pane in panes:
+            try:
+                harness, harness_pid = HARNESS_CACHE.harness(root / 'proc', int(pane['pane_pid']))
+            except Exception as error:
+                log(f'harness {container}', error)
+                continue
+            walked.add((str(root / 'proc'), int(pane['pane_pid'])))
+            if harness is None:
+                continue
+            pane['harness'] = harness
+            if harness_pid:
+                pane['harness_pid'] = harness_pid
+            if harness == 'kimi' and harness_pid:
+                observers['kimi'][AS.pane_key(pane)] = harness_pid
+            elif harness in GROK_OBSERVED or (harness == 'unknown' and harness_pid):
+                observers['grok'][AS.pane_key(pane)] = pane
+        if observers['kimi']:
+            try:
+                mine['kimi'] = AS.kimi_sessions(root, observers['kimi'], now, home)
+            except Exception as error:
+                log(f'kimi {container}', error)
+        if observers['grok']:
+            titles = GROK_TITLES.setdefault(str(root), {})
+            titled.add(str(root))
+            try:
+                mine['grok'] = AS.grok_sessions(root, observers['grok'], AS.GROK_HOME, home=L.CONTAINER_HOME,
+                                                titles=titles)
+            except Exception as error:
+                log(f'grok {container}', error)
+            for gone in set(titles) - set(observers['grok']):
+                del titles[gone]
+        probe = [pane for pane in panes
+                 if AS.parse_codex_title(pane.get('pane_title'), pane.get('harness')) and AS._shell_foreground(pane)
+                 and pane.get('harness') not in AS.AGENT_HARNESSES and AS.pane_key(pane) not in claude]
         if not probe:
             continue
         try:
@@ -334,7 +427,15 @@ def agent_signals(containers, sessions_by_server):
             known = live.get(int(pane['pane_pid']))
             if known is not None:
                 pane['pane_has_child'] = known
-    return claude
+    HARNESS_CACHE.prune(walked)
+    for gone in set(GROK_TITLES) - titled:
+        del GROK_TITLES[gone]
+    return signals
+
+
+def merged(pane, signals, sidecar):
+    """`agent_state.merge` of one pane over its container's `agent_signals` and its lane's hook sidecar."""
+    return AS.merge(pane, signals.get('claude', {}), sidecar, signals.get('kimi', {}), signals.get('grok', {}))
 
 
 SIDECAR_MAX_AGE = 3600   # a hook sidecar whose last event is older than this loses to the live signals
@@ -532,10 +633,10 @@ def observe(registry, containers, live, previous=None, report=None):
         report['cut'] = len(cut)
     # Task 5: the agent signals of every container, read once per cycle over all of its servers' sessions.
     try:
-        claude = agent_signals(containers, {key: sessions for key, (_o, sessions, _a) in probed.items()})
+        signals = agent_signals(containers, {key: sessions for key, (_o, sessions, _a) in probed.items()}, now)
     except Exception as error:
         log('agents', error)
-        claude = {}
+        signals = {}
     # Task 8: the hook sidecars `lane-signal` writes, one directory listing per container per cycle. They outrank the
     # session file and the title in `agent_state.merge`; a container without hooks costs one failed listdir.
     try:
@@ -620,8 +721,8 @@ def observe(registry, containers, live, previous=None, report=None):
                         entry['note'] = 'session name outside the lane declaration grammar'
                     found = lane_entry(pane, now, activities.get(name), viewers.get(name, []),
                                        settled(previous, entry['id'], now,
-                                               AS.merge(pane, claude.get(container, {}),
-                                                        sidecars.get(container, {}).get(entry['id']))))
+                                               merged(pane, signals.get(container, {}),
+                                                      sidecars.get(container, {}).get(entry['id']))))
                 except Exception as error:
                     log(f'observe {container}/{socket}/{name}', error)
                     continue
@@ -641,8 +742,8 @@ def observe(registry, containers, live, previous=None, report=None):
                 try:
                     lanes[lane['id']] = lane_entry(pane, now, activities.get(session), viewers.get(session, []),
                                                    settled(previous, lane['id'], now,
-                                                           AS.merge(pane, claude.get(container, {}),
-                                                                    sidecars.get(container, {}).get(lane['id']))))
+                                                           merged(pane, signals.get(container, {}),
+                                                                  sidecars.get(container, {}).get(lane['id']))))
                 except Exception as error:   # a torn row: the session is there, its details are not readable
                     log(f"observe {lane['id']}", error)
                     lanes[lane['id']] = keep(previous, lane['id'], 'error')
@@ -1018,7 +1119,8 @@ def new_context():
              and HOST_NAME.match(origin[len('https://'):-len(f':{WEB_PORT}')]))
     return {'registry': None, 'stamp': 0, 'containers': {}, 'containers_at': 0.0,
             'payload': seed, 'published': None, 'status_at': written, 'cut': 0, 'unwalkable': [],
-            'container_status': {}, 'container_status_errors': {}, 'web_origin': origin if valid else None}
+            'container_status': {}, 'container_status_errors': {}, 'web_origin': origin if valid else None,
+            'host_name': read_host_name()}
 
 
 # ------------------------------------------- the in-container projection (LNS-32)
@@ -1124,14 +1226,312 @@ def _container_status_error(errors, container, error):
         log(f'container status {container}', error)
 
 
+# ------------------------------------------ fleet facts and the Throttle pulse (W11)
+# `fleet` (DM-LANES-FLEET) is the host and its biggest workspace containers, `throttle` (DM-LANES-THROTTLE-PULSE) the
+# Throttle pulse's last document. Both are additive keys of status.json; the sidebar's pinned block and `lanes doctor`
+# read them there and nowhere else.
+
+
+def read_host_name(path=None):
+    """`host.name`: the first line of /etc/hostname, else the kernel's node name, published only when it is a DNS host
+    name (it is drawn into terminals) — else None. Read once per daemon start (new_context), never per cycle."""
+    try:
+        name = Path(path or HOSTNAME_FILE).read_text().split('\n', 1)[0].strip()
+    except (OSError, ValueError):
+        name = ''
+    name = name or os.uname().nodename
+    return name if HOST_NAME.match(name) else None
+
+
+def fleet_due(ctx, now):
+    """True when FLEET_INTERVAL has passed since the cycle that took the last fleet sample (or none was taken yet)."""
+    last = ctx.get('fleet_started')
+    return last is None or now - last >= FLEET_INTERVAL
+
+
+def update_fleet(ctx, health, sample, due, now):
+    """ctx['fleet'] after this cycle -> it (None until a first sample). A new sample (compute_health's, taken when
+    `due`) replaces the facts (`stale: false`) and is kept as the previous sample of the next one — in the context,
+    never in a global. A due sample that was not taken (a critical host: compute_health took none; a failed read)
+    keeps the last facts, `stale: true`, until one is."""
+    if sample is not None:
+        ctx['fleet'] = L.fleet_facts(sample, ctx.get('fleet_sample'), health, ctx.get('host_name'))
+        ctx['fleet_sample'], ctx['fleet_started'] = sample, now
+    elif due and ctx.get('fleet') and ctx['fleet'].get('stale') is not True:
+        ctx['fleet'] = dict(ctx['fleet'], stale=True)
+    return ctx.get('fleet')
+
+
+def site_settings():
+    """The server's site.conf LANES_* values, by the enrolment service's reader (read, never executed)."""
+    import enrold
+    return enrold.server_site()
+
+
+def daemon_container(registry, site):
+    """The container the Throttle daemon runs in: site.conf's LANES_THROTTLE_CONTAINER, else the container of the
+    registry's `tools--throttle-monitor` tab (lanes_lib.build_registry: the monitor host most boards name), else
+    None. A name that is no container name is None — never a guess in its place."""
+    name = (site or {}).get('LANES_THROTTLE_CONTAINER')
+    if not name:
+        tools = (registry or {}).get('tools') if isinstance(registry, dict) else None
+        tool = next((t for t in tools or [] if isinstance(t, dict) and t.get('id') == THROTTLE_TOOL), None)
+        name = (tool or {}).get('container')
+    return name if isinstance(name, str) and L.IDENT.match(name) else None
+
+
+def resolve_daemon(ctx, now):
+    """Which container hosts the daemon (site.conf, else the loaded registry) and its init pid: the one statusd
+    tracks for that container each cycle, else — a container no lane runs in — the one `lanes sync` recorded in
+    containers.json at this look. Never per cycle (the pulse adds ONE stat per cycle, nothing more): looked at again
+    when the loaded registry changed, at most every FLEET_INTERVAL while the last snapshot could not open the
+    container's ~/.lz/state (a container restarted under a new init pid), and every DAEMON_REFRESH s otherwise (an
+    edited site.conf)."""
+    last, registry = ctx.get('daemon_at'), ctx.get('registry')
+    if last is not None and ctx.get('daemon_registry') is registry:
+        wait = FLEET_INTERVAL if (ctx.get('pulse') or {}).get('no_dir') else DAEMON_REFRESH
+        if now - last < wait:
+            return
+    ctx['daemon_at'], ctx['daemon_registry'] = now, registry
+    try:
+        site = site_settings()
+    except Exception as error:
+        log('site.conf', error)
+        site = {}
+    name = daemon_container(ctx.get('registry'), site)
+    recorded = None
+    if name and name not in (ctx.get('containers') or {}):
+        known = L.read_json(L.STATE / 'containers.json', {})
+        entry = known.get(name) if isinstance(known, dict) else None
+        recorded = entry.get('pid') if isinstance(entry, dict) else None
+    ctx['daemon'] = {'name': name, 'pid': recorded} if name else None
+
+
+def daemon_info(ctx):
+    """(container name, init pid or None) of the daemon's container, or None when no container is named."""
+    daemon = ctx.get('daemon')
+    if not daemon:
+        return None
+    tracked = (ctx.get('containers') or {}).get(daemon['name'])
+    pid = tracked.get('pid') if isinstance(tracked, dict) else daemon.get('pid')
+    return daemon['name'], pid if isinstance(pid, int) and pid > 0 else None
+
+
+def _number(value):
+    return value if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) else None
+
+
+def _count(value):
+    return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None
+
+
+def _flag(value):
+    return value if isinstance(value, bool) else None
+
+
+def pulse_projection(data):
+    """The pulse's bytes as status.json carries them, or None when they are no v1 pulse document (spec W11 §2.1).
+
+    The daemon's container writes that file, and the sidebar draws what is in it into terminals: nothing is copied as
+    such. Only the spec's keys are, each of its own type — counts non-negative integers; fractions, dollars and
+    timestamps finite numbers; flags booleans; `pressure_phase` one short lowercase word; `error` printable characters
+    only, at most PULSE_ERROR_MAX — anything else null, unknown keys and pools dropped. No escape sequence rides
+    through, and a pool the pulse did not write is not invented."""
+    if data is None:
+        return None
+    try:
+        document = json.loads(data.decode('utf-8'))
+    except (UnicodeDecodeError, ValueError, RecursionError):
+        return None
+    if not isinstance(document, dict) or _count(document.get('v')) != 1 or _number(document.get('at')) is None:
+        return None
+    pools, overflow = {}, {}
+    for pool in PULSE_POOLS:
+        row = (document.get('pools') or {}).get(pool) if isinstance(document.get('pools'), dict) else None
+        if isinstance(row, dict):
+            phase = row.get('pressure_phase')
+            pools[pool] = dict({key: _count(row.get(key)) for key in PULSE_COUNTS},
+                               overflow_ready=_flag(row.get('overflow_ready')),
+                               pressure_phase=phase if isinstance(phase, str) and PULSE_PHASE.match(phase) else None,
+                               spill_fraction=_number(row.get('spill_fraction')),
+                               overflow_in_flight=_count(row.get('overflow_in_flight')),
+                               upstream_429_total=_count(row.get('upstream_429_total')),
+                               worst_7d=_number(row.get('worst_7d')), worst_5h=_number(row.get('worst_5h')),
+                               soonest_reset=_number(row.get('soonest_reset')))
+        money = (document.get('overflow') or {}).get(pool) if isinstance(document.get('overflow'), dict) else None
+        if isinstance(money, dict):
+            overflow[pool] = {key: _number(money.get(key)) for key in PULSE_OVERFLOW}
+    error = document.get('error')
+    error = ''.join(c for c in error if c.isprintable())[:PULSE_ERROR_MAX] if isinstance(error, str) else ''
+    return {'v': 1, 'at': int(document['at']), 'up': _flag(document.get('up')), 'tokened': _flag(document.get('tokened')),
+            'pools': pools, 'overflow': overflow, 'error': error or None}
+
+
+def throttle_snapshot(ctx, now):
+    """The `throttle` block of status.json, or None when no daemon container is named (no Throttle here: no key).
+
+    ONE stat of the pulse file per cycle — through /proc/<daemon container init>/root, every directory down to
+    ~/.lz/state walked with O_NOFOLLOW (lanes_lib.open_real_dir: the container owns it, and a link planted there would
+    resolve against the HOST's root) and the file itself lstat'ed — and a read only when its (init pid, mtime, inode,
+    size) moved: `read_regular_at`, a regular file of at most PULSE_MAX bytes, never a link, never a FIFO. What it
+    holds reaches status.json only through `pulse_projection`. The last values are kept when the file goes away, is
+    refused or does not parse, `stale: true` then — and whenever their `at` is more than PULSE_STALE_S old by this
+    clock; with none ever read the block is `{"stale": true}`. ctx['pulse']['mtime'] (the file's own age, None while
+    it cannot be seen) is what `rearm_pulse` judges."""
+    found = daemon_info(ctx)
+    if found is None:
+        ctx.pop('pulse', None)
+        return None
+    name, pid = found
+    pulse = ctx.get('pulse')
+    if not isinstance(pulse, dict) or pulse.get('name') != name:
+        pulse = ctx['pulse'] = {'name': name}
+    stamp = mtime = fd = None
+    why = None if pid else 'container not running'
+    pulse['no_dir'] = not pid
+    if pid:
+        try:
+            fd = L.open_real_dir(Path(L.proc_root()) / str(pid) / 'root', PULSE_DIR)
+        except FileNotFoundError:   # the init is gone (a restart: resolve_daemon looks again soon), or no ~/.lz/state
+            pulse['no_dir'] = not pid_alive(pid)
+            why = 'container not running' if pulse['no_dir'] else 'no ~/.lz/state'
+        except (OSError, L.LanesError) as error:
+            why = f'~/.lz/state refused ({type(error).__name__})'
+    if fd is not None:
+        try:
+            info = os.stat(PULSE_FILE, dir_fd=fd, follow_symlinks=False)
+            stamp, mtime = (pid, info.st_mtime_ns, info.st_ino, info.st_size), info.st_mtime
+            if stamp != pulse.get('stamp'):
+                document = pulse_projection(L.read_regular_at(fd, PULSE_FILE, PULSE_MAX))
+                pulse['valid'] = document is not None
+                if document is not None:
+                    pulse['doc'] = document
+            if not pulse.get('valid'):
+                why = 'refused (no regular file of at most 64 KiB holding a v1 pulse document)'
+        except FileNotFoundError:
+            why = 'absent'
+        except OSError as error:
+            why = f'unreadable ({type(error).__name__})'
+        finally:
+            os.close(fd)
+    pulse['stamp'], pulse['mtime'] = stamp, mtime
+    if why != pulse.get('why', 'unseen'):
+        note('throttle pulse', f'{name}: {why or "ok"}')
+        pulse['why'] = why
+    document = pulse.get('doc')
+    stale = why is not None or document is None or now - document['at'] > PULSE_STALE_S
+    return dict(document or {}, stale=stale)
+
+
+def pulse_rearm_argv(container):
+    """The one spawn of the fleet feature: the pulse loop started detached in its own tmux server in the container,
+    as the container user (CONTAINER_UID, like every other exec Lanes makes into a container: the loop must write
+    /home/coder/.lz/state, and a later re-arm must find its tmux server) with HOME=CONTAINER_HOME whatever the image
+    sets (the pulse's `~/.lz/state` and `~/.lz/throttle-token` defaults expand against it) — byte for byte the argv
+    `lanes install-container` starts it with (bin/lanes `pulse_argv`). argv only, no shell."""
+    return ['docker', 'exec', '-d', '-u', str(L.CONTAINER_UID), '-e', f'HOME={L.CONTAINER_HOME}', container, 'tmux',
+            '-L', 'lanes-pulse', 'new', '-d', '-s', 'pulse', f'{L.CONTAINER_HOME}/.lz/bin/{PULSE_PROGRAM}', '--loop']
+
+
+def pulse_installed(pid):
+    """True when ~/.lz/bin/throttle-pulse is a regular file in the container (`lanes install-container` shipped it),
+    looked at through /proc/<pid>/root without following a link."""
+    try:
+        fd = L.open_real_dir(Path(L.proc_root()) / str(pid) / 'root', PULSE_BIN)
+    except (OSError, L.LanesError):
+        return False
+    try:
+        return stat.S_ISREG(os.stat(PULSE_PROGRAM, dir_fd=fd, follow_symlinks=False).st_mode)
+    except OSError:
+        return False
+    finally:
+        os.close(fd)
+
+
+def rearm_pulse(ctx, level, now=None):
+    """main()'s restart of a stopped Throttle pulse, between two cycles and never inside one -> True when it spawned.
+
+    Only while a daemon container is named and its init is alive, the host is not `critical`, the pulse file is
+    older than PULSE_REARM_S by its mtime (or cannot be seen at all), and the last attempt is at least
+    PULSE_REARM_EVERY seconds ago. An attempt is counted whatever comes of it, so a failing one is not retried every
+    cycle; it starts nothing when ~/.lz/bin/throttle-pulse is not there. Then ONE bounded `docker exec -d`
+    (pulse_rearm_argv, PULSE_REARM_TIMEOUT s), its outcome journalled."""
+    now = time.time() if now is None else now
+    found = daemon_info(ctx)
+    if level == 'critical' or found is None:
+        return False
+    name, pid = found
+    mtime = (ctx.get('pulse') or {}).get('mtime')
+    if mtime is not None and now - mtime <= PULSE_REARM_S:
+        return False
+    last = ctx.get('pulse_rearm_at')
+    if last is not None and now - last < PULSE_REARM_EVERY:
+        return False
+    if not pid_alive(pid):
+        return False
+    ctx['pulse_rearm_at'] = now
+    if not pulse_installed(pid):
+        note('throttle pulse', f'{name}: quiet, and ~/.lz/bin/{PULSE_PROGRAM} is not installed there '
+                               f'(lanes install-container {name}): not started')
+        return False
+    argv = pulse_rearm_argv(name)
+    rc, _out, err = L.run_bounded(argv, PULSE_REARM_TIMEOUT)
+    age = 'not seen' if mtime is None else f'{int(now - mtime)} s old'
+    if rc is None:
+        outcome = f'no answer within {PULSE_REARM_TIMEOUT} s'
+    else:
+        last_line = err.strip().splitlines()[-1][:160] if rc and err.strip() else ''
+        outcome = f'exit {rc}' + (f': {last_line}' if last_line else '')
+    note('throttle pulse', f'{name}: pulse file {age}, re-armed ({outcome}): {" ".join(argv)}')
+    return True
+
+
+def compared(payload):
+    """`payload` as the rewrite decision sees it: without the sample time of `fleet` (new every FLEET_INTERVAL) and
+    of `throttle` (the pulse's 60-s heartbeat) — the quantized facts and the stale marks are what may move a rewrite
+    (like `health`, which is not in the payload at all); a block whose only news is its `at` waits for HEARTBEAT."""
+    if not isinstance(payload, dict):
+        return payload
+    out = dict(payload)
+    for key in ('fleet', 'throttle'):
+        if isinstance(out.get(key), dict):
+            out[key] = {name: value for name, value in out[key].items() if name != 'at'}
+    return out
+
+
+def with_fleet(ctx, payload, health, sample, due, now):
+    """`payload` plus `fleet` and `throttle`, each failing alone: a fleet that could not be computed is the last one,
+    stale; a pulse that could not be looked at is the last block published, stale."""
+    out = dict(payload)
+    try:
+        fleet = update_fleet(ctx, health, sample, due, now)
+    except Exception as error:
+        log('fleet', error)
+        fleet = dict(ctx['fleet'], stale=True) if ctx.get('fleet') else None
+    if fleet:
+        out['fleet'] = fleet
+    try:
+        resolve_daemon(ctx, now)
+        throttle = throttle_snapshot(ctx, now)
+    except Exception as error:
+        log('throttle pulse', error)
+        last = (ctx.get('payload') or {}).get('throttle')
+        throttle = dict(last, stale=True) if isinstance(last, dict) else None
+    if throttle is not None:
+        out['throttle'] = throttle
+    return out
+
+
 def cycle(ctx, reconcile=True):
     """Observe once and ALWAYS publish -> (health level, payload or None)."""
     started = time.time()
-    try:
-        health = L.compute_health()
+    due = fleet_due(ctx, started)
+    try:   # every FLEET_INTERVAL the fleet counters too (W11; compute_health takes none on a critical host)
+        health = L.compute_health(sample=due)
     except Exception as error:
         log('health', error)
         health = {}
+    sample = health.pop('sample', None)
     level = health.get('level', 'ok')
     try:
         payload = build_payload(ctx, level, reconcile)
@@ -1139,10 +1539,12 @@ def cycle(ctx, reconcile=True):
         log('cycle', error)
         payload = None
     if payload is not None:
+        payload = with_fleet(ctx, payload, health, sample, due, started)
         ctx['payload'] = payload
         # Rewrite only on change (plus a heartbeat) so sidebars do not repaint for nothing. `health` changes every
-        # cycle and is therefore not part of the comparison.
-        if payload != ctx['published'] or started - ctx['status_at'] > HEARTBEAT:
+        # cycle and is therefore not part of the comparison; nor is the fleet's sample time (`compared`).
+        if (ctx['published'] is None or compared(payload) != compared(ctx['published'])
+                or started - ctx['status_at'] > HEARTBEAT):
             try:
                 written = time.time()
                 L.atomic_write(L.STATE / 'status.json', json.dumps(dict(payload, health=health, updated_at=written,
@@ -1239,6 +1641,10 @@ def main():
         level, _payload = cycle(ctx)
         # The docs links' origin (plan W10 W7): a bounded look between two cycles, never inside one.
         refresh_origin(ctx, level)
+        try:   # W11: a pulse file gone quiet -> one bounded re-arm, at most every PULSE_REARM_EVERY s, never critical
+            rearm_pulse(ctx, level)
+        except Exception as error:
+            log('throttle pulse', error)
         # Hot-reload: check if any terminals.json changed since we last synced.
         # Rate-limited to at most one triggered sync per HOT_SYNC_EVERY_S to avoid thrashing.
         if time.time() - last_triggered_sync > HOT_SYNC_EVERY_S and ctx['registry']:

package/payload/lanes/libexec/tokens.py

@@ -93,7 +93,18 @@ MARKS = {'cursor': ('▌', '|', 'accent'), 'viewing': ('›', '=', 'accent'), 'v
          'ok': ('✓', 'OK', 'tone-done'), 'fail': ('✗', 'X', 'tone-urgent'),
          # The docs cockpit on the upload strip (` ↑ uploads · ▤ docs 3  (U)`, plan W10 W7): the strip's own tone.
          # Seven-bit `#` is free on that row (`^` uploads, `v` armed, `-` sep are the others).
-         'docs': ('▤', '#', 'ink-dim')}
+         'docs': ('▤', '#', 'ink-dim'),
+         # The fleet block (W11, spec 2026-10-11 §3), pinned above the upload strip: column 2 of each row says what
+         # the row is about — Throttle ⚙, a provider's subscription pool ◆, the host and its containers ▣ — in the
+         # row's worst tone; then a provider's availability bar, ten cells of usable ▰ · cooling ▱ · benched ▪ ·
+         # dead ·. The seven-bit twins are unique among the block's own marks and against every other mark its rows
+         # draw (`-` sep, `?` stale, `OK` / `!` / `X` the folded line's verdict). `#` stays the docs twin alone; the
+         # unicode `·` of a dead cell is the spec's (it shares the separator's glyph, never its place: a bar has no
+         # blank inside it).
+         'fleet-throttle': ('⚙', '@', 'ink-dim'), 'fleet-provider': ('◆', '*', 'ink-dim'),
+         'fleet-host': ('▣', '[', 'ink-dim'),
+         'bar-usable': ('▰', '=', 'tone-done'), 'bar-cooling': ('▱', '~', 'ink-dim'),
+         'bar-benched': ('▪', ':', 'tone-attention'), 'bar-dead': ('·', '.', 'ink-faint')}
 ECONOMY_TAIL = {'starving': ('⚠', '!', 'tone-attention'), 'downgraded': ('↓', 'v', 'tone-attention'), 'paid': ('$', '$', 'ink-dim')}
 MOBILE_AGENT_LABELS = {'needs_approval': 'Needs approval', 'needs_input': 'Needs you', 'working': 'Working',
                        'done': 'Done', 'errored': 'Error', 'shell': 'Shell', 'idle': 'Idle'}