@vibelord/lanes 1.1.1 → 1.3.0

package/payload/lanes/libexec/statusd.py

@@ -6,16 +6,22 @@ dead sockets are never probed), run one read-only `list-panes -a` per live serve
 at most PROBE_WORKERS at once, the whole round cut at PROBE_TIMEOUT + PROBE_GRACE seconds, the servers the last
 round cut started first — and publish ~/.local/state/lanes/status.json
 atomically, plus the one-line ~/.local/state/lanes/health. The sidebar, `lanes status`, `lanes doctor` and the
-future Throttle web harness all read those files.
+future Throttle web harness all read those files. Each workspace container that has the lane contract also gets its
+own slice as ~/.lz/status.json (LNS-32, `write_container_status`), for readers inside it.
 
 It never INVENTS a state. "No answer" (a tmux, cgroup, docker or zmx read that timed out or failed on an overloaded
 host) is not an observation: the lane keeps its last published state, marked stale, for at most STALE_MAX seconds
 and is 'unknown' after that. 'absent' / 'missing' are only published when a server answered without the session,
 or positively does not exist.
+
+The docs cockpit (plan W10 W7): a lane row carries `docs: {n, at}` while docs/counts.json counts documents for it
+(one stat of that file per cycle), and the document `web_origin`, the https origin the docs page answers on, once
+main() has learnt it between two cycles (refresh_origin).
 """
 import concurrent.futures
 import json
 import os
+import re
 import subprocess
 import sys
 import time
@@ -26,6 +32,7 @@ import lanes_lib as L  # noqa: E402
 import session_actions as A  # noqa: E402
 import agent_state as AS  # noqa: E402
 import guard as G  # noqa: E402
+import docs_store  # noqa: E402
 
 INTERVAL = float(os.environ.get('LANES_STATUS_INTERVAL', '3'))
 SLOWDOWN = {'ok': 1, 'busy': 2, 'critical': 4}   # a host under pressure gets FEWER spawns from us, not the same
@@ -39,6 +46,7 @@ PROBE_WORKERS = 8
 PROBE_GRACE = 2
 CONTAINER_REFRESH = 300  # docker is asked again this often as a safety net; liveness itself comes from /proc/<pid>
 CONTAINER_RETRY = 30     # ... and at most this often while a container is down or docker did not answer
+HOT_SYNC_EVERY_S = 10    # hot_sync starts at most one `lanes sync` per this many seconds (the sidebar's PENDING_S)
 ACTIVE_WINDOW = 20  # seconds since last output for a lane to count as active
 STALE_MAX = 300     # a lane nobody could observe for this long is published as 'unknown', no longer as last seen
 FIELDS = ['session_name', 'pane_pid', 'pane_dead', 'window_activity', 'pane_current_command',
@@ -54,6 +62,16 @@ FIELDS = ['session_name', 'pane_pid', 'pane_dead', 'window_activity', 'pane_curr
           'window_bell_flag', 'window_id', 'pane_id', 'pane_title']
 FORMAT = '\t'.join('#{' + f + '}' for f in FIELDS)
 VIEW_PREFIX = 'view-'   # a Throttle Studio browser attach: a grouped session named view-<studio>-<ch>
+# The docs cockpit (plan W10 W7, LNS-108). A lane row carries `docs: {n, at}` while docs/counts.json
+# (DM-LANES-DOCS-COUNTS) holds documents for it: ONE stat of that file per cycle, the file read only when it moved.
+# `web_origin` is the https origin Tailscale Serve answers on, the one the enrolment service announces (enrold
+# site_facts' host name: site.conf's LANES_SERVER_NAME, else this node's MagicDNS name), with Serve's port: the sidebar
+# builds its OSC 8 links to /lanes-docs/ on it and never forks to learn it. Resolved outside the cycle (main), at most
+# every ORIGIN_TTL seconds — enrold's own cache of the name — and not again on a host under pressure once known.
+WEB_PORT = 8443         # bin/lanes SERVE_PORT: `tailscale serve --https=8443` -> Caddy :8083 (web/Caddyfile)
+ORIGIN_TTL = 300        # enrold.TAILNET_TTL
+HOST_NAME = re.compile(r'(?=.{1,253}\Z)[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?'
+                       r'(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*\Z', re.ASCII)
 
 
 def log(where, error):
@@ -236,8 +254,9 @@ def keep(previous, lane_id, outcome):
 def lane_entry(pane, now, activity=None, viewing=(), agent=None):
     """One live session's published row. `activity` is the NEWEST `window_activity` across the session's panes
     (see `newest_activity`) and `viewing` the rows of the sessions that VIEW it (see `split_viewers`); both
-    default to "this pane alone", which is what a caller with a single row has. `agent` is `agent_state.merge`'s
-    verdict for the session's pane: published as the additive `agent` key, absent when no signal names an agent."""
+    default to "this pane alone", which is what a caller with a single row has. `agent` is the lane's settled
+    agent{} (`settled`: `agent_state.merge`'s verdict debounced by `agent_state.settle`): published as the additive
+    `agent` key, absent when no signal names an agent."""
     if activity is None:
         try:
             activity = int(pane['window_activity'])
@@ -267,6 +286,15 @@ def lane_entry(pane, now, activity=None, viewing=(), agent=None):
     return entry
 
 
+def settled(previous, lane_id, now, raw):
+    """What a live lane publishes as its agent{} this cycle: `merge`'s verdict `raw` debounced against the agent{} the
+    PREVIOUS payload published for the same lane id (`agent_state.settle`, cockpit v3 S1). `previous` is the last
+    published payload — `new_context()` seeds it from status.json, so a restarted daemon keeps every lane's
+    `stable_since`. Called at both merge sites in `observe`: declared lanes and undeclared sessions alike."""
+    old = ((previous.get('lanes') or {}).get(lane_id) or {}) if isinstance(previous, dict) else {}
+    return AS.settle(old.get('agent') if isinstance(old, dict) else None, raw, now)
+
+
 def agent_signals(containers, sessions_by_server):
     """The per-container inputs `agent_state.merge` needs, read ONCE per container per cycle through the
     container's root as the host sees it (`/proc/<init pid>/root`), so liveness is judged in the container's pid
@@ -546,6 +574,11 @@ def observe(registry, containers, live, previous=None, report=None):
                 try:
                     pane = own_sessions[name]
                     old_ws, old = prior.get((container, socket, name), (None, None))
+                    if old is None:
+                        # Listed for the first time: maybe a session `registrations.unregister` just released. It had
+                        # no listing to carry its id over (it was a lane), and its directory may name another board
+                        # now — the hand-back keeps the board and id it was registered under (S7 fix, DEF-1826).
+                        old_ws, old = A.handed_back(container, socket, name, pane)
                     board_ids = {w['id'] for w in registry['workspaces']}
                     workspace_id = A.workspace_for(registry, container, socket, pane['pane_current_path'],
                                                    owners.get((container, socket), set()),
@@ -586,8 +619,9 @@ def observe(registry, containers, live, previous=None, report=None):
                     if not L.TMUX_SESSION.match(name):
                         entry['note'] = 'session name outside the lane declaration grammar'
                     found = lane_entry(pane, now, activities.get(name), viewers.get(name, []),
-                                       AS.merge(pane, claude.get(container, {}),
-                                                sidecars.get(container, {}).get(entry['id'])))
+                                       settled(previous, entry['id'], now,
+                                               AS.merge(pane, claude.get(container, {}),
+                                                        sidecars.get(container, {}).get(entry['id']))))
                 except Exception as error:
                     log(f'observe {container}/{socket}/{name}', error)
                     continue
@@ -606,8 +640,9 @@ def observe(registry, containers, live, previous=None, report=None):
             else:
                 try:
                     lanes[lane['id']] = lane_entry(pane, now, activities.get(session), viewers.get(session, []),
-                                                   AS.merge(pane, claude.get(container, {}),
-                                                            sidecars.get(container, {}).get(lane['id'])))
+                                                   settled(previous, lane['id'], now,
+                                                           AS.merge(pane, claude.get(container, {}),
+                                                                    sidecars.get(container, {}).get(lane['id']))))
                 except Exception as error:   # a torn row: the session is there, its details are not readable
                     log(f"observe {lane['id']}", error)
                     lanes[lane['id']] = keep(previous, lane['id'], 'error')
@@ -833,6 +868,68 @@ def update_containers(ctx, registry, level):
         info['running'] = pid_alive(info.get('pid'))
 
 
+def docs_counts(ctx):
+    """docs/counts.json as the lane rows carry it -> {lane id: {n, at}} ({} when absent or not valid). ONE os.stat of
+    the file per cycle; it is read (docs_store.read_counts: no link followed, no FIFO opened, the shape checked) only
+    when its (mtime, inode, size) moved since the last read — docs_store replaces it on every change."""
+    try:
+        info = os.stat(docs_store.counts_path())
+        stamp = (info.st_mtime_ns, info.st_ino, info.st_size)
+    except OSError:
+        stamp = None
+    if stamp != ctx.get('docs_stamp') or 'docs_counts' not in ctx:
+        ctx['docs_counts'] = docs_store.read_counts() if stamp else {}
+        ctx['docs_stamp'] = stamp
+    return ctx['docs_counts']
+
+
+def with_docs(lanes, counts):
+    """`lanes` with `docs: {n, at}` on every row counts.json gives documents (n > 0) and on no other row: absent at 0,
+    and dropped from a kept row whose lane has none any more. A row that changes is a NEW dict (DEF-1538: a kept row
+    may be the very object the published payload holds); the others are passed on as they are."""
+    out = {}
+    for lane_id, row in lanes.items():
+        entry = counts.get(lane_id) if isinstance(counts, dict) else None
+        want = {'n': entry['n'], 'at': entry['at']} if isinstance(entry, dict) and entry.get('n') else None
+        if not isinstance(row, dict) or row.get('docs') == want:
+            out[lane_id] = row
+            continue
+        row = {key: value for key, value in row.items() if key != 'docs'}
+        if want:
+            row['docs'] = want
+        out[lane_id] = row
+    return out
+
+
+def resolve_origin():
+    """`https://<name>:8443` — the origin the docs page answers on (Tailscale Serve -> Caddy), by the name enrold
+    announces (site_facts: site.conf's LANES_SERVER_NAME, else enrold.tailnet_name(), one bounded `tailscale status
+    --json` cached for its TAILNET_TTL), or None when there is no name or it is no DNS host name: it is written into
+    terminal OSC strings and links, so nothing but `[A-Za-z0-9.-]` ever reaches them."""
+    import enrold   # the enrolment service's own reader of the name; imported only when an origin is wanted
+    name = enrold.server_site().get('LANES_SERVER_NAME') or enrold.tailnet_name()
+    return f'https://{name}:{WEB_PORT}' if isinstance(name, str) and HOST_NAME.match(name) else None
+
+
+def refresh_origin(ctx, level, now=None):
+    """main()'s look for `web_origin`, never inside a cycle: at most every ORIGIN_TTL seconds, and on a host that is
+    not `ok` only while none is known yet (one bounded spawn, then it waits). -> True when it looked. A failure keeps
+    the last origin: an unanswered name is no reason to drop the links."""
+    now = time.time() if now is None else now
+    known = ctx.get('origin_at') is not None
+    if known and (now - ctx['origin_at'] < ORIGIN_TTL or level != 'ok'):
+        return False
+    ctx['origin_at'] = now
+    try:
+        origin = resolve_origin()
+    except Exception as error:
+        log('web origin', error)
+        return True
+    if origin or not known:
+        ctx['web_origin'] = origin
+    return True
+
+
 def build_payload(ctx, level, reconcile=True):
     """One observation pass -> the payload of status.json (None = there is no registry to observe). Every phase
     fails alone: what it could not observe is republished as last seen (stale), the other phases still run.
@@ -896,8 +993,15 @@ def build_payload(ctx, level, reconcile=True):
     # Outside the payload: it changes every cycle, and status.json is rewritten only on change (plus the heartbeat).
     ctx['zmx_listed_at'] = listed
     lanes.update(zmx_lanes)
-    return {'lanes': lanes, 'servers': servers, 'unregistered': unregistered, 'zmx': zmx_groups,
-            'containers': {k: {'running': v.get('running', False)} for k, v in containers.items()}}
+    try:   # plan W10 W7: the docs counts ride on the lane rows (one stat of docs/counts.json per cycle)
+        lanes = with_docs(lanes, docs_counts(ctx))
+    except Exception as error:
+        log('docs', error)
+    payload = {'lanes': lanes, 'servers': servers, 'unregistered': unregistered, 'zmx': zmx_groups,
+               'containers': {k: {'running': v.get('running', False)} for k, v in containers.items()}}
+    if ctx.get('web_origin'):   # additive, absent while no origin is known (refresh_origin, main())
+        payload['web_origin'] = ctx['web_origin']
+    return payload
 
 
 def new_context():
@@ -908,8 +1012,116 @@ def new_context():
     written = float(seed.get('updated_at') or 0)
     for entry in (seed.get('lanes') or {}).values():
         entry.setdefault('stale_since', int(written))
+    # The docs links' origin survives a restart (re-checked by main()'s first refresh_origin); only a valid one.
+    origin = seed.get('web_origin')
+    valid = (isinstance(origin, str) and origin.startswith('https://') and origin.endswith(f':{WEB_PORT}')
+             and HOST_NAME.match(origin[len('https://'):-len(f':{WEB_PORT}')]))
     return {'registry': None, 'stamp': 0, 'containers': {}, 'containers_at': 0.0,
-            'payload': seed, 'published': None, 'status_at': written, 'cut': 0, 'unwalkable': []}
+            'payload': seed, 'published': None, 'status_at': written, 'cut': 0, 'unwalkable': [],
+            'container_status': {}, 'container_status_errors': {}, 'web_origin': origin if valid else None}
+
+
+# ------------------------------------------- the in-container projection (LNS-32)
+# `~/.lz/status.json` in each workspace container: that container's slice of the payload, for the in-container Agent
+# API (`lane-mcp`, plan Track M), which cannot read the host's status.json. v1 = {v, at, lanes{<id>: {state, agent?,
+# attached?, viewers?, activity?, size?, stale?, docs?}}, unregistered[{id, name, workspace, boards, tmux, identity}]}.
+# `docs` (plan W10 W7, additive) is the host row's {n, at} verbatim, absent like there.
+CONTAINER_STATUS_FIELDS = ('state', 'agent', 'attached', 'viewers', 'activity', 'size', 'stale', 'docs')
+CONTAINER_STATUS_DIR = tuple(L.CONTAINER_HOME.strip('/').split('/')) + ('.lz',)
+CONTAINER_STATUS_FILE = 'status.json'
+
+
+def container_status(registry, payload):
+    """{container: {'lanes': …, 'unregistered': […]}} — what each workspace container may read about ITS OWN lanes
+    (pure). `lanes` holds every lane the registry places in the container (declared and registered: the ids its
+    `~/.lz/registry.json` holds) and every unregistered session statusd found there, each reduced to the fields a
+    reader in the container needs; a row a cycle has not observed yet is `unknown`, never invented. The host-side
+    details (cwd, pid, tty, command, where an unanswered probe failed) stay on the host, and `stale` is carried so a
+    kept row is never read as a fresh one. A session listed under several boards is ONE `unregistered` row naming
+    them all in `boards`. zmx lanes are not projected: the container's own `lane ls`/`lane status` read them from
+    zmx and lane-run directly, and their `agent` is lane-run's phase record, not an `agent_state` verdict."""
+    rows = (payload or {}).get('lanes') or {}
+    ids = {}
+    for workspace in (registry or {}).get('workspaces') or []:
+        for lane in workspace.get('lanes') or []:
+            if lane.get('container') and lane.get('id'):
+                ids.setdefault(lane['container'], []).append(lane['id'])
+    ghosts = {}
+    for board, entries in sorted(((payload or {}).get('unregistered') or {}).items()):
+        for entry in entries or []:
+            if not isinstance(entry, dict) or not entry.get('container') or not entry.get('id'):
+                continue
+            ghost = ghosts.setdefault(entry['container'], {}).setdefault(entry['id'], {
+                'id': entry['id'], 'name': entry.get('name'), 'workspace': entry.get('workspace') or board,
+                'boards': [], 'tmux': entry.get('tmux'), 'identity': entry.get('identity')})
+            if board not in ghost['boards']:
+                ghost['boards'].append(board)
+    out = {}
+    for container in sorted(set(ids) | set(ghosts)):
+        found = ghosts.get(container, {})
+        lanes = {}
+        for lane_id in ids.get(container, []) + sorted(found):
+            row = rows.get(lane_id) if isinstance(rows.get(lane_id), dict) else {}
+            lanes[lane_id] = {key: row[key] for key in CONTAINER_STATUS_FIELDS if key in row}
+            lanes[lane_id].setdefault('state', 'unknown')
+        out[container] = {'lanes': lanes, 'unregistered': [found[lane_id] for lane_id in sorted(found)]}
+    return out
+
+
+def write_container_status(ctx, payload, now=None):
+    """Write `container_status` into each running workspace container that has the lane contract (`~/.lz/bin/lane`,
+    the rule `lanes_lib.export_container_registries` follows) -> the containers whose file was written.
+
+    At most one small write per container per cycle, and none while that container's slice is unchanged: `at` is
+    the time of the last write, renewed on a change or once HEARTBEAT has passed, so an unchanged slice renders the
+    same bytes and `atomic_write_if_changed_at` skips it, while a reader can still tell a live statusd from a dead
+    one by the age of `at`. Written through /proc/<init pid>/root like every other read of the container here, into
+    a directory the container owns: `lanes_lib.open_real_dir` refuses a symlinked component and the write replaces
+    whatever is at the name without following or reading it (a planted link or FIFO). A container without the
+    contract is skipped silently; any other failure is logged once per distinct error and costs only that container."""
+    now = time.time() if now is None else now
+    last, errors = ctx.setdefault('container_status', {}), ctx.setdefault('container_status_errors', {})
+    written = []
+    slices = container_status(ctx.get('registry'), payload)
+    for gone in set(last) - set(slices):
+        del last[gone]
+    for container, content in slices.items():
+        info = (ctx.get('containers') or {}).get(container) or {}
+        if not info.get('pid') or not info.get('running'):
+            continue
+        before = last.get(container)
+        fresh = before and before['content'] == content and now - before['at'] <= HEARTBEAT
+        at = before['at'] if fresh else now
+        text = json.dumps({'v': 1, 'at': at, **content}, ensure_ascii=False, indent=1) + '\n'
+        try:
+            fd = L.open_real_dir(Path(L.proc_root()) / str(info['pid']) / 'root', CONTAINER_STATUS_DIR)
+        except FileNotFoundError:
+            continue                              # no ~/.lz: the lane contract is not installed there
+        except (OSError, L.LanesError) as error:
+            _container_status_error(errors, container, error)
+            continue
+        try:
+            try:
+                os.stat('bin/lane', dir_fd=fd)
+            except FileNotFoundError:
+                continue                          # a ~/.lz without `lane`: not installed either
+            if L.atomic_write_if_changed_at(fd, CONTAINER_STATUS_FILE, text, mode=0o644):
+                written.append(container)
+            last[container] = {'content': content, 'at': at}
+            errors.pop(container, None)
+        except (OSError, L.LanesError) as error:
+            _container_status_error(errors, container, error)
+        finally:
+            os.close(fd)
+    return written
+
+
+def _container_status_error(errors, container, error):
+    """Log a container's projection failure once per distinct error, not every INTERVAL."""
+    message = f'{type(error).__name__}: {error}'
+    if errors.get(container) != message:
+        errors[container] = message
+        log(f'container status {container}', error)
 
 
 def cycle(ctx, reconcile=True):
@@ -938,6 +1150,10 @@ def cycle(ctx, reconcile=True):
                 ctx['published'], ctx['status_at'] = payload, written
             except Exception as error:
                 log('status.json', error)
+        try:   # LNS-32: each container's own slice, into its ~/.lz (one small write per container at most)
+            write_container_status(ctx, payload)
+        except Exception as error:
+            log('container status', error)
     try:   # "servers alive, sockets deleted" as a flag file: view.sh, the sidebar and doctor read it without parsing
         lost = sorted(key for key, server in ((payload or {}).get('servers') or {}).items() if server.get('outcome') == 'nosocket')
         flag = L.STATE / 'flags' / 'sockets-lost'
@@ -1014,15 +1230,18 @@ def hot_sync(ctx, source_mtimes):
 
 
 def main():
+    L.record_tree('statusd')   # W9 U0: the tree this daemon runs from (`lanes version`, doctor); --once never records
     ctx = new_context()
     last_triggered_sync = 0.0
     source_mtimes = {}   # host path of a terminals.json -> its mtime_ns at last check
     while True:
         started = time.time()
         level, _payload = cycle(ctx)
+        # The docs links' origin (plan W10 W7): a bounded look between two cycles, never inside one.
+        refresh_origin(ctx, level)
         # Hot-reload: check if any terminals.json changed since we last synced.
-        # Rate-limited to at most one triggered sync per 10 s to avoid thrashing.
-        if time.time() - last_triggered_sync > 10 and ctx['registry']:
+        # Rate-limited to at most one triggered sync per HOT_SYNC_EVERY_S to avoid thrashing.
+        if time.time() - last_triggered_sync > HOT_SYNC_EVERY_S and ctx['registry']:
             if hot_sync(ctx, source_mtimes):
                 last_triggered_sync = time.time()
         ctx['children'] = [p for p in ctx.get('children', []) if p.poll() is None]   # reap finished syncs

package/payload/lanes/libexec/studio_host.py

@@ -39,6 +39,9 @@ import tempfile
 import time
 from pathlib import Path
 
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import lanes_lib as L  # noqa: E402
+
 UNIT = 'lanes-studio-host.service'
 ENROL_SERVICE = 'lanes-studio-host-enrol.service'
 ENROL_TIMER = 'lanes-studio-host-enrol.timer'
@@ -1040,7 +1043,10 @@ def _enrol(layout, options, plan, record, receipt, key_file):
 
 
 def _units(layout, plan):
-    """Step 6: render every lanes unit for this checkout (services.py), then enable the Host and its retry timer."""
+    """Step 6: render every lanes unit (services.py), then enable the Host and its retry timer. A re-render, never an
+    install: services.register(install=False) renders the release `current` names and moves no link, so this step run
+    from a tree an update has moved past cannot put the old release back (DEF-1835); it links `current` to this tree
+    only where it names none yet."""
     repo = Path(__file__).resolve().parent.parent
 
     def render():
@@ -1169,6 +1175,7 @@ def enable(argv, layout=None):
     backup = _ensure_config(layout, options, plan)
     if not options['dry_run']:
         _remember_paths(layout, options, record)
+        L.record_tree('studio_host', layout.state)   # W9 U0: the enrol unit's tree, past every static refusal
     replaced = []   # set once the hand-built unit is saved, disabled and stopped
 
     def give_back():