@vibelord/lanes 1.3.0 → 1.5.0

package/payload/lanes/libexec/lanes_lib.py

@@ -39,6 +39,22 @@ def current_link(home=None):
     return (Path(home) if home is not None else Path.home()) / CURRENT_LINK
 
 
+# The server's site facts (~/.config/lanes/site.conf): LANES_* KEY=VALUE lines, read and never executed. bin/lanes
+# site_profile and update.daemon_container read them here; the same line is what `lanes client-config` hands out.
+SITE_CONF = Path('.config/lanes/site.conf')
+SITE_LINE = re.compile(r'^(LANES_[A-Z_]+)=([A-Za-z0-9_.:@/-]*)$')
+
+
+def site_conf(home=None):
+    """`<home>/.config/lanes/site.conf`'s LANES_* facts (default: this account's home) -> {key: value}; the last line
+    naming a key wins; {} when there is no file or it cannot be read."""
+    try:
+        text = ((Path(home) if home is not None else Path.home()) / SITE_CONF).read_text()
+    except (OSError, UnicodeDecodeError, KeyError, RuntimeError):
+        return {}
+    return dict(match.groups() for match in map(SITE_LINE.match, text.splitlines()) if match)
+
+
 def staged_tree(path):
     """Whether `path` is a tree the npm wrapper staged, not a checkout: it sits in the default pkg root
     (`…/pkg/<version>`), or it carries the wrapper's stamp (PACKAGE_STAMP, a regular file lanes-install.mjs stage()
@@ -855,6 +871,7 @@ def tmux_query(info, socket_name, args, timeout=5):
 # -------------------------------------------------------------------- health
 # Host pressure from PLAIN FILES only — no subprocess: this is read exactly when spawning is what hurts.
 
+GIB = 2 ** 30
 HEALTH_LINE = ('at', 'level', 'mem_some10', 'mem_full10', 'io_some10', 'cpu_some10', 'swap_pct', 'load1', 'status_at')
 
 
@@ -874,12 +891,17 @@ def _psi10(text, kind):
         return 0
 
 
-def _container_memory(proc, cgroup_root, pid):
-    """cgroup-v2 memory of the container whose init is `pid`; the cgroup is found the way the kernel names it."""
+def _container_cgroup(proc, cgroup_root, pid):
+    """The cgroup-v2 directory of the container whose init is `pid`, found the way the kernel names it (or None)."""
     match = re.search(r'^0::(/\S*)$', _text(proc / str(pid) / 'cgroup'), re.M) if pid else None
     if not match or '..' in match.group(1).split('/'):
         return None
-    base = cgroup_root / match.group(1).lstrip('/')
+    return cgroup_root / match.group(1).lstrip('/')
+
+
+def _container_memory(base):
+    """cgroup-v2 memory of one container (`base` = its cgroup directory) -> (the health row, memory.current in bytes),
+    or None when the cgroup has no readable memory.current (a container that is down)."""
     current = _text(base / 'memory.current').strip()
     if not current.isdigit():
         return None
@@ -889,13 +911,34 @@ def _container_memory(proc, cgroup_root, pid):
         found = re.search(r'^' + key + r' (\d+)$', _text(base / name), re.M)
         return int(found.group(1)) if found else 0
 
-    gib = 2 ** 30
-    return {'cur_gib': round(int(current) / gib, 1), 'max_gib': round(int(limit) / gib, 1) if limit.isdigit() else None,
-            'shmem_gib': round(counter('memory.stat', 'shmem') / gib, 1),   # shmem = RAM-backed /tmp and /dev/shm
-            # anon + shmem is what cannot be given back; `file` minus shmem is page cache, which always grows to the
-            # limit on a busy container ("207.9 of 208" is normal and says nothing about pressure)
-            'anon_gib': round(counter('memory.stat', 'anon') / gib, 1),
-            'oom_kill': counter('memory.events', 'oom_kill')}
+    return ({'cur_gib': round(int(current) / GIB, 1), 'max_gib': round(int(limit) / GIB, 1) if limit.isdigit() else None,
+             'shmem_gib': round(counter('memory.stat', 'shmem') / GIB, 1),   # shmem = RAM-backed /tmp and /dev/shm
+             # anon + shmem is what cannot be given back; `file` minus shmem is page cache, which always grows to the
+             # limit on a busy container ("207.9 of 208" is normal and says nothing about pressure)
+             'anon_gib': round(counter('memory.stat', 'anon') / GIB, 1),
+             'oom_kill': counter('memory.events', 'oom_kill')}, int(current))
+
+
+def _cpu_usage_usec(base):
+    """The container's cumulative cpu time (cgroup-v2 `cpu.stat` usage_usec), or None when the file has none."""
+    found = re.search(r'^usage_usec (\d+)$', _text(base / 'cpu.stat'), re.M)
+    return int(found.group(1)) if found else None
+
+
+def _proc_stat_cpu(proc):
+    """[busy, total] jiffies of the aggregate `cpu` line of /proc/stat (user nice system idle iowait irq softirq steal;
+    guest time is already inside user and nice), or None. Cumulative since boot: only a difference is a utilization."""
+    match = re.search(r'^cpu[ \t]+([0-9 \t]+)$', _text(proc / 'stat'), re.M)
+    fields = [int(word) for word in match.group(1).split()][:8] if match else []
+    if len(fields) < 4:
+        return None
+    total = sum(fields)
+    return [total - fields[3] - (fields[4] if len(fields) > 4 else 0), total]
+
+
+def _int_file(path):
+    text = _text(path).strip()
+    return int(text) if text.isdigit() else None
 
 
 def health_level(health):
@@ -909,32 +952,109 @@ def health_level(health):
     return 'ok'
 
 
-def compute_health(paths=None):
+def compute_health(paths=None, sample=False):
     """paths (tests): 'proc' (else $LANES_PROC_ROOT, else /proc), 'cgroup' (/sys/fs/cgroup), 'containers'
-    ($STATE/containers.json: whose init pids to look up), 'ncpu'. A file that is missing reads as 0."""
+    ($STATE/containers.json: whose init pids to look up), 'ncpu', 'now'. A file that is missing reads as 0.
+
+    `sample` (statusd, every FLEET_INTERVAL; spec W11 §2.2): also read the fleet's raw counters into
+    health['sample'] — the aggregate `cpu` line of /proc/stat, MemTotal/MemAvailable (from the meminfo this reads
+    anyway), the inotify watch CEILING (never the watches in use: counting those walks every process's fdinfo), and
+    per container its memory.current in bytes and its cgroup `cpu.stat` usage_usec. Not on a `critical` host: no
+    `sample` key then, and nothing more is read where reading is what hurts. The counters are cumulative; `fleet_facts`
+    turns two samples into rates, and the previous sample is kept by the CALLER (statusd's context), never here."""
     paths = paths or {}
     proc = Path(paths.get('proc') or proc_root())
     pressure = {name: _text(proc / 'pressure' / name) for name in ('memory', 'io', 'cpu')}
-    meminfo = {key: int(value) for key, value in re.findall(r'^(SwapTotal|SwapFree):\s+(\d+)', _text(proc / 'meminfo'), re.M)}
+    meminfo = {key: int(value) for key, value in re.findall(r'^(SwapTotal|SwapFree|MemTotal|MemAvailable):\s+(\d+)',
+                                                            _text(proc / 'meminfo'), re.M)}
     try:
         load1 = float(_text(proc / 'loadavg').split()[0])
     except (IndexError, ValueError):
         load1 = 0.0
     swap_total = meminfo.get('SwapTotal', 0)
-    health = {'at': time.time(), 'level': 'ok',
+    now = paths.get('now')
+    health = {'at': time.time() if now is None else now, 'level': 'ok',
               'mem_some10': _psi10(pressure['memory'], 'some'), 'mem_full10': _psi10(pressure['memory'], 'full'),
               'io_some10': _psi10(pressure['io'], 'some'), 'cpu_some10': _psi10(pressure['cpu'], 'some'),
               'swap_pct': int(round(100 * (swap_total - meminfo.get('SwapFree', 0)) / swap_total)) if swap_total else 0,
               'load1': load1, 'ncpu': int(paths.get('ncpu') or os.cpu_count() or 1), 'containers': {}}
     health['level'] = health_level(health)
+    raw = None
+    if sample and health['level'] != 'critical':
+        kib = {key: meminfo[key] * 1024 if key in meminfo else None for key in ('MemTotal', 'MemAvailable')}
+        raw = health['sample'] = {'at': health['at'], 'cpu': _proc_stat_cpu(proc), 'mem_total': kib['MemTotal'],
+                                  'mem_avail': kib['MemAvailable'],
+                                  'inotify_max': _int_file(proc / 'sys/fs/inotify/max_user_watches'), 'containers': {}}
     recorded = read_json(paths.get('containers') or STATE / 'containers.json', {}) or {}
+    cgroup_root = Path(paths.get('cgroup') or '/sys/fs/cgroup')
     for name, info in recorded.items():
-        memory = _container_memory(proc, Path(paths.get('cgroup') or '/sys/fs/cgroup'), (info or {}).get('pid'))
+        pid = (info or {}).get('pid')
+        base = _container_cgroup(proc, cgroup_root, pid)
+        memory = _container_memory(base) if base else None
         if memory:
-            health['containers'][name] = memory
+            health['containers'][name] = memory[0]
+            if raw is not None:
+                raw['containers'][name] = {'pid': pid, 'mem': memory[1], 'usec': _cpu_usage_usec(base)}
     return health
 
 
+# The `fleet` block of status.json (spec W11 §2.2, DM-LANES-FLEET): OpenTelemetry semantic-convention names as the
+# keys, verbatim. Every figure is quantized — fractions to 1 %, percents to whole percents, bytes to 0.1 GiB, the load
+# to 0.1 — so a sample that moved nothing a reader can see renders the same bytes and does not rewrite status.json.
+FLEET_VERSION = 1
+FLEET_TOP = 3     # containers published, the biggest by memory first
+
+
+def gib_tenths(value):
+    """A byte count quantized to 0.1 GiB, still in bytes (an int); None stays None."""
+    return None if value is None else int(round(round(value * 10 / GIB) * GIB / 10))
+
+
+def _rate(after, before):
+    """after - before when both are counts and the counter did not go back (a restart), else None."""
+    if isinstance(after, int) and isinstance(before, int) and after >= before:
+        return after - before
+    return None
+
+
+def fleet_facts(sample, previous, health, host_name):
+    """The `fleet` document from this `compute_health(sample=True)` sample and the `previous` one (None or {} on the
+    first; the caller keeps it). A rate needs two samples: on the first, `system.cpu.utilization` and every
+    `container.cpu.percent` are null — never a since-boot average. A container whose init pid changed since the
+    previous sample (restarted) or whose counter went back has a null `container.cpu.percent` for this sample.
+    `container.cpu.percent` is of the whole host: usage_usec delta ÷ interval ÷ logical cpus × 100. `containers` are
+    the FLEET_TOP biggest by memory.current (ties by name) of every container whose cgroup answered."""
+    previous = previous or {}
+    ncpu = int(health.get('ncpu') or 1)
+    seconds = sample['at'] - previous['at'] if isinstance(previous.get('at'), (int, float)) else None
+    utilization = None
+    if sample.get('cpu') and previous.get('cpu'):
+        busy, total = (_rate(sample['cpu'][i], previous['cpu'][i]) for i in (0, 1))
+        if busy is not None and total and busy <= total:
+            utilization = round(busy / total, 2)
+    total, available = sample.get('mem_total'), sample.get('mem_avail')
+    used = total - available if total and available is not None and available <= total else None
+    rows = []
+    for name, now in (sample.get('containers') or {}).items():
+        before = (previous.get('containers') or {}).get(name) or {}
+        spent = _rate(now.get('usec'), before.get('usec')) if before.get('pid') == now.get('pid') else None
+        percent = float(round(100 * spent / 1e6 / seconds / ncpu)) if spent is not None and seconds and seconds > 0 else None
+        rows.append((-now['mem'], name, {'container.name': name, 'container.memory.usage.total': gib_tenths(now['mem']),
+                                         'container.cpu.percent': percent}))
+    return {'v': FLEET_VERSION, 'at': int(sample['at']), 'host.name': host_name, 'stale': False,
+            'system.cpu.utilization': utilization, 'system.cpu.logical.count': ncpu,
+            'system.cpu.load_average.1m': round(float(health.get('load1') or 0.0), 1),
+            'system.memory.utilization': round(used / total, 2) if used is not None else None,
+            'system.memory.usage': {'used': gib_tenths(used), 'total': gib_tenths(total)},
+            'system.linux.memory.available': gib_tenths(available),
+            'system.pressure': {'cpu.some.avg10': float(health.get('cpu_some10') or 0),
+                                'memory.some.avg10': float(health.get('mem_some10') or 0),
+                                'memory.full.avg10': float(health.get('mem_full10') or 0),
+                                'io.some.avg10': float(health.get('io_some10') or 0)},
+            'system.inotify.max_user_watches': sample.get('inotify_max'),
+            'containers': [row for _mem, _name, row in sorted(rows, key=lambda r: r[:2])[:FLEET_TOP]]}
+
+
 def health_line(health, status_at):
     """The one line of $STATE/health (sh and python read it without JSON). Key order is part of the format."""
     values = dict(health, at=int(health['at']), load1=f"{health['load1']:.2f}", status_at=int(status_at))
@@ -1262,9 +1382,11 @@ def lane_trace(lane_id, ws):
 
 def canonical_recipe(lane_id, ws, socket, session, folder, cmd='bash -l'):
     """The one create recipe that gives a new tmux lane its identity: LANE_ID / LANE_WS for scripts and hooks, the
-    lanes/<ws>/<id> trace for Throttle (ANTHROPIC_CUSTOM_HEADERS for Claude Code, THROTTLE_TRACE for the rest).
-    `tmux new -A` attaches when the session already exists, so the env lands only on a real create — never through
-    `set-environment`, which would leak into unrelated windows. `cmd` is a shell command string, appended as is."""
+    lanes/<ws>/<id> trace for Throttle (ANTHROPIC_CUSTOM_HEADERS for Claude Code, KIMI_CODE_CUSTOM_HEADERS for Kimi
+    Code, which blanks the Anthropic one, LANES_TRACE for Grok Build's `env_http_headers`, THROTTLE_TRACE for the
+    rest) and GROK_HOME (Grok Build's home, never grok-cli's ~/.grok; W12). `tmux new -A` attaches when the session
+    already exists, so the env lands only on a real create — never through `set-environment`, which would leak into
+    unrelated windows. `cmd` is a shell command string, appended as is."""
     trace = lane_trace(lane_id, ws)
     if trace is None:
         raise LanesError(f'Invalid lane trace: lanes/{ws}/{lane_id} (allowed: A-Z a-z 0-9 . _ - /, at most 256)')
@@ -1277,7 +1399,8 @@ def canonical_recipe(lane_id, ws, socket, session, folder, cmd='bash -l'):
     header = shlex.quote(f'x-throttle-trace: {trace}')
     return (f'tmux -L {shlex.quote(socket)} new -ADs {shlex.quote(session)} -c {shlex.quote(folder)} '
             f'-e LANE_ID={lane_id} -e LANE_WS={ws} -e ANTHROPIC_CUSTOM_HEADERS={header} '
-            f'-e THROTTLE_TRACE={trace} {cmd}')
+            f'-e THROTTLE_TRACE={trace} -e KIMI_CODE_CUSTOM_HEADERS={header} -e LANES_TRACE={trace} '
+            f'-e GROK_HOME={CONTAINER_HOME}/.grok-build {cmd}')
 
 
 # What `lane open` gets when it runs a lane's recorded `create` for a lane that does not exist (plan Task 18; the