@vibelord/lanes 1.2.0 → 1.5.0

package/payload/lanes/container/throttle-pulse

@@ -0,0 +1,456 @@
+#!/usr/bin/env python3
+"""throttle-pulse --once|--loop [--control URL] [--state-dir DIR] [--token-file FILE] — the Throttle daemon's live facts,
+read where the daemon runs (W11, spec docs/superpowers/specs/2026-10-11-lanes-fleet-dashboard-design.md §2.1).
+
+The daemon binds its own container's 127.0.0.1 (DEF-1525): a host poller cannot reach it. This script is shipped by
+`lanes install-container` into ~/.lz/bin/ of every workspace container and runs, as a loop, only in the daemon's
+container (`tmux -L lanes-pulse` session `pulse`, started by install-container). Every PULSE_INTERVAL seconds it GETs,
+on the control port (`--control`, else http://127.0.0.1:$LANES_THROTTLE_CONTROL_PORT, else :19181):
+
+  open      /pools/<pool>/ready for each of POOLS     (a 503 still carries the JSON body: the pool is not ready)
+            /metrics                                  (Prometheus text: four `throttle_*{pool="…"}` series)
+  tokened   /pools/<pool>/status for each of POOLS with `Authorization: Bearer <token>` — only while the token
+            file exists (~/.lz/throttle-token, mode 0600; `lanes throttle adopt-token <container>` writes it). The
+            token is the LOCAL proxy bearer (`throttle_…`), never a provider credential. Every guarded control route
+            goes through the daemon's ControlAuth with no loopback exemption, so without it /status answers 401.
+
+and writes ~/.lz/state/throttle.json (the shape below, by rename) only when a value changed or HEARTBEAT seconds passed
+since the file's `at` — `at` is the liveness heartbeat lanes-statusd ages:
+
+  {"v": 1, "at": <epoch s>, "up": bool, "tokened": bool,
+   "pools": {"claude"|"codex"|"kimi": {"configured", "usable", "cooling", "exhausted", "benched", "auth_failed",
+             "overflow_ready", "pressure_phase", "spill_fraction", "overflow_in_flight", "upstream_429_total",
+             "worst_7d", "worst_5h", "soonest_reset"}},
+   "overflow": {"claude"|"codex"|"kimi": {"spend_today_usd", "budget_usd", "fraction", "alert_level_pct"}},
+   "error": null | "<why the daemon did not answer>"}
+
+`cooling` is the remainder configured − usable − exhausted − auth_failed − disabled − billing/org-blocked − unhealthy,
+floored at 0 (the daemon's buckets can overlap); `benched` is `model_benched_subscriptions`. `pressure_phase` is the
+daemon's own word for the gauge (0 `closed` — calm —, 1 `spilling`, 2 `elevated`). `worst_7d` / `worst_5h` are the highest measured
+utilization (0–1) over the pool's subscription accounts, `soonest_reset` the earliest future reset (epoch s) of either
+window; they and `overflow` stay null without the token. `up` is false only when no open GET answered: the previous
+pools are then kept, `error` says why. Why a source failed goes to stderr (the tmux pane, the journal) each time it
+changes, never into the file and never the token.
+
+POOLS are claude, codex and (W12, Lanes 1.5.0) kimi: the daemon's `/pools/:pool/ready` and `/status` routes are
+generic (`src/proxy.rs` `scoped_pool_state`), and a daemon with no Kimi account answers `/pools/kimi/ready` with
+`configured_subscriptions: 0` — a pool with nothing configured, not a failure. Kimi has no paid overflow lane (the
+daemon keeps its `api_overflow` at the default, and its /status still renders a zero overflow block), so only
+MONEY_POOLS carry overflow money: `overflow.kimi` is always nulls.
+
+GET only, 3 s timeout, 1 MiB body cap, no redirect followed (one would carry the bearer elsewhere), no proxy from the
+environment. Stdlib only. Exit: --once 0 when the daemon answered, 1 when not, 2 on a usage error; --loop runs until
+SIGTERM/SIGINT.
+"""
+import argparse
+import json
+import os
+import re
+import signal
+import stat
+import sys
+import tempfile
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+SCHEMA = 1
+PULSE_INTERVAL = 10          # seconds between two cycles of --loop
+HEARTBEAT = 60               # throttle.json is rewritten at least this often (its `at`)
+TIMEOUT = 3                  # seconds per GET
+MAX_BODY = 1 << 20           # bytes: a bigger answer is not the daemon's
+TOKEN_MAX = 4096             # bytes: a token file is one short line
+DEFAULT_PORT = 19181
+POOLS = ('claude', 'codex', 'kimi')
+MONEY_POOLS = frozenset({'claude', 'codex'})   # the pools with a paid overflow lane; kimi's money is always null
+# The gauge's numbers (src/proxy.rs render_metrics: 0=calm,1=spilling,2=elevated) as the words the daemon's own JSON
+# uses for them (src/pressure.rs PressureSnapshot.phase: "closed" is calm), which economy.json and the sidebar read.
+PHASES = {0: 'closed', 1: 'spilling', 2: 'elevated'}
+SUBSCRIPTION_PROVIDERS = frozenset({'anthropic', 'openai', 'kimi'})   # src/proxy.rs is_subscription_provider
+SERIES = {'throttle_pool_spill_fraction': 'spill_fraction', 'throttle_overflow_in_flight': 'overflow_in_flight',
+          'throttle_upstream_429_total': 'upstream_429_total', 'throttle_pool_pressure_phase': 'pressure_phase'}
+POOL_KEYS = ('configured', 'usable', 'cooling', 'exhausted', 'benched', 'auth_failed', 'overflow_ready',
+             'pressure_phase', 'spill_fraction', 'overflow_in_flight', 'upstream_429_total', 'worst_7d', 'worst_5h',
+             'soonest_reset')
+OVERFLOW_KEYS = ('spend_today_usd', 'budget_usd', 'fraction', 'alert_level_pct')
+TOKENED_POOL_KEYS = ('worst_7d', 'worst_5h', 'soonest_reset')
+STATE_FILE = 'throttle.json'
+METRIC_LINE = re.compile(r'^([a-zA-Z_:][a-zA-Z0-9_:]*)\{([^}]*)\}\s+(\S+)')
+LABEL = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="((?:[^"\\]|\\.)*)"')
+
+
+def empty_pool():
+    return dict.fromkeys(POOL_KEYS)
+
+
+def empty_overflow():
+    return dict.fromkeys(OVERFLOW_KEYS)
+
+
+def empty_document():
+    return {'v': SCHEMA, 'at': None, 'up': False, 'tokened': False,
+            'pools': {pool: empty_pool() for pool in POOLS},
+            'overflow': {pool: empty_overflow() for pool in POOLS}, 'error': None}
+
+
+# ------------------------------------------------------------------ HTTP
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, *_args, **_kwargs):   # a redirect would carry the bearer elsewhere: refuse it
+        return None
+
+
+def default_opener():
+    """urllib with no proxy from the environment and no redirect followed."""
+    return urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirect()).open
+
+
+class Failed(Exception):
+    """A GET that gave no usable answer; str() is a short, secret-free reason."""
+
+
+def failure_class(error):
+    if isinstance(error, urllib.error.HTTPError):
+        return f'HTTP {error.code}'
+    if isinstance(error, urllib.error.URLError):
+        reason = error.reason
+        return failure_class(reason) if isinstance(reason, BaseException) else f'unreachable: {reason}'
+    if isinstance(error, ConnectionRefusedError):
+        return 'connection refused'
+    if isinstance(error, (TimeoutError,)) or type(error).__name__ == 'timeout':
+        return 'timed out'
+    if isinstance(error, OSError):
+        return f'unreachable: {error.strerror or type(error).__name__}'
+    return type(error).__name__
+
+
+def _capped(stream):
+    body = stream.read(MAX_BODY + 1)
+    if len(body) > MAX_BODY:
+        raise Failed(f'body over {MAX_BODY >> 20} MiB')
+    return body
+
+
+def get(control, path, token=None, opener=None, accept_status=()):
+    """GET <control><path> -> the body bytes. An HTTP status in `accept_status` (the 503 of a pool that is not ready)
+    still yields its body; any other failure raises Failed. The token goes into the header only."""
+    request = urllib.request.Request(control + path, method='GET')
+    if token:
+        request.add_header('Authorization', f'Bearer {token}')
+    try:
+        with (opener or default_opener())(request, timeout=TIMEOUT) as response:
+            return _capped(response)
+    except urllib.error.HTTPError as error:
+        if error.code in accept_status:
+            try:
+                return _capped(error)
+            finally:
+                error.close()
+        error.close()
+        raise Failed(failure_class(error)) from None
+    except Failed:
+        raise
+    except Exception as error:   # the daemon may be down, restarting or slow: the pulse degrades, never dies
+        raise Failed(failure_class(error)) from None
+
+
+def get_json(control, path, token=None, opener=None, accept_status=()):
+    body = get(control, path, token, opener, accept_status)
+    try:
+        doc = json.loads(body)
+    except ValueError:
+        raise Failed('not JSON') from None
+    if not isinstance(doc, dict):
+        raise Failed('not a JSON object')
+    return doc
+
+
+# ------------------------------------------------------------------ parsing
+
+def _count(value):
+    return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0
+
+
+def _number(value):
+    if isinstance(value, bool) or not isinstance(value, (int, float)) or value != value:
+        return None
+    return value
+
+
+def _fraction(value):
+    value = _number(value)
+    return None if value is None else round(float(value), 3)
+
+
+def from_ready(doc):
+    """The counts of one /pools/<pool>/ready body."""
+    configured, usable = _count(doc.get('configured_subscriptions')), _count(doc.get('usable_subscriptions'))
+    exhausted, auth_failed = _count(doc.get('exhausted_subscriptions')), _count(doc.get('auth_failed_subscriptions'))
+    others = sum(_count(doc.get(key)) for key in ('disabled_subscriptions', 'billing_blocked_subscriptions',
+                                                  'organization_blocked_subscriptions', 'unhealthy_subscriptions'))
+    pressure = doc.get('pressure') if isinstance(doc.get('pressure'), dict) else {}
+    phase = pressure.get('phase')
+    return {'configured': configured, 'usable': usable,
+            'cooling': max(0, configured - usable - exhausted - auth_failed - others),
+            'exhausted': exhausted, 'benched': _count(doc.get('model_benched_subscriptions')),
+            'auth_failed': auth_failed, 'overflow_ready': doc.get('overflow_ready') is True,
+            'pressure_phase': phase if isinstance(phase, str) and phase in PHASES.values() else None}
+
+
+def parse_metrics(text):
+    """{pool: {key: value}} of the four series this pulse reads (SERIES); any other line is ignored."""
+    found = {}
+    for line in text.splitlines():
+        match = METRIC_LINE.match(line)
+        if not match or match.group(1) not in SERIES:
+            continue
+        labels = dict(LABEL.findall(match.group(2)))
+        pool = labels.get('pool')
+        if pool not in POOLS:
+            continue
+        try:
+            value = float(match.group(3))
+        except ValueError:
+            continue
+        if value != value:   # NaN
+            continue
+        key = SERIES[match.group(1)]
+        if key == 'pressure_phase':
+            value = PHASES.get(int(value)) if value.is_integer() else None
+        elif key == 'spill_fraction':
+            value = round(value, 3)
+        else:
+            value = int(value) if value.is_integer() and value >= 0 else None
+        found.setdefault(pool, {})[key] = value
+    return found
+
+
+def from_status(doc, now):
+    """(pool facts, overflow facts) of one tokened /pools/<pool>/status body. Worst = the highest MEASURED utilization
+    over the pool's subscription accounts (a field the daemon lists in `unknown_fields` is a placeholder 0, skipped);
+    soonest reset = the earliest reset of either window still in the future."""
+    worst = {'5h': None, '7d': None}
+    resets = []
+    for account in doc.get('accounts') or []:
+        if not isinstance(account, dict) or account.get('provider') not in SUBSCRIPTION_PROVIDERS:
+            continue
+        unknown = account.get('unknown_fields') if isinstance(account.get('unknown_fields'), list) else []
+        for window in ('5h', '7d'):
+            value = _number(account.get(f'utilization_{window}'))
+            if value is not None and f'utilization_{window}' not in unknown:
+                worst[window] = value if worst[window] is None else max(worst[window], value)
+            reset = account.get(f'reset_{window}')
+            if isinstance(reset, int) and not isinstance(reset, bool) and reset > now:
+                resets.append(reset)
+    overflow = doc.get('overflow') if isinstance(doc.get('overflow'), dict) else {}
+    alert = overflow.get('budget_alert_level_pct')
+    return ({'worst_7d': _fraction(worst['7d']), 'worst_5h': _fraction(worst['5h']),
+             'soonest_reset': min(resets) if resets else None},
+            {'spend_today_usd': _number(overflow.get('spend_today_usd')),
+             'budget_usd': _number(overflow.get('effective_daily_budget_usd')),
+             'fraction': _fraction(overflow.get('spent_fraction')),
+             'alert_level_pct': alert if isinstance(alert, int) and not isinstance(alert, bool) else None})
+
+
+# ------------------------------------------------------------------ token and file
+
+def read_token(path):
+    """(token or None, why or None). Absent is (None, None): the opt-in was never made. A file that is not a regular
+    file, is readable by group/other, is empty or too long is (None, why) — never used, never printed."""
+    try:
+        fd = os.open(str(path), os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, 'O_CLOEXEC', 0))
+    except FileNotFoundError:
+        return None, None
+    except OSError as error:
+        return None, f'token file unreadable ({error.strerror or type(error).__name__})'
+    try:
+        info = os.fstat(fd)
+        if not stat.S_ISREG(info.st_mode):
+            return None, 'token file is not a regular file'
+        if info.st_mode & 0o077:
+            return None, f'token file mode {stat.S_IMODE(info.st_mode):04o}, want 0600'
+        data = os.read(fd, TOKEN_MAX + 1)
+    except OSError as error:
+        return None, f'token file unreadable ({error.strerror or type(error).__name__})'
+    finally:
+        os.close(fd)
+    try:
+        token = data.decode('ascii').strip()
+    except UnicodeDecodeError:
+        return None, 'token file is not ASCII'
+    if not token or len(data) > TOKEN_MAX or any(c.isspace() for c in token):
+        return None, 'token file holds no single token'
+    return token, None
+
+
+def read_previous(path):
+    try:
+        doc = json.loads(Path(path).read_text())
+    except (OSError, ValueError):
+        return None
+    return doc if isinstance(doc, dict) and doc.get('v') == SCHEMA else None
+
+
+def write_atomic(path, document):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix=f'.{path.name}.', dir=str(path.parent))
+    try:
+        with os.fdopen(fd, 'w') as handle:
+            handle.write(json.dumps(document, separators=(',', ':')) + '\n')
+            os.fchmod(handle.fileno(), 0o600)
+        os.replace(temporary, path)
+    except BaseException:
+        try:
+            os.unlink(temporary)
+        except OSError:
+            pass
+        raise
+
+
+def _without_at(document):
+    return {key: value for key, value in (document or {}).items() if key != 'at'}
+
+
+# ------------------------------------------------------------------ one cycle
+
+class Pulse:
+    """One pulse: its last document (seeded from the file, so a restart keeps the pools) and its last journal line."""
+
+    def __init__(self, control, state_dir, token_file, opener=None, clock=time.time, journal=None):
+        self.control, self.path, self.token_file = control.rstrip('/'), Path(state_dir) / STATE_FILE, token_file
+        self.opener, self.clock = opener, clock
+        self.journal = journal or (lambda line: print(f'throttle-pulse: {line}', file=sys.stderr, flush=True))
+        previous = read_previous(self.path)
+        self.document = previous if previous else empty_document()
+        self.said = None
+
+    def observe(self):
+        """GET every source once -> (document without `at`, [problems]). A pool whose source did not answer keeps its
+        previous facts (the file's age tells a reader how old they are)."""
+        now = int(self.clock())
+        old_pools, old_overflow = self.document.get('pools') or {}, self.document.get('overflow') or {}
+        pools = {pool: dict(empty_pool(), **{k: v for k, v in (old_pools.get(pool) or {}).items() if k in POOL_KEYS})
+                 for pool in POOLS}
+        overflow = {pool: dict(empty_overflow(), **{k: v for k, v in (old_overflow.get(pool) or {}).items()
+                                                    if k in OVERFLOW_KEYS}) for pool in POOLS}
+        problems, answered, first_failure = [], False, None
+        for pool in POOLS:
+            path = f'/pools/{pool}/ready'
+            try:
+                pools[pool].update(from_ready(get_json(self.control, path, opener=self.opener, accept_status=(503,))))
+                answered = True
+            except Failed as error:
+                first_failure = first_failure or str(error)
+                problems.append(f'{path}: {error}')
+        try:
+            series = parse_metrics(get(self.control, '/metrics', opener=self.opener).decode('utf-8', 'replace'))
+            answered = True
+            for pool in POOLS:
+                for key in SERIES.values():
+                    if key == 'pressure_phase' and series.get(pool, {}).get(key) is None:
+                        continue   # the ready body's own phase (when it carries one) stands
+                    pools[pool][key] = series.get(pool, {}).get(key)
+        except Failed as error:
+            first_failure = first_failure or str(error)
+            problems.append(f'/metrics: {error}')
+        token, why = read_token(self.token_file)
+        if why:
+            problems.append(why)
+        tokened = False
+        for pool in POOLS:
+            if not token:   # no opt-in, or a token file that is not usable: burn and money are never shown
+                pools[pool].update(dict.fromkeys(TOKENED_POOL_KEYS))
+                overflow[pool] = empty_overflow()
+            elif answered:
+                path = f'/pools/{pool}/status'
+                try:
+                    facts, money = from_status(get_json(self.control, path, token, self.opener), now)
+                except Failed as error:   # the last burn and money of this pool are kept
+                    problems.append(f'{path}: {error}')
+                    continue
+                pools[pool].update(facts)
+                overflow[pool] = money if pool in MONEY_POOLS else empty_overflow()
+                tokened = True
+        if not answered:
+            tokened = bool(token) and self.document.get('tokened') is True
+        document = {'v': SCHEMA, 'up': answered, 'tokened': tokened, 'pools': pools, 'overflow': overflow,
+                    'error': None if answered else first_failure}
+        return document, problems
+
+    def cycle(self):
+        """One observation; throttle.json rewritten when something but `at` changed or HEARTBEAT seconds passed since
+        the file's `at` -> True when it wrote."""
+        document, problems = self.observe()
+        line = '; '.join(problems) or 'ok'
+        if line != self.said and (self.said is not None or problems):
+            self.journal(line)
+        self.said = line
+        now = int(self.clock())
+        on_disk = read_previous(self.path)
+        at = on_disk.get('at') if on_disk else None
+        if (isinstance(at, int) and 0 <= now - at < HEARTBEAT
+                and _without_at(on_disk) == _without_at(json.loads(json.dumps(document)))):
+            self.document = on_disk
+            return False
+        full = {'v': SCHEMA, 'at': now, 'up': document['up'], 'tokened': document['tokened'],
+                'pools': document['pools'], 'overflow': document['overflow'], 'error': document['error']}
+        write_atomic(self.path, full)
+        self.document = full
+        return True
+
+
+def run_loop(pulse, interval=PULSE_INTERVAL, sleep=time.sleep, clock=time.monotonic, cycles=None):
+    """Run `pulse.cycle()` every `interval` seconds (the time a cycle took is subtracted); `cycles` bounds it (tests)."""
+    done = 0
+    while cycles is None or done < cycles:
+        started = clock()
+        try:
+            pulse.cycle()
+        except OSError as error:   # an unwritable state dir is journalled; the loop keeps going
+            pulse.journal(f'state not written ({error.strerror or type(error).__name__})')
+        done += 1
+        if cycles is not None and done >= cycles:
+            break
+        sleep(max(0.0, interval - (clock() - started)))
+    return done
+
+
+def default_control(env):
+    raw = env.get('LANES_THROTTLE_CONTROL_PORT') or str(DEFAULT_PORT)
+    if not raw.isdigit() or not 0 < int(raw) < 65536:
+        raise ValueError(f'LANES_THROTTLE_CONTROL_PORT must be a port number, not {raw!r}')
+    return f'http://127.0.0.1:{int(raw)}'
+
+
+def main(argv=None, env=None):
+    env = os.environ if env is None else env
+    parser = argparse.ArgumentParser(prog='throttle-pulse', description=__doc__.split('\n\n', 1)[0])
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument('--once', action='store_true', help='one cycle: write when changed, print the document')
+    mode.add_argument('--loop', action='store_true', help=f'a cycle every {PULSE_INTERVAL} s until SIGTERM')
+    parser.add_argument('--control', help='control base URL (default http://127.0.0.1:$LANES_THROTTLE_CONTROL_PORT, '
+                                          f'else :{DEFAULT_PORT})')
+    parser.add_argument('--state-dir', default='~/.lz/state')
+    parser.add_argument('--token-file', default='~/.lz/throttle-token')
+    args = parser.parse_args(argv)
+    try:
+        control = args.control or default_control(env)
+    except ValueError as error:
+        parser.error(str(error))
+    if not re.match(r'^https?://[^/?#\s]+$', control.rstrip('/')):
+        parser.error(f'--control must be an http(s) base URL, not {control!r}')
+    pulse = Pulse(control, os.path.expanduser(args.state_dir), os.path.expanduser(args.token_file))
+    if args.once:
+        pulse.cycle()
+        print(json.dumps(pulse.document, indent=1))
+        return 0 if pulse.document.get('up') else 1
+    for signum in (signal.SIGTERM, signal.SIGINT):
+        signal.signal(signum, lambda *_: sys.exit(0))
+    run_loop(pulse)
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())