okstra 0.164.0 → 0.165.0

package/runtime/python/okstra_ctl/worker_runner.py

@@ -0,0 +1,622 @@
+"""Run one worker CLI and record what happened.
+
+Shared by every provider: the entrypoint scripts parse arguments and hand over
+here. What differs per provider is the command (an ``ExecutionStrategy``); what
+differs per surface is the presentation.
+
+Idle is measured from stream arrival, never from the log file's mtime. The
+screen deliberately drops thinking events, so an mtime-based watchdog would
+SIGTERM a healthy worker in the middle of a long reasoning stretch.
+"""
+from __future__ import annotations
+
+import json
+import os
+import selectors
+import signal
+import subprocess
+import sys
+import time
+from functools import partial
+from pathlib import Path
+from typing import Any, Callable, Mapping
+
+from .domain.worker_exec import (
+    STREAM_JSON,
+    ExecCommand,
+    ExecutionStrategy,
+    WorkerExecRequest,
+)
+from .domain.worker_stream import Normalise, final_text, format_live, format_log
+
+LIVE = "live"
+QUIET = "quiet"
+
+_SELECT_TIMEOUT_SECONDS = 0.25
+_TERM_GRACE_SECONDS = 5
+# How long the streams stay open after the worker process itself is gone. A pipe
+# holds at most its capacity of already-written output when its writer exits, so
+# this only has to cover a drain, never a producer.
+_DRAIN_AFTER_EXIT_SECONDS = 2
+_TIMEOUT_EXIT_CODE = 124
+_READ_SIZE = 8192
+_WRITE_SIZE = 8192
+_NO_STATUS_EXTRA: Mapping[str, Any] = {}
+
+# How many progress lines reach the log copy before it starts eliding. Progress
+# is where a text CLI's bulk is — a dispatch that only reads one file already
+# puts kilobytes of tool echo on stderr, and observed sidecars reach 8MB and
+# dominate a project's `.okstra/` bytes. The cap is run-wide rather than
+# per-block because block boundaries are a provider's own vocabulary and this
+# runner has none; the cost is that a very long run keeps its opening rather
+# than a sample throughout, which the elision notices make visible.
+_LOG_PROGRESS_LINE_CAP = 5000
+_ELISION_NOTICE_EVERY = 500
+
+# The signals that end this process without raising anything Python can catch on
+# the way out. SIGINT is absent on purpose: it arrives as KeyboardInterrupt and
+# the exception path already closes the sidecar.
+_ABNORMAL_SIGNALS = (signal.SIGTERM, signal.SIGHUP)
+
+
+def run_worker(
+    strategy: ExecutionStrategy,
+    request: WorkerExecRequest,
+    *,
+    presentation: str,
+    log_path: Path,
+    status_path: Path,
+    status_extra: Mapping[str, Any] = _NO_STATUS_EXTRA,
+) -> int:
+    command = strategy.build_command(request)
+    started_monotonic = time.monotonic()
+    status = _started_status(status_extra, log_path)
+    _write_status(status_path, status)
+    guard = _AbnormalExit(status_path, status, started_monotonic)
+
+    try:
+        with guard:
+            exit_code, timed_out, idle_seconds = _launch(
+                command,
+                log_path,
+                presentation=presentation,
+                idle_timeout_seconds=request.idle_timeout_seconds,
+                on_spawn=guard.watch,
+            )
+    except BaseException as exc:
+        # Whatever ended this run — an OS error, a Ctrl-C, a bug in this file —
+        # the sidecar has to stop saying `started`. Nothing downstream rewrites
+        # it, so `worker_liveness` would read the worker as still working
+        # forever. No exit code is invented — its absence is already how every
+        # reader spells a run that failed. The exits that never raise at all are
+        # the guard's to close.
+        guard.close(f"{type(exc).__name__}: {exc}")
+        raise
+
+    status = _closed(status, started_monotonic)
+    status["exit_code"] = exit_code
+    if timed_out:
+        status.update(
+            timeout=True,
+            idle_at_ts=status["ended_ts"],
+            idle_seconds=idle_seconds,
+            terminated_by="idle-watchdog",
+        )
+    _write_status(status_path, status)
+    return exit_code
+
+
+def _launch(
+    command: ExecCommand,
+    log_path: Path,
+    *,
+    presentation: str,
+    idle_timeout_seconds: int,
+    on_spawn: Callable[[subprocess.Popen[bytes]], None],
+) -> tuple[int, bool, int]:
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    with log_path.open("w", encoding="utf-8") as log_file:
+        # The strategy decided where this provider runs — some CLIs work in the
+        # stage tree, others in the project root and reach the tree by flag.
+        process = subprocess.Popen(
+            list(command.argv),
+            cwd=str(command.cwd),
+            stdin=subprocess.PIPE if command.stdin_text is not None else None,
+            stdout=subprocess.PIPE,
+            stderr=_stderr_target(command.stream_format),
+            start_new_session=True,
+            env=_child_env(),
+        )
+        on_spawn(process)
+        return _pump(
+            process,
+            log_file,
+            stream_format=command.stream_format,
+            normalise=command.normalise,
+            presentation=presentation,
+            idle_timeout_seconds=idle_timeout_seconds,
+            stdin_text=command.stdin_text,
+        )
+
+
+class _AbnormalExit:
+    """Close the status sidecar for the exits that raise nothing at all.
+
+    ``except BaseException`` around the run covers an exception, including the
+    ``KeyboardInterrupt`` a SIGINT raises. It does not cover SIGTERM or SIGHUP:
+    their default disposition ends the process outright, and nothing in this
+    file runs (measured — a bash ``trap … EXIT`` does fire on SIGTERM, which is
+    why the shell wrappers needed no equivalent of this class). Those two are
+    the common abnormal exits: a pane kill, ``okstra-trace-cleanup.sh``, session
+    teardown. Without this the sidecar stays at ``started`` and
+    ``worker_liveness`` reads a dead worker as a working one.
+
+    SIGKILL and a host crash remain uncovered because nothing can cover them. A
+    sidecar still reading ``started`` is the residue they leave, and no reader
+    should take it as proof the worker is alive.
+    """
+
+    def __init__(
+        self, status_path: Path, status: Mapping[str, Any], started_monotonic: float
+    ) -> None:
+        self._status_path = status_path
+        self._status = status
+        self._started_monotonic = started_monotonic
+        self._process: subprocess.Popen[bytes] | None = None
+        self._restore: dict[int, Any] = {}
+
+    def watch(self, process: subprocess.Popen[bytes]) -> None:
+        """Adopt the child, so a signal tears down its group rather than orphan it."""
+        self._process = process
+
+    def close(self, failure: str) -> None:
+        _write_status(
+            self._status_path,
+            {**_closed(self._status, self._started_monotonic), "failure": failure},
+        )
+
+    def __enter__(self) -> _AbnormalExit:
+        for number in _ABNORMAL_SIGNALS:
+            try:
+                self._restore[number] = signal.signal(number, self._handle)
+            except ValueError:
+                # Handlers install from the main thread only. A caller running
+                # the runner off-thread keeps the exception path and nothing
+                # more, which is what it had before this class existed.
+                break
+        return self
+
+    def __exit__(self, *_exception: Any) -> bool:
+        for number, previous in self._restore.items():
+            signal.signal(number, previous)
+        self._restore.clear()
+        return False
+
+    def _handle(self, number: int, _frame: Any) -> None:
+        if self._process is not None:
+            _terminate(self._process)
+        self.close(f"signal {signal.Signals(number).name}")
+        # Then die the way the sender asked, so the exit code still names the
+        # signal instead of reporting a clean stop this run did not make.
+        signal.signal(number, signal.SIG_DFL)
+        os.kill(os.getpid(), number)
+
+
+def _closed(status: Mapping[str, Any], started_monotonic: float) -> dict[str, Any]:
+    """The sidecar's terminal shape, whatever it was that ended the run.
+
+    ``stage`` is the field every reader keys on — ``wrapper_status.is_terminal``,
+    the pane reclaim and the dispatch record all ask whether it reads ``exited``.
+    """
+    return {
+        **status,
+        "stage": "exited",
+        "ended_ts": int(time.time()),
+        "duration_ms": int((time.monotonic() - started_monotonic) * 1000),
+    }
+
+
+def _started_status(status_extra: Mapping[str, Any], log_path: Path) -> dict[str, Any]:
+    """The sidecar as it looks before the CLI has produced anything.
+
+    ``status_extra`` carries what only the entrypoint knows — which wrapper the
+    caller invoked and which role it dispatched — and is placed where the
+    wrapper-written sidecars already carry those keys.
+    """
+    return {
+        "schemaVersion": 1,
+        **dict(status_extra),
+        "pid": os.getpid(),
+        "started_ts": int(time.time()),
+        "log_path": str(log_path),
+        "stage": "started",
+    }
+
+
+def _stderr_target(stream_format: str) -> int:
+    """Whether the child's two streams stay apart.
+
+    A JSON-stream CLI puts events on stdout and only its own error text on
+    stderr, so folding the two loses nothing and leaves one reader. A text CLI
+    splits meaning across them — the result on stdout, progress on stderr — and
+    merging destroys the only way to tell the answer from the noise.
+    """
+    return subprocess.STDOUT if stream_format == STREAM_JSON else subprocess.PIPE
+
+
+def _pump(
+    process: subprocess.Popen[bytes],
+    log_file,
+    *,
+    stream_format: str,
+    normalise: Normalise,
+    presentation: str,
+    idle_timeout_seconds: int,
+    stdin_text: str | None = None,
+) -> tuple[int, bool, int]:
+    selector = selectors.DefaultSelector()
+    readers, finalize_log = _register_output(
+        selector,
+        process,
+        log_file,
+        stream_format=stream_format,
+        normalise=normalise,
+        presentation=presentation,
+    )
+    outgoing = _register_prompt(selector, process, stdin_text)
+    closing_text: str | None = None
+    last_output = time.monotonic()
+    timed_out = False
+    idle_seconds = 0
+    drain_deadline: float | None = None
+
+    while selector.get_map():
+        for key, events in selector.select(timeout=_SELECT_TIMEOUT_SECONDS):
+            if events & selectors.EVENT_WRITE:
+                outgoing = _push_prompt(selector, key, outgoing)
+                continue
+            chunk = os.read(key.fd, _READ_SIZE)
+            if not chunk:
+                selector.unregister(key.fileobj)
+                continue
+            # Either stream is proof of life: a text CLI reports progress for
+            # minutes before a result exists to send.
+            last_output = time.monotonic()
+            closing_text = readers[key.fd].feed(chunk) or closing_text
+        idle_seconds = int(time.monotonic() - last_output)
+        if (
+            idle_timeout_seconds
+            and idle_seconds >= idle_timeout_seconds
+            and process.poll() is None
+        ):
+            timed_out = True
+            _terminate(process)
+        drain_deadline = _drain_deadline(process, drain_deadline)
+        if drain_deadline is not None and time.monotonic() >= drain_deadline:
+            _stop_reading(selector)
+
+    for reader in readers.values():
+        closing_text = reader.flush() or closing_text
+    finalize_log()
+
+    exit_code = process.wait()
+    if closing_text is None:
+        # A JSON stream is supposed to end in a result event. Ending without one
+        # means the CLI stopped without saying what it concluded, and the caller
+        # would otherwise read an empty stdout under `quiet` as a run that
+        # simply had nothing to report. A text CLI has no result event to miss.
+        if stream_format == STREAM_JSON:
+            print(
+                f"okstra worker: no result event in the CLI's output — "
+                f"see {log_file.name}",
+                file=sys.stderr,
+                flush=True,
+            )
+    elif presentation == QUIET:
+        print(closing_text, flush=True)
+    return (_TIMEOUT_EXIT_CODE if timed_out else exit_code), timed_out, idle_seconds
+
+
+def _drain_deadline(
+    process: subprocess.Popen[bytes], deadline: float | None
+) -> float | None:
+    """When to stop reading, once the worker process itself is gone.
+
+    Without this the loop ends only at EOF on every stream, and a worker that
+    leaves a background process behind — a dev server, a file watcher, the
+    watcher the container-build skill starts on purpose — leaves that process
+    holding the write end of the pipe open for as long as it lives. The shell
+    wrappers returned as soon as `wait <pid>` did, so blocking on a grandchild is
+    a regression rather than a policy, and the exit code the caller gets stays
+    the worker's own.
+
+    The deadline is set once and never pushed back: output arriving after the
+    worker exited is the grandchild's, and letting it extend the wait would
+    restore exactly the hang this bounds.
+    """
+    if deadline is not None or process.poll() is None:
+        return deadline
+    return time.monotonic() + _DRAIN_AFTER_EXIT_SECONDS
+
+
+def _stop_reading(selector: selectors.BaseSelector) -> None:
+    """Drop every remaining stream without signalling whoever still holds it.
+
+    A process the worker deliberately left running is not this runner's to kill.
+    The idle watchdog is what ends a run that went wrong; this only ends the
+    reading of a run that already finished.
+    """
+    for key in list(selector.get_map().values()):
+        selector.unregister(key.fileobj)
+
+
+def _register_output(
+    selector: selectors.BaseSelector,
+    process: subprocess.Popen[bytes],
+    log_file,
+    *,
+    stream_format: str,
+    normalise: Normalise,
+    presentation: str,
+) -> tuple[dict[int, _LineReader], Callable[[], None]]:
+    """Register every stream this child speaks, each with its own destination.
+
+    Returns the readers plus the finalizer a capped sink needs to report what it
+    dropped once the streams are done. A JSON stream caps nothing.
+    """
+    assert process.stdout is not None
+    if stream_format == STREAM_JSON:
+        sinks = [
+            (process.stdout, partial(_emit_event, log_file, presentation, normalise))
+        ]
+        finalize_log: Callable[[], None] = _nothing_to_finalize
+    else:
+        assert process.stderr is not None
+        progress = _ProgressSink(log_file, presentation)
+        sinks = [
+            (process.stdout, partial(_emit_result, log_file)),
+            (process.stderr, progress),
+        ]
+        finalize_log = progress.report_elided
+    readers: dict[int, _LineReader] = {}
+    for stream, emit in sinks:
+        selector.register(stream, selectors.EVENT_READ)
+        readers[stream.fileno()] = _LineReader(emit)
+    return readers, finalize_log
+
+
+def _nothing_to_finalize() -> None:
+    """A JSON stream elides nothing, so it has nothing to report at the end."""
+
+
+class _LineReader:
+    """Whole lines out of one byte stream, handed to that stream's sink.
+
+    One per stream rather than one shared buffer: the two streams of a text CLI
+    arrive interleaved, and a shared buffer would splice half a progress line
+    onto the front of the result.
+    """
+
+    def __init__(self, emit: Callable[[str], str | None]) -> None:
+        self._emit = emit
+        self._pending = b""
+
+    def feed(self, chunk: bytes) -> str | None:
+        self._pending += chunk
+        *complete, self._pending = self._pending.split(b"\n")
+        return self._drain(complete)
+
+    def flush(self) -> str | None:
+        """Whatever the stream ended on without a closing newline."""
+        trailing, self._pending = self._pending, b""
+        return self._drain([trailing] if trailing else [])
+
+    def _drain(self, raw_lines: list[bytes]) -> str | None:
+        closing: str | None = None
+        for raw in raw_lines:
+            closing = self._emit(raw.decode("utf-8", "replace")) or closing
+        return closing
+
+
+def _register_prompt(
+    selector: selectors.BaseSelector,
+    process: subprocess.Popen[bytes],
+    stdin_text: str | None,
+) -> bytes:
+    """Queue the prompt for a CLI that reads it from stdin.
+
+    Fed inside the pump rather than written whole before it. A prompt larger
+    than the pipe buffer — worker prompts are tens of kilobytes — would
+    otherwise block this process while the CLI blocks writing the stdout that
+    nobody is draining yet, and the idle watchdog could not even reach that
+    stall because it only runs once the pump is looping.
+    """
+    if stdin_text is None or process.stdin is None:
+        return b""
+    os.set_blocking(process.stdin.fileno(), False)
+    selector.register(process.stdin, selectors.EVENT_WRITE)
+    return stdin_text.encode("utf-8")
+
+
+def _push_prompt(
+    selector: selectors.BaseSelector, key: selectors.SelectorKey, payload: bytes
+) -> bytes:
+    """Hand over as much of the prompt as the pipe will take right now."""
+    try:
+        written = os.write(key.fd, payload[:_WRITE_SIZE]) if payload else 0
+    except BlockingIOError:
+        return payload
+    except OSError:
+        # The CLI exited before reading its prompt. Its exit code is the story;
+        # this half-delivered write is not.
+        _close_prompt(selector, key)
+        return b""
+    payload = payload[written:]
+    if not payload:
+        # EOF is what tells the CLI its prompt is complete.
+        _close_prompt(selector, key)
+    return payload
+
+
+def _close_prompt(selector: selectors.BaseSelector, key: selectors.SelectorKey) -> None:
+    selector.unregister(key.fileobj)
+    key.fileobj.close()
+
+
+def _emit_result(log_file, line: str) -> None:
+    """A text CLI's stdout is its answer, so the caller gets it in either mode.
+
+    ``quiet`` withholds progress, not the result — and unlike a JSON stream
+    there is no result event to hold back and print at the end, so the answer
+    passes through as it arrives.
+    """
+    _write_log(log_file, [line])
+    print(line, flush=True)
+
+
+class _ProgressSink:
+    """A text CLI's stderr: shown live, archived up to a cap.
+
+    Progress goes to this process's stderr rather than its stdout so the
+    caller's stdout stays the result alone. A pane shows both, and a pane is
+    exactly where ``live`` lands.
+
+    Only the log copy is capped, never the screen and never the result stream.
+    That asymmetry is the point: a truncated tool echo costs detail, while a
+    truncated answer costs the whole post-mortem — and the caller's streams are
+    read by a person or a subagent that was promised the CLI's output verbatim.
+    """
+
+    def __init__(self, log_file, presentation: str) -> None:
+        self._log_file = log_file
+        self._presentation = presentation
+        self._archived = 0
+        self._elided = 0
+
+    def __call__(self, line: str) -> None:
+        if self._presentation == LIVE:
+            print(line, file=sys.stderr, flush=True)
+        if self._archived < _LOG_PROGRESS_LINE_CAP:
+            self._archived += 1
+            _write_log(self._log_file, [line])
+            return
+        self._elided += 1
+        # Periodic rather than only at the end: a reader tailing the log has to
+        # see that the run is still producing progress, not a file that stopped.
+        if self._elided % _ELISION_NOTICE_EVERY == 0:
+            self._note_elision()
+
+    def report_elided(self) -> None:
+        """Close the archive with the exact total, if the last notice missed some."""
+        if self._elided % _ELISION_NOTICE_EVERY:
+            self._note_elision()
+
+    def _note_elision(self) -> None:
+        _write_log(
+            self._log_file,
+            [f"  [okstra log-cap] {self._elided} progress line(s) elided"],
+        )
+
+
+def _emit_event(
+    log_file, presentation: str, normalise: Normalise, line: str
+) -> str | None:
+    """Write one line of a JSON stream out; return closing text if this is it.
+
+    The provider's own wire shape gets no further than ``normalise`` — what
+    reaches the projections is the normalised vocabulary, which is the only
+    thing this runner and the formatter are allowed to know.
+    """
+    stripped = line.strip()
+    if not stripped:
+        return None
+    try:
+        event = json.loads(stripped)
+    except ValueError:
+        # Not an event. The CLI's stderr is folded into this stream, so this is
+        # where its own error text arrives — the caller has to see it, and the
+        # archive alone does not show it to anyone. stderr rather than stdout so
+        # it reaches the caller without posing as progress, in either mode.
+        _write_log(log_file, [stripped])
+        print(stripped, file=sys.stderr, flush=True)
+        return None
+    if not isinstance(event, dict):
+        return None
+
+    closing: str | None = None
+    for entry in normalise(event):
+        _write_log(log_file, format_log(entry))
+        if presentation == LIVE:
+            for row in format_live(entry):
+                print(row, flush=True)
+        text = final_text(entry)
+        if text is not None:
+            closing = text
+    if closing is not None:
+        # Whatever the presentation, the archive ends with the worker's own
+        # conclusion — otherwise it stops at the last tool call and the reader
+        # never learns what the run decided.
+        _write_log(log_file, closing.splitlines())
+    return closing
+
+
+def _write_log(log_file, lines) -> None:
+    for line in lines:
+        log_file.write(line + "\n")
+    log_file.flush()
+
+
+def _terminate(process: subprocess.Popen[bytes]) -> None:
+    """SIGTERM the worker's process group, then SIGKILL whatever survives.
+
+    The group, not the process: a CLI spawns shells and build tools, and
+    signalling only the direct child leaves those running with the pipe open.
+    ``start_new_session=True`` at spawn is what makes the group addressable by
+    the child's own pid.
+    """
+    try:
+        os.killpg(process.pid, signal.SIGTERM)
+    except ProcessLookupError:
+        return
+    try:
+        process.wait(timeout=_TERM_GRACE_SECONDS)
+    except subprocess.TimeoutExpired:
+        try:
+            os.killpg(process.pid, signal.SIGKILL)
+        except ProcessLookupError:
+            pass
+        process.wait()
+
+
+def _child_env() -> dict[str, str]:
+    """The worker's environment with git's fsmonitor disabled.
+
+    The main worktree's fsmonitor daemon leaks its IPC socket into task
+    worktrees, so git status/commit there intermittently fail with
+    `fsmonitor_ipc__send_query`. GIT_CONFIG_* scopes the override to this
+    process tree without touching the user's repo config, appending after any
+    GIT_CONFIG_* the caller already set.
+    """
+    env = dict(os.environ)
+    index = int(env.get("GIT_CONFIG_COUNT", "0") or "0")
+    env[f"GIT_CONFIG_KEY_{index}"] = "core.fsmonitor"
+    env[f"GIT_CONFIG_VALUE_{index}"] = "false"
+    env["GIT_CONFIG_COUNT"] = str(index + 1)
+    return env
+
+
+def _write_status(path: Path, status: Mapping[str, Any]) -> None:
+    """Best-effort status write; a sidecar failure must not break the run.
+
+    Replaced whole rather than rewritten in place: readers poll this file while
+    the worker is still running and must never see a half-written document.
+    """
+    temporary = Path(f"{path}.tmp")
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temporary.write_text(
+            json.dumps(status, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+        )
+        os.replace(temporary, path)
+    except OSError:
+        return

package/runtime/python/okstra_token_usage/collect.py

@@ -704,14 +704,17 @@ def _populate_usage_summary(
     workers = state.get("workers", [])
     lead = state.get("leadUsage") or {}
     lead_total = lead.get("totalTokens", 0) or 0
+    lead_cache_read = lead.get("cacheReadTokens", 0) or 0
     lead_billable = lead.get("billableEquivalentTokens", 0) or 0
     lead_cost = lead.get("estimatedCostUsd", 0) or 0
     worker_total = sum((w.get("usage") or {}).get("totalTokens", 0) or 0 for w in workers)
+    worker_cache_read = sum((w.get("usage") or {}).get("cacheReadTokens", 0) or 0 for w in workers)
     worker_billable = sum((w.get("usage") or {}).get("billableEquivalentTokens", 0) or 0 for w in workers)
     worker_cost = sum((w.get("usage") or {}).get("estimatedCostUsd", 0) or 0 for w in workers)
     cli_cost = sum((w.get("usage") or {}).get("cliEstimatedCostUsd", 0) or 0 for w in workers)
     if unattributed_usage is not None:
         worker_total += unattributed_usage.get("totalTokens", 0) or 0
+        worker_cache_read += unattributed_usage.get("cacheReadTokens", 0) or 0
         worker_billable += unattributed_usage.get("billableEquivalentTokens", 0) or 0
         worker_cost += unattributed_usage.get("estimatedCostUsd", 0) or 0
 
@@ -733,6 +736,9 @@ def _populate_usage_summary(
         "leadTotalTokens": lead_total,
         "workerTotalTokens": worker_total,
         "grandTotalTokens": lead_total + worker_total,
+        "leadCacheReadTokens": lead_cache_read,
+        "workerCacheReadTokens": worker_cache_read,
+        "grandCacheReadTokens": lead_cache_read + worker_cache_read,
         "leadBillableEquivalentTokens": lead_billable,
         "workerBillableEquivalentTokens": worker_billable,
         "grandBillableEquivalentTokens": lead_billable + worker_billable,
@@ -750,7 +756,8 @@ def _populate_usage_summary(
         "unattributedTeamSessions": unattributed_sessions or [],
         "unattributedWorkerUsage": unattributed_usage,
         "definitions": {
-            "totalTokens": "Sum of input + output + cache_creation + cache_read tokens (raw processed volume; matches Anthropic API breakdown). Cache reads are 95%+ in long sessions.",
+            "totalTokens": "Sum of input + output + cache_creation tokens — the volume the session put through the model once. cache_read is excluded and reported separately as cacheReadTokens: a session re-reads its whole context from cache every turn, so folding it in here would count the same tokens once per turn.",
+            "cacheReadTokens": "Context re-read from cache. Billed at 0.1x base input, so it lands in billableEquivalentTokens and in the cost even though it is not part of totalTokens.",
             "billableEquivalentTokens": "Tokens normalized to base-input-price units (cache_creation_5m x1.25, cache_creation_1h x2.0, cache_read x0.1, output x5). 5m vs 1h is split from usage.cache_creation when the API breakdown is present; otherwise all cache_creation falls into 5m.",
             "estimatedCostUsd": "USD cost using public list pricing for the model recorded in the session. cliWorkers covers attributable registered-provider CLI calls.",
         },