cctally 1.116.0 → 1.117.0

package/bin/_lib_selector_state.py

@@ -26,9 +26,16 @@ nothing, and one that conflicts is detected from the retained whole-record
 digest, which taints the batch and forces a rebuild that re-derives from the
 journal.
 
-This module imports nothing outside the stdlib and `_lib_journal`, and in
-particular never imports `_cctally_journal`, so it is unit-testable without a
-journal on disk — the same rule `bin/_lib_journal_router.py` follows.
+This module imports nothing outside the stdlib, `_lib_journal` and the pure
+`_lib_quota_lineage` kernel, and in particular never imports
+`_cctally_journal`, so it is unit-testable without a journal on disk — the same
+rule `bin/_lib_journal_router.py` follows.
+
+#1015 (spec 5.1): the batch-completion step also gives each newly completed
+batch's lineage footprint (:func:`completed_batch_footprints`) while it still
+holds every action core, before compaction drops them. The footprint is kept
+off :class:`SelectorRows` — those rows are compared for validation and owned by
+the selector — and handed to the caller through ``lineage`` out-dicts.
 """
 from __future__ import annotations
 
@@ -36,6 +43,7 @@ import json
 from dataclasses import dataclass, replace
 
 import _lib_journal as _jl
+import _lib_quota_lineage
 
 
 #: Bumped whenever the durable row shapes or the merge semantics change. A
@@ -800,6 +808,107 @@ def advance_counter(
     )
 
 
+def _revision_of(winner, owner_of):
+    """The `_lib_quota_lineage.Revision` of one winning candidate."""
+    if winner.status != "active" or winner.record is None:
+        return _lib_quota_lineage.Revision("tombstone")
+    return _lib_quota_lineage.Revision(
+        "active", owner_of((winner.record or {}).get("payload")))
+
+
+def _takes_winner(prior, candidate) -> bool:
+    """The selector's winner rule: the highest revision, and at the same
+    revision the lowest sequence (a durable winner with no sequence keeps its
+    place)."""
+    if prior is None or candidate.rev > prior.rev:
+        return True
+    if candidate.rev < prior.rev:
+        return False
+    return (prior.sequence is not None and candidate.sequence is not None
+            and candidate.sequence < prior.sequence)
+
+
+def completed_batch_footprints(fold, batch_ids, *, owner_of, seed=None):
+    """#1015 spec 5.1: the footprint of each completed batch in ``batch_ids``.
+
+    Returns ``{batch_id: frozenset((provider, account_key), ...)}`` through
+    `_lib_quota_lineage.batch_footprint`. Batches are walked in commit
+    order, and each action's predecessor is the revision effective
+    immediately before its batch commits: the winner over ``seed`` (the
+    durable winner of an event id before ``fold``'s stream, None on a full
+    pass) advanced by every base evt folded before that commit and by every
+    earlier completed batch — dominated and tombstoned revisions included —
+    then the revision the batch's own earlier action produced.
+
+    ``owner_of(payload)`` is the `(provider, account_key)` the evt fold stores
+    for a payload, or None when it cannot be resolved. ``batch_ids`` names
+    batches whose cores ``fold`` still holds (newly completed on this pass);
+    any other id is ignored.
+    """
+    targets = []
+    for batch_id in batch_ids:
+        if batch_id not in fold.completed:
+            continue
+        commit = fold.markers.get(batch_id, {}).get("commit")
+        actions = fold.actions.get(batch_id, {})
+        if commit is None or any(
+                actions[seq][0] is None for seq in actions):
+            continue
+        targets.append((commit[3], batch_id))
+    if not targets:
+        return {}
+    targets.sort()
+    base = sorted(
+        (candidate for candidate in fold.candidates
+         if candidate.batch_id is None),
+        key=lambda item: item.sequence)
+    winners: dict = {}
+    seeded: set = set()
+
+    def current(event_id):
+        if event_id not in seeded:
+            seeded.add(event_id)
+            if event_id not in winners and seed is not None:
+                prior = seed(event_id)
+                if prior is not None:
+                    winners[event_id] = prior
+        return winners.get(event_id)
+
+    def offer(candidate):
+        prior = current(candidate.event_id)
+        if _takes_winner(prior, candidate):
+            winners[candidate.event_id] = candidate
+
+    footprints: dict = {}
+    position = 0
+    for commit_sequence, batch_id in targets:
+        while position < len(base) and base[position].sequence < commit_sequence:
+            offer(base[position])
+            position += 1
+        batch_actions = fold.actions.get(batch_id, {})
+        ordered = [batch_actions[seq] for seq in sorted(batch_actions)]
+        actions = []
+        for normalized, _digest, _sequence in ordered:
+            tombstone = normalized.get("action") == "tombstone"
+            actions.append(_lib_quota_lineage.Action(
+                event_id=normalized["id"],
+                rev=int(normalized["rev"]),
+                kind="tombstone" if tombstone else "replace",
+                payload_owner=(None if tombstone
+                               else owner_of(normalized.get("payload"))),
+            ))
+
+        def predecessor_of(event_id, _rev):
+            winner = current(event_id)
+            return None if winner is None else _revision_of(winner, owner_of)
+
+        footprints[batch_id] = _lib_quota_lineage.batch_footprint(
+            actions, predecessor_of)
+        for normalized, _digest, sequence in ordered:
+            offer(_jl._candidate_from_correction(normalized, sequence))
+    return footprints
+
+
 def merge_delta(
     rows: SelectorRows,
     new_records,
@@ -807,6 +916,7 @@ def merge_delta(
     next_sequence: int,
     coordinates=None,
     covered=None,
+    lineage=None,
 ):
     """Continue the fold over ``new_records`` from durable state.
 
@@ -826,6 +936,13 @@ def merge_delta(
     `journal_protocol_resolution` op: acknowledging a violation authenticates an
     exact length-framed raw-prefix SHA-256, a claimed hash is never accepted
     without recomputation, and no durable summary can reconstruct it.
+
+    ``lineage`` (#1015) is an optional out-dict. When it carries an
+    ``"owner_of"`` callable, it receives under ``"footprints"`` the
+    footprint of every batch this delta completed, computed here, in the
+    batch-completion step, before the cores are compacted away (see
+    :func:`completed_batch_footprints`); the predecessor of an action is
+    seeded from the durable winner of its event id.
     """
     fold = _seed_fold(rows)
     seeded_completed = {
@@ -903,6 +1020,17 @@ def merge_delta(
     # this one converges through the narrowest prefix.
     transitions.sort(key=lambda item: (item.causal_sequence, item.batch_id))
 
+    if lineage is not None and lineage.get("owner_of") is not None:
+        prior_rows = {row.event_id: row for row in rows.effective}
+
+        def seed(event_id):
+            row = prior_rows.get(event_id)
+            return None if row is None else _event_from_row(row)
+
+        lineage["footprints"] = completed_batch_footprints(
+            fold, sorted(fold.completed - seeded_completed),
+            owner_of=lineage["owner_of"], seed=seed)
+
     winners, conflicts = _merge_candidates(rows, fold.candidates)
     batches, batch_records = _batch_rows(fold, coordinates)
     merged_batches = _carry_coordinates(rows, batches)

package/bin/_lib_stale_proof.py

@@ -20,6 +20,14 @@ Both axes are decided here. The weekly axis earns by (i) or (ii); the
 five-hour axis earns by (i) only, against the `statusline` source's detector
 baseline under the current five-hour credit generation (D11), and its
 validity follows the five-hour rows of the section 4.4 table.
+
+#1019/#1014 (spec 4.1) adds the receipt-free DROP proof beside it: a
+member's LOW is proven stale by an admitted provider sample in its exact
+account and window, requested after the member's bound (its frozen supported
+``t_m``, else its ``received_at`` second), reading more than `TOLERANCE_PP`
+ABOVE it and captured before the member's advertised reset
+(`proven_stale_low`). Its facts are independent of the rise facts: a member
+may be eligible for one direction and refused for the other.
 """
 from __future__ import annotations
 
@@ -124,14 +132,36 @@ class Member:
 
 @dataclasses.dataclass(frozen=True)
 class MemberFacts:
+    """One member's refusal facts. ``refused``, ``proof_observation_id`` and
+    ``receipt_valid`` decide its RISE (#797 spec 4.4); ``drop_refused`` and
+    ``drop_proof_observation_id`` decide its DROP (#1019 spec 4.1),
+    independently and without a receipt."""
+
     refused: bool
     proof_observation_id: str | None
     receipt_valid: bool
+    drop_refused: bool = False
+    drop_proof_observation_id: str | None = None
 
 
+#: Eligible in both directions.
 ALL_ELIGIBLE = MemberFacts(False, None, False)
 
 
+@dataclasses.dataclass(frozen=True)
+class DropMember:
+    """A member's reading as the drop proof sees it (#1019 spec 4.1): its
+    contributor token, its bound (`drop_bound`), its axis value, its window
+    and the raw reset it advertised."""
+
+    token: str
+    bound_utc: str
+    axis: str
+    percent: float
+    window: int
+    advertised_reset_epoch: int | None = None
+
+
 def sample_identity(token: str, witness, *, axis: str, percent: float,
                     window: int) -> SampleIdentity | None:
     """σ for a member, or None when its evidence is unknown: no witness, an
@@ -147,6 +177,45 @@ def sample_identity(token: str, witness, *, axis: str, percent: float,
                           witness.evidence_revision, witness.parser_revision)
 
 
+def drop_bound(token: str, witness, received_at_epoch: int | None, *, axis: str,
+               percent: float, window: int) -> str | None:
+    """The bound a member's LOW is proven stale after (#1019 spec 4.1): the
+    frozen, supported ``t_m`` when `sample_identity` accepts its witness;
+    otherwise its ``received_at`` as a whole-second UTC instant, an
+    assumption-free upper bound on when its already-composed reading
+    existed; otherwise None (always eligible). The fallback is used for this
+    proof alone: it is never a σ, a receipt or a provenance field."""
+    sigma = sample_identity(token, witness, axis=axis, percent=percent, window=window)
+    if sigma is not None:
+        return sigma.activity_at_utc
+    return _received_bound(received_at_epoch)
+
+
+def _received_bound(received_at_epoch) -> str | None:
+    if (isinstance(received_at_epoch, bool) or not isinstance(received_at_epoch, int)
+            or received_at_epoch < 0):
+        return None
+    import datetime as _dt
+    return _dt.datetime.fromtimestamp(received_at_epoch, tz=_dt.timezone.utc).strftime(
+        "%Y-%m-%dT%H:%M:%SZ")
+
+
+def witness_bound(witness, received_at_epoch: int | None) -> str | None:
+    """`drop_bound` of a retained support: whether `sample_identity` accepts
+    a witness depends on the witness alone, so the bound needs neither the
+    member's token nor its axis, value or window."""
+    if sample_identity("", witness, axis=WEEKLY, percent=0.0, window=0) is not None:
+        return witness.instant_utc
+    return _received_bound(received_at_epoch)
+
+
+def bound_end(bound_utc: str | None) -> int | None:
+    """The end of a bound's precision interval in nanoseconds: a sample
+    follows the bound when its request starts at or after it (`after`)."""
+    interval = _interval(bound_utc)
+    return None if interval is None else interval[1]
+
+
 def _window_text(window: int) -> str:
     return _samples.normalize_weekly_reset(int(window))
 
@@ -217,6 +286,31 @@ def proven_stale(sigma: SampleIdentity, samples: Iterable, *, after_utc: str,
     return _latest(proving)
 
 
+def proven_stale_low(bound_utc: str, samples: Iterable, *, axis: str, window: int,
+                     percent: float, advertised_reset_epoch: int | None = None):
+    """The latest provider sample proving a member's LOW stale, or None
+    (#1019 spec 4.1, Q1).
+
+    A sample proves it when it is in the member's exact window (weekly: the
+    normalized reset; five-hour: the canonical key; the caller passes only
+    the member's own account's samples), its request instant ``q`` follows
+    ``bound_utc`` (`drop_bound`) under the precision intervals, its raw axis
+    value is strictly more than `TOLERANCE_PP` ABOVE ``percent``, and its
+    response was captured before the member's advertised reset (the
+    window's reset when it advertised none). No receipt, no ``e_σ``, and no
+    baseline or threshold cutoff."""
+    if axis not in AXES or _interval(bound_utc) is None:
+        return None
+    reset = int(window) if advertised_reset_epoch is None else int(advertised_reset_epoch)
+    proving = [
+        s for s in _axis_window(samples, axis, window)
+        if _axis_value(s, axis) > float(percent) + TOLERANCE_PP
+        and after(s.requested_at_utc, bound_utc)
+        and _before_epoch(s.captured_at_utc, reset)
+    ]
+    return _latest(proving)
+
+
 # --- accounting ------------------------------------------------------------------------
 
 
@@ -392,7 +486,8 @@ def receipt_valid(receipt: Receipt, transitions: Iterable[CreditTransition], *,
 
 def refusal_facts(axis: str, members: Sequence[Member], samples: Iterable,
                   receipts: Iterable[Receipt], transitions: Iterable[CreditTransition], *,
-                  index_identity: str, supports: Collection[str] | None = None
+                  index_identity: str, supports: Collection[str] | None = None,
+                  drop_members: Sequence[DropMember] = ()
                   ) -> dict[str, MemberFacts]:
     """Per-member facts for one axis, keyed by contributor token.
 
@@ -400,7 +495,12 @@ def refusal_facts(axis: str, members: Sequence[Member], samples: Iterable,
     and a sample requested after both ``t_m`` and ``e_σ`` proves it stale.
     A member without a receipt is eligible with no proof reported; one whose
     receipt is invalid is eligible with its proof reported. Each axis is
-    decided on its own members, samples and credits."""
+    decided on its own members, samples and credits.
+
+    #1019 (spec 4.1): every ``drop_members`` entry gets its drop facts from
+    `proven_stale_low`, independently of ``receipts``, so one call per axis
+    yields complete facts; a token with drop facts only keeps the rise
+    fields of `ALL_ELIGIBLE`."""
     if axis not in AXES:
         return {}
     samples = list(samples)
@@ -433,6 +533,15 @@ def refusal_facts(axis: str, members: Sequence[Member], samples: Iterable,
             refused=bool(valid) and strict is not None,
             proof_observation_id=None if proof is None else proof.observation_id,
             receipt_valid=bool(valid))
+    for d in drop_members:
+        if d.axis != axis:
+            continue
+        low = proven_stale_low(d.bound_utc, samples, axis=axis, window=d.window,
+                               percent=d.percent,
+                               advertised_reset_epoch=d.advertised_reset_epoch)
+        facts[d.token] = dataclasses.replace(
+            facts.get(d.token, ALL_ELIGIBLE), drop_refused=low is not None,
+            drop_proof_observation_id=None if low is None else low.observation_id)
     return facts
 
 
@@ -535,8 +644,8 @@ def floor_microsecond(text: str) -> str | None:
 
 def compact_samples(samples: Iterable, *, window: int,
                     floor_epoch: int | None, axis: str = WEEKLY) -> tuple:
-    """A subset of the window's samples that yields the same `proven_stale`
-    and `earn_receipts` answers for every member (both pick the latest
+    """A subset of the window's samples that yields the same `proven_stale`,
+    `proven_stale_low` and `earn_receipts` answers for every member (all pick the latest
     qualifying sample, and qualification is monotone in ``q`` within one
     value): per weekly value, the latest sample and the latest captured at
     or after the floor, plus every sample captured within an hour of the

package/bin/_lib_statusline.py

@@ -10,6 +10,7 @@ for the full design.
 from __future__ import annotations
 
 import json
+import math
 import pathlib
 import re
 import sys
@@ -88,6 +89,12 @@ class StatuslineInput:
     rate_limits_7d_pct: Optional[float] = None
     rate_limits_7d_resets_at: Optional[int] = None  # unix epoch
     raw: dict = field(default_factory=dict)  # full parsed JSON for diagnostics
+    # #1002. Claude Code's own `context_window`, already validated (see
+    # `_to_count`): the session's window and the last request's input-side
+    # token sum (input + cache_read + cache_creation). `None` when absent or
+    # malformed; the `context_pct` port then falls back per value.
+    context_window_size: Optional[int] = None
+    context_tokens: Optional[int] = None
 
 
 # ---- CLI args (post-config-resolution) -------------------------------------
@@ -143,8 +150,13 @@ class StatuslineInjections:
         [],
         "Optional[tuple[Optional[float], Optional[int], Optional[float], Optional[int]]]",
     ]
-    # transcript_path → context % (0.0..100.0) or None if unreadable/unknown.
-    context_pct: Callable[[Optional[str], Optional[str]], Optional[float]]
+    # (transcript_path, model_id, window_size, context_tokens) → context %
+    # or None if unavailable. The two payload values win when not None; the
+    # port falls back to the model table / transcript tail independently.
+    context_pct: Callable[
+        [Optional[str], Optional[str], Optional[int], Optional[int]],
+        Optional[float],
+    ]
     # Emits one-shot stderr warnings (deduped by message — caller maintains set).
     warn_once: Callable[[str], None]
     # #661 S2 §9. The stored metering regimes for this install, oldest
@@ -230,6 +242,13 @@ def parse_statusline_stdin(raw: "bytes | str") -> "StatuslineInput | ParseError"
     def _to_str(v) -> Optional[str]:
         return v if isinstance(v, str) and v else None
 
+    context_window = parsed.get("context_window")
+    if not isinstance(context_window, dict):
+        context_window = {}
+    window_size = _to_count(context_window.get("context_window_size"))
+    if window_size is not None and window_size <= 0:
+        window_size = None
+
     return StatuslineInput(
         session_id=_to_str(parsed.get("session_id")),
         model_id=_to_str(_get(parsed, "model", "id")),
@@ -250,9 +269,56 @@ def parse_statusline_stdin(raw: "bytes | str") -> "StatuslineInput | ParseError"
             _get(parsed, "rate_limits", "seven_day", "resets_at")
         ),
         raw=parsed,
+        context_window_size=window_size,
+        context_tokens=_context_tokens(context_window.get("current_usage")),
     )
 
 
+# The input-side `current_usage` fields that occupy the context window.
+# `output_tokens` is deliberately excluded (#1002 spec §5.1).
+_CONTEXT_USAGE_FIELDS = (
+    "input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens",
+)
+
+
+def _to_count(v) -> Optional[int]:
+    """A token count: a finite, integer-valued, non-negative JSON number.
+
+    Never a bool (an int subclass) or a string; an integral float such as
+    ``1000000.0`` normalizes to ``int``. Anything else → ``None``.
+    """
+    if isinstance(v, bool) or not isinstance(v, (int, float)):
+        return None
+    if isinstance(v, float):
+        if not math.isfinite(v) or not v.is_integer():
+            return None
+        v = int(v)
+    return v if v >= 0 else None
+
+
+def _context_tokens(usage) -> Optional[int]:
+    """Sum of the input-side ``current_usage`` fields, or ``None``.
+
+    A missing or null field contributes zero, but at least one must be
+    present and non-null; an explicit zero is a valid observation. Any
+    invalid present field voids the whole sum rather than being skipped.
+    """
+    if not isinstance(usage, dict):
+        return None
+    total = 0
+    observed = False
+    for key in _CONTEXT_USAGE_FIELDS:
+        value = usage.get(key)
+        if value is None:
+            continue
+        count = _to_count(value)
+        if count is None:
+            return None
+        total += count
+        observed = True
+    return total if observed else None
+
+
 # ---- Segment 1: model -----------------------------------------------------
 
 
@@ -407,7 +473,10 @@ def resolve_context_pct(
     this function only returns the plain `🧠 X%` form. ``render_statusline``
     wraps the result in ANSI color codes per ``args.color``.
     """
-    pct = inj.context_pct(inp.transcript_path, inp.model_id)
+    pct = inj.context_pct(
+        inp.transcript_path, inp.model_id,
+        inp.context_window_size, inp.context_tokens,
+    )
     if pct is None:
         return "🧠 N/A"
     return f"🧠 {int(round(pct))}%"