cctally 1.115.0 → 1.116.0

package/bin/_lib_journal_router.py

@@ -155,12 +155,37 @@ class PrefixHashAccumulator:
 
     The framing is `journal_prefix_hash`'s, verbatim: per segment, the 4-byte
     big-endian name length, the name, the 8-byte big-endian data length, then
-    the data. Completed segments are absorbed into a running `sha256`; only the
-    segment currently being streamed is buffered, so residency is bounded by one
-    segment. At a segment transition the caller passes the boundary the next
-    record's `prior_high_water` will name, which is the ONE offset in the
-    outgoing segment that can still be asked for; its digest is precomputed
-    before the buffer is released.
+    the data. The data length precedes the data, so `begin_segment` takes the
+    exact byte count the pass will feed (`size`) and frames the segment header
+    with it; every chunk is then hashed as it arrives and NO segment bytes are
+    retained (#997 D12). The buffered form held the whole current segment in a
+    `bytearray` grown by `extend` — 410 MB on the maintainer's largest segment
+    — and the freed growth steps stayed resident in the macOS allocator.
+
+    A digest at a fully fed segment end (every transition, and every boundary
+    not short of a torn tail) comes from the streamed state. A digest at any
+    other offset — an evidence point inside the current segment, a boundary
+    short of a torn trailing line, a feed that ended short of its declared
+    size — re-reads the segment through `open_segment(name)` in 1 MiB chunks,
+    which the production callers bind to the journal's one segment-read seam,
+    so an evidence re-read is visible to it.
+
+    Consumed extent (#997 D12, revision 10). Per segment the accumulator also
+    tracks the bytes actually fed (`consumed`) and a raw SHA-256 of those bytes
+    alone (the witness). An offset beyond `consumed` raises. Every re-read reads
+    through `consumed`, not only through the requested offset: one pass frames
+    the prefix digest at the offset and hashes `[0, consumed)` raw, and the
+    digest is returned only when the read was not short and that raw hash
+    equals the witness. A torn tail an appender truncated and replaced after
+    the pass read it, or a line the reader assembled from old and replacement
+    bytes across two reads, can therefore never be certified. A registered
+    boundary keeps its segment's `consumed` and witness with its saved hash
+    state. A segment fed short of its declared size is re-framed from its
+    consumed bytes through the same verified re-read at the next
+    `begin_segment`; when that re-read is unavailable or fails verification
+    the accumulator is poisoned and every later digest raises. A missing
+    opener, a short re-read or a failed verification raises
+    `PrefixEvidenceUnavailable`.
     """
 
     # A transition can only ever register one boundary, and consecutive empty
@@ -176,10 +201,28 @@ class PrefixHashAccumulator:
     # entry that is still live.
     _MAX_BOUNDARIES = 16
 
-    def __init__(self) -> None:
+    #: Chunk size of an evidence re-read through the opener.
+    _REREAD_CHUNK = 1 << 20
+
+    def __init__(self, *, open_segment=None) -> None:
+        self._open_segment = open_segment
+        #: The hash of every segment before the current one, absorbed.
         self._running = hashlib.sha256()
         self._current_name = None
-        self._current = bytearray()
+        #: `_running` plus the current segment's header (framed with its
+        #: declared size) and every byte fed so far.
+        self._segment = None
+        self._declared = 0
+        #: Bytes of the current segment actually fed (its consumed extent),
+        #: and the raw SHA-256 of exactly those bytes (the witness).
+        self._fed = 0
+        self._witness = hashlib.sha256()
+        #: Set when a short segment could not be re-framed from verified
+        #: bytes: the running state no longer describes the journal.
+        self._poisoned = None
+        #: `(segment, offset) -> (state before that segment, final digest or
+        #: None, consumed, witness digest)`; a `None` final digest is re-read
+        #: through the opener, verified against the witness, when read.
         self._boundaries: dict = {}
         self._boundary_order: list = []
         #: Bytes fed into an evidence digest, reported as the `protocol_evidence`
@@ -189,55 +232,92 @@ class PrefixHashAccumulator:
 
     # -- feeding ----------------------------------------------------------
 
-    def begin_segment(self, name: str, boundary=None) -> None:
-        """Start `name`, absorbing whatever segment was open before it.
+    def begin_segment(self, name: str, boundary=None, *, size: int) -> None:
+        """Start `name`, which the pass will feed `size` bytes of, absorbing
+        whatever segment was open before it.
 
         `boundary` is the `(segment, offset)` the next record's evidence would
-        name — i.e. the streaming loop's current `prior_high_water`. It is
-        resolved and cached HERE because the outgoing segment's bytes are gone
-        immediately afterwards.
+        name — i.e. the streaming loop's current `prior_high_water`. Its hash
+        state is kept HERE because the outgoing segment's state is folded into
+        the running hash immediately afterwards.
         """
         if boundary is not None:
             self._register_boundary(boundary)
         if self._current_name is not None:
-            # `hashlib.update` accepts the buffer directly, so the outgoing
-            # segment is framed WITHOUT a copy of itself. The maintainer's
-            # largest segment is 410 MB, and copying it here briefly doubled
-            # that at every transition.
-            with memoryview(self._current) as data:
-                self._absorb(self._current_name, data)
+            self._running = self._completed_segment()
         self._current_name = name
-        self._current = bytearray()
-
-    def extend(self, data: bytes) -> None:
-        """Absorb raw bytes read from the segment currently open."""
-        self._current.extend(data)
+        self._declared = int(size)
+        self._fed = 0
+        self._witness = hashlib.sha256()
+        self._segment = self._running.copy()
+        self._header(self._segment, name, self._declared)
+
+    def extend(self, data) -> None:
+        """Hash raw bytes read from the segment currently open."""
+        self._segment.update(data)
+        self._witness.update(data)
+        self._fed += len(data)
 
     # -- reading ----------------------------------------------------------
 
     def digest_at(self, high_water) -> "str | None":
-        """`journal_prefix_hash(high_water)`, from the streamed bytes."""
+        """`journal_prefix_hash(high_water)`, from the streamed state."""
         if high_water is None:
             return None
         segment, offset = high_water
-        if segment == self._current_name:
-            digest = self._running.copy()
-            with memoryview(self._current)[:offset] as data:
-                self._frame(digest, segment, data)
-                self.bytes_hashed += len(data)
-            self.digests_computed += 1
-            return "sha256:" + digest.hexdigest()
-        cached = self._boundaries.get((segment, offset))
-        if cached is None:
+        if self._poisoned is not None:
             raise PrefixEvidenceUnavailable(
-                f"no streamed prefix evidence for {segment}@{offset}"
+                f"no verified prefix evidence for {segment}@{offset}: "
+                f"{self._poisoned}"
             )
-        self.bytes_hashed += cached[1]
+        if segment == self._current_name:
+            if self._streamed_end(offset):
+                digest = self._segment.copy()
+            else:
+                digest = self._verified_rehash(
+                    self._running, segment, offset, self._fed,
+                    self._witness.digest())
+        else:
+            cached = self._boundaries.get((segment, offset))
+            if cached is None:
+                raise PrefixEvidenceUnavailable(
+                    f"no streamed prefix evidence for {segment}@{offset}"
+                )
+            before, final, consumed, witness = cached
+            digest = (final.copy() if final is not None
+                      else self._verified_rehash(
+                          before, segment, offset, consumed, witness))
+        self.bytes_hashed += offset
         self.digests_computed += 1
-        return cached[0]
+        return "sha256:" + digest.hexdigest()
 
     # -- internals --------------------------------------------------------
 
+    def _streamed_end(self, offset) -> bool:
+        """Whether the streamed state IS the digest at `offset`: the current
+        segment was fed exactly its declared size and `offset` is its end."""
+        return offset == self._fed == self._declared
+
+    def _completed_segment(self):
+        """The running hash with the current segment absorbed as fed."""
+        if self._fed == self._declared or self._poisoned is not None:
+            return self._segment
+        # The feed ended short of the declared size (the segment shrank under
+        # the pass): frame the bytes actually fed, as the buffered form did,
+        # from a re-read verified against them. Bytes that cannot be verified
+        # leave no running state that describes the journal, so every later
+        # digest raises rather than certifying one.
+        try:
+            return self._verified_rehash(
+                self._running, self._current_name, self._fed, self._fed,
+                self._witness.digest())
+        except (PrefixEvidenceUnavailable, OSError) as exc:
+            self._poisoned = (
+                f"short segment {self._current_name} could not be re-framed "
+                f"from its consumed bytes ({exc})"
+            )
+            return self._segment
+
     def _register_boundary(self, boundary) -> None:
         segment, offset = boundary
         if segment != self._current_name:
@@ -247,23 +327,61 @@ class PrefixHashAccumulator:
         key = (segment, offset)
         if key in self._boundaries:
             return
-        digest = self._running.copy()
-        with memoryview(self._current)[:offset] as data:
-            self._frame(digest, segment, data)
-            size = len(data)
-        self._boundaries[key] = ("sha256:" + digest.hexdigest(), size)
+        final = (self._segment.copy() if self._streamed_end(offset)
+                 else None)
+        self._boundaries[key] = (self._running.copy(), final, self._fed,
+                                 self._witness.digest())
         self._boundary_order.append(key)
         while len(self._boundary_order) > self._MAX_BOUNDARIES:
             self._boundaries.pop(self._boundary_order.pop(0), None)
 
-    def _absorb(self, name, data) -> None:
-        self._frame(self._running, name, data)
+    def _verified_rehash(self, before, name: str, offset: int,
+                         consumed: int, witness: bytes):
+        """`before` plus `name`'s first `offset` bytes, re-read from the
+        segment through the opener in `_REREAD_CHUNK` pieces.
+
+        The read always runs through `consumed`, the bytes the pass fed, and
+        its raw SHA-256 must equal `witness`; only then are the bytes the
+        digest frames the bytes the pass read."""
+        if offset > consumed:
+            raise PrefixEvidenceUnavailable(
+                f"prefix evidence {name}@{offset} lies beyond the "
+                f"{consumed} bytes the pass consumed"
+            )
+        digest = before.copy()
+        self._header(digest, name, offset)
+        if consumed <= 0:
+            return digest
+        if self._open_segment is None:
+            raise PrefixEvidenceUnavailable(
+                f"no segment opener to re-read prefix evidence for "
+                f"{name}@{offset}"
+            )
+        raw = hashlib.sha256()
+        pos = 0
+        with self._open_segment(name) as handle:
+            while pos < consumed:
+                chunk = handle.read(min(self._REREAD_CHUNK, consumed - pos))
+                if not chunk:
+                    raise PrefixEvidenceUnavailable(
+                        f"short re-read of prefix evidence for {name}@{offset}"
+                    )
+                raw.update(chunk)
+                if pos < offset:
+                    with memoryview(chunk)[:offset - pos] as framed:
+                        digest.update(framed)
+                pos += len(chunk)
+        if raw.digest() != witness:
+            raise PrefixEvidenceUnavailable(
+                f"segment {name} no longer holds the {consumed} bytes the "
+                f"pass consumed; prefix evidence {name}@{offset} is not "
+                f"certified"
+            )
+        return digest
 
     @staticmethod
-    def _frame(digest, name: str, data) -> None:
-        """`data` is any bytes-like buffer; `hashlib` consumes it in place."""
+    def _header(digest, name: str, length: int) -> None:
         encoded = name.encode("utf-8")
         digest.update(len(encoded).to_bytes(4, "big"))
         digest.update(encoded)
-        digest.update(len(data).to_bytes(8, "big"))
-        digest.update(data)
+        digest.update(int(length).to_bytes(8, "big"))

package/bin/_lib_oauth_samples.py

@@ -0,0 +1,238 @@
+"""Provider samples: the raw api observation as an OAuth sample (#797/#823/#840
+spec 4.3, D8 and D9).
+
+Pure-fn layer (no I/O, no sibling imports). Every OAuth fetch this binary
+performs records ``q``, the wall-clock instant immediately before the request
+starts, and publishes it as the additive raw api ``obs`` payload key
+``oauth_request_started_at`` (:data:`REQUEST_STARTED_KEY`). The provider
+sampled the response at some instant at or after ``q`` (assumption A-Q), so a
+sample whose ``q`` is after a contributor's provider-activity instant is a
+reading the contributor could not have seen. This module turns one raw
+payload into one :class:`OAuthSample` — the row shape of the stats table
+``claude_oauth_samples`` — and states window identity once.
+
+Values are the RAW payload values, never the MAX-clamped stored snapshot
+values: the weekly percent and its reset (nearest-hour normalized exactly as
+the week-boundary code does), the five-hour percent and its raw reset, and the
+authoritative five-hour window key. That key is the producer's frozen planned
+key when the payload carries a valid one (an OAuth publication always does
+when it carries five-hour fields), else the pure canonical floor; a planned
+key that contradicts its own raw reset is conflicting identity and yields no
+key, so it can never supply a five-hour proof.
+
+Instants: ``requested_at_utc`` is stored in one canonical full-precision form
+(``YYYY-MM-DDTHH:MM:SS.ffffffZ``), so lexicographic order is chronological and
+a strict ``q > t`` comparison is exact. ``captured_at_utc`` is the payload's
+capture stamp verbatim (seconds precision, truncated): for an integral reset
+``R``, ``captured < R`` holds exactly when the true completion instant is
+before ``R``, so the truncation never weakens the reset bound.
+"""
+from __future__ import annotations
+
+import dataclasses
+import datetime as dt
+import math
+import re
+
+#: The raw api observation payload key carrying ``q``.
+REQUEST_STARTED_KEY = "oauth_request_started_at"
+#: The one source an OAuth publication records.
+API_SOURCE = "api"
+#: `_lib_five_hour._FIVE_HOUR_JITTER_FLOOR_SECONDS`, restated because this
+#: module imports no sibling; `tests/test_oauth_sample_projection_797.py`
+#: pins the equality.
+FIVE_HOUR_JITTER_FLOOR_SECONDS = 600
+#: The tolerance a planned five-hour key may sit from its raw reset — the
+#: same bound `_pipeline_claude_usage` validates a planned key against.
+PLANNED_KEY_TOLERANCE_SECONDS = FIVE_HOUR_JITTER_FLOOR_SECONDS * 3
+#: The capture stamp is truncated to the second, so a genuine ``q`` can sit
+#: up to one second after it.
+_CAPTURE_TRUNCATION_SECONDS = 1
+
+_REQUEST_STARTED = re.compile(
+    r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d{1,6})?Z$")
+_CANONICAL_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+_SECONDS_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+
+
+@dataclasses.dataclass(frozen=True)
+class OAuthSample:
+    """One provider sample: a row of ``claude_oauth_samples``."""
+
+    account_key: str
+    observation_id: str
+    requested_at_utc: str
+    captured_at_utc: str
+    weekly_percent: "float | None"
+    weekly_resets_at_utc: "str | None"
+    five_hour_percent: "float | None"
+    five_hour_resets_at_utc: "str | None"
+    five_hour_window_key: "int | None"
+
+    def row(self) -> tuple:
+        """The column values in table order."""
+        return tuple(getattr(self, f.name) for f in dataclasses.fields(self))
+
+
+def format_request_started(instant: dt.datetime) -> str:
+    """``instant`` (tz-aware) in the carrier's canonical full-precision UTC
+    form. A naive datetime is a programming error and raises."""
+    if instant.tzinfo is None:
+        raise ValueError("the request-start instant must be timezone-aware")
+    return instant.astimezone(dt.timezone.utc).strftime(_CANONICAL_FORMAT)
+
+
+def _parse_aware(value) -> "dt.datetime | None":
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        parsed = dt.datetime.fromisoformat(
+            value[:-1] + "+00:00" if value.endswith("Z") else value)
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        return None
+    return parsed.astimezone(dt.timezone.utc)
+
+
+def canonical_instant(value) -> str:
+    """Any tz-aware ISO-8601 instant (or tz-aware datetime) in the canonical
+    full-precision form used for ``requested_at_utc``; raises ``ValueError``
+    on anything else. Callers compare their own instants against stored
+    ``q`` values through this form."""
+    if isinstance(value, dt.datetime):
+        return format_request_started(value)
+    parsed = _parse_aware(value)
+    if parsed is None:
+        raise ValueError(f"not a timezone-aware ISO-8601 instant: {value!r}")
+    return parsed.strftime(_CANONICAL_FORMAT)
+
+
+def parse_request_started(value) -> "str | None":
+    """The canonical ``q`` of a raw payload value, or None when it is missing
+    or unparsable. Only the carrier's own shape is accepted (UTC with ``Z``,
+    up to microsecond precision); a bool, a number or a naive instant is not
+    a key."""
+    if not isinstance(value, str) or not _REQUEST_STARTED.match(value):
+        return None
+    parsed = _parse_aware(value)
+    return None if parsed is None else parsed.strftime(_CANONICAL_FORMAT)
+
+
+def normalize_weekly_reset(resets_at_epoch: int) -> str:
+    """The weekly window identity of a raw ``resets_at`` epoch: rounded to the
+    nearest hour exactly as `_cctally_core._normalize_week_boundary_dt` does
+    (minutes 0..29 down, 30..59 up), as ``YYYY-MM-DDTHH:00:00Z``."""
+    value = dt.datetime.fromtimestamp(int(resets_at_epoch), tz=dt.timezone.utc)
+    normalized = value.replace(minute=0, second=0, microsecond=0)
+    if value.minute >= 30:
+        normalized += dt.timedelta(hours=1)
+    return normalized.strftime(_SECONDS_FORMAT)
+
+
+def weekly_window(window) -> str:
+    """A weekly window argument (a raw or normalized reset as an epoch int or
+    a tz-aware ISO instant) as its normalized identity."""
+    if isinstance(window, bool):
+        raise ValueError("a weekly window is a reset instant, not a bool")
+    if isinstance(window, int):
+        return normalize_weekly_reset(window)
+    parsed = _parse_aware(window)
+    if parsed is None:
+        raise ValueError(f"not a weekly reset instant: {window!r}")
+    return normalize_weekly_reset(int(parsed.timestamp()))
+
+
+def five_hour_window_key(resets_at_epoch: int, planned) -> "int | None":
+    """The authoritative five-hour window key: the producer's planned key
+    when it is an int within the planned-key tolerance of the raw reset,
+    None when a planned key is present but contradicts it, else the pure
+    canonical floor (`_canonical_5h_window_key` without an anchor)."""
+    if planned is not None:
+        if (type(planned) is not int
+                or abs(planned - resets_at_epoch)
+                > PLANNED_KEY_TOLERANCE_SECONDS):
+            return None
+        return planned
+    return resets_at_epoch - (resets_at_epoch % FIVE_HOUR_JITTER_FLOOR_SECONDS)
+
+
+def _percent(value) -> "float | None":
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    value = float(value)
+    return value if math.isfinite(value) else None
+
+
+def _bound_account(payload: dict) -> "str | None":
+    binding = payload.get("account_binding")
+    if not isinstance(binding, dict):
+        return None
+    key = binding.get("accountKey")
+    return key if isinstance(key, str) and key else None
+
+
+def sample_from_observation(payload: dict, *, account_key: str,
+                            observation_id: str,
+                            captured_at_utc: str) -> "OAuthSample | None":
+    """The provider sample of one raw Claude usage observation payload, or
+    None: for a non-api source, a missing or unparsable request-start key, a
+    request instant after the capture, a missing or unparsable capture
+    instant, or no Claude account binding naming ``account_key``.
+
+    Each axis is independent: an unparsable weekly or five-hour field leaves
+    that axis's columns None (no identity, so no proof on that axis), and a
+    sample with neither axis is no sample."""
+    if not isinstance(payload, dict) or payload.get("source") != API_SOURCE:
+        return None
+    if not isinstance(account_key, str) or not account_key:
+        return None
+    if not isinstance(observation_id, str) or not observation_id:
+        return None
+    if _bound_account(payload) != account_key:
+        return None
+    requested = parse_request_started(payload.get(REQUEST_STARTED_KEY))
+    captured = _parse_aware(captured_at_utc)
+    if requested is None or captured is None:
+        return None
+    requested_dt = _parse_aware(requested)
+    if (requested_dt - captured).total_seconds() > _CAPTURE_TRUNCATION_SECONDS:
+        return None
+
+    weekly_percent = _percent(payload.get("weekly_percent"))
+    weekly_reset = None
+    resets_at = payload.get("resets_at")
+    if not isinstance(resets_at, bool):
+        try:
+            weekly_reset = normalize_weekly_reset(int(resets_at))
+        except (TypeError, ValueError, OverflowError, OSError):
+            weekly_reset = None
+    if weekly_reset is None:
+        weekly_percent = None
+
+    five_percent = _percent(payload.get("five_hour_percent"))
+    five_reset = None
+    five_key = None
+    five_reset_dt = _parse_aware(payload.get("five_hour_resets_at"))
+    if five_reset_dt is not None:
+        five_reset = five_reset_dt.strftime(_SECONDS_FORMAT)
+        five_key = five_hour_window_key(
+            int(five_reset_dt.timestamp()),
+            payload.get("five_hour_window_key"))
+    if five_percent is None:
+        five_reset = None
+        five_key = None
+
+    if weekly_percent is None and five_percent is None:
+        return None
+    return OAuthSample(
+        account_key=account_key,
+        observation_id=observation_id,
+        requested_at_utc=requested,
+        captured_at_utc=captured_at_utc,
+        weekly_percent=weekly_percent,
+        weekly_resets_at_utc=weekly_reset,
+        five_hour_percent=five_percent,
+        five_hour_resets_at_utc=five_reset,
+        five_hour_window_key=five_key,
+    )

package/bin/_lib_pricing_check.py

@@ -232,15 +232,3 @@ def check_table_shapes(claude_tbl, codex_tbl, zero_sentinels) -> list:
     for model, body in codex_tbl.items():
         _check(model, body, _CODEX_REQUIRED, allow_zero=model in zero_sentinels)
     return problems
-
-
-def pricing_issue_action(findings_present: bool, existing_open: bool) -> str:
-    """Decide the cron's GitHub-issue action. Pure; the YAML executes it.
-
-    ``findings_present`` is ANY actionable finding the cron tracks — value
-    drift, missing-from-us, OR (as of #279 S7 W7) a stale/expired allowlist
-    suppression. The name generalizes the former ``drift_present``; the
-    two-state machine is unchanged."""
-    if findings_present:
-        return "update" if existing_open else "create"
-    return "close" if existing_open else "noop"