cctally 1.117.2 → 1.118.0

package/bin/_lib_doctor.py

@@ -281,7 +281,7 @@ class DoctorState:
     conversations_db_quick_check: Optional[str] = None
     # #279 S2 (F5c): non-blocking flock probes on the two sync lock files
     # (name -> True held / False free / None unreadable). Probe never
-    # creates files (doctor read-only contract). None = probe errored.
+    # creates files (doctor never changes your data). None = probe errored.
     locks_held: Optional[dict] = None
     # #297: size in bytes of cache.db-wal, gathered read-only in
     # doctor_gather_state (getsize, OSError/absent -> None). Warns above
@@ -485,6 +485,14 @@ class DoctorState:
     # launch ledger, process identities probed before a re-read, pending
     # marker). None = not gathered; the check then reads both as never run.
     rebuild_workers: Optional[dict] = None
+    # #1018: the newest admitted Claude weekly reading (the observation
+    # index), no later than `now_utc`. `latest_snapshot_at` keeps its
+    # held-inclusive materialized meaning; `data.latest_snapshot_age` takes
+    # the newer of the two.
+    latest_reading_at: Optional[dt.datetime] = None
+    # #1018 C6: why freshness evidence could not be read — None,
+    # "stats_maintenance" or "stats_unreadable".
+    freshness_unavailable_reason: Optional[str] = None
 
 
 @dataclasses.dataclass(frozen=True)
@@ -1059,7 +1067,7 @@ def _check_db_version_ahead(s: DoctorState) -> CheckResult:
             # current index as a mismatch; keep this in lockstep with core
             # (#496 S5b §6.1). It stays a literal because this kernel is pure and
             # must not import `_cctally_core`.
-            epoch = 1023
+            epoch = 1024
         mismatch = uv > legacy_head and uv != epoch
         return {"user_version": uv, "legacy_head": legacy_head, "epoch": epoch,
                 "mismatch": mismatch}
@@ -1376,29 +1384,86 @@ def rebuild_worker_check(rebuild_workers, now_utc) -> CheckResult:
     )
 
 
+_FRESHNESS_UNAVAILABLE = {
+    "stats_maintenance": (
+        "unavailable (stats.db maintenance in progress)",
+        "Retry after the stats.db maintenance finishes.",
+    ),
+    "stats_unreadable": (
+        "unavailable (stats.db could not be read)",
+        "See the stats.db check.",
+    ),
+}
+
+
 def _check_data_latest_snapshot_age(s: DoctorState) -> CheckResult:
-    if s.latest_snapshot_at is None:
-        return CheckResult(
-            id="data.latest_snapshot_age", title="Latest snapshot",
-            severity="fail", summary="never",
-            remediation="Check hooks are installed and Claude Code is running",
-            details={"latest_snapshot_at": None},
-        )
-    age_s = int((s.now_utc - s.latest_snapshot_at).total_seconds())
-    if age_s <= 300:
-        sev, rem = "ok", None
-    elif age_s <= 3600:
+    """Is the usage write path alive? (#1018, spec 5.4.)
+
+    The evidence is the newer of the held-inclusive materialized snapshot and
+    the newest admitted reading, each counted only when no later than now.
+    Fresh positive evidence decides even when a source failed; stale or absent
+    evidence with a failed source is "unavailable", never silence.
+    """
+    now = s.now_utc
+
+    def _valid(moment):
+        return moment if moment is not None and moment <= now else None
+
+    def _age(moment):
+        return int((now - moment).total_seconds())
+
+    snapshot = _valid(s.latest_snapshot_at)
+    reading = _valid(s.latest_reading_at)
+    valid = [moment for moment in (snapshot, reading) if moment is not None]
+    candidate = max(valid) if valid else None
+    cand_age = _age(candidate) if candidate is not None else None
+    reason = s.freshness_unavailable_reason
+
+    effective = candidate
+    if candidate is not None and cand_age <= 300:
+        sev, summary, rem = "ok", f"{cand_age}s ago", None
+    elif reason:
         sev = "warn"
+        summary, rem = _FRESHNESS_UNAVAILABLE.get(
+            reason, _FRESHNESS_UNAVAILABLE["stats_unreadable"])
+        effective = None
+    elif candidate is not None and cand_age <= 3600:
+        sev, summary = "warn", f"{cand_age}s ago"
         rem = "Recent but not current. Check Claude Code session is active."
-    else:
-        sev = "fail"
+    elif candidate is not None:
+        sev, summary = "fail", f"{cand_age}s ago"
         rem = "No snapshot in >1h. Hooks may be broken — check `cctally setup --status`."
+    else:
+        sev, summary = "fail", "never"
+        rem = "Check hooks are installed and Claude Code is running"
+
+    details: dict = {
+        "latest_snapshot_at": (
+            s.latest_snapshot_at.isoformat()
+            if s.latest_snapshot_at is not None else None
+        ),
+    }
+    if s.latest_snapshot_at is not None:
+        details["latest_snapshot_age_s"] = (
+            _age(snapshot) if snapshot is not None else None
+        )
+    details.update({
+        "latest_reading_at": (
+            s.latest_reading_at.isoformat()
+            if s.latest_reading_at is not None else None
+        ),
+        "latest_reading_age_s": _age(reading) if reading is not None else None,
+        "effective_freshness_at": (
+            effective.isoformat() if effective is not None else None
+        ),
+        "effective_freshness_age_s": (
+            _age(effective) if effective is not None else None
+        ),
+        "freshness_unavailable_reason": reason,
+    })
     return CheckResult(
         id="data.latest_snapshot_age", title="Latest snapshot",
-        severity=sev, summary=f"{age_s}s ago",
-        remediation=rem,
-        details={"latest_snapshot_at": s.latest_snapshot_at.isoformat(),
-                 "latest_snapshot_age_s": age_s},
+        severity=sev, summary=summary, remediation=rem, details=details,
     )
 
 
@@ -2386,7 +2451,7 @@ def _check_data_conversation_sessions_rollup(s: DoctorState) -> CheckResult:
     ``doctor_gather_state`` from a NON-BLOCKING ``conversations.db.lock`` flock probe
     (lock held ⇒ a writer is mid-flight) AND the presence of any pending
     ``cache_meta`` reingest/split/backfill flag; if either says in-progress, this
-    stays OK. Doctor remains read-only and never blocks on the lock.
+    stays OK. Doctor never changes your data and never blocks on the lock.
 
     Informational only (no remediation): the next conversation sync re-derives
     the rollup via its incremental DELETE+INSERT.

package/bin/_lib_doctor_scan_index.py

@@ -0,0 +1,321 @@
+"""The deep doctor journal scan's checkpoint — the pure kernel (#1054).
+
+`cctally doctor` decodes every journal line on every deep run to compute its
+journal legs: the malformed and torn counts, the same-revision conflicts, the
+structural protocol violations and acknowledgements, the cutover value and the
+excluded population. On a production-sized journal that is about 2.6 million
+decodes, of which roughly 5 % are records the scan keeps; the rest are
+observations that only consume a selector position.
+
+Doctor therefore keeps one disposable file beside the segments,
+``JOURNAL_DIR/.doctor-scan-index``. Per segment it records a SHA-256 of the
+complete-line prefix it scanned, the number of decoded records and malformed
+lines in that prefix, and the line offset and decoded ordinal of every record
+that matters to the scan (a "hit"). A later run proves the prefix against the
+current bytes, decodes only the hits, fills every other position with the
+``None`` placeholder the selector expects, and decodes the fresh suffix. The
+excluded population is cached the same way, under its own digest of the range
+it classified and an explicit fingerprint of the admission boundary.
+
+**Nothing here is trusted without a content proof.** There is no stat-identity
+shortcut: every reuse hashes the bytes it skips. A document whose version or
+checksum does not match is ignored as a whole, and an entry that fails the
+structural rules below is dropped while the others are kept. Torn status is
+never stored, because a torn tail can be repaired and re-torn at the same size.
+
+**The version is derived, not stamped.** `index_version` hashes `SHAPE_VERSION`
+with the descriptors the glue supplies (the retained record types, the special
+ids, the protocol-resolution kinds, every constant the admission predicate
+reads and the stats epoch), so a change to any of those invalidates every
+checkpoint. `SHAPE_VERSION` is bumped by hand for any change those inputs do
+not capture: the meaning of a stored field, the decoder, or the scan's
+per-record steps.
+
+This module performs no I/O and imports nothing from the journal modules, the
+same rule `bin/_lib_segment_summary.py` and `bin/_lib_journal_router.py` follow,
+so every rule here is unit-testable without a journal on disk. The glue — the
+read, the per-record step, the exclusion read and the atomic publication —
+lives in `bin/_cctally_doctor.py`.
+"""
+from __future__ import annotations
+
+import dataclasses
+import hashlib
+import json
+import re
+from typing import Iterable
+
+#: Bumped by hand whenever a stored field's meaning, the decoder semantics or
+#: the scan's per-record steps change in a way the derived descriptors do not
+#: capture (see `docs/journal-gotchas.md`).
+SHAPE_VERSION = 1
+
+_HEX_DIGEST = re.compile(r"\A[0-9a-f]{64}\Z")
+_ENTRY_KEYS = frozenset(
+    ("name", "covered", "sha256", "decoded", "malformed", "hits", "exclusion"))
+_EXCLUSION_KEYS = frozenset(("boundary", "lo", "covered", "sha256", "excluded"))
+
+
+@dataclasses.dataclass(frozen=True)
+class Exclusion:
+    """One segment's cached excluded population over ``[lo, covered)``.
+
+    ``boundary`` is the `boundary_fingerprint` it was computed under and
+    ``sha256`` the hex digest of exactly those bytes; ``excluded`` lists each
+    excluded record's line-start offset and admission family, in offset
+    order."""
+
+    boundary: str
+    lo: int
+    covered: int
+    sha256: str
+    excluded: tuple
+
+
+@dataclasses.dataclass(frozen=True)
+class Entry:
+    """One segment's main-scan checkpoint over ``[0, covered)``.
+
+    ``covered`` is a complete-line end or 0; ``sha256`` the hex digest of
+    those bytes; ``decoded`` and ``malformed`` the decoded records and the
+    malformed non-blank complete lines in that prefix; ``hits`` the
+    ``(line-start offset, 0-based decoded ordinal)`` of every record that
+    matters to the scan, in offset order."""
+
+    name: str
+    covered: int
+    sha256: str
+    decoded: int
+    malformed: int
+    hits: tuple
+    exclusion: "Exclusion | None"
+
+
+def _canonical(obj) -> bytes:
+    return json.dumps(obj, sort_keys=True, separators=(",", ":"),
+                      allow_nan=False).encode("utf-8")
+
+
+def _digest(obj) -> str:
+    return "sha256:" + hashlib.sha256(_canonical(obj)).hexdigest()
+
+
+def index_version(descriptors: dict) -> str:
+    """The checkpoint version: `SHAPE_VERSION` plus every derived input."""
+    return _digest({"shape": SHAPE_VERSION, "descriptors": descriptors})
+
+
+def boundary_fingerprint(fields: dict) -> str:
+    """The digest of the admission-boundary fields the glue serializes."""
+    return _digest(fields)
+
+
+def _exclusion_document(exclusion: "Exclusion | None"):
+    if exclusion is None:
+        return None
+    return {"boundary": exclusion.boundary, "lo": exclusion.lo,
+            "covered": exclusion.covered, "sha256": exclusion.sha256,
+            "excluded": [[offset, family]
+                         for offset, family in exclusion.excluded]}
+
+
+def _entry_document(entry: Entry) -> dict:
+    return {"name": entry.name, "covered": entry.covered,
+            "sha256": entry.sha256, "decoded": entry.decoded,
+            "malformed": entry.malformed,
+            "hits": [[offset, ordinal] for offset, ordinal in entry.hits],
+            "exclusion": _exclusion_document(entry.exclusion)}
+
+
+def encode(version: str, entries: Iterable[Entry]) -> bytes:
+    """The canonical checkpoint document: segments in the given order."""
+    segments = [_entry_document(entry) for entry in entries]
+    return _canonical({"version": version, "checksum": _digest(segments),
+                       "segments": segments})
+
+
+def _count(value) -> bool:
+    return type(value) is int and value >= 0
+
+
+def _hex(value) -> bool:
+    return isinstance(value, str) and _HEX_DIGEST.match(value) is not None
+
+
+def _pairs(value, second_ok, lo: int, hi: int, second_hi=None):
+    """``value`` as a tuple of ``(offset, second)`` pairs with offsets
+    strictly increasing in ``[lo, hi)`` — and, when ``second_hi`` is given,
+    integer seconds strictly increasing in ``[0, second_hi)`` — else None."""
+    if not isinstance(value, list):
+        return None
+    out = []
+    last_offset = lo - 1
+    last_second = -1
+    for item in value:
+        if not isinstance(item, list) or len(item) != 2:
+            return None
+        offset, second = item
+        if not _count(offset) or not (last_offset < offset < hi):
+            return None
+        if not second_ok(second):
+            return None
+        if second_hi is not None:
+            if not (last_second < second < second_hi):
+                return None
+            last_second = second
+        last_offset = offset
+        out.append((offset, second))
+    return tuple(out)
+
+
+def _exclusion_from_document(doc):
+    """``(valid, Exclusion | None)``."""
+    if doc is None:
+        return True, None
+    if not isinstance(doc, dict) or set(doc) != _EXCLUSION_KEYS:
+        return False, None
+    boundary, lo, covered = doc["boundary"], doc["lo"], doc["covered"]
+    if not isinstance(boundary, str) or not boundary:
+        return False, None
+    if not (_count(lo) and _count(covered) and lo <= covered):
+        return False, None
+    if not _hex(doc["sha256"]):
+        return False, None
+    excluded = _pairs(doc["excluded"], lambda family: isinstance(family, str),
+                      lo, covered)
+    if excluded is None:
+        return False, None
+    return True, Exclusion(boundary, lo, covered, doc["sha256"], excluded)
+
+
+def _entry_from_document(doc) -> "Entry | None":
+    if not isinstance(doc, dict) or set(doc) != _ENTRY_KEYS:
+        return None
+    name, covered = doc["name"], doc["covered"]
+    decoded, malformed = doc["decoded"], doc["malformed"]
+    if not isinstance(name, str) or not name:
+        return None
+    if not (_count(covered) and _count(decoded) and _count(malformed)):
+        return None
+    # Each counted line occupies at least one byte of `[0, covered)`: its
+    # newline.
+    if malformed + decoded > covered:
+        return None
+    if not _hex(doc["sha256"]):
+        return None
+    hits = _pairs(doc["hits"], _count, 0, covered, second_hi=decoded)
+    if hits is None:
+        return None
+    valid, exclusion = _exclusion_from_document(doc["exclusion"])
+    if not valid:
+        return None
+    return Entry(name, covered, doc["sha256"], decoded, malformed, hits,
+                 exclusion)
+
+
+def decode(raw: bytes, version: str) -> "dict[str, Entry] | None":
+    """The valid entries of a checkpoint document, keyed by segment name.
+
+    None when the document does not parse, names another version or fails
+    its checksum. A structurally invalid entry is dropped and the others are
+    kept; a segment named twice keeps neither entry."""
+    return decode_document(raw, version)[0]
+
+
+def decode_document(raw: bytes, version: str):
+    """``(decode(raw, version), clean)``: ``clean`` is True only when the
+    document is valid and every one of its entries was kept, so a writer
+    holding the same entries has nothing to rewrite."""
+    try:
+        doc = json.loads(raw)
+    except (ValueError, TypeError, RecursionError):
+        return None, False
+    if not isinstance(doc, dict) or doc.get("version") != version:
+        return None, False
+    segments = doc.get("segments")
+    if not isinstance(segments, list):
+        return None, False
+    try:
+        if doc.get("checksum") != _digest(segments):
+            return None, False
+    except (ValueError, TypeError, RecursionError):
+        return None, False
+    out: dict = {}
+    duplicated = set()
+    for item in segments:
+        entry = _entry_from_document(item)
+        if entry is None:
+            continue
+        if entry.name in out:
+            duplicated.add(entry.name)
+        out[entry.name] = entry
+    for name in duplicated:
+        del out[name]
+    return out, len(out) == len(segments)
+
+
+def verify_prefix(entry_sha: str, covered: int, data: bytes):
+    """The running SHA-256 after ``data[:covered]`` when that prefix is a
+    complete-line prefix of ``data`` whose digest is ``entry_sha``, else None.
+
+    The caller continues the returned object over the fresh suffix, so every
+    byte is hashed once."""
+    if len(data) < covered:
+        return None
+    if covered and data[covered - 1] != 0x0A:
+        return None
+    hasher = hashlib.sha256()
+    hasher.update(memoryview(data)[:covered])
+    if hasher.hexdigest() != entry_sha:
+        return None
+    return hasher
+
+
+class RangeHasher:
+    """An ``on_bytes`` callback hashing every chunk of one ``[lo, hi)`` read.
+
+    With ``covered`` set it also proves the cached prefix ``[lo, covered)``:
+    the chunk that crosses ``covered`` is split, the prefix digest compared
+    with ``expected_prefix``, and ``prefix_state`` settled before the call
+    returns — so before the reader yields any line starting at or after
+    ``covered``. With ``covered == lo`` the empty prefix is compared at
+    construction. ``prefix_state`` is ``"none"`` without ``covered``,
+    otherwise ``"pending"`` until settled as ``"match"`` or ``"mismatch"``;
+    a read that ends while it is still pending never reached ``covered``."""
+
+    def __init__(self, lo: int, covered: "int | None",
+                 expected_prefix: "str | None"):
+        if covered is not None and covered < lo:
+            raise ValueError("covered precedes lo")
+        self._lo = lo
+        self._covered = covered
+        self._expected = expected_prefix
+        self._hash = hashlib.sha256()
+        self.consumed = 0
+        if covered is None:
+            self.prefix_state = "none"
+        elif covered == lo:
+            self.prefix_state = self._settle()
+        else:
+            self.prefix_state = "pending"
+
+    def _settle(self) -> str:
+        return ("match" if self._hash.hexdigest() == self._expected
+                else "mismatch")
+
+    def __call__(self, chunk: bytes) -> None:
+        if self.prefix_state == "pending":
+            need = self._covered - self._lo - self.consumed
+            if len(chunk) >= need:
+                view = memoryview(chunk)
+                self._hash.update(view[:need])
+                self.prefix_state = self._settle()
+                self._hash.update(view[need:])
+                self.consumed += len(chunk)
+                return
+        self._hash.update(chunk)
+        self.consumed += len(chunk)
+
+    def digest(self) -> str:
+        """The hex digest of every byte consumed so far."""
+        return self._hash.hexdigest()

package/bin/_lib_ingest_frontier.py

@@ -79,6 +79,13 @@ _COMPLETE_KEYS = {
 #: clean except for pending prune debt writes a fresh one; any other
 #: unbudgeted or certifying full walk deletes it.
 CODEX_FULL_WALK_DEBT_ONLY_KEY = "dashboard_codex_full_walk_debt_only"
+#: #916. cache.db's residual-only token: the most recent exhaustive Claude
+#: walk visited every on-disk source cleanly and its only defect was tracked
+#: rows whose transcripts no longer exist (orphans the prune gates refused).
+#: Mutually exclusive with ``claude_ingest_walk_complete``, which keeps its
+#: D5a meaning for the upgrade gate; only this frontier reads the token, as a
+#: second way for the Claude store to be complete. Its JSON value is debug-only.
+CLAUDE_WALK_ORPHAN_RESIDUAL_KEY = "claude_ingest_walk_orphan_residual"
 CODEX_CONVERSATION_WALK_DEBT_ONLY_KEY = "dashboard_codex_conversation_walk_debt_only"
 #: How long a frontier holds a debt-only store before walking it again: the
 #: certificate's own staleness bound, so a store whose certificate is
@@ -1872,6 +1879,15 @@ def _pending_identity(conn) -> tuple[str, ...]:
 
 
 def _provider_complete(conn, provider: str) -> bool:
+    if provider == "claude":
+        # #916: a residual-only walk certifies every on-disk source as fully
+        # as a clean one; the orphan rows it leaves are never read or written
+        # by a walk, so they cannot hide pending work.
+        row = conn.execute(
+            "SELECT 1 FROM cache_meta WHERE key IN (?, ?) LIMIT 1",
+            (_COMPLETE_KEYS[provider], CLAUDE_WALK_ORPHAN_RESIDUAL_KEY),
+        ).fetchone()
+        return row is not None
     row = conn.execute(
         "SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
         (_COMPLETE_KEYS[provider],),
@@ -2228,7 +2244,10 @@ def provider_sync_certifiable(mode: str, stats: object) -> bool:
     if mode == "targeted":
         return bool(getattr(stats, "targeted_clean", True))
     if mode == "full":
-        return bool(getattr(stats, "full_walk_complete", False))
+        # #916: ``full_walk_residual_only`` exists only on the Claude
+        # ``IngestStats``; Codex stats and ``_DebtCleared`` read False.
+        return bool(getattr(stats, "full_walk_complete", False)
+                    or getattr(stats, "full_walk_residual_only", False))
     return False
 
 

package/bin/_lib_oauth_samples.py

@@ -164,6 +164,28 @@ def _percent(value) -> "float | None":
     return value if math.isfinite(value) else None
 
 
+def finite_percent(value) -> "float | None":
+    """A raw payload percent as a finite float, or None (a bool, a string, a
+    missing or non-finite value). The one parse both the samples projection
+    and the weekly observation index (#1018 twin spec 5.1) apply."""
+    return _percent(value)
+
+
+def weekly_reset_identity(resets_at) -> "str | None":
+    """The normalized weekly window identity of a raw payload ``resets_at``
+    (:func:`normalize_weekly_reset`), or None when it is missing, a bool or
+    unparsable. `claude_oauth_samples.weekly_resets_at_utc` and
+    `claude_weekly_observation_index.weekly_resets_at_utc` (#1018 twin spec
+    5.1) are both spelled by this one helper, so window equality holds
+    between them."""
+    if resets_at is None or isinstance(resets_at, bool):
+        return None
+    try:
+        return normalize_weekly_reset(int(resets_at))
+    except (TypeError, ValueError, OverflowError, OSError):
+        return None
+
+
 def _bound_account(payload: dict) -> "str | None":
     binding = payload.get("account_binding")
     if not isinstance(binding, dict):
@@ -200,13 +222,7 @@ def sample_from_observation(payload: dict, *, account_key: str,
         return None
 
     weekly_percent = _percent(payload.get("weekly_percent"))
-    weekly_reset = None
-    resets_at = payload.get("resets_at")
-    if not isinstance(resets_at, bool):
-        try:
-            weekly_reset = normalize_weekly_reset(int(resets_at))
-        except (TypeError, ValueError, OverflowError, OSError):
-            weekly_reset = None
+    weekly_reset = weekly_reset_identity(payload.get("resets_at"))
     if weekly_reset is None:
         weekly_percent = None