cctally 1.101.0 → 1.103.0

package/bin/_cctally_refresh.py

@@ -918,6 +918,58 @@ def cmd_refresh_usage(args: argparse.Namespace) -> int:
 # Hook-tick OAuth refresh path
 # =========================================================================
 
+def _hook_tick_oauth_skip_status(c, *, throttle_seconds: float) -> str | None:
+    """Return the automatic-refresh suppression reason under selected lock."""
+    now_epoch = int(time.time())
+    needs_repair = c._authoritative_repair_required(now_epoch=now_epoch)
+    obs_age = c._statusline_observe_age_seconds()
+    if (not needs_repair
+            and obs_age < float(_cctally_core.OAUTH_BACKFILL_STALE_SECONDS)):
+        return f"skipped(statusline-fresh:{int(obs_age)}s)"
+    backoff_remaining = c._oauth_backoff_remaining_seconds()
+    if backoff_remaining > 0:
+        return f"skipped(backoff:{int(backoff_remaining)}s)"
+    age_s = c._newest_snapshot_age_seconds()
+    if age_s is not None and age_s < throttle_seconds:
+        return f"skipped(fresh:{int(age_s)}s)"
+    return None
+
+
+def _hook_tick_parse_oauth_payload(c, api):
+    """Validate one OAuth response outside the selected-state lock."""
+    try:
+        seven = api["seven_day"]
+        seven_pct = c._normalize_percent(float(seven["utilization"]))
+        seven_resets_epoch = _iso_to_epoch(seven["resets_at"])
+    except (TypeError, ValueError, KeyError):
+        return None
+    five = api.get("five_hour") if isinstance(api.get("five_hour"), dict) else None
+    five_pct: float | None = None
+    five_resets_epoch: int | None = None
+    if (five is not None and "utilization" in five
+            and isinstance(five.get("resets_at"), str)):
+        try:
+            five_pct = c._normalize_percent(float(five["utilization"]))
+            five_resets_epoch = _iso_to_epoch(five["resets_at"])
+        except (TypeError, ValueError):
+            five_pct = None
+            five_resets_epoch = None
+    record_args = argparse.Namespace(
+        percent=seven_pct,
+        resets_at=str(seven_resets_epoch),
+        five_hour_percent=five_pct,
+        five_hour_resets_at=(
+            str(five_resets_epoch) if five_resets_epoch is not None else None
+        ),
+        source="api",
+    )
+    axes = {"sevenDay", *({"fiveHour"} if five_pct is not None else set())}
+    parts = [f"7d={int(round(seven_pct))}"]
+    if five_pct is not None:
+        parts.append(f"5h={int(round(five_pct))}")
+    return record_args, axes, parts
+
+
 def _hook_tick_oauth_refresh(
     timeout_seconds: float = 5.0,
     throttle_seconds: float | None = None,
@@ -950,115 +1002,75 @@ def _hook_tick_oauth_refresh(
             throttle_seconds = float(_get_oauth_usage_config(c.load_config())["throttle_seconds"])
         except OauthUsageConfigError:
             throttle_seconds = float(c.HOOK_TICK_DEFAULT_THROTTLE_SECONDS)
-    # The automatic writer takes the selected-state lock *before* rechecking
-    # its suppression conditions.  A concurrent publisher can otherwise make
-    # this tick issue an unnecessary OAuth request between the initial gate
-    # and the request itself.
-    # #583 S2: the nudge is deferred out of the critical section. This frame
-    # owns the lock across the OAuth fetch AND the authoritative record, so
-    # `_authoritative_record_usage` cannot release it — the deferral has to
-    # reach the frame that acquired it. The nudge is a loopback POST with a
-    # multi-second timeout and this lock is an `fcntl.flock` every cctally
-    # process contends on, so one unresponsive listener would otherwise stall
-    # every other process's selected-state writes at status-line cadence.
-    deferred = []
+    # Check freshness under the selected-state lock, release it for the
+    # potentially five-second network request, then recheck after reacquiring
+    # before publication. A concurrent winner can therefore suppress this
+    # response without the high-cadence hook holding the cross-process flock
+    # during network I/O.
     try:
         with c._selected_state_lock():
-            out = _hook_tick_oauth_refresh_locked(
-                c,
-                token=token,
-                timeout_seconds=timeout_seconds,
-                throttle_seconds=throttle_seconds,
-                nudge_sink=lambda: deferred.append(1),
+            skipped = _hook_tick_oauth_skip_status(
+                c, throttle_seconds=throttle_seconds,
             )
     except OSError:
         return "err(record-usage=exc)", None
-    if deferred:
-        c._nudge_dashboard_repaint()
-    return out
-
-
-def _hook_tick_oauth_refresh_locked(
-    c,
-    *,
-    token: str,
-    timeout_seconds: float,
-    throttle_seconds: float,
-    nudge_sink=None,
-) -> tuple[str, dict | None]:
-    """Automatic OAuth path with the selected-state lock already held.
+    if skipped is not None:
+        return skipped, None
 
-    ``nudge_sink`` records that a dashboard nudge is warranted; the caller
-    fires it after releasing the lock (#583 S2).
-    """
-    # Backfill gate: an inflight/invalid tombstone bypasses only selected-age
-    # suppression so a later authoritative result can repair it.  The normal
-    # throttle and 429 deadline continue to bound OAuth traffic.
-    now_epoch = int(time.time())
-    needs_repair = c._authoritative_repair_required(now_epoch=now_epoch)
-    obs_age = c._statusline_observe_age_seconds()
-    if (not needs_repair
-            and obs_age < float(_cctally_core.OAUTH_BACKFILL_STALE_SECONDS)):
-        return f"skipped(statusline-fresh:{int(obs_age)}s)", None
-    backoff_remaining = c._oauth_backoff_remaining_seconds()
-    if backoff_remaining > 0:
-        return f"skipped(backoff:{int(backoff_remaining)}s)", None
-    age_s = c._newest_snapshot_age_seconds()
-    if age_s is not None and age_s < throttle_seconds:
-        return f"skipped(fresh:{int(age_s)}s)", None
     try:
         api = c._fetch_oauth_usage(token=token, timeout_seconds=timeout_seconds)
     except RefreshUsageRateLimitError as exc:
-        c._oauth_backoff_register_429(
-            retry_after_deadline=getattr(exc, "retry_after_deadline", None),
-            now=time.time(),
-        )
+        try:
+            with c._selected_state_lock():
+                c._oauth_backoff_register_429(
+                    retry_after_deadline=getattr(
+                        exc, "retry_after_deadline", None,
+                    ),
+                    now=time.time(),
+                )
+        except OSError:
+            return "err(record-usage=exc)", None
         return "err(rate-limit)", None
     except RefreshUsageNetworkError:
         return "err(network)", None
     except RefreshUsageMalformedError:
         return "err(parse)", None
-    seven = api["seven_day"]
-    try:
-        seven_pct = c._normalize_percent(float(seven["utilization"]))
-        seven_resets_epoch = _iso_to_epoch(seven["resets_at"])
-    except (TypeError, ValueError, KeyError):
+
+    parsed = _hook_tick_parse_oauth_payload(c, api)
+    if parsed is None:
         return "err(parse)", None
-    five = api.get("five_hour") if isinstance(api.get("five_hour"), dict) else None
-    five_pct: float | None = None
-    five_resets_epoch: int | None = None
-    if (five is not None and "utilization" in five
-            and isinstance(five.get("resets_at"), str)):
-        try:
-            five_pct = c._normalize_percent(float(five["utilization"]))
-            five_resets_epoch = _iso_to_epoch(five["resets_at"])
-        except (TypeError, ValueError):
-            five_pct = None
-            five_resets_epoch = None
-    record_args = argparse.Namespace(
-        percent=seven_pct,
-        resets_at=str(seven_resets_epoch),
-        five_hour_percent=five_pct,
-        five_hour_resets_at=str(five_resets_epoch) if five_resets_epoch is not None else None,
-        source="api",
-    )
-    authoritative = c._authoritative_record_usage(
-        record_args,
-        {"sevenDay", *({"fiveHour"} if five_pct is not None else set())},
-        lock_held=True,
-        nudge_sink=nudge_sink,
-    )
-    if authoritative.status != "ok":
-        reason = authoritative.reason or ""
-        if reason.startswith("exit "):
-            return f"err(record-usage={reason[5:]})", None
+    record_args, axes, parts = parsed
+
+    # #583 S2: the nudge remains deferred until after the publication lock is
+    # released. The sink contract is now enforced by the authority kernel, so
+    # every lock-owning caller has to make the deferral explicit.
+    deferred = []
+    try:
+        with c._selected_state_lock():
+            skipped = _hook_tick_oauth_skip_status(
+                c,
+                throttle_seconds=throttle_seconds,
+            )
+            if skipped is not None:
+                return skipped, None
+            authoritative = c._authoritative_record_usage(
+                record_args,
+                axes,
+                lock_held=True,
+                nudge_sink=lambda: deferred.append(1),
+            )
+            if authoritative.status != "ok":
+                reason = authoritative.reason or ""
+                if reason.startswith("exit "):
+                    return f"err(record-usage={reason[5:]})", None
+                return "err(record-usage=exc)", None
+            # A success reaches shared backoff only after tombstones, control,
+            # and freshness are authoritative under this same lock.
+            c._oauth_backoff_reset()
+    except OSError:
         return "err(record-usage=exc)", None
-    # Success is acknowledged to the shared OAuth backoff only after the
-    # authoritative writer has published tombstones, control, and freshness.
-    c._oauth_backoff_reset()
-    parts = [f"7d={int(round(seven_pct))}"]
-    if five_pct is not None:
-        parts.append(f"5h={int(round(five_pct))}")
+    if deferred:
+        c._nudge_dashboard_repaint()
     return f"ok({','.join(parts)})", api
 
 

package/bin/_cctally_statusline.py

@@ -1008,8 +1008,15 @@ def _authoritative_record_usage(
     selected-state writes. This frame owns the lock only on the
     ``lock_held=False`` path, so that is the only path that can fire the
     deferred nudge itself; a caller passing ``lock_held=True`` owns the lock
-    and must supply its own sink, or the nudge stays inside its section.
+    and must supply its own sink. The fail-closed guard below enforces that
+    contract before any tombstone or journal write begins.
     """
+    if lock_held and nudge_dashboard and nudge_sink is None:
+        return _AuthoritativeRecordResult(
+            "record_failed",
+            "lock_held authoritative record requires nudge_sink",
+        )
+
     if not lock_held:
         deferred = []
         try:

package/bin/_lib_blocks.py

@@ -105,6 +105,7 @@ def _group_entries_into_blocks(
         dt.datetime, tuple[dt.datetime, dt.datetime]
     ] | None = None,
     now: dt.datetime | None = None,
+    _entry_membership: dict[int, list[UsageEntry]] | None = None,
 ) -> list[Block]:
     """Group sorted UsageEntry objects into 5-hour blocks with gap detection.
 
@@ -138,6 +139,13 @@ def _group_entries_into_blocks(
 
     `now` pins the current instant (typically via `_command_as_of()`). When
     omitted, falls back to wall clock so existing callers are unaffected.
+
+    ``_entry_membership`` is an internal detail-route seam. When supplied,
+    each non-gap result object's identity maps to the exact entries assigned
+    by this grouping pass. That ownership is stricter than a later
+    ``start <= timestamp < end`` filter when adjacent recorded windows overlap
+    after a reset shift; consumers that render per-entry samples must use the
+    grouping decision rather than independently reconstructing it.
     """
     if not entries:
         return []
@@ -264,6 +272,8 @@ def _group_entries_into_blocks(
             anchor="recorded",
         )
         first_entry_ts_by_block[id(blk)] = bucket_sorted[0].timestamp
+        if _entry_membership is not None:
+            _entry_membership[id(blk)] = bucket_sorted
         recorded_block_objs.append(blk)
 
     # Phase 2: Build heuristic Block objects from raw_blocks using _aggregate_block
@@ -278,6 +288,8 @@ def _group_entries_into_blocks(
         )
         if block_entries:
             first_entry_ts_by_block[id(blk)] = block_entries[0].timestamp
+        if _entry_membership is not None:
+            _entry_membership[id(blk)] = block_entries
         heuristic_block_objs.append(blk)
 
     # Merge + sort by start_time

package/bin/_lib_codex_conversation.py

@@ -2142,6 +2142,20 @@ def infer_codex_event_turns(
     return turns, current
 
 
+def fold_codex_event_turns(events) -> dict:
+    """Map each prefetched physical event's ``line_offset`` to its turn.
+
+    The pure half of ``_file_turn_map`` (#620 S3 X3): the caller supplies the
+    ordered ``codex_conversation_events`` batch and this returns
+    ``{line_offset: turn_id | None}`` over the same lifecycle inference cost
+    attribution and normalized prose already share. Opens no database.
+    """
+    materialized = list(events)
+    turns, _terminal = infer_codex_event_turns(materialized)
+    return {event.line_offset: turn
+            for event, turn in zip(materialized, turns)}
+
+
 def _is_late_turn_anchor(
     record_type: str | None, payload_type: str | None, explicit_turn: str | None,
 ) -> bool:

package/bin/_lib_codex_conversation_query.py

@@ -1272,15 +1272,18 @@ def _tokens_union(tokens: dict) -> dict:
 # ── cost attribution (§5.4) ───────────────────────────────────────────────────
 
 
-def _file_turn_map(conn: sqlite3.Connection, source_path: str) -> dict[int, str | None]:
-    """Exact physical-offset → canonical logical turn for one retained file.
-
-    Cost attribution and normalized prose use the same pure lifecycle inference,
-    including resumed segments whose native proof arrives on task completion.
+def _codex_file_events(conn: sqlite3.Connection, source_path: str) -> list:
+    """One retained file's physical events, in offset order.
+
+    Split out of ``_file_turn_map`` (#620 S3 X3) so the turn attribution can
+    run as a pure fold over a prefetched batch. This walk is UNBOUNDED by
+    construction: it reads every event the file retains, however far the
+    in-window entry sits behind that history, which is why the diagnosis
+    budgets its own use of it rather than calling this directly.
     """
     from _lib_jsonl import CodexPhysicalEvent
 
-    events = [
+    return [
         CodexPhysicalEvent(*row)
         for row in conn.execute(
             "SELECT source_path, line_offset, source_root_key, conversation_key, "
@@ -1291,8 +1294,19 @@ def _file_turn_map(conn: sqlite3.Connection, source_path: str) -> dict[int, str
             (source_path,),
         )
     ]
-    turns, _terminal = kern.infer_codex_event_turns(events)
-    return {event.line_offset: turn for event, turn in zip(events, turns)}
+
+
+def _file_turn_map(conn: sqlite3.Connection, source_path: str) -> dict[int, str | None]:
+    """Exact physical-offset → canonical logical turn for one retained file.
+
+    Cost attribution and normalized prose use the same pure lifecycle inference,
+    including resumed segments whose native proof arrives on task completion.
+
+    Since #620 S3 this is a thin wrapper: it fetches and delegates to
+    ``fold_codex_event_turns``. Its signature, return shape and every caller
+    are unchanged.
+    """
+    return kern.fold_codex_event_turns(_codex_file_events(conn, source_path))
 
 
 def _attribute_costs(conn: sqlite3.Connection, conversation_key: str, effective_speed: str):

package/bin/_lib_conversation.py

@@ -55,10 +55,13 @@ def _join_text_blocks(blocks):
     migration-011 consumer rebuilds the marker text from blocks_json to feed
     _extract_command_invocation, so this lives in the parser kernel (the one
     _cctally_cache already imports) to keep the two derivations identical."""
-    if not blocks:
+    if not isinstance(blocks, list) or not blocks:
         return ""
-    return "\n".join(b.get("text", "") or ""
-                     for b in blocks if b.get("kind") == "text")
+    return "\n".join(
+        text for b in blocks
+        if isinstance(b, dict) and b.get("kind") == "text"
+        and isinstance((text := b.get("text")), str)
+    )
 
 
 def _extract_command_invocation(blocks, text):