cctally 1.73.0 → 1.74.0

package/bin/_lib_aggregators.py

@@ -42,7 +42,7 @@ import os
 import pathlib
 import sys
 from dataclasses import dataclass
-from typing import Any, Callable
+from typing import Any, Callable, Iterable
 
 
 def _cctally():
@@ -414,6 +414,17 @@ class CodexSessionUsage:
     codex_root: str = ""
 
 
+@dataclass(frozen=True)
+class CodexSessionIdentity:
+    """Caller-supplied identity for the pure Codex session accumulator."""
+
+    group_key: tuple[str, str]
+    session_id_path: str
+    session_file: str
+    directory: str
+    codex_root: str
+
+
 @dataclass
 class ClaudeSessionUsage:
     """Aggregated Claude usage for one sessionId (may span multiple JSONL files)."""
@@ -619,41 +630,19 @@ def _codex_home_root_from_prefix(root_prefix: str) -> str:
     return s
 
 
-def _aggregate_codex_sessions(entries: list[CodexEntry], speed: str = "standard") -> list[CodexSessionUsage]:
-    """Group by session file path (upstream-compatible).
-
-    Sessions are keyed by the full relative-path-without-.jsonl rather than
-    the inner UUID. Result is sorted by last_activity descending (most
-    recent first), matching upstream's default view.
-
-    Per-model breakdowns include `isFallback: bool` — true when the model is
-    absent from CODEX_MODEL_PRICING.
-    """
+def _aggregate_codex_sessions_keyed(
+    entries: Iterable[tuple[CodexEntry, CodexSessionIdentity]],
+    speed: str = "standard",
+) -> list[CodexSessionUsage]:
+    """Pure Codex session arithmetic over caller-owned file identities."""
     by_session: dict[tuple[str, str], dict[str, Any]] = {}
-    for entry in entries:
-        id_path, file_name, directory = _session_path_parts(entry.source_path)
-        # Disambiguate identical relative paths under DIFFERENT $CODEX_HOME
-        # roots (issue #108). _session_path_parts strips the matched root, so
-        # <rootA>/sessions/2026/04/17/rollout-x.jsonl and the same relative
-        # path under <rootB> both yield id_path "2026/04/17/rollout-x";
-        # grouping on id_path alone would silently merge two distinct sessions
-        # (summed tokens, one UUID). Key on (root_prefix, id_path), where
-        # root_prefix is source_path with the id_path tail removed. Single-root
-        # data — and the bare-relative fixture form — has a constant prefix, so
-        # the grouping, insertion order, and every golden stay byte-identical;
-        # only a genuine cross-root collision splits into separate rows.
-        suffix = id_path + ".jsonl"
-        sp = entry.source_path
-        root_prefix = sp[: -len(suffix)] if sp.endswith(suffix) else sp
-        sess = by_session.setdefault((root_prefix, id_path), {
+    for entry, identity in entries:
+        sess = by_session.setdefault(identity.group_key, {
             "session_id_uuid": entry.session_id,
-            "session_id_path": id_path,
-            "session_file": file_name,
-            "directory": directory,
-            # Matched $CODEX_HOME root (home-root form) — issue #110 display
-            # disambiguator. Derived from the same root_prefix that keys the
-            # group, so it's constant per group.
-            "codex_root": _codex_home_root_from_prefix(root_prefix),
+            "session_id_path": identity.session_id_path,
+            "session_file": identity.session_file,
+            "directory": identity.directory,
+            "codex_root": identity.codex_root,
             "input": 0, "cached_input": 0, "output": 0, "reasoning": 0,
             "cost": 0.0, "models": {}, "models_order": [],
             "last": entry.timestamp,
@@ -724,6 +713,31 @@ def _aggregate_codex_sessions(entries: list[CodexEntry], speed: str = "standard"
     return result
 
 
+def _aggregate_codex_sessions(entries: list[CodexEntry], speed: str = "standard") -> list[CodexSessionUsage]:
+    """Group CLI Codex entries by their existing filesystem-derived identity.
+
+    The wrapper preserves the historic CLI path parsing and byte-stable output;
+    only the accumulation itself lives in the keyed pure seam above.
+    """
+    keyed: list[tuple[CodexEntry, CodexSessionIdentity]] = []
+    for entry in entries:
+        id_path, file_name, directory = _session_path_parts(entry.source_path)
+        # Disambiguate equal relative paths under separate configured roots.
+        suffix = id_path + ".jsonl"
+        source_path = entry.source_path
+        root_prefix = (
+            source_path[: -len(suffix)] if source_path.endswith(suffix) else source_path
+        )
+        keyed.append((entry, CodexSessionIdentity(
+            group_key=(root_prefix, id_path),
+            session_id_path=id_path,
+            session_file=file_name,
+            directory=directory,
+            codex_root=_codex_home_root_from_prefix(root_prefix),
+        )))
+    return _aggregate_codex_sessions_keyed(keyed, speed=speed)
+
+
 def _aggregate_claude_sessions(
     entries: list["_JoinedClaudeEntry"],
     mode: str = "auto",

package/bin/_lib_dashboard_sources.py

@@ -272,8 +272,9 @@ def unavailable_source_state(
 
 def _coherent_provider(state: SourceDashboardState) -> bool:
     return (
-        state.availability in ("ok", "empty")
+        state.availability in ("ok", "empty", "partial")
         and state.freshness == "fresh"
+        and bool(state.data_version)
         and state.data is not None
     )
 
@@ -302,6 +303,10 @@ def _combined_metrics(
 ) -> Mapping[str, object] | None:
     if not (_coherent_provider(claude) and _coherent_provider(codex)):
         return None
+    for state in (claude, codex):
+        hero_capability = state.capabilities.get("hero")
+        if hero_capability is None or hero_capability.status not in {"supported", "derived"}:
+            return None
     try:
         claude_hero = claude.data["hero"]
         codex_hero = codex.data["hero"]
@@ -360,19 +365,27 @@ def compose_all_state(
     if claude.source != "claude" or codex.source != "codex":
         raise ValueError("all composition requires Claude and Codex provider states")
     combined = _combined_metrics(claude, codex)
-    coherent = combined is not None
-    if coherent:
+    providers_coherent = _coherent_provider(claude) and _coherent_provider(codex)
+    if providers_coherent:
         availability: Availability = (
-            "empty"
-            if claude.availability == "empty" and codex.availability == "empty"
-            else "ok"
+            "partial"
+            if combined is None or "partial" in (claude.availability, codex.availability)
+            else (
+                "empty"
+                if claude.availability == "empty" and codex.availability == "empty"
+                else "ok"
+            )
         )
         freshness: Freshness = "fresh"
     else:
         availability = "partial"
         freshness = "stale"
     version_material = json.dumps(
-        [claude.data_version, codex.data_version, coherent],
+        [
+            claude.data_version, claude.availability, claude.freshness,
+            codex.data_version, codex.availability, codex.freshness,
+            combined is not None,
+        ],
         separators=(",", ":"),
     ).encode("utf-8")
     data_version = "all:" + hashlib.sha256(version_material).hexdigest()[:24]

package/bin/_lib_doctor.py

@@ -218,6 +218,12 @@ class DoctorState:
     # arbitration pipeline.  None means the gather could not inspect it;
     # absent transport/tombstones are normal when Claude is closed.
     statusline_pipeline: Optional[dict] = None
+    # #312: all-history, root-qualified Codex accounting metadata partition.
+    # The I/O layer reports an explicit error instead of silently converting a
+    # failed health query into zero rows, so doctor can distinguish an empty
+    # retained corpus from unreadable metadata.
+    codex_project_metadata_health: Optional[dict] = None
+    codex_project_metadata_error: Optional[str] = None
 
 
 @dataclasses.dataclass(frozen=True)
@@ -876,6 +882,43 @@ def _check_data_codex_cache(s: DoctorState) -> CheckResult:
     )
 
 
+def _check_data_codex_project_metadata(s: DoctorState) -> CheckResult:
+    """Report the identity-safe all-history Codex metadata partition."""
+    if s.codex_project_metadata_error is not None:
+        return CheckResult(
+            id="data.codex_project_metadata", title="Codex project metadata",
+            severity="fail", summary="metadata health could not be read",
+            remediation="Run `cctally doctor --verbose` after checking cache.db.",
+            details={"error": s.codex_project_metadata_error},
+        )
+
+    health = s.codex_project_metadata_health or {}
+    total_rows = int(health.get("total_rows", 0))
+    qualified_rows = int(health.get("qualified_rows", 0))
+    missing_key_rows = int(health.get("missing_conversation_key_rows", 0))
+    missing_join_rows = int(health.get("missing_thread_join_rows", 0))
+    incomplete_rows = missing_key_rows + missing_join_rows
+    details = {
+        "total_rows": total_rows,
+        "qualified_rows": qualified_rows,
+        "missing_conversation_key_rows": missing_key_rows,
+        "missing_thread_join_rows": missing_join_rows,
+        "incomplete_rows": incomplete_rows,
+    }
+    if incomplete_rows:
+        return CheckResult(
+            id="data.codex_project_metadata", title="Codex project metadata",
+            severity="warn",
+            summary=f"{incomplete_rows} accounting row(s) need metadata repair",
+            remediation="Run `cctally cache-sync --source codex --rebuild`.",
+            details=details,
+        )
+    return CheckResult(
+        id="data.codex_project_metadata", title="Codex project metadata",
+        severity="ok", summary="qualified", remediation=None, details=details,
+    )
+
+
 def _check_data_codex_quota(s: DoctorState) -> CheckResult:
     """Report local physical Codex quota evidence without fabricating a window."""
     raw_windows = s.codex_quota_windows or []
@@ -1722,6 +1765,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
         ("data.statusline_pipeline", "_check_statusline_pipeline"),
         ("data.cache_sync_state", "_check_data_cache_sync_state"),
         ("data.codex_cache", "_check_data_codex_cache"),
+        ("data.codex_project_metadata", "_check_data_codex_project_metadata"),
         ("data.codex_quota", "_check_data_codex_quota"),
         ("data.parse_health", "_check_data_parse_health"),
         ("data.forked_buckets", "_check_data_forked_buckets"),

package/bin/_lib_jsonl.py

@@ -720,6 +720,14 @@ def _accounting_from_record(
         _codex_skip(state, "no-last-token-usage")
         return None, last_total_tokens, filename_session_id_warned
 
+    # Forked/subagent rollouts can begin with a copied slice of their parent's
+    # physical history before the child's first model-bearing turn_context.
+    # Those records remain in the physical event stream, but projecting them
+    # into accounting would duplicate the parent's usage under model="unknown".
+    if (state.model is None and state.thread is not None
+            and state.thread.parent_thread_id is not None):
+        return None, last_total_tokens, filename_session_id_warned
+
     total_token_usage = info.get("total_token_usage")
     if isinstance(total_token_usage, dict):
         try:
@@ -839,6 +847,9 @@ def _iter_codex_fused_records_with_offsets(
             session_id = _codex_string(payload.get("id"))
             if session_id is not None:
                 state.session_id = session_id
+            session_model = _codex_string(payload.get("model"))
+            if session_model is not None:
+                state.model = session_model.strip()
             thread_update = _thread_metadata_from_session_meta(
                 payload, path_str, source_root_key
             )

package/bin/_lib_pricing.py

@@ -53,7 +53,7 @@ def _chip_for_model(name: str) -> str:
 # Date the embedded pricing snapshots below were last verified against
 # vendor sources. Bump whenever CLAUDE_MODEL_PRICING / CODEX_MODEL_PRICING
 # is synced. Read by `pricing-check` + the release pre-flight staleness nudge.
-PRICING_SNAPSHOT_DATE = "2026-07-10"
+PRICING_SNAPSHOT_DATE = "2026-07-19"
 PRICING_STALENESS_DAYS = 60  # release pre-flight WARNs past this age
 
 # Canonical machine-readable pricing source (Claude values + Codex values).
@@ -99,7 +99,7 @@ PRICING_DRIFT_ALLOWLIST: list[dict] = [
 
 # Anthropic API pricing snapshot:
 # - Source: https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json
-# - Captured: 2026-07-01 (see PRICING_SNAPSHOT_DATE)
+# - Captured: 2026-07-19 (see PRICING_SNAPSHOT_DATE)
 # - Verified by maintainer against docs.claude.com/en/docs/about-claude/pricing;
 #   update in PRs touching this table.
 #   2026-06-10: added claude-fable-5 ($10/$50 per MTok; 1M context, no
@@ -321,8 +321,8 @@ _unknown_model_warnings: set[str] = set()
 #
 # Codex (OpenAI) API pricing snapshot:
 # - Source: https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json
-# - Captured: 2026-05-30 (see PRICING_SNAPSHOT_DATE)
-# - As of the 2026-05-30 sync (issue #123) this carries every openai-provider
+# - Captured: 2026-07-19 (see PRICING_SNAPSHOT_DATE)
+# - As of the 2026-07-19 sync this carries every openai-provider
 #   gpt-5* model the LiteLLM snapshot lists, so `pricing-check`'s scope finds
 #   nothing missing. Models absent from this table still fall back to `gpt-5`
 #   pricing with isFallback=true (matches upstream's LEGACY_FALLBACK_MODEL
@@ -332,6 +332,9 @@ _unknown_model_warnings: set[str] = set()
 #   `pricing-check` as missing_from_us (gpt-5.6-sol had real usage falling back
 #   to gpt-5). Exact model_prices_and_context_window.json values, each with the
 #   above-272k tier (max_input_tokens 1050000).
+#   2026-07-19: verified all tracked values and the complete openai-provider
+#   gpt-5* model set against the live LiteLLM snapshot; no named table entries
+#   were missing. Official OpenAI model pages were used as the vendor cross-check.
 #
 # Billing rules:
 # - reasoning_output_tokens is billed at the *output* rate (matches

package/bin/_lib_pricing_check.py

@@ -34,11 +34,17 @@ class DriftResult:
     ahead_of_litellm: list[str]   # informational; never actionable
 
 
+_CODEX_UNATTRIBUTED_MODEL = "unknown"
+
+
 def classify_coverage(observed, resolve_claude, is_codex_fallback) -> list[CoverageGap]:
     """observed: iterable of (provider, model, entry_count, token_total).
 
     Claude model with resolve_claude(model) is None -> kind="unpriced".
     Codex model with is_codex_fallback(model) True  -> kind="fallback".
+    The ingest-only Codex ``unknown`` sentinel is skipped because it records
+    missing model metadata, not a model identifier that can be added to the
+    embedded pricing table.
     Priced models produce no gap. Order preserved.
     """
     gaps: list[CoverageGap] = []
@@ -46,7 +52,7 @@ def classify_coverage(observed, resolve_claude, is_codex_fallback) -> list[Cover
         if provider == "claude":
             if resolve_claude(model) is None:
                 gaps.append(CoverageGap("claude", model, "unpriced", entry_count, token_total))
-        elif provider == "codex":
+        elif provider == "codex" and model != _CODEX_UNATTRIBUTED_MODEL:
             if is_codex_fallback(model):
                 gaps.append(CoverageGap("codex", model, "fallback", entry_count, token_total))
     return gaps

package/bin/_lib_quota.py

@@ -407,7 +407,7 @@ def _single_identity_history(observations: Iterable[QuotaObservation]) -> QuotaH
 def forecast_quota(
     observations: Iterable[QuotaObservation], as_of: dt.datetime,
 ) -> QuotaForecast:
-    """Fit a reset-scoped quota forecast from positive adjacent intervals only."""
+    """Forecast reset-scoped quota use from elapsed native-cycle pace."""
     _require_aware(as_of, "as_of")
     history = _single_identity_history(observations)
     if history is None:
@@ -423,28 +423,29 @@ def forecast_quota(
         if observation.resets_at == baseline.resets_at and observation.captured_at <= as_of
     )
     points = tuple(sorted(points, key=physical_order_key))
-    usable_elapsed_seconds = 0.0
-    positive_delta_percent = 0.0
     sample_count = 0
     for prior, current in zip(points, points[1:]):
         elapsed_seconds = (current.captured_at - prior.captured_at).total_seconds()
         delta_percent = current.used_percent - prior.used_percent
         if elapsed_seconds > 0 and delta_percent > 0:
-            usable_elapsed_seconds += elapsed_seconds
-            positive_delta_percent += delta_percent
             sample_count += 1
 
     remaining_seconds = max(0, int((baseline.resets_at - as_of).total_seconds()))
-    if usable_elapsed_seconds > 0:
-        rate = positive_delta_percent / (usable_elapsed_seconds / 3600)
+    native_window_seconds = baseline.identity.window_minutes * 60
+    cycle_start = baseline.resets_at - dt.timedelta(minutes=baseline.identity.window_minutes)
+    cycle_elapsed_seconds = min(
+        float(native_window_seconds),
+        max(0.0, (as_of - cycle_start).total_seconds()),
+    )
+    if sample_count > 0 and cycle_elapsed_seconds > 0:
+        rate = baseline.used_percent / (cycle_elapsed_seconds / 3600)
         projected = min(100.0, max(
             baseline.used_percent,
             baseline.used_percent + rate * (remaining_seconds / 3600),
         ))
-        native_window_seconds = baseline.identity.window_minutes * 60
-        if sample_count >= 4 and usable_elapsed_seconds >= native_window_seconds * 0.25:
+        if sample_count >= 4 and cycle_elapsed_seconds >= native_window_seconds * 0.25:
             confidence: Literal["high", "medium", "low"] = "high"
-        elif sample_count >= 2 and usable_elapsed_seconds >= native_window_seconds * 0.10:
+        elif sample_count >= 2 and cycle_elapsed_seconds >= native_window_seconds * 0.10:
             confidence = "medium"
         else:
             confidence = "low"
@@ -469,7 +470,7 @@ def forecast_quota(
         resets_at=baseline.resets_at,
         remaining_seconds=remaining_seconds,
         sample_count=sample_count,
-        sample_span_seconds=int(usable_elapsed_seconds),
+        sample_span_seconds=int(cycle_elapsed_seconds) if sample_count > 0 else 0,
         confidence=confidence,
     )
 

package/bin/_lib_view_models.py

@@ -1884,20 +1884,8 @@ def build_codex_weekly_view(entries, *, now_utc, tz_name=None,
     )
 
 
-def build_codex_session_view(entries, *, now_utc, tz_name=None, speed="standard"):
-    """Build a ``CodexSessionView`` from a list of ``CodexEntry`` (issue #58).
-
-    ``rows`` order mirrors the aggregator: descending by
-    ``last_activity`` (upstream parity).
-    ``cmd_codex_session`` reverses to ASC when ``--order asc``.
-
-    ``period_start`` is set to ``min(s.last_activity)`` across emitted
-    sessions when any exist — best-available approximation since
-    ``CodexSessionUsage`` doesn't carry a ``first_activity`` field (the
-    aggregator only tracks ``last`` per session). ``None`` on empty.
-    """
-    _agg = _load_lib("_lib_aggregators")
-    sessions = _agg._aggregate_codex_sessions(entries, speed=speed)
+def _build_codex_session_view_from_rows(sessions, *, now_utc, tz_name):
+    """Wrap already-aggregated Codex sessions in the public view contract."""
     total_cost = 0.0
     total_tok = 0
     earliest = None
@@ -1915,3 +1903,51 @@ def build_codex_session_view(entries, *, now_utc, tz_name=None, speed="standard"
         period_end=now_utc,
         display_tz_label=_codex_tz_label(tz_name),
     )
+
+
+def build_codex_session_view(entries, *, now_utc, tz_name=None, speed="standard"):
+    """Build a ``CodexSessionView`` from CLI ``CodexEntry`` rows (issue #58)."""
+    _agg = _load_lib("_lib_aggregators")
+    return _build_codex_session_view_from_rows(
+        _agg._aggregate_codex_sessions(entries, speed=speed),
+        now_utc=now_utc,
+        tz_name=tz_name,
+    )
+
+
+def build_rooted_codex_session_view(entries, *, now_utc, tz_name=None, speed="standard"):
+    """Build dashboard sessions from cache-rooted accounting without I/O.
+
+    The file identity is lexical: ``(source_root_key, source_path)``.  It
+    deliberately does not inspect configured roots or touch the filesystem,
+    and a native session id remains payload rather than a grouping key.
+    """
+    _agg = _load_lib("_lib_aggregators")
+    keyed = []
+    for entry in entries:
+        source_path = str(getattr(entry, "source_path"))
+        source_root_key = str(getattr(entry, "source_root_key"))
+        path = pathlib.PurePosixPath(source_path)
+        codex_entry = _agg.CodexEntry(
+            timestamp=getattr(entry, "timestamp"),
+            session_id=str(getattr(entry, "session_id") or ""),
+            model=str(getattr(entry, "model")),
+            input_tokens=int(getattr(entry, "input_tokens")),
+            cached_input_tokens=int(getattr(entry, "cached_input_tokens")),
+            output_tokens=int(getattr(entry, "output_tokens")),
+            reasoning_output_tokens=int(getattr(entry, "reasoning_output_tokens")),
+            total_tokens=int(getattr(entry, "total_tokens")),
+            source_path=source_path,
+        )
+        keyed.append((codex_entry, _agg.CodexSessionIdentity(
+            group_key=(source_root_key, source_path),
+            session_id_path=source_path,
+            session_file=path.stem,
+            directory=str(path.parent),
+            codex_root=source_root_key,
+        )))
+    return _build_codex_session_view_from_rows(
+        _agg._aggregate_codex_sessions_keyed(keyed, speed=speed),
+        now_utc=now_utc,
+        tz_name=tz_name,
+    )