cctally 1.73.0 → 1.75.0

package/bin/_lib_dashboard_sources.py

@@ -272,8 +272,9 @@ def unavailable_source_state(
 
 def _coherent_provider(state: SourceDashboardState) -> bool:
     return (
-        state.availability in ("ok", "empty")
+        state.availability in ("ok", "empty", "partial")
         and state.freshness == "fresh"
+        and bool(state.data_version)
         and state.data is not None
     )
 
@@ -302,6 +303,10 @@ def _combined_metrics(
 ) -> Mapping[str, object] | None:
     if not (_coherent_provider(claude) and _coherent_provider(codex)):
         return None
+    for state in (claude, codex):
+        hero_capability = state.capabilities.get("hero")
+        if hero_capability is None or hero_capability.status not in {"supported", "derived"}:
+            return None
     try:
         claude_hero = claude.data["hero"]
         codex_hero = codex.data["hero"]
@@ -360,19 +365,27 @@ def compose_all_state(
     if claude.source != "claude" or codex.source != "codex":
         raise ValueError("all composition requires Claude and Codex provider states")
     combined = _combined_metrics(claude, codex)
-    coherent = combined is not None
-    if coherent:
+    providers_coherent = _coherent_provider(claude) and _coherent_provider(codex)
+    if providers_coherent:
         availability: Availability = (
-            "empty"
-            if claude.availability == "empty" and codex.availability == "empty"
-            else "ok"
+            "partial"
+            if combined is None or "partial" in (claude.availability, codex.availability)
+            else (
+                "empty"
+                if claude.availability == "empty" and codex.availability == "empty"
+                else "ok"
+            )
         )
         freshness: Freshness = "fresh"
     else:
         availability = "partial"
         freshness = "stale"
     version_material = json.dumps(
-        [claude.data_version, codex.data_version, coherent],
+        [
+            claude.data_version, claude.availability, claude.freshness,
+            codex.data_version, codex.availability, codex.freshness,
+            combined is not None,
+        ],
         separators=(",", ":"),
     ).encode("utf-8")
     data_version = "all:" + hashlib.sha256(version_material).hexdigest()[:24]

package/bin/_lib_doctor.py

@@ -218,6 +218,12 @@ class DoctorState:
     # arbitration pipeline.  None means the gather could not inspect it;
     # absent transport/tombstones are normal when Claude is closed.
     statusline_pipeline: Optional[dict] = None
+    # #312: all-history, root-qualified Codex accounting metadata partition.
+    # The I/O layer reports an explicit error instead of silently converting a
+    # failed health query into zero rows, so doctor can distinguish an empty
+    # retained corpus from unreadable metadata.
+    codex_project_metadata_health: Optional[dict] = None
+    codex_project_metadata_error: Optional[str] = None
 
 
 @dataclasses.dataclass(frozen=True)
@@ -876,6 +882,43 @@ def _check_data_codex_cache(s: DoctorState) -> CheckResult:
     )
 
 
+def _check_data_codex_project_metadata(s: DoctorState) -> CheckResult:
+    """Report the identity-safe all-history Codex metadata partition."""
+    if s.codex_project_metadata_error is not None:
+        return CheckResult(
+            id="data.codex_project_metadata", title="Codex project metadata",
+            severity="fail", summary="metadata health could not be read",
+            remediation="Run `cctally doctor --verbose` after checking cache.db.",
+            details={"error": s.codex_project_metadata_error},
+        )
+
+    health = s.codex_project_metadata_health or {}
+    total_rows = int(health.get("total_rows", 0))
+    qualified_rows = int(health.get("qualified_rows", 0))
+    missing_key_rows = int(health.get("missing_conversation_key_rows", 0))
+    missing_join_rows = int(health.get("missing_thread_join_rows", 0))
+    incomplete_rows = missing_key_rows + missing_join_rows
+    details = {
+        "total_rows": total_rows,
+        "qualified_rows": qualified_rows,
+        "missing_conversation_key_rows": missing_key_rows,
+        "missing_thread_join_rows": missing_join_rows,
+        "incomplete_rows": incomplete_rows,
+    }
+    if incomplete_rows:
+        return CheckResult(
+            id="data.codex_project_metadata", title="Codex project metadata",
+            severity="warn",
+            summary=f"{incomplete_rows} accounting row(s) need metadata repair",
+            remediation="Run `cctally cache-sync --source codex --rebuild`.",
+            details=details,
+        )
+    return CheckResult(
+        id="data.codex_project_metadata", title="Codex project metadata",
+        severity="ok", summary="qualified", remediation=None, details=details,
+    )
+
+
 def _check_data_codex_quota(s: DoctorState) -> CheckResult:
     """Report local physical Codex quota evidence without fabricating a window."""
     raw_windows = s.codex_quota_windows or []
@@ -1607,10 +1650,10 @@ def _check_db_integrity(s: DoctorState) -> CheckResult:
             summary=f"stats.db quick_check: {s.stats_db_quick_check}",
             remediation=(
                 "stats.db (the non-re-derivable DB) reports corruption. "
-                "Back up the file FIRST, then try "
-                "`sqlite3 <path> \".recover\"` into a fresh file; "
-                "`cctally db recover` does not repair corruption. Please "
-                "file a bug. Do not delete the file."),
+                "Stop the dashboard and other cctally processes, then run "
+                "`cctally db repair --db stats --yes`. The command preserves "
+                "a backup of the corrupt original before replacing anything. "
+                "Do not copy, restore, move, or delete the live DB by hand."),
             details=details,
         )
     if s.cache_db_quick_check is not None and s.cache_db_quick_check != "ok":
@@ -1722,6 +1765,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
         ("data.statusline_pipeline", "_check_statusline_pipeline"),
         ("data.cache_sync_state", "_check_data_cache_sync_state"),
         ("data.codex_cache", "_check_data_codex_cache"),
+        ("data.codex_project_metadata", "_check_data_codex_project_metadata"),
         ("data.codex_quota", "_check_data_codex_quota"),
         ("data.parse_health", "_check_data_parse_health"),
         ("data.forked_buckets", "_check_data_forked_buckets"),

package/bin/_lib_jsonl.py

@@ -720,6 +720,14 @@ def _accounting_from_record(
         _codex_skip(state, "no-last-token-usage")
         return None, last_total_tokens, filename_session_id_warned
 
+    # Forked/subagent rollouts can begin with a copied slice of their parent's
+    # physical history before the child's first model-bearing turn_context.
+    # Those records remain in the physical event stream, but projecting them
+    # into accounting would duplicate the parent's usage under model="unknown".
+    if (state.model is None and state.thread is not None
+            and state.thread.parent_thread_id is not None):
+        return None, last_total_tokens, filename_session_id_warned
+
     total_token_usage = info.get("total_token_usage")
     if isinstance(total_token_usage, dict):
         try:
@@ -839,6 +847,9 @@ def _iter_codex_fused_records_with_offsets(
             session_id = _codex_string(payload.get("id"))
             if session_id is not None:
                 state.session_id = session_id
+            session_model = _codex_string(payload.get("model"))
+            if session_model is not None:
+                state.model = session_model.strip()
             thread_update = _thread_metadata_from_session_meta(
                 payload, path_str, source_root_key
             )

package/bin/_lib_pricing.py

@@ -53,7 +53,7 @@ def _chip_for_model(name: str) -> str:
 # Date the embedded pricing snapshots below were last verified against
 # vendor sources. Bump whenever CLAUDE_MODEL_PRICING / CODEX_MODEL_PRICING
 # is synced. Read by `pricing-check` + the release pre-flight staleness nudge.
-PRICING_SNAPSHOT_DATE = "2026-07-10"
+PRICING_SNAPSHOT_DATE = "2026-07-19"
 PRICING_STALENESS_DAYS = 60  # release pre-flight WARNs past this age
 
 # Canonical machine-readable pricing source (Claude values + Codex values).
@@ -99,7 +99,7 @@ PRICING_DRIFT_ALLOWLIST: list[dict] = [
 
 # Anthropic API pricing snapshot:
 # - Source: https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json
-# - Captured: 2026-07-01 (see PRICING_SNAPSHOT_DATE)
+# - Captured: 2026-07-19 (see PRICING_SNAPSHOT_DATE)
 # - Verified by maintainer against docs.claude.com/en/docs/about-claude/pricing;
 #   update in PRs touching this table.
 #   2026-06-10: added claude-fable-5 ($10/$50 per MTok; 1M context, no
@@ -321,8 +321,8 @@ _unknown_model_warnings: set[str] = set()
 #
 # Codex (OpenAI) API pricing snapshot:
 # - Source: https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json
-# - Captured: 2026-05-30 (see PRICING_SNAPSHOT_DATE)
-# - As of the 2026-05-30 sync (issue #123) this carries every openai-provider
+# - Captured: 2026-07-19 (see PRICING_SNAPSHOT_DATE)
+# - As of the 2026-07-19 sync this carries every openai-provider
 #   gpt-5* model the LiteLLM snapshot lists, so `pricing-check`'s scope finds
 #   nothing missing. Models absent from this table still fall back to `gpt-5`
 #   pricing with isFallback=true (matches upstream's LEGACY_FALLBACK_MODEL
@@ -332,6 +332,9 @@ _unknown_model_warnings: set[str] = set()
 #   `pricing-check` as missing_from_us (gpt-5.6-sol had real usage falling back
 #   to gpt-5). Exact model_prices_and_context_window.json values, each with the
 #   above-272k tier (max_input_tokens 1050000).
+#   2026-07-19: verified all tracked values and the complete openai-provider
+#   gpt-5* model set against the live LiteLLM snapshot; no named table entries
+#   were missing. Official OpenAI model pages were used as the vendor cross-check.
 #
 # Billing rules:
 # - reasoning_output_tokens is billed at the *output* rate (matches

package/bin/_lib_pricing_check.py

@@ -34,11 +34,17 @@ class DriftResult:
     ahead_of_litellm: list[str]   # informational; never actionable
 
 
+_CODEX_UNATTRIBUTED_MODEL = "unknown"
+
+
 def classify_coverage(observed, resolve_claude, is_codex_fallback) -> list[CoverageGap]:
     """observed: iterable of (provider, model, entry_count, token_total).
 
     Claude model with resolve_claude(model) is None -> kind="unpriced".
     Codex model with is_codex_fallback(model) True  -> kind="fallback".
+    The ingest-only Codex ``unknown`` sentinel is skipped because it records
+    missing model metadata, not a model identifier that can be added to the
+    embedded pricing table.
     Priced models produce no gap. Order preserved.
     """
     gaps: list[CoverageGap] = []
@@ -46,7 +52,7 @@ def classify_coverage(observed, resolve_claude, is_codex_fallback) -> list[Cover
         if provider == "claude":
             if resolve_claude(model) is None:
                 gaps.append(CoverageGap("claude", model, "unpriced", entry_count, token_total))
-        elif provider == "codex":
+        elif provider == "codex" and model != _CODEX_UNATTRIBUTED_MODEL:
             if is_codex_fallback(model):
                 gaps.append(CoverageGap("codex", model, "fallback", entry_count, token_total))
     return gaps

package/bin/_lib_quota.py

@@ -407,7 +407,7 @@ def _single_identity_history(observations: Iterable[QuotaObservation]) -> QuotaH
 def forecast_quota(
     observations: Iterable[QuotaObservation], as_of: dt.datetime,
 ) -> QuotaForecast:
-    """Fit a reset-scoped quota forecast from positive adjacent intervals only."""
+    """Forecast reset-scoped quota use from elapsed native-cycle pace."""
     _require_aware(as_of, "as_of")
     history = _single_identity_history(observations)
     if history is None:
@@ -423,28 +423,29 @@ def forecast_quota(
         if observation.resets_at == baseline.resets_at and observation.captured_at <= as_of
     )
     points = tuple(sorted(points, key=physical_order_key))
-    usable_elapsed_seconds = 0.0
-    positive_delta_percent = 0.0
     sample_count = 0
     for prior, current in zip(points, points[1:]):
         elapsed_seconds = (current.captured_at - prior.captured_at).total_seconds()
         delta_percent = current.used_percent - prior.used_percent
         if elapsed_seconds > 0 and delta_percent > 0:
-            usable_elapsed_seconds += elapsed_seconds
-            positive_delta_percent += delta_percent
             sample_count += 1
 
     remaining_seconds = max(0, int((baseline.resets_at - as_of).total_seconds()))
-    if usable_elapsed_seconds > 0:
-        rate = positive_delta_percent / (usable_elapsed_seconds / 3600)
+    native_window_seconds = baseline.identity.window_minutes * 60
+    cycle_start = baseline.resets_at - dt.timedelta(minutes=baseline.identity.window_minutes)
+    cycle_elapsed_seconds = min(
+        float(native_window_seconds),
+        max(0.0, (as_of - cycle_start).total_seconds()),
+    )
+    if sample_count > 0 and cycle_elapsed_seconds > 0:
+        rate = baseline.used_percent / (cycle_elapsed_seconds / 3600)
         projected = min(100.0, max(
             baseline.used_percent,
             baseline.used_percent + rate * (remaining_seconds / 3600),
         ))
-        native_window_seconds = baseline.identity.window_minutes * 60
-        if sample_count >= 4 and usable_elapsed_seconds >= native_window_seconds * 0.25:
+        if sample_count >= 4 and cycle_elapsed_seconds >= native_window_seconds * 0.25:
             confidence: Literal["high", "medium", "low"] = "high"
-        elif sample_count >= 2 and usable_elapsed_seconds >= native_window_seconds * 0.10:
+        elif sample_count >= 2 and cycle_elapsed_seconds >= native_window_seconds * 0.10:
             confidence = "medium"
         else:
             confidence = "low"
@@ -469,7 +470,7 @@ def forecast_quota(
         resets_at=baseline.resets_at,
         remaining_seconds=remaining_seconds,
         sample_count=sample_count,
-        sample_span_seconds=int(usable_elapsed_seconds),
+        sample_span_seconds=int(cycle_elapsed_seconds) if sample_count > 0 else 0,
         confidence=confidence,
     )
 

package/bin/_lib_view_models.py

@@ -1884,20 +1884,8 @@ def build_codex_weekly_view(entries, *, now_utc, tz_name=None,
     )
 
 
-def build_codex_session_view(entries, *, now_utc, tz_name=None, speed="standard"):
-    """Build a ``CodexSessionView`` from a list of ``CodexEntry`` (issue #58).
-
-    ``rows`` order mirrors the aggregator: descending by
-    ``last_activity`` (upstream parity).
-    ``cmd_codex_session`` reverses to ASC when ``--order asc``.
-
-    ``period_start`` is set to ``min(s.last_activity)`` across emitted
-    sessions when any exist — best-available approximation since
-    ``CodexSessionUsage`` doesn't carry a ``first_activity`` field (the
-    aggregator only tracks ``last`` per session). ``None`` on empty.
-    """
-    _agg = _load_lib("_lib_aggregators")
-    sessions = _agg._aggregate_codex_sessions(entries, speed=speed)
+def _build_codex_session_view_from_rows(sessions, *, now_utc, tz_name):
+    """Wrap already-aggregated Codex sessions in the public view contract."""
     total_cost = 0.0
     total_tok = 0
     earliest = None
@@ -1915,3 +1903,51 @@ def build_codex_session_view(entries, *, now_utc, tz_name=None, speed="standard"
         period_end=now_utc,
         display_tz_label=_codex_tz_label(tz_name),
     )
+
+
+def build_codex_session_view(entries, *, now_utc, tz_name=None, speed="standard"):
+    """Build a ``CodexSessionView`` from CLI ``CodexEntry`` rows (issue #58)."""
+    _agg = _load_lib("_lib_aggregators")
+    return _build_codex_session_view_from_rows(
+        _agg._aggregate_codex_sessions(entries, speed=speed),
+        now_utc=now_utc,
+        tz_name=tz_name,
+    )
+
+
+def build_rooted_codex_session_view(entries, *, now_utc, tz_name=None, speed="standard"):
+    """Build dashboard sessions from cache-rooted accounting without I/O.
+
+    The file identity is lexical: ``(source_root_key, source_path)``.  It
+    deliberately does not inspect configured roots or touch the filesystem,
+    and a native session id remains payload rather than a grouping key.
+    """
+    _agg = _load_lib("_lib_aggregators")
+    keyed = []
+    for entry in entries:
+        source_path = str(getattr(entry, "source_path"))
+        source_root_key = str(getattr(entry, "source_root_key"))
+        path = pathlib.PurePosixPath(source_path)
+        codex_entry = _agg.CodexEntry(
+            timestamp=getattr(entry, "timestamp"),
+            session_id=str(getattr(entry, "session_id") or ""),
+            model=str(getattr(entry, "model")),
+            input_tokens=int(getattr(entry, "input_tokens")),
+            cached_input_tokens=int(getattr(entry, "cached_input_tokens")),
+            output_tokens=int(getattr(entry, "output_tokens")),
+            reasoning_output_tokens=int(getattr(entry, "reasoning_output_tokens")),
+            total_tokens=int(getattr(entry, "total_tokens")),
+            source_path=source_path,
+        )
+        keyed.append((codex_entry, _agg.CodexSessionIdentity(
+            group_key=(source_root_key, source_path),
+            session_id_path=source_path,
+            session_file=path.stem,
+            directory=str(path.parent),
+            codex_root=source_root_key,
+        )))
+    return _build_codex_session_view_from_rows(
+        _agg._aggregate_codex_sessions_keyed(keyed, speed=speed),
+        now_utc=now_utc,
+        tz_name=tz_name,
+    )

package/bin/cctally

@@ -933,6 +933,9 @@ Migration = _cctally_db.Migration
 DowngradeDetected = _cctally_db.DowngradeDetected
 ProdMigrationRefused = _cctally_db.ProdMigrationRefused
 StatsDbCorruptError = _cctally_db.StatsDbCorruptError
+StatsDbMaintenanceError = _cctally_db.StatsDbMaintenanceError
+_is_sqlite_corruption_error = _cctally_db._is_sqlite_corruption_error
+_stats_corruption_guidance = _cctally_db._stats_corruption_guidance
 _STATS_MIGRATIONS = _cctally_db._STATS_MIGRATIONS
 _CACHE_MIGRATIONS = _cctally_db._CACHE_MIGRATIONS
 _make_migration_decorator = _cctally_db._make_migration_decorator
@@ -957,6 +960,8 @@ _db_path_for_label = _cctally_db._db_path_for_label
 cmd_db_skip = _cctally_db.cmd_db_skip
 cmd_db_unskip = _cctally_db.cmd_db_unskip
 cmd_db_recover = _cctally_db.cmd_db_recover
+cmd_db_repair = _cctally_db.cmd_db_repair
+cmd_db_backup = _cctally_db.cmd_db_backup
 cmd_db_checkpoint = _cctally_db.cmd_db_checkpoint
 cmd_db_vacuum = _cctally_db.cmd_db_vacuum
 
@@ -1384,6 +1389,7 @@ cmd_codex_quota_statusline = _cctally_quota.cmd_codex_quota_statusline
 cmd_codex_quota_forecast = _cctally_quota.cmd_codex_quota_forecast
 cmd_codex_quota_blocks = _cctally_quota.cmd_codex_quota_blocks
 cmd_codex_quota_breakdown = _cctally_quota.cmd_codex_quota_breakdown
+cmd_codex_percent_breakdown = _cctally_quota.cmd_codex_percent_breakdown
 
 # S3's qualified Codex accounting adapter is intentionally cache-only: callers
 # that need project identity must not downgrade to an unqualified rollout parse.
@@ -2926,6 +2932,7 @@ _is_reset_drop                      = _cctally_weekrefs._is_reset_drop
 # handler (C8, #125 Batch B). Parser dispatch reaches c.cmd_percent_breakdown.
 _cctally_percent_breakdown = _load_sibling("_cctally_percent_breakdown")
 cmd_percent_breakdown = _cctally_percent_breakdown.cmd_percent_breakdown
+_render_percent_breakdown_terminal = _cctally_percent_breakdown._render_percent_breakdown_terminal
 
 
 # _is_cctally_hook_command, SetupError, _load_claude_settings,
@@ -3207,7 +3214,26 @@ def main(argv: list[str] | None = None) -> int:
         # + recovery guidance, exit 2. Caught before the generic handler so it
         # never renders as a raw traceback.
         eprint(f"cctally: {exc}")
-        return 2
+        return 3
+    except StatsDbMaintenanceError as exc:
+        eprint(f"cctally: {exc}")
+        return 3
+    except sqlite3.DatabaseError as exc:
+        if _is_sqlite_corruption_error(exc):
+            eprint(
+                "cctally: a SQLite database is corrupt — run `cctally doctor` "
+                "to identify it. For stats.db run `cctally db repair --db "
+                "stats --yes`; cache.db is re-derivable via `cctally "
+                "cache-sync --rebuild`."
+            )
+            return 3
+        if _lib_log.debug_enabled():
+            _lib_log.get_logger("cli").debug(
+                "unhandled exception in %r",
+                getattr(args, "command", None), exc_info=True,
+            )
+        eprint(f"Error: {exc}")
+        rc = 1
     except Exception as exc:  # pragma: no cover
         # #279 S2 F2: CCTALLY_DEBUG=1 surfaces the full traceback on stderr
         # via the _lib_log chokepoint; default mode stays byte-identical.