cctally 1.72.0 → 1.74.0

package/bin/_lib_conversation_query.py

@@ -2540,6 +2540,94 @@ def build_anon_plan_for_db(conn, *, home_dir):
         home_dirs=sorted(home_dirs), usernames=sorted(usernames))
 
 
+def build_anon_plan_for_sources(conn, *, home_dir, sources):
+    """Provider-aware :class:`AnonPlan` for QUALIFIED conversation surfaces
+    (#294 S7, §3.6). A SEPARATE builder from :func:`build_anon_plan_for_db`, which
+    stays byte-untouched together with every bare-Claude call site — unioning Codex
+    roots into the legacy builder would renumber its ``project-N`` tokens and change
+    bare-Claude anonymized bytes. This builder is used ONLY by ``v1.``-qualified
+    requests (both providers), which are new and carry no byte-compat debt.
+
+    ``sources`` selects the provider legs to include (e.g. ``{"codex"}`` or
+    ``{"claude"}``). The Codex leg draws ONLY from the authoritative tables:
+    provider root dirs from ``codex_source_roots.canonical_root_path``, observed
+    project paths from validated ``codex_conversation_threads.cwd`` (absolute paths
+    only), display labels from ``codex_conversation_rollups.project_label``.
+    ``git_json`` is NOT a path source (its fields are branch/repository metadata,
+    not filesystem roots). The existing home-dir/username machinery and
+    ``SECRET_PATTERNS`` apply unchanged; coverage stays best-effort-over-known-tokens
+    and the per-token replacement stays fail-closed."""
+    from _lib_conversation_anon import build_anon_plan
+    want = set(sources or ())
+    project_roots: dict = {}
+    paths: set = set()
+    if "claude" in want:
+        cwds: set = set()
+        try:
+            for (cwd,) in conn.execute(
+                    "SELECT DISTINCT cwd FROM conversation_messages "
+                    "WHERE cwd IS NOT NULL AND cwd != ''"):
+                if cwd:
+                    cwds.add(cwd)
+        except sqlite3.OperationalError:
+            pass
+        try:
+            for (pp,) in conn.execute(
+                    "SELECT DISTINCT project_path FROM session_files "
+                    "WHERE project_path IS NOT NULL AND project_path != ''"):
+                if pp:
+                    cwds.add(pp)
+        except sqlite3.OperationalError:
+            pass
+        for c in cwds:
+            project_roots[c] = _project_label(c)
+            paths.add(c)
+    if "codex" in want:
+        labels: dict = {}
+        try:
+            for ck, pl in conn.execute(
+                    "SELECT conversation_key, project_label FROM codex_conversation_rollups "
+                    "WHERE project_label IS NOT NULL AND project_label != ''"):
+                labels[ck] = pl
+        except sqlite3.OperationalError:
+            pass
+        try:
+            for ck, cwd in conn.execute(
+                    "SELECT conversation_key, cwd FROM codex_conversation_threads "
+                    "WHERE cwd IS NOT NULL AND cwd != ''"):
+                if isinstance(cwd, str) and cwd and os.path.isabs(cwd):
+                    project_roots[cwd] = labels.get(ck) or _project_label(cwd)
+                    paths.add(cwd)
+        except sqlite3.OperationalError:
+            pass
+        try:
+            for (root_path,) in conn.execute(
+                    "SELECT canonical_root_path FROM codex_source_roots "
+                    "WHERE canonical_root_path IS NOT NULL AND canonical_root_path != ''"):
+                if isinstance(root_path, str) and root_path and os.path.isabs(root_path):
+                    project_roots.setdefault(root_path, _project_label(root_path))
+                    paths.add(root_path)
+        except sqlite3.OperationalError:
+            pass
+    home_dirs: set = set()
+    usernames: set = set()
+    if home_dir:
+        home_dirs.add(home_dir)
+        u = os.path.basename(home_dir.rstrip("/"))
+        if u:
+            usernames.add(u)
+    for p in paths:
+        for prefix in ("/Users/", "/home/"):
+            if p.startswith(prefix):
+                user = p[len(prefix):].split("/", 1)[0]
+                if user:
+                    home_dirs.add(prefix + user)
+                    usernames.add(user)
+    return build_anon_plan(
+        project_roots=project_roots,
+        home_dirs=sorted(home_dirs), usernames=sorted(usernames))
+
+
 _TASK_TRIO = ("TaskCreate", "TaskUpdate", "TaskList")
 
 

package/bin/_lib_dashboard_sources.py

@@ -272,8 +272,9 @@ def unavailable_source_state(
 
 def _coherent_provider(state: SourceDashboardState) -> bool:
     return (
-        state.availability in ("ok", "empty")
+        state.availability in ("ok", "empty", "partial")
         and state.freshness == "fresh"
+        and bool(state.data_version)
         and state.data is not None
     )
 
@@ -302,6 +303,10 @@ def _combined_metrics(
 ) -> Mapping[str, object] | None:
     if not (_coherent_provider(claude) and _coherent_provider(codex)):
         return None
+    for state in (claude, codex):
+        hero_capability = state.capabilities.get("hero")
+        if hero_capability is None or hero_capability.status not in {"supported", "derived"}:
+            return None
     try:
         claude_hero = claude.data["hero"]
         codex_hero = codex.data["hero"]
@@ -360,19 +365,27 @@ def compose_all_state(
     if claude.source != "claude" or codex.source != "codex":
         raise ValueError("all composition requires Claude and Codex provider states")
     combined = _combined_metrics(claude, codex)
-    coherent = combined is not None
-    if coherent:
+    providers_coherent = _coherent_provider(claude) and _coherent_provider(codex)
+    if providers_coherent:
         availability: Availability = (
-            "empty"
-            if claude.availability == "empty" and codex.availability == "empty"
-            else "ok"
+            "partial"
+            if combined is None or "partial" in (claude.availability, codex.availability)
+            else (
+                "empty"
+                if claude.availability == "empty" and codex.availability == "empty"
+                else "ok"
+            )
         )
         freshness: Freshness = "fresh"
     else:
         availability = "partial"
         freshness = "stale"
     version_material = json.dumps(
-        [claude.data_version, codex.data_version, coherent],
+        [
+            claude.data_version, claude.availability, claude.freshness,
+            codex.data_version, codex.availability, codex.freshness,
+            combined is not None,
+        ],
         separators=(",", ":"),
     ).encode("utf-8")
     data_version = "all:" + hashlib.sha256(version_material).hexdigest()[:24]

package/bin/_lib_doctor.py

@@ -15,6 +15,7 @@ import dataclasses
 import datetime as dt
 import hashlib
 import json
+import math
 import pathlib
 import sys
 from typing import Optional
@@ -213,6 +214,16 @@ class DoctorState:
     # never-WARN posture (the hooks.installed / settings warnings already
     # surface a genuinely-unreadable settings.json — no double-WARN here).
     statusline_refresh_state: str = "unavailable"
+    # #318: read-only evidence for the per-session statusline candidate
+    # arbitration pipeline.  None means the gather could not inspect it;
+    # absent transport/tombstones are normal when Claude is closed.
+    statusline_pipeline: Optional[dict] = None
+    # #312: all-history, root-qualified Codex accounting metadata partition.
+    # The I/O layer reports an explicit error instead of silently converting a
+    # failed health query into zero rows, so doctor can distinguish an empty
+    # retained corpus from unreadable metadata.
+    codex_project_metadata_health: Optional[dict] = None
+    codex_project_metadata_error: Optional[str] = None
 
 
 @dataclasses.dataclass(frozen=True)
@@ -737,6 +748,62 @@ def _check_data_latest_snapshot_age(s: DoctorState) -> CheckResult:
     )
 
 
+def _check_statusline_pipeline(s: DoctorState) -> CheckResult:
+    """Report transport, selected, control, and authority health separately."""
+    pipeline = s.statusline_pipeline if isinstance(s.statusline_pipeline, dict) else {}
+    tombstones = pipeline.get("tombstones")
+    tombstones = tombstones if isinstance(tombstones, dict) else {}
+    details = {
+        "transport_age_seconds": pipeline.get("transport_age_seconds"),
+        "selected_age_seconds": pipeline.get("selected_age_seconds"),
+        "active_candidate_count": pipeline.get("active_candidate_count", 0),
+        "control_db_agrees": pipeline.get("control_db_agrees"),
+        "tombstones": tombstones,
+    }
+    if any(value in {"invalid", "inflight"} for value in tombstones.values()):
+        return CheckResult(
+            id="data.statusline_pipeline", title="Statusline pipeline",
+            severity="warn", summary="authoritative state needs repair",
+            remediation="Run `cctally refresh-usage`", details=details,
+        )
+    if pipeline.get("control_db_agrees") is False:
+        return CheckResult(
+            id="data.statusline_pipeline", title="Statusline pipeline",
+            severity="warn", summary="selected control disagrees with database",
+            remediation="Run `cctally refresh-usage`, then inspect active sessions",
+            details=details,
+        )
+    transport_age = pipeline.get("transport_age_seconds")
+    selected_age = pipeline.get("selected_age_seconds")
+    transport_age = (
+        float(transport_age)
+        if isinstance(transport_age, (int, float)) and not isinstance(transport_age, bool)
+        else math.inf
+    )
+    selected_age = (
+        float(selected_age)
+        if isinstance(selected_age, (int, float)) and not isinstance(selected_age, bool)
+        else math.inf
+    )
+    if transport_age < 90 and selected_age >= 300:
+        return CheckResult(
+            id="data.statusline_pipeline", title="Statusline pipeline",
+            severity="warn", summary="timer active; selected usage stale",
+            remediation="Run `cctally refresh-usage`, then inspect active sessions",
+            details=details,
+        )
+    if transport_age >= 90:
+        summary = "no recent regular-pool timer observed"
+    elif selected_age < 300:
+        summary = "timer and selected usage fresh"
+    else:
+        summary = "selected usage awaiting next active timer"
+    return CheckResult(
+        id="data.statusline_pipeline", title="Statusline pipeline",
+        severity="ok", summary=summary, remediation=None, details=details,
+    )
+
+
 def _check_data_cache_sync_state(s: DoctorState) -> CheckResult:
     count = s.cache_entries_count or 0
     if count == 0:
@@ -815,6 +882,43 @@ def _check_data_codex_cache(s: DoctorState) -> CheckResult:
     )
 
 
+def _check_data_codex_project_metadata(s: DoctorState) -> CheckResult:
+    """Report the identity-safe all-history Codex metadata partition."""
+    if s.codex_project_metadata_error is not None:
+        return CheckResult(
+            id="data.codex_project_metadata", title="Codex project metadata",
+            severity="fail", summary="metadata health could not be read",
+            remediation="Run `cctally doctor --verbose` after checking cache.db.",
+            details={"error": s.codex_project_metadata_error},
+        )
+
+    health = s.codex_project_metadata_health or {}
+    total_rows = int(health.get("total_rows", 0))
+    qualified_rows = int(health.get("qualified_rows", 0))
+    missing_key_rows = int(health.get("missing_conversation_key_rows", 0))
+    missing_join_rows = int(health.get("missing_thread_join_rows", 0))
+    incomplete_rows = missing_key_rows + missing_join_rows
+    details = {
+        "total_rows": total_rows,
+        "qualified_rows": qualified_rows,
+        "missing_conversation_key_rows": missing_key_rows,
+        "missing_thread_join_rows": missing_join_rows,
+        "incomplete_rows": incomplete_rows,
+    }
+    if incomplete_rows:
+        return CheckResult(
+            id="data.codex_project_metadata", title="Codex project metadata",
+            severity="warn",
+            summary=f"{incomplete_rows} accounting row(s) need metadata repair",
+            remediation="Run `cctally cache-sync --source codex --rebuild`.",
+            details=details,
+        )
+    return CheckResult(
+        id="data.codex_project_metadata", title="Codex project metadata",
+        severity="ok", summary="qualified", remediation=None, details=details,
+    )
+
+
 def _check_data_codex_quota(s: DoctorState) -> CheckResult:
     """Report local physical Codex quota evidence without fabricating a window."""
     raw_windows = s.codex_quota_windows or []
@@ -1658,8 +1762,10 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
     )),
     ("data", "Data", (
         ("data.latest_snapshot_age", "_check_data_latest_snapshot_age"),
+        ("data.statusline_pipeline", "_check_statusline_pipeline"),
         ("data.cache_sync_state", "_check_data_cache_sync_state"),
         ("data.codex_cache", "_check_data_codex_cache"),
+        ("data.codex_project_metadata", "_check_data_codex_project_metadata"),
         ("data.codex_quota", "_check_data_codex_quota"),
         ("data.parse_health", "_check_data_parse_health"),
         ("data.forked_buckets", "_check_data_forked_buckets"),

package/bin/_lib_jsonl.py

@@ -720,6 +720,14 @@ def _accounting_from_record(
         _codex_skip(state, "no-last-token-usage")
         return None, last_total_tokens, filename_session_id_warned
 
+    # Forked/subagent rollouts can begin with a copied slice of their parent's
+    # physical history before the child's first model-bearing turn_context.
+    # Those records remain in the physical event stream, but projecting them
+    # into accounting would duplicate the parent's usage under model="unknown".
+    if (state.model is None and state.thread is not None
+            and state.thread.parent_thread_id is not None):
+        return None, last_total_tokens, filename_session_id_warned
+
     total_token_usage = info.get("total_token_usage")
     if isinstance(total_token_usage, dict):
         try:
@@ -839,6 +847,9 @@ def _iter_codex_fused_records_with_offsets(
             session_id = _codex_string(payload.get("id"))
             if session_id is not None:
                 state.session_id = session_id
+            session_model = _codex_string(payload.get("model"))
+            if session_model is not None:
+                state.model = session_model.strip()
             thread_update = _thread_metadata_from_session_meta(
                 payload, path_str, source_root_key
             )

package/bin/_lib_pricing.py

@@ -53,7 +53,7 @@ def _chip_for_model(name: str) -> str:
 # Date the embedded pricing snapshots below were last verified against
 # vendor sources. Bump whenever CLAUDE_MODEL_PRICING / CODEX_MODEL_PRICING
 # is synced. Read by `pricing-check` + the release pre-flight staleness nudge.
-PRICING_SNAPSHOT_DATE = "2026-07-10"
+PRICING_SNAPSHOT_DATE = "2026-07-19"
 PRICING_STALENESS_DAYS = 60  # release pre-flight WARNs past this age
 
 # Canonical machine-readable pricing source (Claude values + Codex values).
@@ -99,7 +99,7 @@ PRICING_DRIFT_ALLOWLIST: list[dict] = [
 
 # Anthropic API pricing snapshot:
 # - Source: https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json
-# - Captured: 2026-07-01 (see PRICING_SNAPSHOT_DATE)
+# - Captured: 2026-07-19 (see PRICING_SNAPSHOT_DATE)
 # - Verified by maintainer against docs.claude.com/en/docs/about-claude/pricing;
 #   update in PRs touching this table.
 #   2026-06-10: added claude-fable-5 ($10/$50 per MTok; 1M context, no
@@ -321,8 +321,8 @@ _unknown_model_warnings: set[str] = set()
 #
 # Codex (OpenAI) API pricing snapshot:
 # - Source: https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json
-# - Captured: 2026-05-30 (see PRICING_SNAPSHOT_DATE)
-# - As of the 2026-05-30 sync (issue #123) this carries every openai-provider
+# - Captured: 2026-07-19 (see PRICING_SNAPSHOT_DATE)
+# - As of the 2026-07-19 sync this carries every openai-provider
 #   gpt-5* model the LiteLLM snapshot lists, so `pricing-check`'s scope finds
 #   nothing missing. Models absent from this table still fall back to `gpt-5`
 #   pricing with isFallback=true (matches upstream's LEGACY_FALLBACK_MODEL
@@ -332,6 +332,9 @@ _unknown_model_warnings: set[str] = set()
 #   `pricing-check` as missing_from_us (gpt-5.6-sol had real usage falling back
 #   to gpt-5). Exact model_prices_and_context_window.json values, each with the
 #   above-272k tier (max_input_tokens 1050000).
+#   2026-07-19: verified all tracked values and the complete openai-provider
+#   gpt-5* model set against the live LiteLLM snapshot; no named table entries
+#   were missing. Official OpenAI model pages were used as the vendor cross-check.
 #
 # Billing rules:
 # - reasoning_output_tokens is billed at the *output* rate (matches

package/bin/_lib_pricing_check.py

@@ -34,11 +34,17 @@ class DriftResult:
     ahead_of_litellm: list[str]   # informational; never actionable
 
 
+_CODEX_UNATTRIBUTED_MODEL = "unknown"
+
+
 def classify_coverage(observed, resolve_claude, is_codex_fallback) -> list[CoverageGap]:
     """observed: iterable of (provider, model, entry_count, token_total).
 
     Claude model with resolve_claude(model) is None -> kind="unpriced".
     Codex model with is_codex_fallback(model) True  -> kind="fallback".
+    The ingest-only Codex ``unknown`` sentinel is skipped because it records
+    missing model metadata, not a model identifier that can be added to the
+    embedded pricing table.
     Priced models produce no gap. Order preserved.
     """
     gaps: list[CoverageGap] = []
@@ -46,7 +52,7 @@ def classify_coverage(observed, resolve_claude, is_codex_fallback) -> list[Cover
         if provider == "claude":
             if resolve_claude(model) is None:
                 gaps.append(CoverageGap("claude", model, "unpriced", entry_count, token_total))
-        elif provider == "codex":
+        elif provider == "codex" and model != _CODEX_UNATTRIBUTED_MODEL:
             if is_codex_fallback(model):
                 gaps.append(CoverageGap("codex", model, "fallback", entry_count, token_total))
     return gaps

package/bin/_lib_quota.py

@@ -407,7 +407,7 @@ def _single_identity_history(observations: Iterable[QuotaObservation]) -> QuotaH
 def forecast_quota(
     observations: Iterable[QuotaObservation], as_of: dt.datetime,
 ) -> QuotaForecast:
-    """Fit a reset-scoped quota forecast from positive adjacent intervals only."""
+    """Forecast reset-scoped quota use from elapsed native-cycle pace."""
     _require_aware(as_of, "as_of")
     history = _single_identity_history(observations)
     if history is None:
@@ -423,28 +423,29 @@ def forecast_quota(
         if observation.resets_at == baseline.resets_at and observation.captured_at <= as_of
     )
     points = tuple(sorted(points, key=physical_order_key))
-    usable_elapsed_seconds = 0.0
-    positive_delta_percent = 0.0
     sample_count = 0
     for prior, current in zip(points, points[1:]):
         elapsed_seconds = (current.captured_at - prior.captured_at).total_seconds()
         delta_percent = current.used_percent - prior.used_percent
         if elapsed_seconds > 0 and delta_percent > 0:
-            usable_elapsed_seconds += elapsed_seconds
-            positive_delta_percent += delta_percent
             sample_count += 1
 
     remaining_seconds = max(0, int((baseline.resets_at - as_of).total_seconds()))
-    if usable_elapsed_seconds > 0:
-        rate = positive_delta_percent / (usable_elapsed_seconds / 3600)
+    native_window_seconds = baseline.identity.window_minutes * 60
+    cycle_start = baseline.resets_at - dt.timedelta(minutes=baseline.identity.window_minutes)
+    cycle_elapsed_seconds = min(
+        float(native_window_seconds),
+        max(0.0, (as_of - cycle_start).total_seconds()),
+    )
+    if sample_count > 0 and cycle_elapsed_seconds > 0:
+        rate = baseline.used_percent / (cycle_elapsed_seconds / 3600)
         projected = min(100.0, max(
             baseline.used_percent,
             baseline.used_percent + rate * (remaining_seconds / 3600),
         ))
-        native_window_seconds = baseline.identity.window_minutes * 60
-        if sample_count >= 4 and usable_elapsed_seconds >= native_window_seconds * 0.25:
+        if sample_count >= 4 and cycle_elapsed_seconds >= native_window_seconds * 0.25:
             confidence: Literal["high", "medium", "low"] = "high"
-        elif sample_count >= 2 and usable_elapsed_seconds >= native_window_seconds * 0.10:
+        elif sample_count >= 2 and cycle_elapsed_seconds >= native_window_seconds * 0.10:
             confidence = "medium"
         else:
             confidence = "low"
@@ -469,7 +470,7 @@ def forecast_quota(
         resets_at=baseline.resets_at,
         remaining_seconds=remaining_seconds,
         sample_count=sample_count,
-        sample_span_seconds=int(usable_elapsed_seconds),
+        sample_span_seconds=int(cycle_elapsed_seconds) if sample_count > 0 else 0,
         confidence=confidence,
     )