cctally 1.72.0 → 1.74.0

package/bin/_cctally_db.py

@@ -876,6 +876,10 @@ def _run_pending_migrations(
             # absent must still be treated as non-fresh so a flag-only
             # conversation migration's consumer actually runs (e.g. 011's
             # command-args promotion) instead of being stamped without it.
+            # codex_session_entries joins them (#312): an accounting-bearing
+            # cache written before conversation-key enrichment but missing
+            # schema_migrations must not be stamped fresh, or 026 would never
+            # clear it for the required byte-zero Codex replay.
             # codex_conversation_events joins them (#294 S6): a Codex-bearing
             # cache written by S1's fused ingest but missing schema_migrations
             # would otherwise be classified fresh and stamp 025 WITHOUT replaying
@@ -883,7 +887,7 @@ def _run_pending_migrations(
             # normalized corpus. Probing it forces 025's handler to run and derive
             # the normalized rows from the retained events.
             "cache.db": (
-                "session_entries", "conversation_messages",
+                "session_entries", "conversation_messages", "codex_session_entries",
                 "codex_conversation_events",
             ),
         }.get(db_label, ())
@@ -4583,6 +4587,105 @@ def _025_codex_conversation_normalization(conn: sqlite3.Connection) -> None:
             lock_fh.close()
 
 
+@cache_migration("026_codex_conversation_key_backfill")
+def _026_codex_conversation_key_backfill(conn: sqlite3.Connection) -> None:
+    """Arm a byte-zero Codex replay for missing conversation-key metadata (#312).
+
+    Historical accounting rows predate the source-derived conversation-key
+    enrichment.  The cache cannot truthfully recover those keys on its own, so
+    clear exactly the existing Codex-derived families and let the next Codex
+    sync replay retained rollout data from byte zero.  This shares the runtime
+    clear helper with ``sync_codex_cache(rebuild=True)`` so new or future Codex
+    families cannot drift from the migration's destructive scope.
+
+    The Codex flock is acquired before ``BEGIN IMMEDIATE``.  Contention defers
+    through ``MigrationGateNotMet`` before any DML; after a handler/data commit
+    but before the dispatcher's central stamp, a rerun against the empty state
+    is a byte-idempotent no-op.  The physical mutation sequence advances only
+    when the shared clear actually changed persisted Codex state.  The handler
+    never self-stamps.
+    """
+    lock_path = _cache_db_codex_lock_path_for_conn(conn)
+    lock_fh = None
+    if lock_path is not None:
+        lock_fh = open(lock_path, "w")
+        try:
+            fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            lock_fh.close()
+            raise MigrationGateNotMet(
+                "cache.db.codex.lock held by a concurrent sync_codex_cache; "
+                "deferring cache 026 conversation-key backfill (#312)"
+            )
+    try:
+        conn.execute("BEGIN IMMEDIATE")
+        try:
+            import _cctally_cache
+
+            if _cctally_cache._clear_codex_derived_rows(conn):
+                _cctally_cache._bump_codex_physical_mutation_seq(conn)
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+    finally:
+        if lock_fh is not None:
+            try:
+                fcntl.flock(lock_fh, fcntl.LOCK_UN)
+            except OSError:
+                pass
+            lock_fh.close()
+
+
+@cache_migration("027_codex_fork_preamble_rebuild")
+def _027_codex_fork_preamble_rebuild(conn: sqlite3.Connection) -> None:
+    """Replay Codex rollouts after suppressing copied fork-preamble accounting.
+
+    Forked/subagent JSONL can retain parent ``token_count`` records before the
+    child's first model-bearing ``turn_context``.  Older ingest projected those
+    copied records into ``codex_session_entries`` as ``model='unknown'``, double
+    counting usage already present in the parent rollout.  The source records
+    remain authoritative, so clear all re-derivable Codex families and let the
+    corrected fused parser replay them from byte zero.
+
+    The Codex flock precedes ``BEGIN IMMEDIATE``.  Contention defers before DML;
+    a markerless retry against already-cleared state is a no-op.  The shared
+    clear helper keeps this migration aligned with runtime rebuild scope and
+    advances the physical mutation sequence only when state actually changed.
+    The dispatcher owns the migration marker.
+    """
+    lock_path = _cache_db_codex_lock_path_for_conn(conn)
+    lock_fh = None
+    if lock_path is not None:
+        lock_fh = open(lock_path, "w")
+        try:
+            fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            lock_fh.close()
+            raise MigrationGateNotMet(
+                "cache.db.codex.lock held by a concurrent sync_codex_cache; "
+                "deferring cache 027 fork-preamble rebuild"
+            )
+    try:
+        conn.execute("BEGIN IMMEDIATE")
+        try:
+            import _cctally_cache
+
+            if _cctally_cache._clear_codex_derived_rows(conn):
+                _cctally_cache._bump_codex_physical_mutation_seq(conn)
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+    finally:
+        if lock_fh is not None:
+            try:
+                fcntl.flock(lock_fh, fcntl.LOCK_UN)
+            except OSError:
+                pass
+            lock_fh.close()
+
+
 # === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
 
 @stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")

package/bin/_cctally_doctor.py

@@ -42,6 +42,54 @@ def _cctally():
     return sys.modules["cctally"]
 
 
+def _gather_statusline_pipeline(c, *, now_utc: dt.datetime) -> dict:
+    """Read the #318 statusline pipeline without creating or pruning files."""
+    now_epoch = int(now_utc.timestamp())
+    result = {
+        "transport_age_seconds": None,
+        "selected_age_seconds": None,
+        "active_candidate_count": 0,
+        "control_db_agrees": None,
+        "tombstones": {"fiveHour": "absent", "sevenDay": "absent"},
+    }
+    try:
+        result["transport_age_seconds"] = c._statusline_transport_age_seconds()
+    except Exception:
+        pass
+    try:
+        result["selected_age_seconds"] = c._statusline_observe_age_seconds()
+    except Exception:
+        pass
+    try:
+        result["active_candidate_count"] = len(
+            c._scan_active_candidate_spool(now_epoch=now_epoch)
+        )
+    except Exception:
+        pass
+    try:
+        result["control_db_agrees"] = c._statusline_control_db_agreement(
+            now_epoch=now_epoch
+        )
+    except Exception:
+        pass
+    for axis, path in (
+        ("fiveHour", _cctally_core.STATUSLINE_AUTHORITATIVE_5H_PATH),
+        ("sevenDay", _cctally_core.STATUSLINE_AUTHORITATIVE_7D_PATH),
+    ):
+        try:
+            if not path.exists():
+                continue
+            tombstone = c._read_tombstone(
+                axis, now_epoch=now_epoch, fail_closed=False
+            )
+            result["tombstones"][axis] = (
+                tombstone.state if tombstone is not None else "invalid"
+            )
+        except Exception:
+            result["tombstones"][axis] = "invalid"
+    return result
+
+
 def _codex_lifecycle_activity_24h(
     *, root_keys: set[str], now_utc: dt.datetime,
 ) -> dict[str, dict]:
@@ -357,6 +405,16 @@ def doctor_gather_state(
     except Exception:
         pass
 
+    # ── Statusline candidate arbitration (#318) ──────────────────────
+    # This inspection is deliberately independent of SQLite mutation: marker
+    # mtime, candidate/control files, and tombstones are all read fail-soft.
+    # In particular it uses the scan-only candidate helper, never the reducer
+    # loader that prunes expired or malformed spool files.
+    try:
+        statusline_pipeline = _gather_statusline_pipeline(c, now_utc=now_utc)
+    except Exception:
+        statusline_pipeline = None
+
     # Conversation-sessions rollup consistency (#217 S1 / U9). Two cheap COUNTs
     # (graceful None on a missing table / unreadable DB) + an in-progress signal
     # so a transient mid-sync mismatch never WARNs. The in-progress signal is a
@@ -432,6 +490,8 @@ def doctor_gather_state(
 
     codex_entries_count = None
     codex_last_entry_at = None
+    codex_project_metadata_health = None
+    codex_project_metadata_error = None
     try:
         if _cctally_core.CACHE_DB_PATH.exists():
             conn = sqlite3.connect(str(_cctally_core.CACHE_DB_PATH))
@@ -445,12 +505,31 @@ def doctor_gather_state(
                         codex_last_entry_at = parse_iso_datetime(
                             row[1], "codex_session_entries.timestamp_utc",
                         ).astimezone(dt.timezone.utc)
-            except sqlite3.OperationalError:
-                pass
+                # Keep the health probe on the existing read-only cache
+                # connection.  A failed probe is health evidence, not an
+                # empty corpus: the kernel renders it as a distinct FAIL.
+                try:
+                    import _cctally_source_analytics
+
+                    health = _cctally_source_analytics.load_codex_project_metadata_health(
+                        cache_conn=conn,
+                    )
+                    codex_project_metadata_health = {
+                        "total_rows": health.total_rows,
+                        "qualified_rows": health.qualified_rows,
+                        "missing_conversation_key_rows": health.missing_conversation_key_rows,
+                        "missing_thread_join_rows": health.missing_thread_join_rows,
+                    }
+                except Exception as exc:
+                    codex_project_metadata_error = type(exc).__name__
+            except sqlite3.OperationalError as exc:
+                # Pre-Codex cache shapes still produce the established Codex
+                # cache result, while the new health check fails explicitly.
+                codex_project_metadata_error = type(exc).__name__
             finally:
                 conn.close()
-    except Exception:
-        pass
+    except Exception as exc:
+        codex_project_metadata_error = type(exc).__name__
 
     # Issue #109: probe every $CODEX_HOME session root (not the single
     # hardcoded ~/.codex/sessions), matching the multi-root ingestion path
@@ -744,6 +823,8 @@ def doctor_gather_state(
         codex_entries_count=codex_entries_count,
         codex_last_entry_at=codex_last_entry_at,
         codex_jsonl_present=codex_jsonl_present,
+        codex_project_metadata_health=codex_project_metadata_health,
+        codex_project_metadata_error=codex_project_metadata_error,
         dashboard_bind_stored=dashboard_bind_stored,
         runtime_bind=runtime_bind,
         # Conversation viewer (Plan 2, spec §5): only consulted on a LAN bind.
@@ -787,6 +868,7 @@ def doctor_gather_state(
         codex_lifecycle_activity_24h=codex_lifecycle_activity_24h,
         # #311: precomputed statusLine.refreshInterval classification.
         statusline_refresh_state=statusline_refresh_state,
+        statusline_pipeline=statusline_pipeline,
     )
 
 

package/bin/_cctally_parser.py

@@ -2650,15 +2650,21 @@ def _build_transcript_parser(subparsers, name, *, help_text, xref=None):
     t_export = t_sub.add_parser(
         "export", help="Export a whole session as Markdown (anonymized by default)",
         formatter_class=CLIHelpFormatter)
-    t_export.add_argument("session_id", metavar="SESSION_ID",
-                          help="Claude sessionId to export")
+    t_export.add_argument("session_id", metavar="ID",
+                          help="A Claude sessionId or a v1. conversation key "
+                               "(Codex conversations use the v1. key)")
     t_export.add_argument(
         "--scope", choices=("all", "prompts", "chat", "recipe"), default="all",
-        help="Which slice to export (default: all)")
+        help="Which slice to export (default: all; Codex accepts only 'all')")
     t_export.add_argument(
         "--raw", action="store_true",
         help="Disable the whole scrub (identity + secrets); byte-identical to "
              "the dashboard raw export")
+    t_export.add_argument(
+        "--speed", choices=("auto", "standard", "fast"), default=None,
+        help="Codex service tier for per-turn cost (default: auto). Applies only "
+             "to Codex (v1.) conversations; an explicit value on any other ref "
+             "is a usage error")
     t_export.add_argument(
         "-o", "--output", metavar="PATH", default=None,
         help="Write to PATH instead of stdout (same exact bytes)")
@@ -2668,6 +2674,9 @@ def _build_transcript_parser(subparsers, name, *, help_text, xref=None):
         "search", help="Search transcripts across sessions (raw output)",
         formatter_class=CLIHelpFormatter)
     t_search.add_argument("query", metavar="QUERY", help="Search text")
+    t_search.add_argument(
+        "--source", choices=("claude", "codex"), default="claude",
+        help="Which provider's conversations to search (default: claude)")
     t_search.add_argument(
         "--kind",
         choices=("all", "prompts", "assistant", "tools", "thinking",
@@ -2676,7 +2685,11 @@ def _build_transcript_parser(subparsers, name, *, help_text, xref=None):
     t_search.add_argument("--limit", type=int, default=50,
                           help="Max results (default: 50)")
     t_search.add_argument("--offset", type=int, default=0,
-                          help="Result offset for pagination (default: 0)")
+                          help="Result offset for pagination (default: 0; "
+                               "Claude only — Codex paginates with --cursor)")
+    t_search.add_argument("--cursor", default=None, metavar="TOKEN",
+                          help="Codex pagination cursor from a prior nextCursor "
+                               "(requires --source codex)")
     t_search.add_argument("--project", action="append", default=None,
                           metavar="LABEL",
                           help="Filter by project label (repeatable)")

package/bin/_cctally_record.py

@@ -2548,6 +2548,73 @@ def _credit_json(plan, *, applied, dry_run, forced, stale_replays, hwm_before):
     }
 
 
+def _revalidate_credit_plan(conn, args, *, now, at_dt, expected_plan):
+    """Recompute the confirmed credit plan from locked, current DB truth.
+
+    The caller has already completed every preview/refusal/confirmation path.
+    Returning ``None`` is deliberately side-effect free: it means a concurrent
+    writer changed the requested credit's basis and the user must retry rather
+    than authorizing a different mutation than the preview showed.
+    """
+    try:
+        if getattr(args, "week", None):
+            week_start_date = args.week
+            ws_at, we_at = _get_canonical_boundary_for_date(conn, week_start_date)
+            if not ws_at or not we_at:
+                return None
+        else:
+            fetched = _fetch_current_week_snapshots(conn, at_dt)
+            if fetched is None:
+                return None
+            ws_at, we_at, _samples = fetched
+            ws_at = ws_at if isinstance(ws_at, str) else ws_at.isoformat(timespec="seconds")
+            we_at = we_at if isinstance(we_at, str) else we_at.isoformat(timespec="seconds")
+            week_start_date = parse_iso_datetime(ws_at, "ws_at").date().isoformat()
+        existing = conn.execute(
+            "SELECT id, effective_at_utc, observed_pre_credit_pct "
+            "FROM weekly_credit_floors WHERE week_start_date=? "
+            "ORDER BY unixepoch(effective_at_utc) DESC, id DESC LIMIT 1",
+            (week_start_date,),
+        ).fetchone()
+        is_force = bool(getattr(args, "force", False))
+        if getattr(args, "from_pct", None) is not None:
+            from_pct, from_source = float(args.from_pct), "explicit"
+        elif existing is not None and existing[2] is not None:
+            from_pct, from_source = float(existing[2]), "prior_credit"
+        else:
+            from_pct = _resolve_reset_aware_hwm(conn, week_start_date, ws_at, we_at)
+            if from_pct is None:
+                return None
+            from_source = "hwm"
+        is_completion = False
+        if existing is not None and not is_force:
+            owned = conn.execute(
+                "SELECT 1 FROM weekly_usage_snapshots "
+                " WHERE week_start_date=? AND source='record-credit' "
+                "   AND unixepoch(captured_at_utc) >= unixepoch(?) LIMIT 1",
+                (week_start_date, existing[1]),
+            ).fetchone()
+            is_completion = owned is None
+        if existing is not None and not is_force and not is_completion:
+            return None
+        plan = _build_credit_plan(
+            week_start_date=week_start_date,
+            week_start_at=ws_at,
+            week_end_at=we_at,
+            from_pct=from_pct,
+            from_source=from_source,
+            to_pct=args.to,
+            at_dt=at_dt,
+            now=now,
+            effective_override=existing[1] if is_completion else None,
+        )
+    except (sqlite3.DatabaseError, ValueError, TypeError):
+        return None
+    if plan != expected_plan:
+        return None
+    return plan, existing, is_completion
+
+
 def cmd_record_credit(args) -> int:
     now = _command_as_of()
     try:
@@ -2704,23 +2771,68 @@ def cmd_record_credit(args) -> int:
                 print("aborted — nothing written")
                 return 0
 
-        # 4a. Existing-floor handling (M2; state classified at step 2a). The
-        #     fully-applied refuse was hoisted above the confirm prompt (#212
-        #     N2), so only two cases remain here: `--force` clears the prior
-        #     floor + re-records at a fresh effective; a half-applied credit
-        #     (floor row, no command-owned snapshot — a crash between
-        #     _apply_credit's floor commit and the synthetic snapshot) falls
-        #     through to a completion (the plan reuses the existing effective
-        #     and the apply steps are idempotent). `existing`/`is_completion`
-        #     were resolved up front keyed on week_start_date /
-        #     weekly_credit_floors.
-        forced = False
-        if existing is not None and is_force:
-            _force_clear_credit(conn, plan.week_start_date)
-            forced = True
-
-        five_hour = _resolve_prior_5h(conn, at_dt)
-        _apply_credit(conn, plan, five_hour=five_hour)
+        # Every non-mutating exit is above this point.  Re-open under the
+        # selected-state writer lock, recompute from current DB truth, and
+        # refuse plan drift before any durable pipeline artifact is visible.
+        # This is intentionally after confirmation: preview/no/TTY refusal
+        # must not create an inflight tombstone merely by inspecting a credit.
+        conn.close()
+        conn = None
+        c = _cctally()
+        with c._selected_state_lock():
+            conn = open_db()
+            revalidated = c._revalidate_credit_plan(
+                conn,
+                args,
+                now=now,
+                at_dt=at_dt,
+                expected_plan=plan,
+            )
+            if revalidated is None:
+                eprint("record-credit: plan changed while awaiting confirmation; retry")
+                return 2
+            plan, existing, is_completion = revalidated
+            stale_replays = _count_stale_replays(conn, plan)
+            hwm_before = _resolve_reset_aware_hwm(
+                conn, plan.week_start_date, plan.week_start_at, plan.week_end_at
+            )
+            if hwm_before is None:
+                hwm_before = plan.from_pct
+            try:
+                handles = c._authoritative_begin(
+                    {"sevenDay"}, now_epoch=int(time.time())
+                )
+            except (OSError, ValueError) as exc:
+                eprint(f"record-credit: could not prepare authoritative state: {exc}")
+                return 3
+
+            # Existing-floor handling remains exactly scoped to this week, but
+            # the weekly tombstone is already fail-closed immediately before
+            # either mutation kernel runs.
+            forced = False
+            if existing is not None and is_force:
+                _force_clear_credit(conn, plan.week_start_date)
+                forced = True
+            five_hour = _resolve_prior_5h(conn, at_dt)
+            _apply_credit(conn, plan, five_hour=five_hour)
+
+            # A successful credit is authoritative only after the post-credit
+            # DB state has a stable projection and all selected artifacts are
+            # atomically committed.  Leave inflight on any failure so a later
+            # authority repair remains fail-closed.
+            try:
+                projection = c._read_db_projection_stable()
+                completion_epoch = int(time.time())
+                c._authoritative_commit(
+                    handles, completion_epoch=completion_epoch
+                )
+                c._reconcile_selected_control(
+                    projection, now_epoch=completion_epoch, observed_axes={"sevenDay"}
+                )
+                c._statusline_observe_touch()
+            except Exception as exc:
+                eprint(f"record-credit: authoritative state incomplete: {exc}")
+                return 3
 
         if is_json:
             print(json.dumps(_credit_json(
@@ -3757,13 +3869,14 @@ def _hook_tick_throttle_touch() -> None:
 
 
 # =========================================================================
-# Statusline observation marker + OAuth backoff deadline (spec 2026-07-17)
+# Statusline selected/transport markers + OAuth backoff deadline (#318)
 # =========================================================================
 #
 # Two independent markers with two DIFFERENT time encodings:
-#   - The observation marker is MTIME-based (mirrors _hook_tick_throttle_*):
-#     it answers "how long since the statusline last fed?", so its age is
-#     the signal. It is touched even on a dedup no-op.
+#   - The selected-observation marker is MTIME-based and represents an actual
+#     selected DB change or authoritative OAuth confirmation.
+#   - The transport marker is also MTIME-based and represents an eligible
+#     regular-pool candidate reaching the spool; it never throttles OAuth.
 #   - The OAuth backoff marker is CONTENT-based: it stores a FUTURE absolute
 #     epoch deadline as text. Its mtime is meaningless (~now). Encoding the
 #     deadline as mtime would future-date the file and, if ever confused
@@ -3771,11 +3884,9 @@ def _hook_tick_throttle_touch() -> None:
 #     deliberate split (Codex P1-3).
 
 
-def _statusline_observe_age_seconds() -> float:
-    """Seconds since the statusline last successfully fed usage; +inf if
-    never. Mtime-based, mirroring ``_hook_tick_throttle_age_seconds``."""
+def _marker_age(path) -> float:
     try:
-        mtime = _cctally_core.STATUSLINE_OBSERVE_MARKER_PATH.stat().st_mtime
+        mtime = path.stat().st_mtime
     except FileNotFoundError:
         return float("inf")
     except OSError:
@@ -3783,20 +3894,38 @@ def _statusline_observe_age_seconds() -> float:
     return max(0.0, time.time() - mtime)
 
 
-def _statusline_observe_touch() -> None:
-    """Mark the statusline as alive (mtime := now, creating if missing).
-
-    Called after every persist attempt that fed valid 7d input through
-    ``cmd_record_usage`` successfully — INCLUDING a dedup (no-insert)
-    return, so the throttles reflect liveness rather than snapshot age."""
+def _touch_marker(path) -> None:
     try:
         _cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
-        _cctally_core.STATUSLINE_OBSERVE_MARKER_PATH.touch(exist_ok=True)
-        os.utime(_cctally_core.STATUSLINE_OBSERVE_MARKER_PATH, None)
+        path.touch(exist_ok=True)
+        os.utime(path, None)
     except OSError:
         pass
 
 
+def _statusline_observe_age_seconds() -> float:
+    """Seconds since selected/authoritative usage changed; +inf if never."""
+    return _marker_age(_cctally_core.STATUSLINE_OBSERVE_MARKER_PATH)
+
+
+def _statusline_observe_touch() -> None:
+    """Mark selected usage freshness after a proven selected transition.
+
+    The historical public helper names remain aliases for selected freshness.
+    """
+    _touch_marker(_cctally_core.STATUSLINE_OBSERVE_MARKER_PATH)
+
+
+def _statusline_transport_age_seconds() -> float:
+    """Seconds since an eligible regular-pool candidate reached the spool."""
+    return _marker_age(_cctally_core.STATUSLINE_TRANSPORT_MARKER_PATH)
+
+
+def _statusline_transport_touch() -> None:
+    """Mark regular-pool statusline transport after an atomic candidate write."""
+    _touch_marker(_cctally_core.STATUSLINE_TRANSPORT_MARKER_PATH)
+
+
 def _oauth_backoff_remaining_seconds() -> float:
     """Seconds until the shared OAuth 429 backoff deadline; ``0.0`` when the
     marker is absent, empty, malformed, or already elapsed.