cctally 1.73.0 → 1.74.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/bin/_cctally_cache.py +26 -5
- package/bin/_cctally_dashboard.py +35 -1
- package/bin/_cctally_dashboard_share.py +38 -3
- package/bin/_cctally_dashboard_sources.py +812 -74
- package/bin/_cctally_db.py +104 -1
- package/bin/_cctally_doctor.py +27 -4
- package/bin/_cctally_source_analytics.py +288 -3
- package/bin/_cctally_tui.py +17 -14
- package/bin/_lib_aggregators.py +48 -34
- package/bin/_lib_dashboard_sources.py +20 -7
- package/bin/_lib_doctor.py +44 -0
- package/bin/_lib_jsonl.py +11 -0
- package/bin/_lib_pricing.py +7 -4
- package/bin/_lib_pricing_check.py +7 -1
- package/bin/_lib_quota.py +12 -11
- package/bin/_lib_view_models.py +50 -14
- package/dashboard/static/assets/index-DeQjEMO6.js +80 -0
- package/dashboard/static/assets/index-ZiPO8veo.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +1 -1
- package/dashboard/static/assets/index-BWadAHxC.js +0 -80
- package/dashboard/static/assets/index-D2nwo_ln.css +0 -1
package/bin/_cctally_db.py
CHANGED
|
@@ -876,6 +876,10 @@ def _run_pending_migrations(
|
|
|
876
876
|
# absent must still be treated as non-fresh so a flag-only
|
|
877
877
|
# conversation migration's consumer actually runs (e.g. 011's
|
|
878
878
|
# command-args promotion) instead of being stamped without it.
|
|
879
|
+
# codex_session_entries joins them (#312): an accounting-bearing
|
|
880
|
+
# cache written before conversation-key enrichment but missing
|
|
881
|
+
# schema_migrations must not be stamped fresh, or 026 would never
|
|
882
|
+
# clear it for the required byte-zero Codex replay.
|
|
879
883
|
# codex_conversation_events joins them (#294 S6): a Codex-bearing
|
|
880
884
|
# cache written by S1's fused ingest but missing schema_migrations
|
|
881
885
|
# would otherwise be classified fresh and stamp 025 WITHOUT replaying
|
|
@@ -883,7 +887,7 @@ def _run_pending_migrations(
|
|
|
883
887
|
# normalized corpus. Probing it forces 025's handler to run and derive
|
|
884
888
|
# the normalized rows from the retained events.
|
|
885
889
|
"cache.db": (
|
|
886
|
-
"session_entries", "conversation_messages",
|
|
890
|
+
"session_entries", "conversation_messages", "codex_session_entries",
|
|
887
891
|
"codex_conversation_events",
|
|
888
892
|
),
|
|
889
893
|
}.get(db_label, ())
|
|
@@ -4583,6 +4587,105 @@ def _025_codex_conversation_normalization(conn: sqlite3.Connection) -> None:
|
|
|
4583
4587
|
lock_fh.close()
|
|
4584
4588
|
|
|
4585
4589
|
|
|
4590
|
+
@cache_migration("026_codex_conversation_key_backfill")
|
|
4591
|
+
def _026_codex_conversation_key_backfill(conn: sqlite3.Connection) -> None:
|
|
4592
|
+
"""Arm a byte-zero Codex replay for missing conversation-key metadata (#312).
|
|
4593
|
+
|
|
4594
|
+
Historical accounting rows predate the source-derived conversation-key
|
|
4595
|
+
enrichment. The cache cannot truthfully recover those keys on its own, so
|
|
4596
|
+
clear exactly the existing Codex-derived families and let the next Codex
|
|
4597
|
+
sync replay retained rollout data from byte zero. This shares the runtime
|
|
4598
|
+
clear helper with ``sync_codex_cache(rebuild=True)`` so new or future Codex
|
|
4599
|
+
families cannot drift from the migration's destructive scope.
|
|
4600
|
+
|
|
4601
|
+
The Codex flock is acquired before ``BEGIN IMMEDIATE``. Contention defers
|
|
4602
|
+
through ``MigrationGateNotMet`` before any DML; after a handler/data commit
|
|
4603
|
+
but before the dispatcher's central stamp, a rerun against the empty state
|
|
4604
|
+
is a byte-idempotent no-op. The physical mutation sequence advances only
|
|
4605
|
+
when the shared clear actually changed persisted Codex state. The handler
|
|
4606
|
+
never self-stamps.
|
|
4607
|
+
"""
|
|
4608
|
+
lock_path = _cache_db_codex_lock_path_for_conn(conn)
|
|
4609
|
+
lock_fh = None
|
|
4610
|
+
if lock_path is not None:
|
|
4611
|
+
lock_fh = open(lock_path, "w")
|
|
4612
|
+
try:
|
|
4613
|
+
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
4614
|
+
except BlockingIOError:
|
|
4615
|
+
lock_fh.close()
|
|
4616
|
+
raise MigrationGateNotMet(
|
|
4617
|
+
"cache.db.codex.lock held by a concurrent sync_codex_cache; "
|
|
4618
|
+
"deferring cache 026 conversation-key backfill (#312)"
|
|
4619
|
+
)
|
|
4620
|
+
try:
|
|
4621
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
4622
|
+
try:
|
|
4623
|
+
import _cctally_cache
|
|
4624
|
+
|
|
4625
|
+
if _cctally_cache._clear_codex_derived_rows(conn):
|
|
4626
|
+
_cctally_cache._bump_codex_physical_mutation_seq(conn)
|
|
4627
|
+
conn.commit()
|
|
4628
|
+
except Exception:
|
|
4629
|
+
conn.rollback()
|
|
4630
|
+
raise
|
|
4631
|
+
finally:
|
|
4632
|
+
if lock_fh is not None:
|
|
4633
|
+
try:
|
|
4634
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
4635
|
+
except OSError:
|
|
4636
|
+
pass
|
|
4637
|
+
lock_fh.close()
|
|
4638
|
+
|
|
4639
|
+
|
|
4640
|
+
@cache_migration("027_codex_fork_preamble_rebuild")
|
|
4641
|
+
def _027_codex_fork_preamble_rebuild(conn: sqlite3.Connection) -> None:
|
|
4642
|
+
"""Replay Codex rollouts after suppressing copied fork-preamble accounting.
|
|
4643
|
+
|
|
4644
|
+
Forked/subagent JSONL can retain parent ``token_count`` records before the
|
|
4645
|
+
child's first model-bearing ``turn_context``. Older ingest projected those
|
|
4646
|
+
copied records into ``codex_session_entries`` as ``model='unknown'``, double
|
|
4647
|
+
counting usage already present in the parent rollout. The source records
|
|
4648
|
+
remain authoritative, so clear all re-derivable Codex families and let the
|
|
4649
|
+
corrected fused parser replay them from byte zero.
|
|
4650
|
+
|
|
4651
|
+
The Codex flock precedes ``BEGIN IMMEDIATE``. Contention defers before DML;
|
|
4652
|
+
a markerless retry against already-cleared state is a no-op. The shared
|
|
4653
|
+
clear helper keeps this migration aligned with runtime rebuild scope and
|
|
4654
|
+
advances the physical mutation sequence only when state actually changed.
|
|
4655
|
+
The dispatcher owns the migration marker.
|
|
4656
|
+
"""
|
|
4657
|
+
lock_path = _cache_db_codex_lock_path_for_conn(conn)
|
|
4658
|
+
lock_fh = None
|
|
4659
|
+
if lock_path is not None:
|
|
4660
|
+
lock_fh = open(lock_path, "w")
|
|
4661
|
+
try:
|
|
4662
|
+
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
4663
|
+
except BlockingIOError:
|
|
4664
|
+
lock_fh.close()
|
|
4665
|
+
raise MigrationGateNotMet(
|
|
4666
|
+
"cache.db.codex.lock held by a concurrent sync_codex_cache; "
|
|
4667
|
+
"deferring cache 027 fork-preamble rebuild"
|
|
4668
|
+
)
|
|
4669
|
+
try:
|
|
4670
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
4671
|
+
try:
|
|
4672
|
+
import _cctally_cache
|
|
4673
|
+
|
|
4674
|
+
if _cctally_cache._clear_codex_derived_rows(conn):
|
|
4675
|
+
_cctally_cache._bump_codex_physical_mutation_seq(conn)
|
|
4676
|
+
conn.commit()
|
|
4677
|
+
except Exception:
|
|
4678
|
+
conn.rollback()
|
|
4679
|
+
raise
|
|
4680
|
+
finally:
|
|
4681
|
+
if lock_fh is not None:
|
|
4682
|
+
try:
|
|
4683
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
4684
|
+
except OSError:
|
|
4685
|
+
pass
|
|
4686
|
+
lock_fh.close()
|
|
4687
|
+
|
|
4688
|
+
|
|
4586
4689
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
4587
4690
|
|
|
4588
4691
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|
package/bin/_cctally_doctor.py
CHANGED
|
@@ -490,6 +490,8 @@ def doctor_gather_state(
|
|
|
490
490
|
|
|
491
491
|
codex_entries_count = None
|
|
492
492
|
codex_last_entry_at = None
|
|
493
|
+
codex_project_metadata_health = None
|
|
494
|
+
codex_project_metadata_error = None
|
|
493
495
|
try:
|
|
494
496
|
if _cctally_core.CACHE_DB_PATH.exists():
|
|
495
497
|
conn = sqlite3.connect(str(_cctally_core.CACHE_DB_PATH))
|
|
@@ -503,12 +505,31 @@ def doctor_gather_state(
|
|
|
503
505
|
codex_last_entry_at = parse_iso_datetime(
|
|
504
506
|
row[1], "codex_session_entries.timestamp_utc",
|
|
505
507
|
).astimezone(dt.timezone.utc)
|
|
506
|
-
|
|
507
|
-
|
|
508
|
+
# Keep the health probe on the existing read-only cache
|
|
509
|
+
# connection. A failed probe is health evidence, not an
|
|
510
|
+
# empty corpus: the kernel renders it as a distinct FAIL.
|
|
511
|
+
try:
|
|
512
|
+
import _cctally_source_analytics
|
|
513
|
+
|
|
514
|
+
health = _cctally_source_analytics.load_codex_project_metadata_health(
|
|
515
|
+
cache_conn=conn,
|
|
516
|
+
)
|
|
517
|
+
codex_project_metadata_health = {
|
|
518
|
+
"total_rows": health.total_rows,
|
|
519
|
+
"qualified_rows": health.qualified_rows,
|
|
520
|
+
"missing_conversation_key_rows": health.missing_conversation_key_rows,
|
|
521
|
+
"missing_thread_join_rows": health.missing_thread_join_rows,
|
|
522
|
+
}
|
|
523
|
+
except Exception as exc:
|
|
524
|
+
codex_project_metadata_error = type(exc).__name__
|
|
525
|
+
except sqlite3.OperationalError as exc:
|
|
526
|
+
# Pre-Codex cache shapes still produce the established Codex
|
|
527
|
+
# cache result, while the new health check fails explicitly.
|
|
528
|
+
codex_project_metadata_error = type(exc).__name__
|
|
508
529
|
finally:
|
|
509
530
|
conn.close()
|
|
510
|
-
except Exception:
|
|
511
|
-
|
|
531
|
+
except Exception as exc:
|
|
532
|
+
codex_project_metadata_error = type(exc).__name__
|
|
512
533
|
|
|
513
534
|
# Issue #109: probe every $CODEX_HOME session root (not the single
|
|
514
535
|
# hardcoded ~/.codex/sessions), matching the multi-root ingestion path
|
|
@@ -802,6 +823,8 @@ def doctor_gather_state(
|
|
|
802
823
|
codex_entries_count=codex_entries_count,
|
|
803
824
|
codex_last_entry_at=codex_last_entry_at,
|
|
804
825
|
codex_jsonl_present=codex_jsonl_present,
|
|
826
|
+
codex_project_metadata_health=codex_project_metadata_health,
|
|
827
|
+
codex_project_metadata_error=codex_project_metadata_error,
|
|
805
828
|
dashboard_bind_stored=dashboard_bind_stored,
|
|
806
829
|
runtime_bind=runtime_bind,
|
|
807
830
|
# Conversation viewer (Plan 2, spec §5): only consulted on a LAN bind.
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
import datetime as dt
|
|
5
|
+
import dataclasses
|
|
5
6
|
import hashlib
|
|
6
7
|
import json
|
|
7
8
|
import pathlib
|
|
@@ -20,6 +21,7 @@ from _cctally_core import (
|
|
|
20
21
|
)
|
|
21
22
|
from _cctally_cache import _codex_provider_roots
|
|
22
23
|
from _lib_quota import build_blocks
|
|
24
|
+
from _lib_pricing import _calculate_codex_entry_cost
|
|
23
25
|
from _cctally_quota import load_codex_quota_observations
|
|
24
26
|
from _lib_source_analytics import (
|
|
25
27
|
AnalyticsWindow,
|
|
@@ -50,6 +52,37 @@ class SourceUsageError(ValueError):
|
|
|
50
52
|
"""A provider-aware request is syntactically or semantically invalid."""
|
|
51
53
|
|
|
52
54
|
|
|
55
|
+
@dataclasses.dataclass(frozen=True)
|
|
56
|
+
class CodexProjectMetadataHealth:
|
|
57
|
+
"""A root-qualified partition of retained Codex accounting metadata."""
|
|
58
|
+
|
|
59
|
+
total_rows: int
|
|
60
|
+
qualified_rows: int
|
|
61
|
+
missing_conversation_key_rows: int
|
|
62
|
+
missing_thread_join_rows: int
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def incomplete_rows(self) -> int:
|
|
66
|
+
return self.missing_conversation_key_rows + self.missing_thread_join_rows
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclasses.dataclass(frozen=True)
|
|
70
|
+
class RootedCodexAccountingEntry:
|
|
71
|
+
"""One cache-only Codex accounting row with authoritative root identity."""
|
|
72
|
+
|
|
73
|
+
timestamp: dt.datetime
|
|
74
|
+
session_id: str
|
|
75
|
+
source_path: str
|
|
76
|
+
source_root_key: str
|
|
77
|
+
model: str
|
|
78
|
+
input_tokens: int
|
|
79
|
+
cached_input_tokens: int
|
|
80
|
+
output_tokens: int
|
|
81
|
+
reasoning_output_tokens: int
|
|
82
|
+
total_tokens: int
|
|
83
|
+
cost_usd: float
|
|
84
|
+
|
|
85
|
+
|
|
53
86
|
_OPAQUE_PROJECT_KEY_RE = re.compile(r"^project:[0-9a-f]{24}$")
|
|
54
87
|
|
|
55
88
|
|
|
@@ -74,6 +107,18 @@ _QUALIFIED_CODEX_ENTRIES_SQL = """
|
|
|
74
107
|
"""
|
|
75
108
|
|
|
76
109
|
|
|
110
|
+
_INHERITED_CODEX_PROJECT_METADATA_SQL = """
|
|
111
|
+
SELECT files.source_root_key, files.path, inherited.cwd, inherited.git_json
|
|
112
|
+
FROM codex_session_files AS files
|
|
113
|
+
JOIN codex_conversation_threads AS inherited
|
|
114
|
+
ON inherited.source_root_key = files.source_root_key
|
|
115
|
+
AND inherited.native_thread_id = files.last_native_thread_id
|
|
116
|
+
WHERE files.last_native_thread_id IS NOT NULL
|
|
117
|
+
AND files.last_native_thread_id != ''
|
|
118
|
+
ORDER BY inherited.last_seen_utc DESC, inherited.conversation_key DESC
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
|
|
77
122
|
_CODEX_ACCOUNTING_ENTRIES_SQL = """
|
|
78
123
|
SELECT timestamp_utc, source_root_key, conversation_key, model,
|
|
79
124
|
input_tokens, cached_input_tokens, output_tokens,
|
|
@@ -86,6 +131,75 @@ _CODEX_ACCOUNTING_ENTRIES_SQL = """
|
|
|
86
131
|
"""
|
|
87
132
|
|
|
88
133
|
|
|
134
|
+
_ROOTED_CODEX_ACCOUNTING_ENTRIES_SQL = """
|
|
135
|
+
SELECT timestamp_utc, session_id, source_path, source_root_key, model,
|
|
136
|
+
input_tokens, cached_input_tokens, output_tokens,
|
|
137
|
+
reasoning_output_tokens, total_tokens
|
|
138
|
+
FROM codex_session_entries INDEXED BY idx_codex_entries_ts_root_conversation
|
|
139
|
+
WHERE timestamp_utc >= ?
|
|
140
|
+
AND timestamp_utc < ?
|
|
141
|
+
{root_predicate}
|
|
142
|
+
ORDER BY timestamp_utc ASC, source_root_key ASC, conversation_key ASC, id ASC
|
|
143
|
+
"""
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
_CODEX_PROJECT_METADATA_HEALTH_SQL = """
|
|
147
|
+
SELECT
|
|
148
|
+
COUNT(*) AS total_rows,
|
|
149
|
+
COALESCE(SUM(CASE
|
|
150
|
+
WHEN entries.conversation_key IS NULL OR entries.conversation_key = ''
|
|
151
|
+
THEN 1 ELSE 0 END), 0) AS missing_conversation_key_rows,
|
|
152
|
+
COALESCE(SUM(CASE
|
|
153
|
+
WHEN entries.conversation_key IS NOT NULL
|
|
154
|
+
AND entries.conversation_key != ''
|
|
155
|
+
AND threads.conversation_key IS NULL
|
|
156
|
+
AND NOT EXISTS (
|
|
157
|
+
SELECT 1 FROM codex_session_files AS files
|
|
158
|
+
JOIN codex_conversation_threads AS inherited
|
|
159
|
+
ON inherited.source_root_key = entries.source_root_key
|
|
160
|
+
AND inherited.native_thread_id = files.last_native_thread_id
|
|
161
|
+
WHERE files.path = entries.source_path
|
|
162
|
+
AND files.source_root_key = entries.source_root_key
|
|
163
|
+
)
|
|
164
|
+
THEN 1 ELSE 0 END), 0) AS missing_thread_join_rows
|
|
165
|
+
FROM codex_session_entries AS entries
|
|
166
|
+
LEFT JOIN codex_conversation_threads AS threads
|
|
167
|
+
ON threads.conversation_key = entries.conversation_key
|
|
168
|
+
AND threads.source_root_key = entries.source_root_key
|
|
169
|
+
WHERE (? IS NULL OR entries.timestamp_utc >= ?)
|
|
170
|
+
AND (? IS NULL OR entries.timestamp_utc < ?)
|
|
171
|
+
"""
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
_CODEX_PROJECT_METADATA_HEALTH_LEGACY_SQL = """
|
|
175
|
+
SELECT
|
|
176
|
+
COUNT(*) AS total_rows,
|
|
177
|
+
COALESCE(SUM(CASE
|
|
178
|
+
WHEN entries.conversation_key IS NULL OR entries.conversation_key = ''
|
|
179
|
+
THEN 1 ELSE 0 END), 0) AS missing_conversation_key_rows,
|
|
180
|
+
COALESCE(SUM(CASE
|
|
181
|
+
WHEN entries.conversation_key IS NOT NULL
|
|
182
|
+
AND entries.conversation_key != ''
|
|
183
|
+
AND threads.conversation_key IS NULL
|
|
184
|
+
THEN 1 ELSE 0 END), 0) AS missing_thread_join_rows
|
|
185
|
+
FROM codex_session_entries AS entries
|
|
186
|
+
LEFT JOIN codex_conversation_threads AS threads
|
|
187
|
+
ON threads.conversation_key = entries.conversation_key
|
|
188
|
+
AND threads.source_root_key = entries.source_root_key
|
|
189
|
+
WHERE (? IS NULL OR entries.timestamp_utc >= ?)
|
|
190
|
+
AND (? IS NULL OR entries.timestamp_utc < ?)
|
|
191
|
+
"""
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _supports_native_file_aliases(cache_conn: sqlite3.Connection) -> bool:
|
|
195
|
+
"""Return whether this cache generation can link child files to a root task."""
|
|
196
|
+
try:
|
|
197
|
+
columns = cache_conn.execute("PRAGMA table_info(codex_session_files)").fetchall()
|
|
198
|
+
except sqlite3.Error:
|
|
199
|
+
return False
|
|
200
|
+
return any(len(row) > 1 and row[1] == "last_native_thread_id" for row in columns)
|
|
201
|
+
|
|
202
|
+
|
|
89
203
|
def _cctally():
|
|
90
204
|
return sys.modules["cctally"]
|
|
91
205
|
|
|
@@ -100,6 +214,160 @@ def _parse_timestamp(value: object) -> dt.datetime:
|
|
|
100
214
|
return timestamp.astimezone(UTC)
|
|
101
215
|
|
|
102
216
|
|
|
217
|
+
def _metadata_health_bound(value: dt.datetime | None, *, name: str) -> str | None:
|
|
218
|
+
if value is None:
|
|
219
|
+
return None
|
|
220
|
+
if value.tzinfo is None or value.utcoffset() is None:
|
|
221
|
+
raise ValueError(f"{name} must be timezone-aware")
|
|
222
|
+
# Retain the canonical ``+00:00`` representation written by Codex ingest
|
|
223
|
+
# and consumed by the accounting readers: SQLite TEXT bounds must sort in
|
|
224
|
+
# the same encoding for the [start, end) window to remain exact.
|
|
225
|
+
return value.astimezone(UTC).isoformat()
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def load_codex_project_metadata_health(
|
|
229
|
+
*,
|
|
230
|
+
cache_conn: sqlite3.Connection,
|
|
231
|
+
start: dt.datetime | None = None,
|
|
232
|
+
end: dt.datetime | None = None,
|
|
233
|
+
) -> CodexProjectMetadataHealth:
|
|
234
|
+
"""Classify retained Codex accounting in one root-qualified SQL read.
|
|
235
|
+
|
|
236
|
+
Bounds are both timezone-aware or both omitted; the omitted form serves the
|
|
237
|
+
all-history doctor check. The output is an exhaustive, mutually-exclusive
|
|
238
|
+
count partition and contains no source or identity data.
|
|
239
|
+
"""
|
|
240
|
+
if (start is None) != (end is None):
|
|
241
|
+
raise ValueError("start and end must be supplied together or both omitted")
|
|
242
|
+
bound_start = _metadata_health_bound(start, name="start")
|
|
243
|
+
bound_end = _metadata_health_bound(end, name="end")
|
|
244
|
+
if start is not None and end is not None and start > end:
|
|
245
|
+
raise ValueError("start must not be after end")
|
|
246
|
+
|
|
247
|
+
row = cache_conn.execute(
|
|
248
|
+
(_CODEX_PROJECT_METADATA_HEALTH_SQL if _supports_native_file_aliases(cache_conn)
|
|
249
|
+
else _CODEX_PROJECT_METADATA_HEALTH_LEGACY_SQL),
|
|
250
|
+
(bound_start, bound_start, bound_end, bound_end),
|
|
251
|
+
).fetchone()
|
|
252
|
+
if row is None or len(row) != 3:
|
|
253
|
+
raise RuntimeError("Codex project metadata health query returned no partition")
|
|
254
|
+
try:
|
|
255
|
+
total_rows, missing_key_rows, missing_join_rows = (int(value) for value in row)
|
|
256
|
+
except (TypeError, ValueError) as exc:
|
|
257
|
+
raise RuntimeError("Codex project metadata health query returned invalid counts") from exc
|
|
258
|
+
qualified_rows = total_rows - missing_key_rows - missing_join_rows
|
|
259
|
+
values = (total_rows, qualified_rows, missing_key_rows, missing_join_rows)
|
|
260
|
+
if any(value < 0 for value in values) or total_rows != sum(values[1:]):
|
|
261
|
+
raise RuntimeError("Codex project metadata health partition is invalid")
|
|
262
|
+
return CodexProjectMetadataHealth(
|
|
263
|
+
total_rows=total_rows,
|
|
264
|
+
qualified_rows=qualified_rows,
|
|
265
|
+
missing_conversation_key_rows=missing_key_rows,
|
|
266
|
+
missing_thread_join_rows=missing_join_rows,
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def load_cached_rooted_codex_accounting_entries(
|
|
271
|
+
start: dt.datetime,
|
|
272
|
+
end: dt.datetime,
|
|
273
|
+
*,
|
|
274
|
+
speed: str,
|
|
275
|
+
cache_conn: sqlite3.Connection,
|
|
276
|
+
source_root_keys: Iterable[str] | None = None,
|
|
277
|
+
) -> tuple[RootedCodexAccountingEntry, ...]:
|
|
278
|
+
"""Read bounded accounting from one caller-owned cache snapshot only.
|
|
279
|
+
|
|
280
|
+
This reader intentionally never syncs, opens or closes a database, reads
|
|
281
|
+
rollouts, resolves projects, or discovers configured Codex roots. The
|
|
282
|
+
cached root key plus source path are the dashboard's file identity. A
|
|
283
|
+
caller can additionally constrain the read to roots that established one
|
|
284
|
+
native cycle; root identities remain internal to this adapter.
|
|
285
|
+
"""
|
|
286
|
+
if start.tzinfo is None or start.utcoffset() is None:
|
|
287
|
+
raise ValueError("start must be timezone-aware")
|
|
288
|
+
if end.tzinfo is None or end.utcoffset() is None:
|
|
289
|
+
raise ValueError("end must be timezone-aware")
|
|
290
|
+
if end <= start:
|
|
291
|
+
raise ValueError("end must be after start")
|
|
292
|
+
root_keys = (
|
|
293
|
+
tuple(sorted({key for key in source_root_keys if isinstance(key, str) and key}))
|
|
294
|
+
if source_root_keys is not None else ()
|
|
295
|
+
)
|
|
296
|
+
if source_root_keys is not None and not root_keys:
|
|
297
|
+
return ()
|
|
298
|
+
root_predicate = (
|
|
299
|
+
"AND source_root_key IN (" + ", ".join("?" for _ in root_keys) + ")"
|
|
300
|
+
if source_root_keys is not None else ""
|
|
301
|
+
)
|
|
302
|
+
try:
|
|
303
|
+
rows = tuple(cache_conn.execute(
|
|
304
|
+
_ROOTED_CODEX_ACCOUNTING_ENTRIES_SQL.format(root_predicate=root_predicate),
|
|
305
|
+
(
|
|
306
|
+
start.astimezone(UTC).isoformat(), end.astimezone(UTC).isoformat(), *root_keys,
|
|
307
|
+
),
|
|
308
|
+
))
|
|
309
|
+
except sqlite3.Error as exc:
|
|
310
|
+
raise QualifiedMetadataUnavailable("Codex accounting metadata is unavailable") from exc
|
|
311
|
+
|
|
312
|
+
result: list[RootedCodexAccountingEntry] = []
|
|
313
|
+
for row in rows:
|
|
314
|
+
try:
|
|
315
|
+
(
|
|
316
|
+
timestamp_raw, session_id_raw, source_path_raw, source_root_key_raw,
|
|
317
|
+
model_raw, input_raw, cached_input_raw, output_raw, reasoning_raw,
|
|
318
|
+
total_raw,
|
|
319
|
+
) = row
|
|
320
|
+
source_path = source_path_raw if isinstance(source_path_raw, str) else ""
|
|
321
|
+
source_root_key = source_root_key_raw if isinstance(source_root_key_raw, str) else ""
|
|
322
|
+
if not source_path or not source_root_key:
|
|
323
|
+
raise ValueError("rooted accounting identity is absent")
|
|
324
|
+
model = str(model_raw)
|
|
325
|
+
input_tokens = int(input_raw)
|
|
326
|
+
cached_input_tokens = int(cached_input_raw)
|
|
327
|
+
output_tokens = int(output_raw)
|
|
328
|
+
reasoning_output_tokens = int(reasoning_raw)
|
|
329
|
+
total_tokens = int(total_raw)
|
|
330
|
+
cost_usd = _calculate_codex_entry_cost(
|
|
331
|
+
model,
|
|
332
|
+
input_tokens,
|
|
333
|
+
cached_input_tokens,
|
|
334
|
+
output_tokens,
|
|
335
|
+
reasoning_output_tokens,
|
|
336
|
+
speed=speed,
|
|
337
|
+
)
|
|
338
|
+
except (TypeError, ValueError, OverflowError) as exc:
|
|
339
|
+
raise QualifiedMetadataUnavailable("Codex accounting metadata is unavailable") from exc
|
|
340
|
+
result.append(RootedCodexAccountingEntry(
|
|
341
|
+
timestamp=_parse_timestamp(timestamp_raw),
|
|
342
|
+
session_id=str(session_id_raw or ""),
|
|
343
|
+
source_path=source_path,
|
|
344
|
+
source_root_key=source_root_key,
|
|
345
|
+
model=model,
|
|
346
|
+
input_tokens=input_tokens,
|
|
347
|
+
cached_input_tokens=cached_input_tokens,
|
|
348
|
+
output_tokens=output_tokens,
|
|
349
|
+
reasoning_output_tokens=reasoning_output_tokens,
|
|
350
|
+
total_tokens=total_tokens,
|
|
351
|
+
cost_usd=cost_usd,
|
|
352
|
+
))
|
|
353
|
+
return tuple(result)
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def has_cached_codex_accounting_entries(*, cache_conn: sqlite3.Connection) -> bool:
|
|
357
|
+
"""Return whether any retained Codex accounting exists without range filtering.
|
|
358
|
+
|
|
359
|
+
This intentionally tiny caller-owned cache read lets a missing native cycle
|
|
360
|
+
fail closed even when the dashboard's visible or budget range is empty.
|
|
361
|
+
It neither resolves roots nor reads rollout files.
|
|
362
|
+
"""
|
|
363
|
+
try:
|
|
364
|
+
return cache_conn.execute(
|
|
365
|
+
"SELECT 1 FROM codex_session_entries LIMIT 1"
|
|
366
|
+
).fetchone() is not None
|
|
367
|
+
except sqlite3.Error as exc:
|
|
368
|
+
raise QualifiedMetadataUnavailable("Codex accounting metadata is unavailable") from exc
|
|
369
|
+
|
|
370
|
+
|
|
103
371
|
def _project_label(value: object) -> str:
|
|
104
372
|
"""Render only a basename-like label; absolute paths remain internal."""
|
|
105
373
|
raw = str(value).strip()
|
|
@@ -191,6 +459,12 @@ def load_qualified_codex_entries(
|
|
|
191
459
|
_QUALIFIED_CODEX_ENTRIES_SQL,
|
|
192
460
|
(start.astimezone(UTC).isoformat(), end.astimezone(UTC).isoformat()),
|
|
193
461
|
))
|
|
462
|
+
inherited_metadata: dict[tuple[str, str], sqlite3.Row] = {}
|
|
463
|
+
if _supports_native_file_aliases(conn):
|
|
464
|
+
for inherited in conn.execute(_INHERITED_CODEX_PROJECT_METADATA_SQL):
|
|
465
|
+
identity = (str(inherited["source_root_key"] or ""), str(inherited["path"] or ""))
|
|
466
|
+
if all(identity):
|
|
467
|
+
inherited_metadata.setdefault(identity, inherited)
|
|
194
468
|
except sqlite3.Error as exc:
|
|
195
469
|
raise QualifiedMetadataUnavailable("Codex qualified project metadata is unavailable") from exc
|
|
196
470
|
finally:
|
|
@@ -204,8 +478,19 @@ def load_qualified_codex_entries(
|
|
|
204
478
|
resolved_by_git_json: dict[str, str | None] = {}
|
|
205
479
|
result: list[QualifiedCodexEntry] = []
|
|
206
480
|
for row in rows:
|
|
207
|
-
root_key
|
|
208
|
-
|
|
481
|
+
root_key = row["source_root_key"]
|
|
482
|
+
conversation_key = row["conversation_key"]
|
|
483
|
+
inherited = inherited_metadata.get((str(root_key or ""), str(row["source_path"] or "")))
|
|
484
|
+
try:
|
|
485
|
+
root_key, conversation_key = _require_joined_metadata(row)
|
|
486
|
+
except QualifiedMetadataUnavailable:
|
|
487
|
+
if (
|
|
488
|
+
inherited is None
|
|
489
|
+
or not isinstance(root_key, str) or not root_key
|
|
490
|
+
or not isinstance(conversation_key, str) or not conversation_key
|
|
491
|
+
):
|
|
492
|
+
raise
|
|
493
|
+
cwd = row["cwd"] or (inherited["cwd"] if inherited is not None else None)
|
|
209
494
|
if isinstance(cwd, str) and cwd:
|
|
210
495
|
project = resolved_by_cwd.get(cwd)
|
|
211
496
|
if project is None:
|
|
@@ -218,7 +503,7 @@ def load_qualified_codex_entries(
|
|
|
218
503
|
else _project_label(project.display_key)
|
|
219
504
|
)
|
|
220
505
|
else:
|
|
221
|
-
git_json = row["git_json"]
|
|
506
|
+
git_json = row["git_json"] or (inherited["git_json"] if inherited is not None else None)
|
|
222
507
|
if isinstance(git_json, str) and git_json not in resolved_by_git_json:
|
|
223
508
|
resolved_by_git_json[git_json] = _git_resolved_key(git_json)
|
|
224
509
|
git_key = resolved_by_git_json.get(git_json) if isinstance(git_json, str) else None
|
package/bin/_cctally_tui.py
CHANGED
|
@@ -227,6 +227,7 @@ from _lib_fmt import stable_sum
|
|
|
227
227
|
# shared no-op singleton), so the _tui_build_snapshot seam wraps below cost
|
|
228
228
|
# nothing on the default path.
|
|
229
229
|
import _lib_perf as _perf
|
|
230
|
+
import _lib_log
|
|
230
231
|
|
|
231
232
|
import importlib.util as _ilu
|
|
232
233
|
|
|
@@ -261,7 +262,6 @@ from _lib_forecast import _compute_forecast, ForecastInputs, ForecastOutput, Bud
|
|
|
261
262
|
_ensure_sibling_loaded("_lib_dashboard_sources")
|
|
262
263
|
_ensure_sibling_loaded("_cctally_dashboard_sources")
|
|
263
264
|
from _cctally_dashboard_sources import (
|
|
264
|
-
CodexProjectionIncoherent,
|
|
265
265
|
DashboardReadContext,
|
|
266
266
|
build_codex_source_state,
|
|
267
267
|
refresh_codex_source_clock,
|
|
@@ -2520,8 +2520,18 @@ def _tui_build_source_bundle(
|
|
|
2520
2520
|
else unavailable_source_state("codex", warning)
|
|
2521
2521
|
)
|
|
2522
2522
|
else:
|
|
2523
|
-
|
|
2524
|
-
|
|
2523
|
+
# Projection coherence can recover when S2 writes its certificate
|
|
2524
|
+
# without changing physical accounting. A current, hero-scoped
|
|
2525
|
+
# incoherence generation is intentionally not retained through
|
|
2526
|
+
# that repair opportunity; all other fresh partial states remain
|
|
2527
|
+
# eligible for exact reuse.
|
|
2528
|
+
codex = (
|
|
2529
|
+
None if prior_codex is not None and any(
|
|
2530
|
+
warning.code == "codex_projection_incoherent"
|
|
2531
|
+
for warning in prior_codex.warnings
|
|
2532
|
+
) else reuse_coherent_source_state(
|
|
2533
|
+
prior_codex, data_version=codex_version,
|
|
2534
|
+
)
|
|
2525
2535
|
)
|
|
2526
2536
|
if codex is None:
|
|
2527
2537
|
try:
|
|
@@ -2539,18 +2549,11 @@ def _tui_build_source_bundle(
|
|
|
2539
2549
|
),
|
|
2540
2550
|
data_version=codex_version,
|
|
2541
2551
|
)
|
|
2542
|
-
except CodexProjectionIncoherent:
|
|
2543
|
-
warning = SourceDashboardWarning(
|
|
2544
|
-
"codex_projection_incoherent",
|
|
2545
|
-
"Codex quota projection is unavailable.",
|
|
2546
|
-
"quota",
|
|
2547
|
-
)
|
|
2548
|
-
codex = (
|
|
2549
|
-
degrade_source_state(prior_codex, warning)
|
|
2550
|
-
if prior_codex is not None
|
|
2551
|
-
else unavailable_source_state("codex", warning)
|
|
2552
|
-
)
|
|
2553
2552
|
except Exception:
|
|
2553
|
+
_lib_log.get_logger("dashboard").error(
|
|
2554
|
+
"codex_read_model source build failed",
|
|
2555
|
+
exc_info=True,
|
|
2556
|
+
)
|
|
2554
2557
|
warning = SourceDashboardWarning(
|
|
2555
2558
|
"source_build_failed", "Source data could not be built.", "read_model",
|
|
2556
2559
|
)
|