cctally 1.76.0 → 1.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/bin/_cctally_cache.py +918 -143
- package/bin/_cctally_core.py +19 -5
- package/bin/_cctally_dashboard.py +226 -51
- package/bin/_cctally_dashboard_conversation.py +43 -19
- package/bin/_cctally_dashboard_sources.py +28 -9
- package/bin/_cctally_db.py +207 -5
- package/bin/_cctally_doctor.py +56 -10
- package/bin/_cctally_parser.py +1 -1
- package/bin/_cctally_quota.py +23 -24
- package/bin/_cctally_statusline.py +128 -3
- package/bin/_cctally_transcript.py +4 -4
- package/bin/_cctally_tui.py +6 -34
- package/bin/_lib_codex_conversation_query.py +19 -3
- package/bin/_lib_conversation_query.py +8 -3
- package/bin/_lib_conversation_retention.py +3 -3
- package/bin/_lib_dashboard_json.py +43 -0
- package/bin/_lib_doctor.py +53 -4
- package/bin/_lib_source_analytics.py +20 -6
- package/bin/cctally +8 -0
- package/dashboard/static/assets/index-CkTe6VJt.js +87 -0
- package/dashboard/static/assets/index-DsEXuMpq.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +2 -1
- package/dashboard/static/assets/index-CsDAxBT5.js +0 -80
- package/dashboard/static/assets/index-yftBNnLR.css +0 -1
package/bin/_cctally_quota.py
CHANGED
|
@@ -279,33 +279,32 @@ def load_codex_quota_observations(
|
|
|
279
279
|
}
|
|
280
280
|
return required <= columns
|
|
281
281
|
|
|
282
|
-
has_conversation_events = has_columns(
|
|
283
|
-
"codex_conversation_events",
|
|
284
|
-
{"source_path", "line_offset", "record_type", "payload_json"},
|
|
285
|
-
)
|
|
286
282
|
has_session_entries = has_columns(
|
|
287
283
|
"codex_session_entries", {"source_path", "line_offset", "model"},
|
|
288
284
|
)
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
285
|
+
has_observed_model = has_columns(
|
|
286
|
+
"quota_window_snapshots", {"observed_model"},
|
|
287
|
+
)
|
|
288
|
+
entry_lookup = (
|
|
289
|
+
"(SELECT entries.model FROM codex_session_entries AS entries "
|
|
290
|
+
"WHERE entries.source_path=quota_window_snapshots.source_path "
|
|
291
|
+
"AND entries.line_offset<=quota_window_snapshots.line_offset "
|
|
292
|
+
"ORDER BY entries.line_offset DESC LIMIT 1)"
|
|
293
|
+
)
|
|
294
|
+
# A file's terminal model is not evidence for an earlier quota sample:
|
|
295
|
+
# the model may change later in the rollout. Prefer the compact stamp,
|
|
296
|
+
# then only a nearest-prior accounting context for legacy rows. If
|
|
297
|
+
# neither exists, leave the pool unscoped rather than fabricating it.
|
|
298
|
+
fallback_model = entry_lookup if has_session_entries else "NULL"
|
|
299
|
+
selected_model = (
|
|
300
|
+
f"COALESCE(quota_window_snapshots.observed_model,{fallback_model})"
|
|
301
|
+
if has_observed_model and fallback_model != "NULL"
|
|
302
|
+
else (
|
|
303
|
+
"quota_window_snapshots.observed_model"
|
|
304
|
+
if has_observed_model else fallback_model
|
|
305
|
+
)
|
|
306
|
+
)
|
|
307
|
+
model_expr = f"{selected_model} AS observed_model"
|
|
309
308
|
sql = """
|
|
310
309
|
SELECT source, source_root_key, source_path, line_offset,
|
|
311
310
|
captured_at_utc, observed_slot, logical_limit_key, limit_id,
|
|
@@ -275,13 +275,22 @@ def cmd_statusline(args: argparse.Namespace) -> int:
|
|
|
275
275
|
# side effect that runs AFTER the line is printed and is FULLY guarded:
|
|
276
276
|
# persistence must NEVER break or slow rendering, so the whole call is
|
|
277
277
|
# wrapped here (the feeder itself forks detached + swallows its own
|
|
278
|
-
# errors, but this is belt-and-suspenders). Absence of rate_limits
|
|
279
|
-
# lost persist lock
|
|
280
|
-
#
|
|
278
|
+
# errors, but this is belt-and-suspenders). Absence of rate_limits or a
|
|
279
|
+
# lost persist lock degrades to a clean no-op; the authoritative OAuth
|
|
280
|
+
# confirmation below covers the missing/stale-payload case.
|
|
281
281
|
try:
|
|
282
282
|
_statusline_persist(inp)
|
|
283
283
|
except Exception:
|
|
284
284
|
pass
|
|
285
|
+
# Claude Code may replay an unchanged rate_limits object across timer
|
|
286
|
+
# renders even after the provider counters have advanced. Confirm against
|
|
287
|
+
# the authoritative OAuth surface once per account-wide timer cycle. The
|
|
288
|
+
# helper is fully detached, shares hook-tick's throttle lock/marker, and
|
|
289
|
+
# honors the existing selected-freshness and 429-backoff gates.
|
|
290
|
+
try:
|
|
291
|
+
c._statusline_oauth_tick()
|
|
292
|
+
except Exception:
|
|
293
|
+
pass
|
|
285
294
|
return 0
|
|
286
295
|
|
|
287
296
|
|
|
@@ -1229,6 +1238,122 @@ def _statusline_persist(parsed, *, sync_for_test: bool = False) -> None:
|
|
|
1229
1238
|
c._release_persist_lock(lock_fd)
|
|
1230
1239
|
|
|
1231
1240
|
|
|
1241
|
+
def _try_acquire_statusline_oauth_lock() -> "int | None":
|
|
1242
|
+
"""Take hook-tick's account-wide OAuth throttle lock without blocking."""
|
|
1243
|
+
try:
|
|
1244
|
+
_cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
1245
|
+
fd = os.open(
|
|
1246
|
+
_cctally_core.HOOK_TICK_THROTTLE_LOCK_PATH,
|
|
1247
|
+
os.O_WRONLY | os.O_CREAT,
|
|
1248
|
+
0o644,
|
|
1249
|
+
)
|
|
1250
|
+
except OSError:
|
|
1251
|
+
return None
|
|
1252
|
+
try:
|
|
1253
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1254
|
+
except OSError:
|
|
1255
|
+
try:
|
|
1256
|
+
os.close(fd)
|
|
1257
|
+
except OSError:
|
|
1258
|
+
pass
|
|
1259
|
+
return None
|
|
1260
|
+
return fd
|
|
1261
|
+
|
|
1262
|
+
|
|
1263
|
+
def _release_statusline_oauth_lock(fd: int) -> None:
|
|
1264
|
+
try:
|
|
1265
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
1266
|
+
except OSError:
|
|
1267
|
+
pass
|
|
1268
|
+
try:
|
|
1269
|
+
os.close(fd)
|
|
1270
|
+
except OSError:
|
|
1271
|
+
pass
|
|
1272
|
+
|
|
1273
|
+
|
|
1274
|
+
def _run_statusline_oauth_refresh() -> None:
|
|
1275
|
+
"""Run one already-throttled automatic OAuth confirmation."""
|
|
1276
|
+
c = _cctally()
|
|
1277
|
+
status, _ = c._hook_tick_oauth_refresh(
|
|
1278
|
+
throttle_seconds=float(_cctally_core.STATUSLINE_OAUTH_POLL_SECONDS)
|
|
1279
|
+
)
|
|
1280
|
+
if status.startswith("ok("):
|
|
1281
|
+
c._hook_tick_throttle_touch()
|
|
1282
|
+
|
|
1283
|
+
|
|
1284
|
+
def _fork_statusline_oauth_refresh(lock_fd: int) -> bool:
|
|
1285
|
+
"""Detach one OAuth confirmation while keeping ``lock_fd`` in the child."""
|
|
1286
|
+
try:
|
|
1287
|
+
pid = os.fork()
|
|
1288
|
+
except OSError:
|
|
1289
|
+
return False
|
|
1290
|
+
if pid > 0:
|
|
1291
|
+
# Do not explicitly unlock: the child inherited the same open file
|
|
1292
|
+
# description and owns the account-wide lock until its refresh ends.
|
|
1293
|
+
try:
|
|
1294
|
+
os.close(lock_fd)
|
|
1295
|
+
except OSError:
|
|
1296
|
+
pass
|
|
1297
|
+
return True
|
|
1298
|
+
|
|
1299
|
+
try:
|
|
1300
|
+
try:
|
|
1301
|
+
os.setsid()
|
|
1302
|
+
except OSError:
|
|
1303
|
+
pass
|
|
1304
|
+
try:
|
|
1305
|
+
devnull = os.open(os.devnull, os.O_RDWR)
|
|
1306
|
+
os.dup2(devnull, 0)
|
|
1307
|
+
os.dup2(devnull, 1)
|
|
1308
|
+
os.dup2(devnull, 2)
|
|
1309
|
+
if devnull > 2:
|
|
1310
|
+
os.close(devnull)
|
|
1311
|
+
except OSError:
|
|
1312
|
+
pass
|
|
1313
|
+
_run_statusline_oauth_refresh()
|
|
1314
|
+
except BaseException:
|
|
1315
|
+
pass
|
|
1316
|
+
finally:
|
|
1317
|
+
_release_statusline_oauth_lock(lock_fd)
|
|
1318
|
+
os._exit(0)
|
|
1319
|
+
|
|
1320
|
+
|
|
1321
|
+
def _statusline_oauth_tick(*, sync_for_test: bool = False) -> None:
|
|
1322
|
+
"""Schedule one bounded authoritative refresh from a statusline timer.
|
|
1323
|
+
|
|
1324
|
+
Fast prechecks avoid forks while selected data is fresh, a shared 429
|
|
1325
|
+
deadline is active, or another session/hook owns the account-wide poll.
|
|
1326
|
+
Every condition is rechecked under hook-tick's throttle lock before the
|
|
1327
|
+
detached child starts.
|
|
1328
|
+
"""
|
|
1329
|
+
c = _cctally()
|
|
1330
|
+
interval = float(_cctally_core.STATUSLINE_OAUTH_POLL_SECONDS)
|
|
1331
|
+
if c._statusline_observe_age_seconds() < interval:
|
|
1332
|
+
return
|
|
1333
|
+
if c._oauth_backoff_remaining_seconds() > 0:
|
|
1334
|
+
return
|
|
1335
|
+
if c._hook_tick_throttle_age_seconds() < interval:
|
|
1336
|
+
return
|
|
1337
|
+
lock_fd = _try_acquire_statusline_oauth_lock()
|
|
1338
|
+
if lock_fd is None:
|
|
1339
|
+
return
|
|
1340
|
+
try:
|
|
1341
|
+
if c._statusline_observe_age_seconds() < interval:
|
|
1342
|
+
return
|
|
1343
|
+
if c._oauth_backoff_remaining_seconds() > 0:
|
|
1344
|
+
return
|
|
1345
|
+
if c._hook_tick_throttle_age_seconds() < interval:
|
|
1346
|
+
return
|
|
1347
|
+
if sync_for_test:
|
|
1348
|
+
_run_statusline_oauth_refresh()
|
|
1349
|
+
return
|
|
1350
|
+
if _fork_statusline_oauth_refresh(lock_fd):
|
|
1351
|
+
lock_fd = -1
|
|
1352
|
+
finally:
|
|
1353
|
+
if lock_fd >= 0:
|
|
1354
|
+
_release_statusline_oauth_lock(lock_fd)
|
|
1355
|
+
|
|
1356
|
+
|
|
1232
1357
|
def _resolve_context_window(model_id, warn_once) -> "int | None":
|
|
1233
1358
|
"""Look up ``model_id`` in ``CLAUDE_MODEL_CONTEXT_WINDOWS``; fall back
|
|
1234
1359
|
to a family-substring match against
|
|
@@ -86,7 +86,7 @@ def _cmd_transcript_export(args) -> int:
|
|
|
86
86
|
eprint(_SPEED_ONLY_CODEX_MSG)
|
|
87
87
|
return 2
|
|
88
88
|
|
|
89
|
-
conn = c.
|
|
89
|
+
conn = c.open_conversations_db()
|
|
90
90
|
try:
|
|
91
91
|
cq = c._load_sibling("_lib_conversation_query")
|
|
92
92
|
md = cq.get_conversation_export(conn, session_id, scope)
|
|
@@ -118,7 +118,7 @@ def _cmd_transcript_export_qualified(
|
|
|
118
118
|
return 2
|
|
119
119
|
speed = c._resolve_codex_speed(speed_arg or "auto")
|
|
120
120
|
|
|
121
|
-
conn = c.
|
|
121
|
+
conn = c.open_conversations_db()
|
|
122
122
|
try:
|
|
123
123
|
env = disp.neutral_export(
|
|
124
124
|
conn, session_id, scope=scope, effective_speed=speed)
|
|
@@ -189,7 +189,7 @@ def _cmd_transcript_search_claude(args) -> int:
|
|
|
189
189
|
eprint(f"transcript: {exc}")
|
|
190
190
|
return 2
|
|
191
191
|
|
|
192
|
-
conn = c.
|
|
192
|
+
conn = c.open_conversations_db()
|
|
193
193
|
try:
|
|
194
194
|
cq = c._load_sibling("_lib_conversation_query")
|
|
195
195
|
try:
|
|
@@ -264,7 +264,7 @@ def _cmd_transcript_search_codex(args) -> int:
|
|
|
264
264
|
eprint("transcript: invalid --cursor")
|
|
265
265
|
return 2
|
|
266
266
|
|
|
267
|
-
conn = c.
|
|
267
|
+
conn = c.open_conversations_db()
|
|
268
268
|
try:
|
|
269
269
|
result = disp.neutral_search(
|
|
270
270
|
conn, query, source="codex", kind=kind,
|
package/bin/_cctally_tui.py
CHANGED
|
@@ -1772,40 +1772,12 @@ def _tui_build_sessions(
|
|
|
1772
1772
|
(), now_utc=now_utc, limit=limit, display_tz=None,
|
|
1773
1773
|
aggregated_override=aggregated_override,
|
|
1774
1774
|
)
|
|
1775
|
-
|
|
1776
|
-
#
|
|
1777
|
-
#
|
|
1778
|
-
#
|
|
1779
|
-
#
|
|
1780
|
-
|
|
1781
|
-
# bounded indexed query (<= `limit` session ids, rides idx_conv_session_ts)
|
|
1782
|
-
# per snapshot build. Titles are stashed unconditionally on this
|
|
1783
|
-
# server-internal row; the privacy gate is applied later, at envelope
|
|
1784
|
-
# serialization (``snapshot_to_envelope(transcripts_visible=...)``).
|
|
1785
|
-
if rows:
|
|
1786
|
-
session_ids = [r.session_id for r in rows if r.session_id]
|
|
1787
|
-
try:
|
|
1788
|
-
conn = c.open_cache_db()
|
|
1789
|
-
except (sqlite3.DatabaseError, OSError):
|
|
1790
|
-
conn = None
|
|
1791
|
-
if conn is not None:
|
|
1792
|
-
try:
|
|
1793
|
-
titles = c._load_sibling(
|
|
1794
|
-
"_lib_conversation_query"
|
|
1795
|
-
)._session_titles_map(conn, session_ids)
|
|
1796
|
-
rows = [
|
|
1797
|
-
dataclasses.replace(r, title=titles.get(r.session_id))
|
|
1798
|
-
if r.session_id in titles else r
|
|
1799
|
-
for r in rows
|
|
1800
|
-
]
|
|
1801
|
-
except (sqlite3.DatabaseError, OSError):
|
|
1802
|
-
pass # fail-soft: leave titles None
|
|
1803
|
-
finally:
|
|
1804
|
-
try:
|
|
1805
|
-
conn.close()
|
|
1806
|
-
except sqlite3.Error:
|
|
1807
|
-
pass
|
|
1808
|
-
return rows
|
|
1775
|
+
# #320: transcript-derived titles are optional decoration. The core
|
|
1776
|
+
# dashboard/TUI snapshot must never open conversations.db, because even a
|
|
1777
|
+
# fail-soft read pays SQLite's lock timeout before it can fail. Conversation
|
|
1778
|
+
# routes retain title derivation; the accounting Sessions panel renders its
|
|
1779
|
+
# existing em-dash fallback when the independent store is unavailable.
|
|
1780
|
+
return list(view.rows)
|
|
1809
1781
|
|
|
1810
1782
|
|
|
1811
1783
|
def _tui_sessions_cached(
|
|
@@ -71,9 +71,25 @@ _SEARCH_BADGE = {
|
|
|
71
71
|
|
|
72
72
|
|
|
73
73
|
def codex_normalization_authoritative(conn: sqlite3.Connection) -> bool:
|
|
74
|
-
"""True iff
|
|
75
|
-
|
|
76
|
-
|
|
74
|
+
"""True iff the normalized Codex corpus is authoritative (§3.5).
|
|
75
|
+
|
|
76
|
+
Split stores use their provider-local rebuild marker: current schema alone
|
|
77
|
+
is not authority while migration 028's byte-zero replay is pending. Legacy
|
|
78
|
+
monolithic/bare connections retain the migration-025 stamp contract.
|
|
79
|
+
"""
|
|
80
|
+
try:
|
|
81
|
+
split = conn.execute(
|
|
82
|
+
"SELECT 1 FROM cache_meta "
|
|
83
|
+
"WHERE key='conversation_schema_version'"
|
|
84
|
+
).fetchone() is not None
|
|
85
|
+
if split:
|
|
86
|
+
pending = conn.execute(
|
|
87
|
+
"SELECT 1 FROM cache_meta "
|
|
88
|
+
"WHERE key='conversation_rebuild_codex_pending'"
|
|
89
|
+
).fetchone() is not None
|
|
90
|
+
return not pending
|
|
91
|
+
except sqlite3.OperationalError:
|
|
92
|
+
pass
|
|
77
93
|
try:
|
|
78
94
|
row = conn.execute(
|
|
79
95
|
"SELECT 1 FROM schema_migrations WHERE name = ?",
|
|
@@ -2048,7 +2048,9 @@ def _assemble_memo_clear() -> None:
|
|
|
2048
2048
|
def _assemble_memo_key(conn, session_id):
|
|
2049
2049
|
"""Return (session_id, msg_count, last_activity_utc, entry_mutation_seq), or
|
|
2050
2050
|
None to signal 'bypass the memo' — a missing rollup row or an unreadable
|
|
2051
|
-
cache_meta
|
|
2051
|
+
accounting ``cache_meta``. Split-store connections read that watermark
|
|
2052
|
+
from the read-only ``cache_db`` attachment; legacy/bare test connections
|
|
2053
|
+
retain the main-schema fallback. The caller has already confirmed the
|
|
2052
2054
|
rollup is authoritative."""
|
|
2053
2055
|
try:
|
|
2054
2056
|
row = conn.execute(
|
|
@@ -2059,9 +2061,12 @@ def _assemble_memo_key(conn, session_id):
|
|
|
2059
2061
|
if row is None:
|
|
2060
2062
|
return None
|
|
2061
2063
|
try:
|
|
2064
|
+
schemas = {row[1] for row in conn.execute("PRAGMA database_list")}
|
|
2065
|
+
meta_schema = "cache_db" if "cache_db" in schemas else "main"
|
|
2062
2066
|
seq_row = conn.execute(
|
|
2063
|
-
"SELECT value FROM cache_meta "
|
|
2064
|
-
"WHERE key='session_entries_mutation_seq'"
|
|
2067
|
+
f"SELECT value FROM {meta_schema}.cache_meta "
|
|
2068
|
+
"WHERE key='session_entries_mutation_seq'"
|
|
2069
|
+
).fetchone()
|
|
2065
2070
|
except sqlite3.OperationalError:
|
|
2066
2071
|
return None
|
|
2067
2072
|
seq = int(seq_row[0]) if seq_row and seq_row[0] is not None else 0
|
|
@@ -277,7 +277,7 @@ def _maybe_prune_conversation_retention(
|
|
|
277
277
|
core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
278
278
|
except OSError:
|
|
279
279
|
return None
|
|
280
|
-
maint_fh = open(core.
|
|
280
|
+
maint_fh = open(core.CONVERSATIONS_LOCK_MAINTENANCE_PATH, "w")
|
|
281
281
|
try:
|
|
282
282
|
try:
|
|
283
283
|
fcntl.flock(maint_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
@@ -285,13 +285,13 @@ def _maybe_prune_conversation_retention(
|
|
|
285
285
|
return None # another prune holds it; skip cleanly, do NOT stamp
|
|
286
286
|
if not force and not _retention_due(conn, now_utc):
|
|
287
287
|
return None
|
|
288
|
-
claude_fh = open(core.
|
|
288
|
+
claude_fh = open(core.CONVERSATIONS_LOCK_PATH, "w")
|
|
289
289
|
try:
|
|
290
290
|
try:
|
|
291
291
|
fcntl.flock(claude_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
292
292
|
except (BlockingIOError, OSError):
|
|
293
293
|
return None # a Claude sync is mid-flight; retry next cycle
|
|
294
|
-
codex_fh = open(core.
|
|
294
|
+
codex_fh = open(core.CONVERSATIONS_LOCK_CODEX_PATH, "w")
|
|
295
295
|
try:
|
|
296
296
|
try:
|
|
297
297
|
fcntl.flock(codex_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Strict outbound JSON contract shared by dashboard HTTP/SSE and Doctor.
|
|
2
|
+
|
|
3
|
+
Python's ``json.dumps`` defaults to emitting JavaScript-only ``NaN`` and
|
|
4
|
+
``Infinity`` tokens. Browsers reject those tokens in ``JSON.parse`` and
|
|
5
|
+
``Response.json``. Normalize supported JSON containers recursively, then keep
|
|
6
|
+
``allow_nan=False`` as the final fail-loud guard. Unsupported objects and
|
|
7
|
+
non-finite mapping keys are deliberately not coerced.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def normalize_dashboard_json(value: Any) -> Any:
|
|
17
|
+
"""Return a non-mutating JSON value with non-finite floats mapped to null."""
|
|
18
|
+
if isinstance(value, float):
|
|
19
|
+
return value if math.isfinite(value) else None
|
|
20
|
+
if isinstance(value, dict):
|
|
21
|
+
return {
|
|
22
|
+
key: normalize_dashboard_json(item)
|
|
23
|
+
for key, item in value.items()
|
|
24
|
+
}
|
|
25
|
+
if isinstance(value, list):
|
|
26
|
+
return [normalize_dashboard_json(item) for item in value]
|
|
27
|
+
if isinstance(value, tuple):
|
|
28
|
+
return tuple(normalize_dashboard_json(item) for item in value)
|
|
29
|
+
return value
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def encode_dashboard_json(value: Any, **kwargs: Any) -> str:
|
|
33
|
+
"""Serialize one outbound payload with browser-strict JSON semantics."""
|
|
34
|
+
return json.dumps(
|
|
35
|
+
normalize_dashboard_json(value),
|
|
36
|
+
allow_nan=False,
|
|
37
|
+
**kwargs,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def encode_dashboard_json_bytes(value: Any, **kwargs: Any) -> bytes:
|
|
42
|
+
"""UTF-8 bytes companion for HTTP response bodies."""
|
|
43
|
+
return encode_dashboard_json(value, **kwargs).encode("utf-8")
|
package/bin/_lib_doctor.py
CHANGED
|
@@ -206,6 +206,10 @@ class DoctorState:
|
|
|
206
206
|
# cache was absent or unreadable; the check degrades to OK.
|
|
207
207
|
cache_db_page_count: Optional[int] = None
|
|
208
208
|
cache_db_freelist_count: Optional[int] = None
|
|
209
|
+
# #320: the independently large transcript store needs the same reclaim
|
|
210
|
+
# visibility, with remediation targeted at its own VACUUM surface.
|
|
211
|
+
conversations_db_page_count: Optional[int] = None
|
|
212
|
+
conversations_db_freelist_count: Optional[int] = None
|
|
209
213
|
# #294 S2: root-qualified physical Codex quota freshness, per-root native
|
|
210
214
|
# hook state, and lifecycle activity are gathered by _cctally_doctor.
|
|
211
215
|
codex_quota_windows: Optional[list[dict]] = None
|
|
@@ -1235,21 +1239,22 @@ def _check_data_conversation_sessions_rollup(s: DoctorState) -> CheckResult:
|
|
|
1235
1239
|
row per distinct ``conversation_messages.session_id`` (#217 S1 / U9).
|
|
1236
1240
|
|
|
1237
1241
|
OK when the two counts are equal, when either is ``None`` (pre-rollup or an
|
|
1238
|
-
unreadable
|
|
1242
|
+
unreadable conversations.db — consistent with the kernel's graceful-degrade
|
|
1243
|
+
posture),
|
|
1239
1244
|
OR when a sync/reingest/backfill is in progress. WARN ONLY on a mismatch
|
|
1240
1245
|
observed in a QUIESCENT cache.
|
|
1241
1246
|
|
|
1242
|
-
False-WARN avoidance (Codex P2): ``
|
|
1247
|
+
False-WARN avoidance (Codex P2): ``sync_claude_conversations`` commits
|
|
1243
1248
|
``conversation_messages`` per file *before* the ``conversation_sessions``
|
|
1244
1249
|
recompute, and resumable reingest commits per file before its rollup
|
|
1245
1250
|
completes — so an unsynchronized read can transiently mismatch. The
|
|
1246
1251
|
in-progress signal (``conv_rollup_sync_in_progress``) is set by
|
|
1247
|
-
``doctor_gather_state`` from a NON-BLOCKING ``
|
|
1252
|
+
``doctor_gather_state`` from a NON-BLOCKING ``conversations.db.lock`` flock probe
|
|
1248
1253
|
(lock held ⇒ a writer is mid-flight) AND the presence of any pending
|
|
1249
1254
|
``cache_meta`` reingest/split/backfill flag; if either says in-progress, this
|
|
1250
1255
|
stays OK. Doctor remains read-only and never blocks on the lock.
|
|
1251
1256
|
|
|
1252
|
-
Informational only (no remediation): the next
|
|
1257
|
+
Informational only (no remediation): the next conversation sync re-derives
|
|
1253
1258
|
the rollup via its incremental DELETE+INSERT.
|
|
1254
1259
|
"""
|
|
1255
1260
|
rollup = s.conv_sessions_rollup_count
|
|
@@ -1773,6 +1778,49 @@ def _check_db_reclaimable(s: DoctorState) -> CheckResult:
|
|
|
1773
1778
|
)
|
|
1774
1779
|
|
|
1775
1780
|
|
|
1781
|
+
def _check_db_conversations_reclaimable(s: DoctorState) -> CheckResult:
|
|
1782
|
+
"""Surface transcript-store free pages without mutating the DB (#320)."""
|
|
1783
|
+
page_count = s.conversations_db_page_count
|
|
1784
|
+
freelist_count = s.conversations_db_freelist_count
|
|
1785
|
+
ratio = None
|
|
1786
|
+
if (
|
|
1787
|
+
isinstance(page_count, int)
|
|
1788
|
+
and not isinstance(page_count, bool)
|
|
1789
|
+
and page_count > 0
|
|
1790
|
+
and isinstance(freelist_count, int)
|
|
1791
|
+
and not isinstance(freelist_count, bool)
|
|
1792
|
+
and 0 <= freelist_count <= page_count
|
|
1793
|
+
):
|
|
1794
|
+
ratio = freelist_count / page_count
|
|
1795
|
+
details = {
|
|
1796
|
+
"conversations_db_page_count": page_count,
|
|
1797
|
+
"conversations_db_freelist_count": freelist_count,
|
|
1798
|
+
"conversations_db_free_ratio": ratio,
|
|
1799
|
+
"warn_ratio": DOCTOR_RECLAIMABLE_WARN_RATIO,
|
|
1800
|
+
}
|
|
1801
|
+
if ratio is not None and ratio >= DOCTOR_RECLAIMABLE_WARN_RATIO:
|
|
1802
|
+
return CheckResult(
|
|
1803
|
+
id="db.conversations_reclaimable",
|
|
1804
|
+
title="Reclaimable transcript space",
|
|
1805
|
+
severity="warn",
|
|
1806
|
+
summary=(
|
|
1807
|
+
f"high — {ratio * 100:.1f}% of conversations.db pages are free"
|
|
1808
|
+
),
|
|
1809
|
+
remediation=(
|
|
1810
|
+
"Run `cctally db vacuum --db conversations` to reclaim disk space."
|
|
1811
|
+
),
|
|
1812
|
+
details=details,
|
|
1813
|
+
)
|
|
1814
|
+
return CheckResult(
|
|
1815
|
+
id="db.conversations_reclaimable",
|
|
1816
|
+
title="Reclaimable transcript space",
|
|
1817
|
+
severity="ok",
|
|
1818
|
+
summary="below threshold",
|
|
1819
|
+
remediation=None,
|
|
1820
|
+
details=details,
|
|
1821
|
+
)
|
|
1822
|
+
|
|
1823
|
+
|
|
1776
1824
|
# Each entry is (category_id, category_title, ((check_id, evaluator_fn_name), ...)).
|
|
1777
1825
|
# The dotted check_id is the stable JSON-contract ID (spec §5.2) AND the
|
|
1778
1826
|
# fingerprint identity-slice key (spec §5.5). When an evaluator raises,
|
|
@@ -1808,6 +1856,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
|
|
|
1808
1856
|
("db.lock_state", "_check_db_lock_state"),
|
|
1809
1857
|
("db.wal_size", "_check_db_wal_size"),
|
|
1810
1858
|
("db.reclaimable", "_check_db_reclaimable"),
|
|
1859
|
+
("db.conversations_reclaimable", "_check_db_conversations_reclaimable"),
|
|
1811
1860
|
)),
|
|
1812
1861
|
("data", "Data", (
|
|
1813
1862
|
("data.latest_snapshot_age", "_check_data_latest_snapshot_age"),
|
|
@@ -83,9 +83,26 @@ def assign_collision_safe_project_labels(
|
|
|
83
83
|
selection the exact string that terminal, JSON, and share emit.
|
|
84
84
|
"""
|
|
85
85
|
values = tuple(entries)
|
|
86
|
+
assigned = collision_safe_project_label_map(
|
|
87
|
+
(entry.project_key, entry.project_label) for entry in values
|
|
88
|
+
)
|
|
89
|
+
return tuple(
|
|
90
|
+
replace(entry, display_label=assigned.get(entry.project_key, entry.project_label))
|
|
91
|
+
for entry in values
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def collision_safe_project_label_map(
|
|
96
|
+
identities_and_labels: Iterable[tuple[str, str]],
|
|
97
|
+
) -> dict[str, str]:
|
|
98
|
+
"""Allocate the shared deterministic privacy-safe label contract.
|
|
99
|
+
|
|
100
|
+
Callers supply opaque internal identities. Only the returned labels are
|
|
101
|
+
presentation data; the identity keys must remain inside the adapter.
|
|
102
|
+
"""
|
|
86
103
|
keys_by_label: dict[str, set[str]] = defaultdict(set)
|
|
87
|
-
for
|
|
88
|
-
keys_by_label[
|
|
104
|
+
for identity, label in identities_and_labels:
|
|
105
|
+
keys_by_label[label].add(identity)
|
|
89
106
|
# Never let an allocator-owned ``label (N)`` shadow a literal project
|
|
90
107
|
# label. Reserving every raw label also makes a later presentation pass
|
|
91
108
|
# unnecessary, which avoids producing ``label (N) (N)``.
|
|
@@ -107,10 +124,7 @@ def assign_collision_safe_project_labels(
|
|
|
107
124
|
assigned[key] = candidate
|
|
108
125
|
used_labels.add(candidate)
|
|
109
126
|
ordinal += 1
|
|
110
|
-
return
|
|
111
|
-
replace(entry, display_label=assigned.get(entry.project_key, entry.project_label))
|
|
112
|
-
for entry in values
|
|
113
|
-
)
|
|
127
|
+
return assigned
|
|
114
128
|
|
|
115
129
|
|
|
116
130
|
def opaque_project_key(source: str, root_key: str, resolved_key: str) -> str:
|
package/bin/cctally
CHANGED
|
@@ -558,6 +558,10 @@ _lib_log = _load_sibling("_lib_log")
|
|
|
558
558
|
_lib_json_envelope = _load_sibling("_lib_json_envelope")
|
|
559
559
|
stamp_schema_version = _lib_json_envelope.stamp_schema_version
|
|
560
560
|
|
|
561
|
+
# Strict outbound dashboard/Doctor JSON: recursive non-finite normalization
|
|
562
|
+
# followed by allow_nan=False serialization.
|
|
563
|
+
_lib_dashboard_json = _load_sibling("_lib_dashboard_json")
|
|
564
|
+
|
|
561
565
|
_lib_changelog = _load_sibling("_lib_changelog")
|
|
562
566
|
# Backward-compat re-export: callers and tests reach the helper through
|
|
563
567
|
# ``cctally._release_read_latest_release_version`` (incl. monkeypatch
|
|
@@ -1008,6 +1012,9 @@ get_codex_entries = _cctally_cache.get_codex_entries
|
|
|
1008
1012
|
_sum_codex_cost_for_range = _cctally_cache._sum_codex_cost_for_range
|
|
1009
1013
|
get_entries = _cctally_cache.get_entries
|
|
1010
1014
|
open_cache_db = _cctally_cache.open_cache_db
|
|
1015
|
+
open_conversations_db = _cctally_cache.open_conversations_db
|
|
1016
|
+
sync_claude_conversations = _cctally_cache.sync_claude_conversations
|
|
1017
|
+
sync_codex_conversations = _cctally_cache.sync_codex_conversations
|
|
1011
1018
|
_reset_orphan_warning_throttle = _cctally_cache._reset_orphan_warning_throttle
|
|
1012
1019
|
cmd_cache_sync = _cctally_cache.cmd_cache_sync
|
|
1013
1020
|
|
|
@@ -1333,6 +1340,7 @@ _build_statusline_injections = _cctally_statusline._build_statusline_injections
|
|
|
1333
1340
|
# `cctally._statusline_persist` (test surface) + the `_cctally()` accessor
|
|
1334
1341
|
# resolve them.
|
|
1335
1342
|
_statusline_persist = _cctally_statusline._statusline_persist
|
|
1343
|
+
_statusline_oauth_tick = _cctally_statusline._statusline_oauth_tick
|
|
1336
1344
|
_fork_persist = _cctally_statusline._fork_persist
|
|
1337
1345
|
_record_args = _cctally_statusline._record_args
|
|
1338
1346
|
_try_acquire_persist_lock = _cctally_statusline._try_acquire_persist_lock
|