cctally 1.75.1 → 1.77.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/bin/_cctally_cache.py +918 -143
- package/bin/_cctally_core.py +12 -1
- package/bin/_cctally_dashboard.py +49 -13
- package/bin/_cctally_dashboard_conversation.py +40 -17
- package/bin/_cctally_dashboard_sources.py +43 -10
- package/bin/_cctally_db.py +207 -5
- package/bin/_cctally_doctor.py +51 -7
- package/bin/_cctally_parser.py +1 -1
- package/bin/_cctally_quota.py +54 -3
- package/bin/_cctally_statusline.py +0 -2
- package/bin/_cctally_transcript.py +4 -4
- package/bin/_cctally_tui.py +6 -34
- package/bin/_lib_codex_conversation_query.py +19 -3
- package/bin/_lib_conversation_query.py +8 -3
- package/bin/_lib_conversation_retention.py +58 -15
- package/bin/_lib_doctor.py +102 -4
- package/bin/_lib_jsonl.py +23 -6
- package/bin/_lib_statusline.py +0 -26
- package/bin/cctally +3 -0
- package/dashboard/static/assets/index-CvcCeA4L.css +1 -0
- package/dashboard/static/assets/index-DLNXAevv.js +87 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +1 -1
- package/dashboard/static/assets/index-fZRxsSf5.js +0 -80
- package/dashboard/static/assets/index-yftBNnLR.css +0 -1
package/bin/_cctally_db.py
CHANGED
|
@@ -2648,6 +2648,7 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2648
2648
|
plan_type TEXT,
|
|
2649
2649
|
individual_limit_json TEXT,
|
|
2650
2650
|
reached_type TEXT,
|
|
2651
|
+
observed_model TEXT,
|
|
2651
2652
|
UNIQUE(source, source_path, line_offset, logical_limit_key),
|
|
2652
2653
|
CHECK(source != 'codex' OR source_root_key IS NOT NULL)
|
|
2653
2654
|
);
|
|
@@ -2893,6 +2894,12 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2893
2894
|
add_column_if_missing(conn, "conversation_sessions", "git_branch", "TEXT")
|
|
2894
2895
|
add_column_if_missing(conn, "conversation_sessions", "models_json", "TEXT")
|
|
2895
2896
|
add_column_if_missing(conn, "conversation_sessions", "title", "TEXT")
|
|
2897
|
+
# #320: quota pool identity cannot depend on transcript events after the
|
|
2898
|
+
# store split. Stamp the active model directly on each compact physical
|
|
2899
|
+
# quota observation. Existing caches receive the nullable column here; 028
|
|
2900
|
+
# backfills it from the still-present legacy event corpus before dropping
|
|
2901
|
+
# that corpus.
|
|
2902
|
+
add_column_if_missing(conn, "quota_window_snapshots", "observed_model", "TEXT")
|
|
2896
2903
|
conn.execute(
|
|
2897
2904
|
"CREATE INDEX IF NOT EXISTS idx_session_files_session_id "
|
|
2898
2905
|
"ON session_files(session_id)"
|
|
@@ -3014,6 +3021,72 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
3014
3021
|
conn.commit()
|
|
3015
3022
|
|
|
3016
3023
|
|
|
3024
|
+
def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
|
|
3025
|
+
"""Create the #320 transcript/search database schema.
|
|
3026
|
+
|
|
3027
|
+
The historical cache schema remains the migration-fixture source of truth.
|
|
3028
|
+
Reuse it here, then remove the compact accounting families so conversation
|
|
3029
|
+
queries can resolve those names through the read-only ``cache_db``
|
|
3030
|
+
attachment. Empty compatibility tables may still exist in cache.db for old
|
|
3031
|
+
migration handlers, but all live transcript rows belong here.
|
|
3032
|
+
"""
|
|
3033
|
+
# The first open projects the historical monolithic schema into the new
|
|
3034
|
+
# store. Avoid repeating that create-then-drop work on every conversation
|
|
3035
|
+
# endpoint: it writes sqlite_schema and WAL pages even when no transcript
|
|
3036
|
+
# data changed. Future conversation schema revisions must bump this marker.
|
|
3037
|
+
try:
|
|
3038
|
+
current = conn.execute(
|
|
3039
|
+
"SELECT value FROM cache_meta "
|
|
3040
|
+
"WHERE key='conversation_schema_version'"
|
|
3041
|
+
).fetchone()
|
|
3042
|
+
except sqlite3.OperationalError:
|
|
3043
|
+
current = None
|
|
3044
|
+
if current is not None and current[0] == "1":
|
|
3045
|
+
return
|
|
3046
|
+
|
|
3047
|
+
_apply_cache_schema(conn)
|
|
3048
|
+
conn.executescript(
|
|
3049
|
+
"""
|
|
3050
|
+
DROP TABLE IF EXISTS session_entries;
|
|
3051
|
+
DROP TABLE IF EXISTS session_files;
|
|
3052
|
+
DROP TABLE IF EXISTS codex_session_entries;
|
|
3053
|
+
DROP TABLE IF EXISTS codex_session_files;
|
|
3054
|
+
DROP TABLE IF EXISTS quota_window_snapshots;
|
|
3055
|
+
DROP TABLE IF EXISTS codex_conversation_threads;
|
|
3056
|
+
DROP TABLE IF EXISTS codex_source_roots;
|
|
3057
|
+
|
|
3058
|
+
CREATE TABLE IF NOT EXISTS conversation_source_files (
|
|
3059
|
+
path TEXT PRIMARY KEY,
|
|
3060
|
+
size_bytes INTEGER NOT NULL,
|
|
3061
|
+
mtime_ns INTEGER NOT NULL,
|
|
3062
|
+
last_byte_offset INTEGER NOT NULL,
|
|
3063
|
+
last_ingested_at TEXT NOT NULL
|
|
3064
|
+
);
|
|
3065
|
+
CREATE TABLE IF NOT EXISTS codex_conversation_source_files (
|
|
3066
|
+
path TEXT PRIMARY KEY,
|
|
3067
|
+
size_bytes INTEGER NOT NULL,
|
|
3068
|
+
mtime_ns INTEGER NOT NULL,
|
|
3069
|
+
last_byte_offset INTEGER NOT NULL,
|
|
3070
|
+
last_ingested_at TEXT NOT NULL,
|
|
3071
|
+
source_root_key TEXT,
|
|
3072
|
+
last_session_id TEXT,
|
|
3073
|
+
last_model TEXT,
|
|
3074
|
+
last_total_tokens INTEGER,
|
|
3075
|
+
last_native_thread_id TEXT,
|
|
3076
|
+
last_root_thread_id TEXT,
|
|
3077
|
+
last_parent_thread_id TEXT,
|
|
3078
|
+
last_conversation_key TEXT,
|
|
3079
|
+
last_turn_id TEXT
|
|
3080
|
+
);
|
|
3081
|
+
"""
|
|
3082
|
+
)
|
|
3083
|
+
conn.execute(
|
|
3084
|
+
"INSERT INTO cache_meta(key,value) VALUES "
|
|
3085
|
+
"('conversation_schema_version','1') "
|
|
3086
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value"
|
|
3087
|
+
)
|
|
3088
|
+
|
|
3089
|
+
|
|
3017
3090
|
def _fts5_available(conn: sqlite3.Connection) -> bool:
|
|
3018
3091
|
"""True if this sqlite build can create an FTS5 table. Cheap probe on a
|
|
3019
3092
|
temp table that is created then dropped. Hidden test seam: tests monkeypatch
|
|
@@ -4730,6 +4803,118 @@ def _027_codex_fork_preamble_rebuild(conn: sqlite3.Connection) -> None:
|
|
|
4730
4803
|
lock_fh.close()
|
|
4731
4804
|
|
|
4732
4805
|
|
|
4806
|
+
@cache_migration("028_split_conversation_store")
|
|
4807
|
+
def _028_split_conversation_store(conn: sqlite3.Connection) -> None:
|
|
4808
|
+
"""Move the re-derivable transcript corpus out of cache.db (#320).
|
|
4809
|
+
|
|
4810
|
+
The upgrade deliberately does not copy gigabytes of prose/events. It creates
|
|
4811
|
+
the independent current-schema store, arms provider-local byte-zero replay,
|
|
4812
|
+
then drops the legacy transcript tables from the hot cache. Compact Codex
|
|
4813
|
+
thread metadata stays in cache.db because accounting/project attribution
|
|
4814
|
+
joins it directly.
|
|
4815
|
+
"""
|
|
4816
|
+
core = _cctally_core
|
|
4817
|
+
lock_paths = (
|
|
4818
|
+
core.CACHE_LOCK_MAINTENANCE_PATH,
|
|
4819
|
+
core.CACHE_LOCK_PATH,
|
|
4820
|
+
core.CACHE_LOCK_CODEX_PATH,
|
|
4821
|
+
core.CONVERSATIONS_LOCK_MAINTENANCE_PATH,
|
|
4822
|
+
core.CONVERSATIONS_LOCK_PATH,
|
|
4823
|
+
core.CONVERSATIONS_LOCK_CODEX_PATH,
|
|
4824
|
+
)
|
|
4825
|
+
held = []
|
|
4826
|
+
try:
|
|
4827
|
+
core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
4828
|
+
for path in lock_paths:
|
|
4829
|
+
fh = open(path, "w")
|
|
4830
|
+
try:
|
|
4831
|
+
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
4832
|
+
except BlockingIOError:
|
|
4833
|
+
fh.close()
|
|
4834
|
+
raise MigrationGateNotMet(
|
|
4835
|
+
"cache/conversation sync lock held; deferring cache 028 split"
|
|
4836
|
+
)
|
|
4837
|
+
held.append(fh)
|
|
4838
|
+
|
|
4839
|
+
conv = sqlite3.connect(core.CONVERSATIONS_DB_PATH)
|
|
4840
|
+
try:
|
|
4841
|
+
conv.execute("PRAGMA auto_vacuum=INCREMENTAL")
|
|
4842
|
+
conv.execute("PRAGMA journal_mode=WAL")
|
|
4843
|
+
_apply_conversations_schema(conv)
|
|
4844
|
+
_set_cache_meta(conv, "conversation_rebuild_claude_pending", "1")
|
|
4845
|
+
_set_cache_meta(conv, "conversation_rebuild_codex_pending", "1")
|
|
4846
|
+
conv.commit()
|
|
4847
|
+
finally:
|
|
4848
|
+
conv.close()
|
|
4849
|
+
|
|
4850
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
4851
|
+
try:
|
|
4852
|
+
# Preserve exact model-scoped quota-pool identity before discarding
|
|
4853
|
+
# the legacy physical event corpus. The correlated lookup is the
|
|
4854
|
+
# same nearest-prior context rule the quota reader used pre-split.
|
|
4855
|
+
if conn.execute(
|
|
4856
|
+
"SELECT 1 FROM sqlite_master "
|
|
4857
|
+
"WHERE type='table' AND name='codex_conversation_events'"
|
|
4858
|
+
).fetchone() is not None:
|
|
4859
|
+
conn.execute(
|
|
4860
|
+
"UPDATE quota_window_snapshots AS q SET observed_model=("
|
|
4861
|
+
" SELECT json_extract(e.payload_json, '$.payload.model')"
|
|
4862
|
+
" FROM codex_conversation_events AS e"
|
|
4863
|
+
" WHERE e.source_path=q.source_path"
|
|
4864
|
+
" AND e.line_offset<=q.line_offset"
|
|
4865
|
+
" AND e.record_type IN ('turn_context','session_meta')"
|
|
4866
|
+
" AND json_valid(e.payload_json)"
|
|
4867
|
+
" AND json_type(e.payload_json, '$.payload.model')='text'"
|
|
4868
|
+
" ORDER BY e.line_offset DESC LIMIT 1)"
|
|
4869
|
+
" WHERE q.source='codex' AND q.observed_model IS NULL"
|
|
4870
|
+
)
|
|
4871
|
+
for drop in (
|
|
4872
|
+
"DROP TRIGGER IF EXISTS conv_fts_ai",
|
|
4873
|
+
"DROP TRIGGER IF EXISTS conv_fts_ad",
|
|
4874
|
+
"DROP TRIGGER IF EXISTS conv_fts_au",
|
|
4875
|
+
"DROP TRIGGER IF EXISTS conv_fts_aux_ai",
|
|
4876
|
+
"DROP TRIGGER IF EXISTS conv_fts_aux_ad",
|
|
4877
|
+
"DROP TRIGGER IF EXISTS conv_fts_aux_au",
|
|
4878
|
+
"DROP TRIGGER IF EXISTS conv_title_fts_ai",
|
|
4879
|
+
"DROP TRIGGER IF EXISTS conv_title_fts_ad",
|
|
4880
|
+
"DROP TRIGGER IF EXISTS conv_title_fts_au",
|
|
4881
|
+
"DROP TRIGGER IF EXISTS codex_conv_fts_ai",
|
|
4882
|
+
"DROP TRIGGER IF EXISTS codex_conv_fts_ad",
|
|
4883
|
+
"DROP TRIGGER IF EXISTS codex_conv_fts_au",
|
|
4884
|
+
"DROP TABLE IF EXISTS conversation_fts_aux",
|
|
4885
|
+
"DROP TABLE IF EXISTS conversation_fts",
|
|
4886
|
+
"DROP TABLE IF EXISTS conversation_title_fts",
|
|
4887
|
+
"DROP TABLE IF EXISTS codex_conversation_fts",
|
|
4888
|
+
"DROP TABLE IF EXISTS conversation_file_touches",
|
|
4889
|
+
"DROP TABLE IF EXISTS conversation_sessions",
|
|
4890
|
+
"DROP TABLE IF EXISTS conversation_ai_titles",
|
|
4891
|
+
"DROP TABLE IF EXISTS conversation_messages",
|
|
4892
|
+
"DROP TABLE IF EXISTS codex_conversation_file_touches",
|
|
4893
|
+
"DROP TABLE IF EXISTS codex_conversation_rollups",
|
|
4894
|
+
"DROP TABLE IF EXISTS codex_conversation_messages",
|
|
4895
|
+
"DROP TABLE IF EXISTS codex_conversation_events",
|
|
4896
|
+
):
|
|
4897
|
+
conn.execute(drop)
|
|
4898
|
+
conn.commit()
|
|
4899
|
+
except Exception:
|
|
4900
|
+
conn.rollback()
|
|
4901
|
+
raise
|
|
4902
|
+
try:
|
|
4903
|
+
if conn.execute("PRAGMA auto_vacuum").fetchone()[0] == 2:
|
|
4904
|
+
# Python 3.11/Linux may defer this zero-column PRAGMA until the
|
|
4905
|
+
# cursor is exhausted; fetchall drives it to completion.
|
|
4906
|
+
conn.execute("PRAGMA incremental_vacuum").fetchall()
|
|
4907
|
+
except sqlite3.DatabaseError:
|
|
4908
|
+
pass
|
|
4909
|
+
finally:
|
|
4910
|
+
for fh in reversed(held):
|
|
4911
|
+
try:
|
|
4912
|
+
fcntl.flock(fh, fcntl.LOCK_UN)
|
|
4913
|
+
except OSError:
|
|
4914
|
+
pass
|
|
4915
|
+
fh.close()
|
|
4916
|
+
|
|
4917
|
+
|
|
4733
4918
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
4734
4919
|
|
|
4735
4920
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|
|
@@ -7006,7 +7191,12 @@ def _vacuum_one_db(path, label: str, provider_locked: bool) -> int:
|
|
|
7006
7191
|
# Serialize against the retention prune and concurrent vacuums via the
|
|
7007
7192
|
# dedicated maintenance flock (F13/F7), then the provider flocks for
|
|
7008
7193
|
# cache.db. All non-blocking: fail promptly rather than hang.
|
|
7009
|
-
|
|
7194
|
+
conversation_store = path == core.CONVERSATIONS_DB_PATH
|
|
7195
|
+
maintenance_path = (
|
|
7196
|
+
core.CONVERSATIONS_LOCK_MAINTENANCE_PATH
|
|
7197
|
+
if conversation_store else core.CACHE_LOCK_MAINTENANCE_PATH
|
|
7198
|
+
)
|
|
7199
|
+
maint_fh = open(maintenance_path, "w")
|
|
7010
7200
|
try:
|
|
7011
7201
|
try:
|
|
7012
7202
|
fcntl.flock(maint_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
@@ -7019,10 +7209,18 @@ def _vacuum_one_db(path, label: str, provider_locked: bool) -> int:
|
|
|
7019
7209
|
held = []
|
|
7020
7210
|
try:
|
|
7021
7211
|
if provider_locked:
|
|
7022
|
-
|
|
7023
|
-
(
|
|
7024
|
-
|
|
7025
|
-
|
|
7212
|
+
provider_locks = (
|
|
7213
|
+
(
|
|
7214
|
+
(core.CONVERSATIONS_LOCK_PATH, "claude conversations"),
|
|
7215
|
+
(core.CONVERSATIONS_LOCK_CODEX_PATH, "codex conversations"),
|
|
7216
|
+
)
|
|
7217
|
+
if conversation_store else
|
|
7218
|
+
(
|
|
7219
|
+
(core.CACHE_LOCK_PATH, "claude"),
|
|
7220
|
+
(core.CACHE_LOCK_CODEX_PATH, "codex"),
|
|
7221
|
+
)
|
|
7222
|
+
)
|
|
7223
|
+
for lock_path, lname in provider_locks:
|
|
7026
7224
|
fh = open(lock_path, "w")
|
|
7027
7225
|
try:
|
|
7028
7226
|
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
@@ -7063,6 +7261,10 @@ def cmd_db_vacuum(args: argparse.Namespace) -> int:
|
|
|
7063
7261
|
targets = []
|
|
7064
7262
|
if which in ("cache", "all"):
|
|
7065
7263
|
targets.append((_cctally_core.CACHE_DB_PATH, "cache.db", True))
|
|
7264
|
+
if which in ("conversations", "all"):
|
|
7265
|
+
targets.append((
|
|
7266
|
+
_cctally_core.CONVERSATIONS_DB_PATH, "conversations.db", True,
|
|
7267
|
+
))
|
|
7066
7268
|
if which in ("stats", "all"):
|
|
7067
7269
|
targets.append((_cctally_core.DB_PATH, "stats.db", False))
|
|
7068
7270
|
overall = 0
|
package/bin/_cctally_doctor.py
CHANGED
|
@@ -385,10 +385,21 @@ def doctor_gather_state(
|
|
|
385
385
|
|
|
386
386
|
cache_entries_count = None
|
|
387
387
|
cache_last_entry_at = None
|
|
388
|
+
cache_db_page_count = None
|
|
389
|
+
cache_db_freelist_count = None
|
|
388
390
|
try:
|
|
389
391
|
if _cctally_core.CACHE_DB_PATH.exists():
|
|
390
392
|
conn = sqlite3.connect(str(_cctally_core.CACHE_DB_PATH))
|
|
391
393
|
try:
|
|
394
|
+
try:
|
|
395
|
+
row = conn.execute("PRAGMA page_count").fetchone()
|
|
396
|
+
if row and row[0] is not None:
|
|
397
|
+
cache_db_page_count = int(row[0])
|
|
398
|
+
row = conn.execute("PRAGMA freelist_count").fetchone()
|
|
399
|
+
if row and row[0] is not None:
|
|
400
|
+
cache_db_freelist_count = int(row[0])
|
|
401
|
+
except sqlite3.Error:
|
|
402
|
+
pass
|
|
392
403
|
row = conn.execute(
|
|
393
404
|
"SELECT COUNT(*), MAX(timestamp_utc) FROM session_entries"
|
|
394
405
|
).fetchone()
|
|
@@ -418,16 +429,35 @@ def doctor_gather_state(
|
|
|
418
429
|
# Conversation-sessions rollup consistency (#217 S1 / U9). Two cheap COUNTs
|
|
419
430
|
# (graceful None on a missing table / unreadable DB) + an in-progress signal
|
|
420
431
|
# so a transient mid-sync mismatch never WARNs. The in-progress signal is a
|
|
421
|
-
# NON-BLOCKING
|
|
432
|
+
# NON-BLOCKING conversations flock probe (a writer mid-walk holds it) OR the
|
|
422
433
|
# presence of any pending reingest/split/backfill cache_meta flag — doctor
|
|
423
434
|
# stays read-only and never blocks on the lock.
|
|
424
435
|
conv_sessions_rollup_count = None
|
|
425
436
|
conv_messages_distinct_sessions = None
|
|
426
437
|
conv_rollup_sync_in_progress = False
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
438
|
+
conversations_db_page_count = None
|
|
439
|
+
conversations_db_freelist_count = None
|
|
440
|
+
try:
|
|
441
|
+
if _cctally_core.CONVERSATIONS_DB_PATH.exists():
|
|
442
|
+
# This gather also runs inside dashboard snapshot precompute. A
|
|
443
|
+
# transcript writer may hold an exclusive SQLite lock, so use a
|
|
444
|
+
# read-only zero-timeout probe: conversation health can degrade,
|
|
445
|
+
# but it must never delay core snapshot freshness (#320).
|
|
446
|
+
conv_uri = (
|
|
447
|
+
_cctally_core.CONVERSATIONS_DB_PATH.resolve().as_uri()
|
|
448
|
+
+ "?mode=ro"
|
|
449
|
+
)
|
|
450
|
+
conn = sqlite3.connect(conv_uri, uri=True, timeout=0.0)
|
|
430
451
|
try:
|
|
452
|
+
try:
|
|
453
|
+
row = conn.execute("PRAGMA page_count").fetchone()
|
|
454
|
+
if row and row[0] is not None:
|
|
455
|
+
conversations_db_page_count = int(row[0])
|
|
456
|
+
row = conn.execute("PRAGMA freelist_count").fetchone()
|
|
457
|
+
if row and row[0] is not None:
|
|
458
|
+
conversations_db_freelist_count = int(row[0])
|
|
459
|
+
except sqlite3.Error:
|
|
460
|
+
pass
|
|
431
461
|
try:
|
|
432
462
|
row = conn.execute(
|
|
433
463
|
"SELECT COUNT(*) FROM conversation_sessions"
|
|
@@ -461,12 +491,12 @@ def doctor_gather_state(
|
|
|
461
491
|
pass
|
|
462
492
|
finally:
|
|
463
493
|
conn.close()
|
|
464
|
-
# Non-blocking flock probe: if a writer
|
|
465
|
-
#
|
|
494
|
+
# Non-blocking flock probe: if a transcript writer/reingest holds the
|
|
495
|
+
# conversations.db lock, the rollup may be mid-recompute → in progress. We
|
|
466
496
|
# acquire LOCK_EX|LOCK_NB and immediately release; failure (held) is the
|
|
467
497
|
# signal. Never blocks (LOCK_NB), so doctor stays read-only + prompt.
|
|
468
498
|
if not conv_rollup_sync_in_progress:
|
|
469
|
-
lock_path = _cctally_core.
|
|
499
|
+
lock_path = _cctally_core.CONVERSATIONS_LOCK_PATH
|
|
470
500
|
if lock_path is not None and pathlib.Path(lock_path).exists():
|
|
471
501
|
import fcntl as _fcntl
|
|
472
502
|
lock_fh = open(str(lock_path), "w")
|
|
@@ -658,6 +688,15 @@ def doctor_gather_state(
|
|
|
658
688
|
for _name, _lp in (
|
|
659
689
|
("cache.db.lock", _cctally_core.CACHE_LOCK_PATH),
|
|
660
690
|
("cache.db.codex.lock", _cctally_core.CACHE_LOCK_CODEX_PATH),
|
|
691
|
+
("conversations.db.lock", _cctally_core.CONVERSATIONS_LOCK_PATH),
|
|
692
|
+
(
|
|
693
|
+
"conversations.db.codex.lock",
|
|
694
|
+
_cctally_core.CONVERSATIONS_LOCK_CODEX_PATH,
|
|
695
|
+
),
|
|
696
|
+
(
|
|
697
|
+
"conversations.db.maintenance.lock",
|
|
698
|
+
_cctally_core.CONVERSATIONS_LOCK_MAINTENANCE_PATH,
|
|
699
|
+
),
|
|
661
700
|
):
|
|
662
701
|
if not _lp.exists():
|
|
663
702
|
locks_held[_name] = False
|
|
@@ -863,6 +902,11 @@ def doctor_gather_state(
|
|
|
863
902
|
locks_held=locks_held,
|
|
864
903
|
# #297: cache.db WAL size backstop (gathered outside the deep branch).
|
|
865
904
|
cache_db_wal_bytes=cache_db_wal_bytes,
|
|
905
|
+
# #315: read-only cache free-page evidence for the reclaim hint.
|
|
906
|
+
cache_db_page_count=cache_db_page_count,
|
|
907
|
+
cache_db_freelist_count=cache_db_freelist_count,
|
|
908
|
+
conversations_db_page_count=conversations_db_page_count,
|
|
909
|
+
conversations_db_freelist_count=conversations_db_freelist_count,
|
|
866
910
|
codex_quota_windows=codex_quota_windows,
|
|
867
911
|
codex_hook_roots=codex_hook_roots,
|
|
868
912
|
codex_lifecycle_activity_24h=codex_lifecycle_activity_24h,
|
package/bin/_cctally_parser.py
CHANGED
|
@@ -2924,7 +2924,7 @@ def _build_db_parser(subparsers, name, *, help_text, xref=None):
|
|
|
2924
2924
|
)
|
|
2925
2925
|
db_vacuum.add_argument(
|
|
2926
2926
|
"--db",
|
|
2927
|
-
choices=("cache", "stats", "all"),
|
|
2927
|
+
choices=("cache", "conversations", "stats", "all"),
|
|
2928
2928
|
default="cache",
|
|
2929
2929
|
help="Which DB to VACUUM (default: cache)",
|
|
2930
2930
|
)
|
package/bin/_cctally_quota.py
CHANGED
|
@@ -41,10 +41,12 @@ from _lib_quota import (
|
|
|
41
41
|
source_path_key,
|
|
42
42
|
)
|
|
43
43
|
from _lib_json_envelope import stamp_schema_version
|
|
44
|
+
from _lib_jsonl import _codex_logical_limit_key, codex_model_scoped_quota_pool
|
|
44
45
|
|
|
45
46
|
|
|
46
47
|
UTC = dt.timezone.utc
|
|
47
48
|
_DASHBOARD_PROJECTION_CERTIFICATE_KEY = "codex_quota_projection_certificate"
|
|
49
|
+
_CODEX_QUOTA_INTERPRETATION_VERSION = 2
|
|
48
50
|
|
|
49
51
|
|
|
50
52
|
@dataclass(frozen=True)
|
|
@@ -134,6 +136,11 @@ def load_codex_quota_projection_certificate(
|
|
|
134
136
|
if row is None:
|
|
135
137
|
return None
|
|
136
138
|
payload = json.loads(str(row[0]))
|
|
139
|
+
if (
|
|
140
|
+
int(payload["interpretationVersion"])
|
|
141
|
+
!= _CODEX_QUOTA_INTERPRETATION_VERSION
|
|
142
|
+
):
|
|
143
|
+
return None
|
|
137
144
|
sequence = int(payload["sequence"])
|
|
138
145
|
signatures = {
|
|
139
146
|
str(root_key): str(signature)
|
|
@@ -169,6 +176,7 @@ def _store_codex_quota_projection_certificate(
|
|
|
169
176
|
conn.rollback()
|
|
170
177
|
return
|
|
171
178
|
payload = json.dumps({
|
|
179
|
+
"interpretationVersion": _CODEX_QUOTA_INTERPRETATION_VERSION,
|
|
172
180
|
"sequence": sequence,
|
|
173
181
|
"signatures": dict(sorted(signatures.items())),
|
|
174
182
|
}, sort_keys=True, separators=(",", ":"))
|
|
@@ -262,14 +270,50 @@ def load_codex_quota_observations(
|
|
|
262
270
|
previous_row_factory = conn.row_factory
|
|
263
271
|
try:
|
|
264
272
|
conn.row_factory = sqlite3.Row
|
|
273
|
+
|
|
274
|
+
def has_columns(table: str, required: set[str]) -> bool:
|
|
275
|
+
columns = {
|
|
276
|
+
str(row[1]) for row in conn.execute(
|
|
277
|
+
f"PRAGMA table_info({table})"
|
|
278
|
+
)
|
|
279
|
+
}
|
|
280
|
+
return required <= columns
|
|
281
|
+
|
|
282
|
+
has_session_entries = has_columns(
|
|
283
|
+
"codex_session_entries", {"source_path", "line_offset", "model"},
|
|
284
|
+
)
|
|
285
|
+
has_observed_model = has_columns(
|
|
286
|
+
"quota_window_snapshots", {"observed_model"},
|
|
287
|
+
)
|
|
288
|
+
entry_lookup = (
|
|
289
|
+
"(SELECT entries.model FROM codex_session_entries AS entries "
|
|
290
|
+
"WHERE entries.source_path=quota_window_snapshots.source_path "
|
|
291
|
+
"AND entries.line_offset<=quota_window_snapshots.line_offset "
|
|
292
|
+
"ORDER BY entries.line_offset DESC LIMIT 1)"
|
|
293
|
+
)
|
|
294
|
+
# A file's terminal model is not evidence for an earlier quota sample:
|
|
295
|
+
# the model may change later in the rollout. Prefer the compact stamp,
|
|
296
|
+
# then only a nearest-prior accounting context for legacy rows. If
|
|
297
|
+
# neither exists, leave the pool unscoped rather than fabricating it.
|
|
298
|
+
fallback_model = entry_lookup if has_session_entries else "NULL"
|
|
299
|
+
selected_model = (
|
|
300
|
+
f"COALESCE(quota_window_snapshots.observed_model,{fallback_model})"
|
|
301
|
+
if has_observed_model and fallback_model != "NULL"
|
|
302
|
+
else (
|
|
303
|
+
"quota_window_snapshots.observed_model"
|
|
304
|
+
if has_observed_model else fallback_model
|
|
305
|
+
)
|
|
306
|
+
)
|
|
307
|
+
model_expr = f"{selected_model} AS observed_model"
|
|
265
308
|
sql = """
|
|
266
309
|
SELECT source, source_root_key, source_path, line_offset,
|
|
267
310
|
captured_at_utc, observed_slot, logical_limit_key, limit_id,
|
|
268
311
|
limit_name, window_minutes, used_percent, resets_at_utc,
|
|
269
|
-
plan_type, individual_limit_json, reached_type
|
|
312
|
+
plan_type, individual_limit_json, reached_type,
|
|
313
|
+
{model_expr}
|
|
270
314
|
FROM quota_window_snapshots
|
|
271
315
|
WHERE source='codex' AND source_root_key IS NOT NULL
|
|
272
|
-
"""
|
|
316
|
+
""".format(model_expr=model_expr)
|
|
273
317
|
params: list[object] = []
|
|
274
318
|
if requested is not None:
|
|
275
319
|
if not requested:
|
|
@@ -317,10 +361,17 @@ def load_codex_quota_observations(
|
|
|
317
361
|
if any(row[name] is None or not str(row[name]).strip() for name in required_text):
|
|
318
362
|
continue
|
|
319
363
|
try:
|
|
364
|
+
logical_limit_key = str(row["logical_limit_key"])
|
|
365
|
+
if codex_model_scoped_quota_pool(row["observed_model"]) is not None:
|
|
366
|
+
logical_limit_key = _codex_logical_limit_key(
|
|
367
|
+
str(row["source_root_key"]), row["limit_id"],
|
|
368
|
+
str(row["observed_slot"]), int(row["window_minutes"]),
|
|
369
|
+
str(row["observed_model"]),
|
|
370
|
+
)
|
|
320
371
|
identity = QuotaWindowIdentity(
|
|
321
372
|
source=str(row["source"]),
|
|
322
373
|
source_root_key=str(row["source_root_key"]),
|
|
323
|
-
logical_limit_key=
|
|
374
|
+
logical_limit_key=logical_limit_key,
|
|
324
375
|
observed_slot=str(row["observed_slot"]),
|
|
325
376
|
window_minutes=int(row["window_minutes"]),
|
|
326
377
|
limit_id=row["limit_id"],
|
|
@@ -1205,8 +1205,6 @@ def _fork_persist(parent_lock_fd: int) -> None:
|
|
|
1205
1205
|
|
|
1206
1206
|
def _statusline_persist(parsed, *, sync_for_test: bool = False) -> None:
|
|
1207
1207
|
"""Spool an eligible session candidate then reduce it opportunistically."""
|
|
1208
|
-
if _lib_statusline.is_alternate_pool_model_id(parsed.model_id):
|
|
1209
|
-
return
|
|
1210
1208
|
candidate = _candidate_from_input(parsed, received_at=int(time.time()))
|
|
1211
1209
|
if candidate is None:
|
|
1212
1210
|
return
|
|
@@ -86,7 +86,7 @@ def _cmd_transcript_export(args) -> int:
|
|
|
86
86
|
eprint(_SPEED_ONLY_CODEX_MSG)
|
|
87
87
|
return 2
|
|
88
88
|
|
|
89
|
-
conn = c.
|
|
89
|
+
conn = c.open_conversations_db()
|
|
90
90
|
try:
|
|
91
91
|
cq = c._load_sibling("_lib_conversation_query")
|
|
92
92
|
md = cq.get_conversation_export(conn, session_id, scope)
|
|
@@ -118,7 +118,7 @@ def _cmd_transcript_export_qualified(
|
|
|
118
118
|
return 2
|
|
119
119
|
speed = c._resolve_codex_speed(speed_arg or "auto")
|
|
120
120
|
|
|
121
|
-
conn = c.
|
|
121
|
+
conn = c.open_conversations_db()
|
|
122
122
|
try:
|
|
123
123
|
env = disp.neutral_export(
|
|
124
124
|
conn, session_id, scope=scope, effective_speed=speed)
|
|
@@ -189,7 +189,7 @@ def _cmd_transcript_search_claude(args) -> int:
|
|
|
189
189
|
eprint(f"transcript: {exc}")
|
|
190
190
|
return 2
|
|
191
191
|
|
|
192
|
-
conn = c.
|
|
192
|
+
conn = c.open_conversations_db()
|
|
193
193
|
try:
|
|
194
194
|
cq = c._load_sibling("_lib_conversation_query")
|
|
195
195
|
try:
|
|
@@ -264,7 +264,7 @@ def _cmd_transcript_search_codex(args) -> int:
|
|
|
264
264
|
eprint("transcript: invalid --cursor")
|
|
265
265
|
return 2
|
|
266
266
|
|
|
267
|
-
conn = c.
|
|
267
|
+
conn = c.open_conversations_db()
|
|
268
268
|
try:
|
|
269
269
|
result = disp.neutral_search(
|
|
270
270
|
conn, query, source="codex", kind=kind,
|
package/bin/_cctally_tui.py
CHANGED
|
@@ -1772,40 +1772,12 @@ def _tui_build_sessions(
|
|
|
1772
1772
|
(), now_utc=now_utc, limit=limit, display_tz=None,
|
|
1773
1773
|
aggregated_override=aggregated_override,
|
|
1774
1774
|
)
|
|
1775
|
-
|
|
1776
|
-
#
|
|
1777
|
-
#
|
|
1778
|
-
#
|
|
1779
|
-
#
|
|
1780
|
-
|
|
1781
|
-
# bounded indexed query (<= `limit` session ids, rides idx_conv_session_ts)
|
|
1782
|
-
# per snapshot build. Titles are stashed unconditionally on this
|
|
1783
|
-
# server-internal row; the privacy gate is applied later, at envelope
|
|
1784
|
-
# serialization (``snapshot_to_envelope(transcripts_visible=...)``).
|
|
1785
|
-
if rows:
|
|
1786
|
-
session_ids = [r.session_id for r in rows if r.session_id]
|
|
1787
|
-
try:
|
|
1788
|
-
conn = c.open_cache_db()
|
|
1789
|
-
except (sqlite3.DatabaseError, OSError):
|
|
1790
|
-
conn = None
|
|
1791
|
-
if conn is not None:
|
|
1792
|
-
try:
|
|
1793
|
-
titles = c._load_sibling(
|
|
1794
|
-
"_lib_conversation_query"
|
|
1795
|
-
)._session_titles_map(conn, session_ids)
|
|
1796
|
-
rows = [
|
|
1797
|
-
dataclasses.replace(r, title=titles.get(r.session_id))
|
|
1798
|
-
if r.session_id in titles else r
|
|
1799
|
-
for r in rows
|
|
1800
|
-
]
|
|
1801
|
-
except (sqlite3.DatabaseError, OSError):
|
|
1802
|
-
pass # fail-soft: leave titles None
|
|
1803
|
-
finally:
|
|
1804
|
-
try:
|
|
1805
|
-
conn.close()
|
|
1806
|
-
except sqlite3.Error:
|
|
1807
|
-
pass
|
|
1808
|
-
return rows
|
|
1775
|
+
# #320: transcript-derived titles are optional decoration. The core
|
|
1776
|
+
# dashboard/TUI snapshot must never open conversations.db, because even a
|
|
1777
|
+
# fail-soft read pays SQLite's lock timeout before it can fail. Conversation
|
|
1778
|
+
# routes retain title derivation; the accounting Sessions panel renders its
|
|
1779
|
+
# existing em-dash fallback when the independent store is unavailable.
|
|
1780
|
+
return list(view.rows)
|
|
1809
1781
|
|
|
1810
1782
|
|
|
1811
1783
|
def _tui_sessions_cached(
|
|
@@ -71,9 +71,25 @@ _SEARCH_BADGE = {
|
|
|
71
71
|
|
|
72
72
|
|
|
73
73
|
def codex_normalization_authoritative(conn: sqlite3.Connection) -> bool:
|
|
74
|
-
"""True iff
|
|
75
|
-
|
|
76
|
-
|
|
74
|
+
"""True iff the normalized Codex corpus is authoritative (§3.5).
|
|
75
|
+
|
|
76
|
+
Split stores use their provider-local rebuild marker: current schema alone
|
|
77
|
+
is not authority while migration 028's byte-zero replay is pending. Legacy
|
|
78
|
+
monolithic/bare connections retain the migration-025 stamp contract.
|
|
79
|
+
"""
|
|
80
|
+
try:
|
|
81
|
+
split = conn.execute(
|
|
82
|
+
"SELECT 1 FROM cache_meta "
|
|
83
|
+
"WHERE key='conversation_schema_version'"
|
|
84
|
+
).fetchone() is not None
|
|
85
|
+
if split:
|
|
86
|
+
pending = conn.execute(
|
|
87
|
+
"SELECT 1 FROM cache_meta "
|
|
88
|
+
"WHERE key='conversation_rebuild_codex_pending'"
|
|
89
|
+
).fetchone() is not None
|
|
90
|
+
return not pending
|
|
91
|
+
except sqlite3.OperationalError:
|
|
92
|
+
pass
|
|
77
93
|
try:
|
|
78
94
|
row = conn.execute(
|
|
79
95
|
"SELECT 1 FROM schema_migrations WHERE name = ?",
|
|
@@ -2048,7 +2048,9 @@ def _assemble_memo_clear() -> None:
|
|
|
2048
2048
|
def _assemble_memo_key(conn, session_id):
|
|
2049
2049
|
"""Return (session_id, msg_count, last_activity_utc, entry_mutation_seq), or
|
|
2050
2050
|
None to signal 'bypass the memo' — a missing rollup row or an unreadable
|
|
2051
|
-
cache_meta
|
|
2051
|
+
accounting ``cache_meta``. Split-store connections read that watermark
|
|
2052
|
+
from the read-only ``cache_db`` attachment; legacy/bare test connections
|
|
2053
|
+
retain the main-schema fallback. The caller has already confirmed the
|
|
2052
2054
|
rollup is authoritative."""
|
|
2053
2055
|
try:
|
|
2054
2056
|
row = conn.execute(
|
|
@@ -2059,9 +2061,12 @@ def _assemble_memo_key(conn, session_id):
|
|
|
2059
2061
|
if row is None:
|
|
2060
2062
|
return None
|
|
2061
2063
|
try:
|
|
2064
|
+
schemas = {row[1] for row in conn.execute("PRAGMA database_list")}
|
|
2065
|
+
meta_schema = "cache_db" if "cache_db" in schemas else "main"
|
|
2062
2066
|
seq_row = conn.execute(
|
|
2063
|
-
"SELECT value FROM cache_meta "
|
|
2064
|
-
"WHERE key='session_entries_mutation_seq'"
|
|
2067
|
+
f"SELECT value FROM {meta_schema}.cache_meta "
|
|
2068
|
+
"WHERE key='session_entries_mutation_seq'"
|
|
2069
|
+
).fetchone()
|
|
2065
2070
|
except sqlite3.OperationalError:
|
|
2066
2071
|
return None
|
|
2067
2072
|
seq = int(seq_row[0]) if seq_row and seq_row[0] is not None else 0
|