cctally 1.68.0 → 1.69.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +56 -0
- package/bin/_cctally_alerts.py +54 -2
- package/bin/_cctally_cache.py +644 -107
- package/bin/_cctally_cache_report.py +41 -17
- package/bin/_cctally_codex.py +109 -4
- package/bin/_cctally_config.py +368 -2
- package/bin/_cctally_core.py +168 -5
- package/bin/_cctally_dashboard.py +679 -5
- package/bin/_cctally_dashboard_conversation.py +9 -4
- package/bin/_cctally_dashboard_envelope.py +110 -1
- package/bin/_cctally_dashboard_share.py +557 -79
- package/bin/_cctally_db.py +303 -17
- package/bin/_cctally_diff.py +49 -16
- package/bin/_cctally_doctor.py +118 -0
- package/bin/_cctally_forecast.py +12 -4
- package/bin/_cctally_parser.py +298 -92
- package/bin/_cctally_project.py +24 -8
- package/bin/_cctally_quota.py +1381 -0
- package/bin/_cctally_record.py +319 -14
- package/bin/_cctally_refresh.py +105 -3
- package/bin/_cctally_reporting.py +7 -1
- package/bin/_cctally_setup.py +343 -113
- package/bin/_cctally_statusline.py +228 -0
- package/bin/_cctally_tui.py +787 -7
- package/bin/_lib_alert_axes.py +11 -0
- package/bin/_lib_codex_hooks.py +367 -0
- package/bin/_lib_conversation_query.py +156 -67
- package/bin/_lib_doctor.py +202 -0
- package/bin/_lib_jsonl.py +529 -162
- package/bin/_lib_quota.py +566 -0
- package/bin/_lib_share.py +152 -34
- package/bin/_lib_snapshot_cache.py +26 -0
- package/bin/_lib_source_identity.py +74 -0
- package/bin/cctally +324 -10
- package/dashboard/static/assets/index-CBbErI-P.js +80 -0
- package/dashboard/static/assets/index-kDDVOLa_.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +5 -1
- package/dashboard/static/assets/index-BybNp_Di.js +0 -80
- package/dashboard/static/assets/index-DgAmLK65.css +0 -1
package/bin/_cctally_db.py
CHANGED
|
@@ -2331,11 +2331,11 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2331
2331
|
can call it without an import cycle. Idempotent (CREATE ... IF NOT EXISTS +
|
|
2332
2332
|
``add_column_if_missing``). Does NOT run the dispatcher and does NOT include
|
|
2333
2333
|
the Codex ``last_total_tokens`` ALTER, which carries a one-time purge
|
|
2334
|
-
side-effect that stays in ``open_cache_db
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2334
|
+
side-effect that stays in ``open_cache_db``. The eager dispatcher can run
|
|
2335
|
+
024, but that handler acquires its own Codex flock and only deletes
|
|
2336
|
+
Codex-derived rows; it neither reads nor requires ``last_total_tokens``.
|
|
2337
|
+
Normal ``open_cache_db`` remains responsible for the ALTER and its historic
|
|
2338
|
+
purge, so leaving it outside this shared schema is safe.
|
|
2339
2339
|
"""
|
|
2340
2340
|
conn.executescript(
|
|
2341
2341
|
"""
|
|
@@ -2411,6 +2411,19 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2411
2411
|
ON conversation_messages(cwd)
|
|
2412
2412
|
WHERE cwd IS NOT NULL AND cwd != '';
|
|
2413
2413
|
|
|
2414
|
+
-- #301: partial covering index on (model, session_id). list_conversation_facets
|
|
2415
|
+
-- and _model_clause source model facets/filters with `SELECT DISTINCT ... model
|
|
2416
|
+
-- FROM conversation_messages WHERE model IS NOT NULL AND model != ''`; without
|
|
2417
|
+
-- this index those DISTINCTs do an index->heap walk of the whole table (~22s
|
|
2418
|
+
-- cold on a 5.5 GB cache). The partial (model, session_id) index answers all
|
|
2419
|
+
-- three model queries index-only: the facets DISTINCT and distinct-model
|
|
2420
|
+
-- enumeration as ordered index walks, and the ?models= filter as a model seek.
|
|
2421
|
+
-- MIRRORS migration 022 (base schema here for fresh/rebuilt caches, migration
|
|
2422
|
+
-- for existing ones) — same discipline as idx_conversation_messages_cwd (#289).
|
|
2423
|
+
CREATE INDEX IF NOT EXISTS idx_conversation_messages_model_session
|
|
2424
|
+
ON conversation_messages(model, session_id)
|
|
2425
|
+
WHERE model IS NOT NULL AND model != '';
|
|
2426
|
+
|
|
2414
2427
|
-- #193: per-session AI-generated title, isolated from the six places
|
|
2415
2428
|
-- that iterate conversation_messages. The explicit NOT NULL on the
|
|
2416
2429
|
-- non-INTEGER PRIMARY KEY matters (SQLite's legacy NULL-in-PK bug);
|
|
@@ -2429,18 +2442,25 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2429
2442
|
-- render a 50-row page, PLUS three filter columns
|
|
2430
2443
|
-- (project_label/cost_usd/cache_rebuild_count, migration 015) so the
|
|
2431
2444
|
-- Browse list's date/project/cost/cache-rebuild filters are pure-SQL
|
|
2432
|
-
-- predicates
|
|
2433
|
-
--
|
|
2445
|
+
-- predicates, PLUS three DISPLAYED-enrichment columns
|
|
2446
|
+
-- (git_branch/models_json/title, migration 023 / #302) so the rail's
|
|
2447
|
+
-- per-session enrichment (git branch, ordered model list, stable
|
|
2448
|
+
-- first-prompt title) is read straight off the rollup instead of
|
|
2449
|
+
-- re-scanning conversation_messages per session on every cold page.
|
|
2450
|
+
-- The structural columns are recomputed by a COUNT/MIN/MAX GROUP BY; the
|
|
2451
|
+
-- filter + enrichment columns are filled per-session by
|
|
2434
2452
|
-- _fill_conversation_sessions_filter_columns in the same flock-held
|
|
2435
|
-
-- recompute (cost via the query kernel's batch maps,
|
|
2436
|
-
-- via a per-session assemble). The
|
|
2437
|
-
--
|
|
2438
|
-
--
|
|
2439
|
-
--
|
|
2440
|
-
--
|
|
2441
|
-
--
|
|
2442
|
-
--
|
|
2443
|
-
--
|
|
2453
|
+
-- recompute (cost/models/title/branch via the query kernel's batch maps,
|
|
2454
|
+
-- cache_rebuild_count via a per-session assemble). The AI title is NOT
|
|
2455
|
+
-- stored here — it is volatile, so list_conversations overlays it live
|
|
2456
|
+
-- from conversation_ai_titles (#302 Q2-B). The explicit NOT NULL on the
|
|
2457
|
+
-- non-INTEGER PK matters (SQLite's legacy NULL-in-PK bug); the rail keys
|
|
2458
|
+
-- on a concrete session_id and the recompute's GROUP BY already filters
|
|
2459
|
+
-- nulls. The index lets the only paginated ordering (recent)
|
|
2460
|
+
-- early-terminate at LIMIT with no temp B-tree. Re-derivable like the
|
|
2461
|
+
-- rest of cache.db; sync_cache keeps it honest (scoped DELETE+INSERT
|
|
2462
|
+
-- re-derive + flag-gated full recompute) — migration 013 arms the
|
|
2463
|
+
-- one-time history backfill, 023 re-arms it for the enrichment columns.
|
|
2444
2464
|
CREATE TABLE IF NOT EXISTS conversation_sessions (
|
|
2445
2465
|
session_id TEXT NOT NULL PRIMARY KEY,
|
|
2446
2466
|
msg_count INTEGER NOT NULL DEFAULT 0,
|
|
@@ -2448,7 +2468,10 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2448
2468
|
last_activity_utc TEXT,
|
|
2449
2469
|
project_label TEXT,
|
|
2450
2470
|
cost_usd REAL NOT NULL DEFAULT 0,
|
|
2451
|
-
cache_rebuild_count INTEGER NOT NULL DEFAULT 0
|
|
2471
|
+
cache_rebuild_count INTEGER NOT NULL DEFAULT 0,
|
|
2472
|
+
git_branch TEXT,
|
|
2473
|
+
models_json TEXT,
|
|
2474
|
+
title TEXT
|
|
2452
2475
|
);
|
|
2453
2476
|
CREATE INDEX IF NOT EXISTS idx_conv_sessions_recent
|
|
2454
2477
|
ON conversation_sessions(last_activity_utc DESC, session_id DESC);
|
|
@@ -2517,6 +2540,87 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2517
2540
|
CREATE INDEX IF NOT EXISTS idx_codex_entries_source
|
|
2518
2541
|
ON codex_session_entries(source_path);
|
|
2519
2542
|
|
|
2543
|
+
-- #294 S1: physical Codex rollout retention. These tables deliberately
|
|
2544
|
+
-- live in the unconditional base-schema script, before the legacy-FTS
|
|
2545
|
+
-- topology checks below: an existing cache with an old FTS shape must
|
|
2546
|
+
-- still gain the S1 tables on every open, just as conversation_file_touches
|
|
2547
|
+
-- does above.
|
|
2548
|
+
CREATE TABLE IF NOT EXISTS codex_source_roots (
|
|
2549
|
+
source_root_key TEXT NOT NULL PRIMARY KEY,
|
|
2550
|
+
canonical_root_path TEXT NOT NULL UNIQUE,
|
|
2551
|
+
first_seen_utc TEXT NOT NULL,
|
|
2552
|
+
last_seen_utc TEXT NOT NULL
|
|
2553
|
+
);
|
|
2554
|
+
|
|
2555
|
+
CREATE TABLE IF NOT EXISTS codex_conversation_threads (
|
|
2556
|
+
conversation_key TEXT NOT NULL PRIMARY KEY,
|
|
2557
|
+
source_root_key TEXT NOT NULL,
|
|
2558
|
+
native_thread_id TEXT NOT NULL,
|
|
2559
|
+
root_thread_id TEXT NOT NULL,
|
|
2560
|
+
parent_thread_id TEXT,
|
|
2561
|
+
source_path TEXT NOT NULL,
|
|
2562
|
+
cwd TEXT,
|
|
2563
|
+
git_json TEXT,
|
|
2564
|
+
source_kind TEXT,
|
|
2565
|
+
thread_source_json TEXT,
|
|
2566
|
+
model_provider TEXT,
|
|
2567
|
+
context_window INTEGER,
|
|
2568
|
+
first_seen_utc TEXT,
|
|
2569
|
+
last_seen_utc TEXT,
|
|
2570
|
+
UNIQUE(source_root_key, root_thread_id, native_thread_id)
|
|
2571
|
+
);
|
|
2572
|
+
CREATE INDEX IF NOT EXISTS idx_codex_threads_source_root
|
|
2573
|
+
ON codex_conversation_threads(source_root_key);
|
|
2574
|
+
CREATE INDEX IF NOT EXISTS idx_codex_threads_source_path
|
|
2575
|
+
ON codex_conversation_threads(source_path);
|
|
2576
|
+
|
|
2577
|
+
CREATE TABLE IF NOT EXISTS quota_window_snapshots (
|
|
2578
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
2579
|
+
source TEXT NOT NULL CHECK(source IN ('claude','codex')),
|
|
2580
|
+
source_root_key TEXT,
|
|
2581
|
+
source_path TEXT NOT NULL,
|
|
2582
|
+
line_offset INTEGER NOT NULL,
|
|
2583
|
+
captured_at_utc TEXT NOT NULL,
|
|
2584
|
+
observed_slot TEXT,
|
|
2585
|
+
logical_limit_key TEXT NOT NULL,
|
|
2586
|
+
limit_id TEXT,
|
|
2587
|
+
limit_name TEXT,
|
|
2588
|
+
window_minutes INTEGER NOT NULL CHECK(window_minutes > 0),
|
|
2589
|
+
used_percent REAL NOT NULL CHECK(used_percent >= 0 AND used_percent <= 100),
|
|
2590
|
+
resets_at_utc TEXT NOT NULL,
|
|
2591
|
+
plan_type TEXT,
|
|
2592
|
+
individual_limit_json TEXT,
|
|
2593
|
+
reached_type TEXT,
|
|
2594
|
+
UNIQUE(source, source_path, line_offset, logical_limit_key),
|
|
2595
|
+
CHECK(source != 'codex' OR source_root_key IS NOT NULL)
|
|
2596
|
+
);
|
|
2597
|
+
CREATE INDEX IF NOT EXISTS idx_quota_window_source_root
|
|
2598
|
+
ON quota_window_snapshots(source_root_key);
|
|
2599
|
+
CREATE INDEX IF NOT EXISTS idx_quota_window_captured_at
|
|
2600
|
+
ON quota_window_snapshots(captured_at_utc);
|
|
2601
|
+
|
|
2602
|
+
CREATE TABLE IF NOT EXISTS codex_conversation_events (
|
|
2603
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
2604
|
+
source_path TEXT NOT NULL,
|
|
2605
|
+
line_offset INTEGER NOT NULL,
|
|
2606
|
+
source_root_key TEXT NOT NULL,
|
|
2607
|
+
conversation_key TEXT,
|
|
2608
|
+
native_thread_id TEXT,
|
|
2609
|
+
root_thread_id TEXT,
|
|
2610
|
+
parent_thread_id TEXT,
|
|
2611
|
+
timestamp_utc TEXT,
|
|
2612
|
+
record_type TEXT,
|
|
2613
|
+
event_type TEXT,
|
|
2614
|
+
turn_id TEXT,
|
|
2615
|
+
call_id TEXT,
|
|
2616
|
+
payload_json TEXT NOT NULL,
|
|
2617
|
+
UNIQUE(source_path, line_offset)
|
|
2618
|
+
);
|
|
2619
|
+
CREATE INDEX IF NOT EXISTS idx_codex_events_conversation
|
|
2620
|
+
ON codex_conversation_events(conversation_key);
|
|
2621
|
+
CREATE INDEX IF NOT EXISTS idx_codex_events_timestamp
|
|
2622
|
+
ON codex_conversation_events(timestamp_utc);
|
|
2623
|
+
|
|
2520
2624
|
CREATE TABLE IF NOT EXISTS cache_meta (
|
|
2521
2625
|
key TEXT PRIMARY KEY,
|
|
2522
2626
|
value TEXT
|
|
@@ -2528,6 +2632,44 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2528
2632
|
# populated lazily in sync_cache() / _ensure_session_files_row().
|
|
2529
2633
|
add_column_if_missing(conn, "session_files", "session_id", "TEXT")
|
|
2530
2634
|
add_column_if_missing(conn, "session_files", "project_path", "TEXT")
|
|
2635
|
+
# #294 S1: old Codex accounting rows have no truthful provider-root or
|
|
2636
|
+
# conversation identity, so the linkage remains nullable until migration
|
|
2637
|
+
# 024 clears them for a source-derived reingest. Existing report selectors
|
|
2638
|
+
# intentionally keep selecting their shipped columns only.
|
|
2639
|
+
add_column_if_missing(conn, "codex_session_entries", "source_root_key", "TEXT")
|
|
2640
|
+
add_column_if_missing(conn, "codex_session_entries", "conversation_key", "TEXT")
|
|
2641
|
+
conn.execute(
|
|
2642
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_entries_source_root "
|
|
2643
|
+
"ON codex_session_entries(source_root_key)"
|
|
2644
|
+
)
|
|
2645
|
+
conn.execute(
|
|
2646
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_entries_conversation "
|
|
2647
|
+
"ON codex_session_entries(conversation_key)"
|
|
2648
|
+
)
|
|
2649
|
+
# #294 S3: the qualified accounting adapter is bounded by timestamp and
|
|
2650
|
+
# joins only through the S1 root-qualified conversation identity. Keep
|
|
2651
|
+
# this re-derivable index in the unconditional schema path so an existing
|
|
2652
|
+
# cache gains the same scale-safe plan without a data migration.
|
|
2653
|
+
conn.execute(
|
|
2654
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_entries_ts_root_conversation "
|
|
2655
|
+
"ON codex_session_entries(timestamp_utc, source_root_key, conversation_key)"
|
|
2656
|
+
)
|
|
2657
|
+
# The per-file terminal thread facts seed a later append without rereading
|
|
2658
|
+
# the prefix. They are nullable for old cache rows; migration 024 never
|
|
2659
|
+
# fabricates these source facts and instead clears/rederives them.
|
|
2660
|
+
add_column_if_missing(conn, "codex_session_files", "source_root_key", "TEXT")
|
|
2661
|
+
add_column_if_missing(conn, "codex_session_files", "last_native_thread_id", "TEXT")
|
|
2662
|
+
add_column_if_missing(conn, "codex_session_files", "last_root_thread_id", "TEXT")
|
|
2663
|
+
add_column_if_missing(conn, "codex_session_files", "last_parent_thread_id", "TEXT")
|
|
2664
|
+
add_column_if_missing(conn, "codex_session_files", "last_conversation_key", "TEXT")
|
|
2665
|
+
conn.execute(
|
|
2666
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_files_source_root "
|
|
2667
|
+
"ON codex_session_files(source_root_key)"
|
|
2668
|
+
)
|
|
2669
|
+
conn.execute(
|
|
2670
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_files_conversation "
|
|
2671
|
+
"ON codex_session_files(last_conversation_key)"
|
|
2672
|
+
)
|
|
2531
2673
|
# #181: materialize the only-ever-consumed extra `usage` key (`speed`) into
|
|
2532
2674
|
# a real session_entries column so the hot cache read paths (iter_entries /
|
|
2533
2675
|
# get_claude_session_entries) stop json.loads-ing the deeply-nested
|
|
@@ -2604,6 +2746,17 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2604
2746
|
conn, "conversation_messages", "search_tool", "TEXT NOT NULL DEFAULT ''")
|
|
2605
2747
|
add_column_if_missing(
|
|
2606
2748
|
conn, "conversation_messages", "search_thinking", "TEXT NOT NULL DEFAULT ''")
|
|
2749
|
+
# #302: the browse-rail DISPLAYED-enrichment columns on the conversation_sessions
|
|
2750
|
+
# rollup (git_branch / models_json / title). Idempotent column-adds (no marker,
|
|
2751
|
+
# no version — the repo's column-addition rule, NOT a raw ALTER in a migration
|
|
2752
|
+
# handler; Codex P1-2). They MUST land here — after the CREATE TABLE above (so
|
|
2753
|
+
# the table exists on a fresh DB) and BEFORE the legacy-FTS early-return below
|
|
2754
|
+
# (so an old-shape existing cache.db still receives them before any rail read
|
|
2755
|
+
# SELECTs them). Migration 023 arms the one-time full backfill that fills the
|
|
2756
|
+
# values on existing history; a fresh DB fills them at ingest.
|
|
2757
|
+
add_column_if_missing(conn, "conversation_sessions", "git_branch", "TEXT")
|
|
2758
|
+
add_column_if_missing(conn, "conversation_sessions", "models_json", "TEXT")
|
|
2759
|
+
add_column_if_missing(conn, "conversation_sessions", "title", "TEXT")
|
|
2607
2760
|
conn.execute(
|
|
2608
2761
|
"CREATE INDEX IF NOT EXISTS idx_session_files_session_id "
|
|
2609
2762
|
"ON session_files(session_id)"
|
|
@@ -3269,6 +3422,29 @@ def _cache_db_lock_path_for_conn(conn: sqlite3.Connection) -> "pathlib.Path | No
|
|
|
3269
3422
|
return None
|
|
3270
3423
|
|
|
3271
3424
|
|
|
3425
|
+
def _cache_db_codex_lock_path_for_conn(
|
|
3426
|
+
conn: sqlite3.Connection,
|
|
3427
|
+
) -> "pathlib.Path | None":
|
|
3428
|
+
"""Return this connection's ``<cache.db>.codex.lock`` sibling.
|
|
3429
|
+
|
|
3430
|
+
Codex ingest deliberately uses a separate fcntl lock from Claude ingest.
|
|
3431
|
+
Derive it from the connection rather than the global path constant so a
|
|
3432
|
+
migration test or recovery connection never contends on the caller's real
|
|
3433
|
+
cache lock. ``sync_codex_cache`` opens this exact sibling in production.
|
|
3434
|
+
"""
|
|
3435
|
+
try:
|
|
3436
|
+
rows = conn.execute("PRAGMA database_list").fetchall()
|
|
3437
|
+
except sqlite3.DatabaseError:
|
|
3438
|
+
return None
|
|
3439
|
+
for row in rows:
|
|
3440
|
+
if row[1] == "main":
|
|
3441
|
+
db_file = row[2]
|
|
3442
|
+
if not db_file:
|
|
3443
|
+
return None
|
|
3444
|
+
return pathlib.Path(str(db_file) + ".codex.lock")
|
|
3445
|
+
return None
|
|
3446
|
+
|
|
3447
|
+
|
|
3272
3448
|
@cache_migration("001_dedup_highest_wins")
|
|
3273
3449
|
def _001_dedup_highest_wins(conn: sqlite3.Connection) -> None:
|
|
3274
3450
|
"""One-time re-ingest of session_entries with corrected msg_id+req_id dedup.
|
|
@@ -4018,6 +4194,103 @@ def _021_index_conversation_messages_cwd(conn: sqlite3.Connection) -> None:
|
|
|
4018
4194
|
)
|
|
4019
4195
|
|
|
4020
4196
|
|
|
4197
|
+
@cache_migration("022_index_conversation_messages_model")
|
|
4198
|
+
def _022_index_conversation_messages_model(conn: sqlite3.Connection) -> None:
|
|
4199
|
+
"""#301: partial covering index on conversation_messages(model, session_id) to
|
|
4200
|
+
collapse the full-table `SELECT DISTINCT ... model` walks in
|
|
4201
|
+
list_conversation_facets and _model_clause (~22s cold on a 5.5 GB cache) into
|
|
4202
|
+
index-only walks/seeks.
|
|
4203
|
+
|
|
4204
|
+
Fresh installs never run this handler: the dispatcher stamps it WITHOUT running
|
|
4205
|
+
(the fresh-install branch), and `_apply_cache_schema` already created the index
|
|
4206
|
+
on the fresh DB. NO self-stamp — the dispatcher central-stamps on a clean return
|
|
4207
|
+
(#140). Run-twice safe: CREATE INDEX is IF NOT EXISTS. cache.db is re-derivable —
|
|
4208
|
+
`cache-sync --rebuild` is the escape hatch.
|
|
4209
|
+
|
|
4210
|
+
No flock/BEGIN IMMEDIATE (like 021): a single CREATE INDEX IF NOT EXISTS performs
|
|
4211
|
+
no row mutations (it reads the table once to build the index but modifies no
|
|
4212
|
+
rows), so it needs no mutual-exclusion scaffolding.
|
|
4213
|
+
"""
|
|
4214
|
+
conn.execute(
|
|
4215
|
+
"CREATE INDEX IF NOT EXISTS idx_conversation_messages_model_session "
|
|
4216
|
+
"ON conversation_messages(model, session_id) "
|
|
4217
|
+
"WHERE model IS NOT NULL AND model != ''"
|
|
4218
|
+
)
|
|
4219
|
+
|
|
4220
|
+
|
|
4221
|
+
@cache_migration("023_conversation_sessions_enrichment_columns")
|
|
4222
|
+
def _023_conversation_sessions_enrichment_columns(conn: sqlite3.Connection) -> None:
|
|
4223
|
+
"""Flag-only arm for the #302 browse-rail enrichment columns (git_branch,
|
|
4224
|
+
models_json, title). The columns themselves are added by _apply_cache_schema's
|
|
4225
|
+
CREATE TABLE + add_column_if_missing (the repo's column-addition rule, NOT an
|
|
4226
|
+
ALTER here); this migration only arms the SHARED
|
|
4227
|
+
conversation_sessions_backfill_pending flag so the next sync_cache full
|
|
4228
|
+
recompute fills the new (empty) columns via the augmented
|
|
4229
|
+
_fill_conversation_sessions_filter_columns. Central stamp via the dispatcher
|
|
4230
|
+
(#140); the handler does NOT self-stamp.
|
|
4231
|
+
|
|
4232
|
+
A fresh install gets the columns from _apply_cache_schema's CREATE TABLE and
|
|
4233
|
+
stamps 023 WITHOUT running this handler; its empty rollup needs no backfill
|
|
4234
|
+
(the incremental DELETE+INSERT re-derive fills all columns in lockstep).
|
|
4235
|
+
Mirrors 013/015."""
|
|
4236
|
+
_set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
|
|
4237
|
+
conn.commit()
|
|
4238
|
+
|
|
4239
|
+
|
|
4240
|
+
@cache_migration("024_codex_fused_ingest_rebuild")
|
|
4241
|
+
def _024_codex_fused_ingest_rebuild(conn: sqlite3.Connection) -> None:
|
|
4242
|
+
"""Clear stale Codex cache state for #294 S1's fused source reingest.
|
|
4243
|
+
|
|
4244
|
+
Pre-S1 Codex rows lack provider-root identity, qualified conversation
|
|
4245
|
+
linkage, quota observations, and physical event payloads. Those facts are
|
|
4246
|
+
not recoverable from the cache, so this migration clears only the Codex
|
|
4247
|
+
accounting/file surface and partial S1 derived rows; the next normal Codex
|
|
4248
|
+
sync rederives them from rollout source. Claude tables and Claude quota
|
|
4249
|
+
snapshots remain untouched.
|
|
4250
|
+
|
|
4251
|
+
The handler takes ``cache.db.codex.lock`` before ``BEGIN IMMEDIATE`` — the
|
|
4252
|
+
same fcntl-then-SQLite order as ``sync_codex_cache``. Contention raises
|
|
4253
|
+
``MigrationGateNotMet`` before any DML, so both ordinary and eager cache
|
|
4254
|
+
dispatch defer without a partial clear. The DELETE-only transition is safe
|
|
4255
|
+
to retry after a crash between this commit and the dispatcher's separate
|
|
4256
|
+
marker stamp: rerunning against its own empty Codex state is a no-op.
|
|
4257
|
+
Never self-stamp; the dispatcher owns ``schema_migrations`` and
|
|
4258
|
+
``user_version``.
|
|
4259
|
+
"""
|
|
4260
|
+
lock_path = _cache_db_codex_lock_path_for_conn(conn)
|
|
4261
|
+
lock_fh = None
|
|
4262
|
+
if lock_path is not None:
|
|
4263
|
+
lock_fh = open(lock_path, "w")
|
|
4264
|
+
try:
|
|
4265
|
+
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
4266
|
+
except BlockingIOError:
|
|
4267
|
+
lock_fh.close()
|
|
4268
|
+
raise MigrationGateNotMet(
|
|
4269
|
+
"cache.db.codex.lock held by a concurrent sync_codex_cache; "
|
|
4270
|
+
"deferring cache 024 fused-ingest rebuild (#294 S1)"
|
|
4271
|
+
)
|
|
4272
|
+
try:
|
|
4273
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
4274
|
+
try:
|
|
4275
|
+
conn.execute("DELETE FROM codex_session_entries")
|
|
4276
|
+
conn.execute("DELETE FROM codex_session_files")
|
|
4277
|
+
conn.execute("DELETE FROM quota_window_snapshots WHERE source = 'codex'")
|
|
4278
|
+
conn.execute("DELETE FROM codex_conversation_threads")
|
|
4279
|
+
conn.execute("DELETE FROM codex_conversation_events")
|
|
4280
|
+
conn.execute("DELETE FROM codex_source_roots")
|
|
4281
|
+
conn.commit()
|
|
4282
|
+
except Exception:
|
|
4283
|
+
conn.rollback()
|
|
4284
|
+
raise
|
|
4285
|
+
finally:
|
|
4286
|
+
if lock_fh is not None:
|
|
4287
|
+
try:
|
|
4288
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
4289
|
+
except OSError:
|
|
4290
|
+
pass
|
|
4291
|
+
lock_fh.close()
|
|
4292
|
+
|
|
4293
|
+
|
|
4021
4294
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
4022
4295
|
|
|
4023
4296
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|
|
@@ -5001,6 +5274,19 @@ def _migration_unify_budget_milestones_vendor(conn: sqlite3.Connection) -> None:
|
|
|
5001
5274
|
raise
|
|
5002
5275
|
|
|
5003
5276
|
|
|
5277
|
+
@stats_migration("013_codex_quota_projection_state")
|
|
5278
|
+
def _migration_codex_quota_projection_state(conn: sqlite3.Connection) -> None:
|
|
5279
|
+
"""Create the durable provider-neutral quota projection tables.
|
|
5280
|
+
|
|
5281
|
+
All physical evidence stays in cache.db's ``quota_window_snapshots``. This
|
|
5282
|
+
migration deliberately performs no cache read or data backfill: a later
|
|
5283
|
+
full projector transaction creates the interpreted rows from the committed
|
|
5284
|
+
physical set. The schema helper is safe after a crash before central
|
|
5285
|
+
stamping, so the dispatcher can retry without duplicate state.
|
|
5286
|
+
"""
|
|
5287
|
+
_cctally_core._apply_quota_projection_schema(conn)
|
|
5288
|
+
|
|
5289
|
+
|
|
5004
5290
|
# === Region 8: Test-only migration registration (was bin/cctally:12086-12140) ===
|
|
5005
5291
|
|
|
5006
5292
|
# ──────────────────────────────────────────────────────────────────────
|
package/bin/_cctally_diff.py
CHANGED
|
@@ -15,6 +15,7 @@ from __future__ import annotations
|
|
|
15
15
|
|
|
16
16
|
import argparse
|
|
17
17
|
import dataclasses
|
|
18
|
+
import json
|
|
18
19
|
import shutil
|
|
19
20
|
import sqlite3
|
|
20
21
|
import sys
|
|
@@ -104,14 +105,24 @@ def _emit_diff_debug_samples(args, window_a, window_b) -> None:
|
|
|
104
105
|
def cmd_diff(args: argparse.Namespace) -> int:
|
|
105
106
|
"""Compare Claude usage between two windows."""
|
|
106
107
|
c = _cctally()
|
|
108
|
+
c._share_validate_args(args)
|
|
107
109
|
now_utc = _command_as_of()
|
|
108
110
|
if getattr(args, "debug_now", False):
|
|
109
111
|
print(f"now_utc={c._iso_z(now_utc)}")
|
|
110
112
|
return 0
|
|
111
113
|
|
|
112
|
-
#
|
|
113
|
-
#
|
|
114
|
-
|
|
114
|
+
# The source-aware all-provider path resolves calendar week tokens once
|
|
115
|
+
# before dispatching either provider. Reuse those absolute, half-open
|
|
116
|
+
# windows for Claude instead of attempting to reinterpret the original
|
|
117
|
+
# tokens against its subscription-week anchor. Ordinary Claude invocations
|
|
118
|
+
# keep the established anchor/parser path byte-for-byte.
|
|
119
|
+
supplied_windows = getattr(args, "_source_analytics_windows", None)
|
|
120
|
+
if supplied_windows is None:
|
|
121
|
+
# Resolve anchors (None when no snapshots exist; week tokens then
|
|
122
|
+
# raise NoAnchorError in the parser).
|
|
123
|
+
anchor_week_start, anchor_resets_at = dk._diff_resolve_anchor(now_utc)
|
|
124
|
+
else:
|
|
125
|
+
anchor_week_start, anchor_resets_at = None, None
|
|
115
126
|
|
|
116
127
|
# Validation already happened via _argparse_tz; resolve now to a ZoneInfo
|
|
117
128
|
# (or None for "local") and derive the IANA name for window resolution.
|
|
@@ -124,18 +135,35 @@ def cmd_diff(args: argparse.Namespace) -> int:
|
|
|
124
135
|
tz_name = (tz_obj.key if tz_obj is not None else c._local_tz_name())
|
|
125
136
|
|
|
126
137
|
try:
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
138
|
+
if supplied_windows is not None:
|
|
139
|
+
def _from_source_window(window):
|
|
140
|
+
start = window.start_at
|
|
141
|
+
end = window.end_at
|
|
142
|
+
return dk.ParsedWindow(
|
|
143
|
+
label=window.label,
|
|
144
|
+
start_utc=start,
|
|
145
|
+
end_utc=end,
|
|
146
|
+
length_days=(end - start).total_seconds() / 86400,
|
|
147
|
+
kind=window.kind,
|
|
148
|
+
week_aligned=False,
|
|
149
|
+
full_weeks_count=0,
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
window_a = _from_source_window(supplied_windows[0])
|
|
153
|
+
window_b = _from_source_window(supplied_windows[1])
|
|
154
|
+
else:
|
|
155
|
+
window_a = dk._parse_diff_window(
|
|
156
|
+
args.a, now_utc=now_utc,
|
|
157
|
+
anchor_resets_at=anchor_resets_at,
|
|
158
|
+
anchor_week_start=anchor_week_start,
|
|
159
|
+
tz_name=tz_name,
|
|
160
|
+
)
|
|
161
|
+
window_b = dk._parse_diff_window(
|
|
162
|
+
args.b, now_utc=now_utc,
|
|
163
|
+
anchor_resets_at=anchor_resets_at,
|
|
164
|
+
anchor_week_start=anchor_week_start,
|
|
165
|
+
tz_name=tz_name,
|
|
166
|
+
)
|
|
139
167
|
except dk.NoAnchorError as exc:
|
|
140
168
|
print(f"diff: {exc}", file=sys.stderr)
|
|
141
169
|
return 1
|
|
@@ -221,7 +249,12 @@ def cmd_diff(args: argparse.Namespace) -> int:
|
|
|
221
249
|
}
|
|
222
250
|
|
|
223
251
|
if args.emit_json:
|
|
224
|
-
|
|
252
|
+
payload = dk._diff_to_json_payload(result, options=options)
|
|
253
|
+
sink = getattr(args, "_source_result_sink", None)
|
|
254
|
+
if sink is not None:
|
|
255
|
+
sink(payload)
|
|
256
|
+
else:
|
|
257
|
+
print(json.dumps(payload, indent=2))
|
|
225
258
|
return 0
|
|
226
259
|
|
|
227
260
|
# Session A (spec §7.3): route through the new color resolver so
|
package/bin/_cctally_doctor.py
CHANGED
|
@@ -42,6 +42,65 @@ def _cctally():
|
|
|
42
42
|
return sys.modules["cctally"]
|
|
43
43
|
|
|
44
44
|
|
|
45
|
+
def _codex_lifecycle_activity_24h(
|
|
46
|
+
*, root_keys: set[str], now_utc: dt.datetime,
|
|
47
|
+
) -> dict[str, dict]:
|
|
48
|
+
"""Read root-qualified Codex lifecycle outcomes from bounded local logs.
|
|
49
|
+
|
|
50
|
+
The parser intentionally accepts only timestamped token records and retains
|
|
51
|
+
aggregate lifecycle counters. It never loads session, prompt, or response
|
|
52
|
+
content into doctor state.
|
|
53
|
+
"""
|
|
54
|
+
cutoff = now_utc - dt.timedelta(hours=24)
|
|
55
|
+
records: dict[str, dict] = {}
|
|
56
|
+
for path in (
|
|
57
|
+
_cctally_core.HOOK_TICK_LOG_ROTATED_PATH,
|
|
58
|
+
_cctally_core.HOOK_TICK_LOG_PATH,
|
|
59
|
+
):
|
|
60
|
+
try:
|
|
61
|
+
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
62
|
+
except OSError:
|
|
63
|
+
continue
|
|
64
|
+
for line in lines:
|
|
65
|
+
tokens = line.split()
|
|
66
|
+
if not tokens:
|
|
67
|
+
continue
|
|
68
|
+
try:
|
|
69
|
+
captured_at = parse_iso_datetime(tokens[0], "codex lifecycle log timestamp")
|
|
70
|
+
captured_at = captured_at.astimezone(dt.timezone.utc)
|
|
71
|
+
except (IndexError, ValueError, TypeError):
|
|
72
|
+
continue
|
|
73
|
+
if captured_at > now_utc:
|
|
74
|
+
continue
|
|
75
|
+
fields = {
|
|
76
|
+
token.split("=", 1)[0]: token.split("=", 1)[1]
|
|
77
|
+
for token in tokens[1:] if "=" in token
|
|
78
|
+
}
|
|
79
|
+
if fields.get("provider") != "codex":
|
|
80
|
+
continue
|
|
81
|
+
key = fields.get("source_root_key")
|
|
82
|
+
if key not in root_keys:
|
|
83
|
+
continue
|
|
84
|
+
outcome = fields.get("result")
|
|
85
|
+
if outcome not in {"success", "error"}:
|
|
86
|
+
continue
|
|
87
|
+
row = records.setdefault(key, {
|
|
88
|
+
"last_tick_at": None,
|
|
89
|
+
"success_count_24h": 0,
|
|
90
|
+
"error_count_24h": 0,
|
|
91
|
+
})
|
|
92
|
+
if outcome == "success":
|
|
93
|
+
prior = row["last_tick_at"]
|
|
94
|
+
if prior is None or captured_at > prior:
|
|
95
|
+
row["last_tick_at"] = captured_at
|
|
96
|
+
if captured_at >= cutoff:
|
|
97
|
+
if outcome == "success":
|
|
98
|
+
row["success_count_24h"] += 1
|
|
99
|
+
else:
|
|
100
|
+
row["error_count_24h"] += 1
|
|
101
|
+
return records
|
|
102
|
+
|
|
103
|
+
|
|
45
104
|
def doctor_gather_state(
|
|
46
105
|
*,
|
|
47
106
|
now_utc: "dt.datetime | None" = None,
|
|
@@ -398,6 +457,62 @@ def doctor_gather_state(
|
|
|
398
457
|
except Exception:
|
|
399
458
|
pass
|
|
400
459
|
|
|
460
|
+
# ── Codex quota lifecycle (#294 S2) ──────────────────────────────
|
|
461
|
+
# All three probes are read-only and root-qualified. The physical cache
|
|
462
|
+
# adapter preserves S1's per-window degradation, while setup's existing
|
|
463
|
+
# inspector supplies the exact owned-hook state without exposing paths.
|
|
464
|
+
codex_quota_windows: list[dict] = []
|
|
465
|
+
try:
|
|
466
|
+
observations = c._cctally_quota.load_codex_quota_observations()
|
|
467
|
+
by_identity: dict[object, list] = {}
|
|
468
|
+
for observation in observations:
|
|
469
|
+
by_identity.setdefault(observation.identity, []).append(observation)
|
|
470
|
+
for identity in sorted(
|
|
471
|
+
by_identity,
|
|
472
|
+
key=lambda item: (
|
|
473
|
+
item.source, item.source_root_key, item.logical_limit_key,
|
|
474
|
+
item.observed_slot, item.window_minutes,
|
|
475
|
+
),
|
|
476
|
+
):
|
|
477
|
+
freshness = c.quota_freshness(by_identity[identity], now_utc)
|
|
478
|
+
codex_quota_windows.append({
|
|
479
|
+
"identity": {
|
|
480
|
+
"source": identity.source,
|
|
481
|
+
"source_root_key": identity.source_root_key,
|
|
482
|
+
"logical_limit_key": identity.logical_limit_key,
|
|
483
|
+
"observed_slot": identity.observed_slot,
|
|
484
|
+
"window_minutes": identity.window_minutes,
|
|
485
|
+
},
|
|
486
|
+
"latest_capture_at": freshness.captured_at,
|
|
487
|
+
"freshness_state": freshness.state,
|
|
488
|
+
"age_seconds": freshness.age_seconds,
|
|
489
|
+
"stale_after_seconds": freshness.stale_after_seconds,
|
|
490
|
+
})
|
|
491
|
+
except Exception:
|
|
492
|
+
codex_quota_windows = []
|
|
493
|
+
|
|
494
|
+
codex_hook_roots: list[dict] = []
|
|
495
|
+
try:
|
|
496
|
+
codex_binary = str(c._setup_resolve_hook_target(repo_root))
|
|
497
|
+
hook_rows = [
|
|
498
|
+
c._cctally_setup._codex_hook_row(root, codex_binary)
|
|
499
|
+
for root in c._setup_codex_hook_roots()
|
|
500
|
+
]
|
|
501
|
+
codex_hook_roots = [
|
|
502
|
+
{"source_root_key": row["source_root_key"], "state": row["state"]}
|
|
503
|
+
for row in sorted(hook_rows, key=lambda row: row["source_root_key"])
|
|
504
|
+
]
|
|
505
|
+
except Exception:
|
|
506
|
+
codex_hook_roots = []
|
|
507
|
+
|
|
508
|
+
try:
|
|
509
|
+
codex_lifecycle_activity_24h = _codex_lifecycle_activity_24h(
|
|
510
|
+
root_keys={row["source_root_key"] for row in codex_hook_roots},
|
|
511
|
+
now_utc=now_utc,
|
|
512
|
+
)
|
|
513
|
+
except Exception:
|
|
514
|
+
codex_lifecycle_activity_24h = {}
|
|
515
|
+
|
|
401
516
|
# ── Parse health (#279 S2 F5a) ───────────────────────────────────
|
|
402
517
|
parse_health_claude = parse_health_codex = None
|
|
403
518
|
try:
|
|
@@ -659,6 +774,9 @@ def doctor_gather_state(
|
|
|
659
774
|
locks_held=locks_held,
|
|
660
775
|
# #297: cache.db WAL size backstop (gathered outside the deep branch).
|
|
661
776
|
cache_db_wal_bytes=cache_db_wal_bytes,
|
|
777
|
+
codex_quota_windows=codex_quota_windows,
|
|
778
|
+
codex_hook_roots=codex_hook_roots,
|
|
779
|
+
codex_lifecycle_activity_24h=codex_lifecycle_activity_24h,
|
|
662
780
|
)
|
|
663
781
|
|
|
664
782
|
|
package/bin/_cctally_forecast.py
CHANGED
|
@@ -1011,9 +1011,12 @@ def cmd_report(args: argparse.Namespace) -> int:
|
|
|
1011
1011
|
c._share_render_and_emit(snap, args)
|
|
1012
1012
|
return 0
|
|
1013
1013
|
if args.json:
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1014
|
+
payload = c.stamp_schema_version({"current": None, "trend": []})
|
|
1015
|
+
sink = getattr(args, "_source_result_sink", None)
|
|
1016
|
+
if sink is not None:
|
|
1017
|
+
sink(payload)
|
|
1018
|
+
else:
|
|
1019
|
+
print(json.dumps(payload, indent=2))
|
|
1017
1020
|
else:
|
|
1018
1021
|
print("No data yet. Add record-usage to your status line script (see record-usage --help).")
|
|
1019
1022
|
return 0
|
|
@@ -1206,7 +1209,12 @@ def cmd_report(args: argparse.Namespace) -> int:
|
|
|
1206
1209
|
return 0
|
|
1207
1210
|
|
|
1208
1211
|
if args.json:
|
|
1209
|
-
|
|
1212
|
+
payload = c.stamp_schema_version(output)
|
|
1213
|
+
sink = getattr(args, "_source_result_sink", None)
|
|
1214
|
+
if sink is not None:
|
|
1215
|
+
sink(payload)
|
|
1216
|
+
else:
|
|
1217
|
+
print(json.dumps(payload, indent=2))
|
|
1210
1218
|
return 0
|
|
1211
1219
|
|
|
1212
1220
|
if current_row is not None:
|