cctally 1.103.0 → 1.105.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/README.md +6 -6
- package/bin/_cctally_alerts.py +104 -10
- package/bin/_cctally_cache.py +193 -71
- package/bin/_cctally_config.py +62 -2
- package/bin/_cctally_core.py +62 -1
- package/bin/_cctally_dashboard.py +694 -70
- package/bin/_cctally_dashboard_conversation.py +400 -6
- package/bin/_cctally_dashboard_envelope.py +336 -14
- package/bin/_cctally_dashboard_perf.py +93 -0
- package/bin/_cctally_dashboard_share.py +56 -25
- package/bin/_cctally_dashboard_sources.py +988 -94
- package/bin/_cctally_db.py +30 -3
- package/bin/_cctally_diagnosis_sources.py +699 -186
- package/bin/_cctally_doctor.py +77 -0
- package/bin/_cctally_forecast.py +932 -51
- package/bin/_cctally_journal.py +430 -24
- package/bin/_cctally_parser.py +66 -0
- package/bin/_cctally_project.py +535 -10
- package/bin/_cctally_quota.py +28 -0
- package/bin/_cctally_quota_calibration.py +146 -0
- package/bin/_cctally_quota_model.py +2142 -0
- package/bin/_cctally_record.py +204 -30
- package/bin/_cctally_share.py +16 -8
- package/bin/_cctally_statusline.py +34 -0
- package/bin/_cctally_tui.py +531 -115
- package/bin/_lib_codex_conversation_query.py +258 -12
- package/bin/_lib_codex_hooks.py +26 -0
- package/bin/_lib_conversation_query.py +145 -15
- package/bin/_lib_dashboard_json.py +105 -0
- package/bin/_lib_dashboard_settings_contract.py +2 -0
- package/bin/_lib_diagnosis.py +21 -2
- package/bin/_lib_doctor.py +215 -1
- package/bin/_lib_forecast.py +337 -43
- package/bin/_lib_ingest_frontier.py +889 -0
- package/bin/_lib_meter_rate_change.py +360 -0
- package/bin/_lib_perf.py +22 -0
- package/bin/_lib_pricing.py +30 -3
- package/bin/_lib_quota_calibration.py +311 -0
- package/bin/_lib_quota_copy.py +157 -0
- package/bin/_lib_quota_model.py +2520 -0
- package/bin/_lib_rate_change_delivery.py +119 -0
- package/bin/_lib_record.py +50 -0
- package/bin/_lib_rederive.py +10 -0
- package/bin/_lib_render.py +6 -0
- package/bin/_lib_retained_size.py +187 -0
- package/bin/_lib_share_templates.py +37 -5
- package/bin/_lib_snapshot_cache.py +504 -17
- package/bin/_lib_statusline.py +200 -2
- package/bin/_lib_tick_stats.py +28 -4
- package/bin/_lib_view_models.py +30 -12
- package/bin/cctally +32 -0
- package/dashboard/static/assets/ConversationsView-DA4j7Yso.js +72 -0
- package/dashboard/static/assets/DoctorModal-D4PbVnbE.js +1 -0
- package/dashboard/static/assets/ModalRoot-D4oSPxPp.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DkPlB9JH.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-BE7FONsS.js +1 -0
- package/dashboard/static/assets/UpdateModal-BJuI8glf.js +7 -0
- package/dashboard/static/assets/index-klO46NcU.css +1 -0
- package/dashboard/static/assets/index-w_bINkJ8.js +13 -0
- package/dashboard/static/assets/outlineNavigation-zDVm6Hdd.js +9 -0
- package/dashboard/static/assets/useKeymap-CJ-Pi17D.js +1 -0
- package/dashboard/static/dashboard.html +3 -2
- package/package.json +10 -1
- package/dashboard/static/assets/index-Di2hljvB.css +0 -1
- package/dashboard/static/assets/index-XYCIWjVG.js +0 -97
package/bin/_cctally_cache.py
CHANGED
|
@@ -201,6 +201,7 @@ _lib_codex_conversation = _load_lib("_lib_codex_conversation")
|
|
|
201
201
|
# stdlib leaf, same shape as `_lib_codex_pools`, so it loads here rather than
|
|
202
202
|
# through a bare import that would depend on ``bin/`` being on ``sys.path``.
|
|
203
203
|
_lib_codex_account_adoption = _load_lib("_lib_codex_account_adoption")
|
|
204
|
+
_ingest_frontier = _load_lib("_lib_ingest_frontier")
|
|
204
205
|
|
|
205
206
|
# Opt-in backend phase-instrumentation collector (issue #276, Session A). Pure
|
|
206
207
|
# stdlib leaf; near-noop when CCTALLY_PERF_TRACE is unset (phase() returns a
|
|
@@ -2142,6 +2143,10 @@ def _clear_codex_derived_rows(conn: sqlite3.Connection) -> bool:
|
|
|
2142
2143
|
"('codex_accounting_bulk_clear', '1')"
|
|
2143
2144
|
)
|
|
2144
2145
|
try:
|
|
2146
|
+
conn.execute(
|
|
2147
|
+
"DELETE FROM cache_meta WHERE key=?",
|
|
2148
|
+
(_ingest_frontier.CODEX_FULL_WALK_COMPLETE_KEY,),
|
|
2149
|
+
)
|
|
2145
2150
|
conn.execute("DELETE FROM codex_session_entries")
|
|
2146
2151
|
finally:
|
|
2147
2152
|
if _has_accounting_ledger:
|
|
@@ -2272,6 +2277,11 @@ COVERAGE_WRITER_ACTIONS: "dict[str, str]" = {
|
|
|
2272
2277
|
# it is a property of the observation's population rather than of the
|
|
2273
2278
|
# observation.
|
|
2274
2279
|
"_cctally_cache.CodexResetAnchorResolver.apply_pending_merges": "preserve",
|
|
2280
|
+
# Maintainer-only snapshot measurement mutates a copied cache, never the
|
|
2281
|
+
# live store. Its quota-only scenario models the production writer contract
|
|
2282
|
+
# by bumping `codex_physical_mutation_seq` in the same transaction as the
|
|
2283
|
+
# value update, so any copied certificate is invalid on the sequence axis.
|
|
2284
|
+
"cctally-snapshot-measure.mutate_quota_only": "preserve",
|
|
2275
2285
|
# The two attribution-map leaves. Both materialize a decision that was
|
|
2276
2286
|
# journaled fail-closed before the call, and neither deletes, so no caller
|
|
2277
2287
|
# can reach them in a way that breaks coverage. The action belongs to the
|
|
@@ -2646,7 +2656,25 @@ def _insert_codex_normalized_rows(
|
|
|
2646
2656
|
)
|
|
2647
2657
|
|
|
2648
2658
|
|
|
2649
|
-
def
|
|
2659
|
+
def _next_conversation_render_revision(conn: sqlite3.Connection) -> int:
|
|
2660
|
+
"""Reserve one monotonic revision inside the caller's transaction."""
|
|
2661
|
+
row = conn.execute(
|
|
2662
|
+
"SELECT value FROM cache_meta WHERE key='conversation_render_revision'"
|
|
2663
|
+
).fetchone()
|
|
2664
|
+
try:
|
|
2665
|
+
revision = int(row[0]) + 1 if row is not None else 1
|
|
2666
|
+
except (TypeError, ValueError):
|
|
2667
|
+
revision = 1
|
|
2668
|
+
_set_cache_meta(conn, "conversation_render_revision", str(revision))
|
|
2669
|
+
return revision
|
|
2670
|
+
|
|
2671
|
+
|
|
2672
|
+
def _recompute_codex_rollups(
|
|
2673
|
+
conn: sqlite3.Connection,
|
|
2674
|
+
conversation_keys,
|
|
2675
|
+
*,
|
|
2676
|
+
advance_render_revision: bool = True,
|
|
2677
|
+
) -> None:
|
|
2650
2678
|
"""Recompute-affected-or-delete the rollup for each conversation (§3.2).
|
|
2651
2679
|
|
|
2652
2680
|
A rollup is a pure function of surviving codex_conversation_messages (+ thread
|
|
@@ -2656,7 +2684,14 @@ def _recompute_codex_rollups(conn: sqlite3.Connection, conversation_keys) -> Non
|
|
|
2656
2684
|
survives.
|
|
2657
2685
|
"""
|
|
2658
2686
|
kern = _lib_codex_conversation
|
|
2659
|
-
|
|
2687
|
+
keys = {key for key in conversation_keys if key}
|
|
2688
|
+
if not keys:
|
|
2689
|
+
return
|
|
2690
|
+
render_revision = (
|
|
2691
|
+
_next_conversation_render_revision(conn)
|
|
2692
|
+
if advance_render_revision else 0
|
|
2693
|
+
)
|
|
2694
|
+
for conversation_key in keys:
|
|
2660
2695
|
rows = _load_codex_normalized_rows(conn, conversation_key)
|
|
2661
2696
|
if not rows:
|
|
2662
2697
|
conn.execute(
|
|
@@ -2688,25 +2723,34 @@ def _recompute_codex_rollups(conn: sqlite3.Connection, conversation_keys) -> Non
|
|
|
2688
2723
|
"INSERT INTO codex_conversation_rollups "
|
|
2689
2724
|
"(conversation_key, source_root_key, parent_thread_id, item_count, "
|
|
2690
2725
|
" started_utc, last_activity_utc, project_key, project_label, "
|
|
2691
|
-
" models_json, title) VALUES (
|
|
2726
|
+
" models_json, title, render_revision) VALUES (?,?,?,?,?,?,?,?,?,?,?) "
|
|
2692
2727
|
"ON CONFLICT(conversation_key) DO UPDATE SET "
|
|
2693
2728
|
" source_root_key=excluded.source_root_key, "
|
|
2694
2729
|
" parent_thread_id=excluded.parent_thread_id, "
|
|
2695
2730
|
" item_count=excluded.item_count, started_utc=excluded.started_utc, "
|
|
2696
2731
|
" last_activity_utc=excluded.last_activity_utc, "
|
|
2697
2732
|
" project_key=excluded.project_key, project_label=excluded.project_label, "
|
|
2698
|
-
" models_json=excluded.models_json, title=excluded.title"
|
|
2733
|
+
" models_json=excluded.models_json, title=excluded.title, "
|
|
2734
|
+
" render_revision=excluded.render_revision",
|
|
2699
2735
|
(conversation_key, source_root_key, parent_thread_id, item_count,
|
|
2700
|
-
started, last_activity, project_key, project_label, models_json, title
|
|
2736
|
+
started, last_activity, project_key, project_label, models_json, title,
|
|
2737
|
+
render_revision),
|
|
2701
2738
|
)
|
|
2702
2739
|
|
|
2703
2740
|
|
|
2704
|
-
def _replay_codex_normalization(
|
|
2741
|
+
def _replay_codex_normalization(
|
|
2742
|
+
conn: sqlite3.Connection,
|
|
2743
|
+
*,
|
|
2744
|
+
advance_render_revision: bool = True,
|
|
2745
|
+
) -> None:
|
|
2705
2746
|
"""Re-derive ALL normalized rows/touches/rollups from stored
|
|
2706
2747
|
codex_conversation_events, per file in (source_path ASC, line_offset ASC)
|
|
2707
2748
|
order (migration 025). Runs inside the caller's transaction — the caller
|
|
2708
2749
|
full-clears first (§3.4 helper) and owns the commit. Deterministic order +
|
|
2709
|
-
the plain rowid alias make a re-run byte-idempotent.
|
|
2750
|
+
the plain rowid alias make a re-run byte-idempotent. The legacy cache
|
|
2751
|
+
migration passes ``advance_render_revision=False`` because a crash-before-
|
|
2752
|
+
stamp retry must not mutate the new invalidation frontier; ordinary retained
|
|
2753
|
+
replay keeps the default and invalidates readers."""
|
|
2710
2754
|
kern = _lib_codex_conversation
|
|
2711
2755
|
events_by_file: dict[str, list] = {}
|
|
2712
2756
|
accounts_by_file: dict[str, dict[tuple[str, int], str | None]] = {}
|
|
@@ -2743,7 +2787,10 @@ def _replay_codex_normalization(conn: sqlite3.Connection) -> None:
|
|
|
2743
2787
|
accounts_by_file[source_path] if has_account else None,
|
|
2744
2788
|
)
|
|
2745
2789
|
affected.update(r.conversation_key for r in result.rows)
|
|
2746
|
-
_recompute_codex_rollups(
|
|
2790
|
+
_recompute_codex_rollups(
|
|
2791
|
+
conn, affected,
|
|
2792
|
+
advance_render_revision=advance_render_revision,
|
|
2793
|
+
)
|
|
2747
2794
|
# Migration 025 still replays this helper against legacy cache.db fixtures,
|
|
2748
2795
|
# where the conversations-only #482 projection table does not exist. Live
|
|
2749
2796
|
# conversations.db replays must refresh the projection, but the historical
|
|
@@ -3350,6 +3397,8 @@ class IngestStats:
|
|
|
3350
3397
|
lines_malformed: int = 0
|
|
3351
3398
|
assistant_lines_skipped: int = 0
|
|
3352
3399
|
skip_reasons: dict = field(default_factory=dict)
|
|
3400
|
+
full_walk_complete: bool = False
|
|
3401
|
+
maintenance_failed: bool = False
|
|
3353
3402
|
|
|
3354
3403
|
@property
|
|
3355
3404
|
def targeted_clean(self) -> bool:
|
|
@@ -3865,6 +3914,17 @@ def sync_cache(
|
|
|
3865
3914
|
# confirmed-current file still counts as walked.
|
|
3866
3915
|
walk_clean = True
|
|
3867
3916
|
|
|
3917
|
+
# The marker certifies the MOST RECENT exhaustive walk, not merely some
|
|
3918
|
+
# historical clean walk. Retire it before an ordinary full pass too;
|
|
3919
|
+
# any early return or per-file failure must leave the store visibly
|
|
3920
|
+
# incomplete until this same call reaches the clean end-of-walk write.
|
|
3921
|
+
if not targeted and not rebuild:
|
|
3922
|
+
conn.execute(
|
|
3923
|
+
"DELETE FROM cache_meta "
|
|
3924
|
+
"WHERE key='claude_ingest_walk_complete'"
|
|
3925
|
+
)
|
|
3926
|
+
conn.commit()
|
|
3927
|
+
|
|
3868
3928
|
if rebuild:
|
|
3869
3929
|
# Clear INSIDE the lock — a concurrent rebuild that lost the
|
|
3870
3930
|
# race would otherwise have wiped this cache before bailing,
|
|
@@ -4172,20 +4232,21 @@ def sync_cache(
|
|
|
4172
4232
|
_consume_file_touches(conn)
|
|
4173
4233
|
_p_backfills.__exit__(None, None, None)
|
|
4174
4234
|
|
|
4175
|
-
with _perf.phase("
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
|
|
4182
|
-
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
|
|
4186
|
-
|
|
4187
|
-
|
|
4188
|
-
|
|
4235
|
+
with _perf.phase("frontier"):
|
|
4236
|
+
with _perf.phase("discover") as _p_disc:
|
|
4237
|
+
if targeted:
|
|
4238
|
+
# A requested path that vanished (session rotated/deleted
|
|
4239
|
+
# mid-live-tail) is deliberately DROPPED here without flagging
|
|
4240
|
+
# failure: marking files_failed would wedge the watch loop's
|
|
4241
|
+
# targeted_clean advance forever for a file that will never
|
|
4242
|
+
# return; the orphan-prune path owns its stale rows on the next
|
|
4243
|
+
# full sync. Pinned by tests/test_cache_accepted_behaviors.py
|
|
4244
|
+
# (#279 S3 F4).
|
|
4245
|
+
paths = [pathlib.Path(p) for p in only_paths if pathlib.Path(p).is_file()]
|
|
4246
|
+
else:
|
|
4247
|
+
paths = list(_iter_claude_jsonl_files())
|
|
4248
|
+
stats.files_total = len(paths)
|
|
4249
|
+
_p_disc.set_count(len(paths))
|
|
4189
4250
|
|
|
4190
4251
|
# This SELECT does NOT open an implicit transaction (Python's
|
|
4191
4252
|
# sqlite3 module only BEGINs on DML). Do NOT add any INSERT/
|
|
@@ -4694,23 +4755,23 @@ def sync_cache(
|
|
|
4694
4755
|
# ~1 session/tick). Both recomputes derive COUNT/MIN/MAX from the same
|
|
4695
4756
|
# rows the rail's old live aggregate read, so the rollup stays
|
|
4696
4757
|
# byte-identical to that aggregate.
|
|
4697
|
-
with _perf.phase("
|
|
4698
|
-
|
|
4699
|
-
|
|
4700
|
-
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
|
|
4705
|
-
|
|
4706
|
-
|
|
4707
|
-
|
|
4708
|
-
|
|
4709
|
-
|
|
4710
|
-
|
|
4711
|
-
|
|
4712
|
-
|
|
4713
|
-
|
|
4758
|
+
with _perf.phase("accounting"):
|
|
4759
|
+
with _perf.phase("recompute.conversation_sessions"):
|
|
4760
|
+
# #302: auto-invalidate the rollup's MATERIALIZED cost when the
|
|
4761
|
+
# embedded pricing snapshot changed since it was last derived.
|
|
4762
|
+
# Runs BEFORE the pending check so a mismatch arms the same
|
|
4763
|
+
# durable flag the full-recompute path already consumes below.
|
|
4764
|
+
_arm_rollup_backfill_on_pricing_change(conn)
|
|
4765
|
+
if _conversation_sessions_backfill_pending(conn):
|
|
4766
|
+
_recompute_conversation_sessions(conn)
|
|
4767
|
+
conn.execute(
|
|
4768
|
+
"DELETE FROM cache_meta "
|
|
4769
|
+
"WHERE key='conversation_sessions_backfill_pending'"
|
|
4770
|
+
)
|
|
4771
|
+
conn.commit()
|
|
4772
|
+
elif touched_sessions:
|
|
4773
|
+
_recompute_conversation_sessions(conn, touched_sessions)
|
|
4774
|
+
conn.commit()
|
|
4714
4775
|
|
|
4715
4776
|
# Walk-complete sentinel write (cctally-dev#93, D5a). Still inside the
|
|
4716
4777
|
# held fcntl lock, before the finally-unlock. Only when the entire walk
|
|
@@ -4735,6 +4796,7 @@ def sync_cache(
|
|
|
4735
4796
|
(_cctally_db_sib.CACHE_CREATION_SPLIT_REWALK_KEY,),
|
|
4736
4797
|
)
|
|
4737
4798
|
conn.commit()
|
|
4799
|
+
stats.full_walk_complete = True
|
|
4738
4800
|
# #279 S2 F1: rolling parse-health record. Anomaly-delta-gated so
|
|
4739
4801
|
# steady-state (incl. targeted live-tail) syncs stay zero-write;
|
|
4740
4802
|
# targeted syncs still accumulate — they ingest real new bytes.
|
|
@@ -5073,7 +5135,9 @@ def _arm_rollup_backfill_on_pricing_change(conn) -> None:
|
|
|
5073
5135
|
conn.commit()
|
|
5074
5136
|
|
|
5075
5137
|
|
|
5076
|
-
def _recompute_conversation_sessions(
|
|
5138
|
+
def _recompute_conversation_sessions(
|
|
5139
|
+
conn, session_ids=None, *, advance_render_revision: bool = True,
|
|
5140
|
+
) -> None:
|
|
5077
5141
|
"""Recompute the ``conversation_sessions`` browse-rail rollup from
|
|
5078
5142
|
``conversation_messages``. The caller holds the cache.db.lock flock and owns
|
|
5079
5143
|
the commit (this helper never commits).
|
|
@@ -5094,7 +5158,14 @@ def _recompute_conversation_sessions(conn, session_ids=None) -> None:
|
|
|
5094
5158
|
The recomputed COUNT/MIN/MAX are byte-identical to the rail's prior live
|
|
5095
5159
|
aggregate over the same rows — that is the load-bearing invariant
|
|
5096
5160
|
(assert_rollup_matches_live in the maintenance test pins it)."""
|
|
5097
|
-
if session_ids is None
|
|
5161
|
+
ids = None if session_ids is None else [s for s in session_ids if s is not None]
|
|
5162
|
+
if ids == []:
|
|
5163
|
+
return
|
|
5164
|
+
render_revision = (
|
|
5165
|
+
_next_conversation_render_revision(conn)
|
|
5166
|
+
if advance_render_revision else 0
|
|
5167
|
+
)
|
|
5168
|
+
if ids is None:
|
|
5098
5169
|
conn.execute("DELETE FROM conversation_sessions")
|
|
5099
5170
|
conn.execute(
|
|
5100
5171
|
"INSERT INTO conversation_sessions "
|
|
@@ -5102,8 +5173,11 @@ def _recompute_conversation_sessions(conn, session_ids=None) -> None:
|
|
|
5102
5173
|
+ _CONV_SESSIONS_SELECT + " GROUP BY session_id"
|
|
5103
5174
|
)
|
|
5104
5175
|
_fill_conversation_sessions_filter_columns(conn, None)
|
|
5176
|
+
conn.execute(
|
|
5177
|
+
"UPDATE conversation_sessions SET render_revision=?",
|
|
5178
|
+
(render_revision,),
|
|
5179
|
+
)
|
|
5105
5180
|
return
|
|
5106
|
-
ids = [s for s in session_ids if s is not None]
|
|
5107
5181
|
for i in range(0, len(ids), 400):
|
|
5108
5182
|
chunk = ids[i:i + 400]
|
|
5109
5183
|
placeholders = ",".join("?" for _ in chunk)
|
|
@@ -5118,6 +5192,11 @@ def _recompute_conversation_sessions(conn, session_ids=None) -> None:
|
|
|
5118
5192
|
+ f" AND session_id IN ({placeholders}) GROUP BY session_id",
|
|
5119
5193
|
chunk,
|
|
5120
5194
|
)
|
|
5195
|
+
conn.execute(
|
|
5196
|
+
f"UPDATE conversation_sessions SET render_revision=? "
|
|
5197
|
+
f"WHERE session_id IN ({placeholders})",
|
|
5198
|
+
(render_revision, *chunk),
|
|
5199
|
+
)
|
|
5121
5200
|
_fill_conversation_sessions_filter_columns(conn, ids)
|
|
5122
5201
|
|
|
5123
5202
|
|
|
@@ -6019,6 +6098,8 @@ class CodexIngestStats:
|
|
|
6019
6098
|
backlog_files: int = 0
|
|
6020
6099
|
backlog_bytes: int = 0
|
|
6021
6100
|
budget_exhausted: bool = False
|
|
6101
|
+
full_walk_complete: bool = False
|
|
6102
|
+
maintenance_failed: bool = False
|
|
6022
6103
|
|
|
6023
6104
|
@property
|
|
6024
6105
|
def targeted_clean(self) -> bool:
|
|
@@ -6531,6 +6612,16 @@ def sync_codex_cache(
|
|
|
6531
6612
|
raise ValueError(
|
|
6532
6613
|
"sync_codex_cache: budget_seconds is incompatible with rebuild")
|
|
6533
6614
|
|
|
6615
|
+
# As on the Claude side, this sentinel describes the latest exhaustive
|
|
6616
|
+
# pass. Clear it before any full-pass decline or walk can occur, then
|
|
6617
|
+
# restore it only after every file and maintenance leg certifies clean.
|
|
6618
|
+
if not targeted:
|
|
6619
|
+
conn.execute(
|
|
6620
|
+
"DELETE FROM cache_meta WHERE key=?",
|
|
6621
|
+
(_ingest_frontier.CODEX_FULL_WALK_COMPLETE_KEY,),
|
|
6622
|
+
)
|
|
6623
|
+
conn.commit()
|
|
6624
|
+
|
|
6534
6625
|
# A pending byte-zero replay is consumed HERE, not by the migration that
|
|
6535
6626
|
# armed it, so the rebuild path below captures `rebuild_known_identities`
|
|
6536
6627
|
# before clearing. A migration that cleared `codex_session_files`
|
|
@@ -6794,13 +6885,14 @@ def sync_codex_cache(
|
|
|
6794
6885
|
# Pure read (glob + is_file only); safe to run before the SELECT and
|
|
6795
6886
|
# the per-file loop, where no cache.db write lock may be held. Targeted
|
|
6796
6887
|
# mode qualifies ONLY the requested paths — never a tree walk (§5.1).
|
|
6797
|
-
with _perf.phase("
|
|
6798
|
-
|
|
6799
|
-
|
|
6800
|
-
|
|
6801
|
-
|
|
6802
|
-
|
|
6803
|
-
|
|
6888
|
+
with _perf.phase("frontier"):
|
|
6889
|
+
with _perf.phase("discover") as _p_disc:
|
|
6890
|
+
if targeted:
|
|
6891
|
+
files = _qualify_codex_targets(only_paths)
|
|
6892
|
+
else:
|
|
6893
|
+
files = _discover_codex_files_with_roots()
|
|
6894
|
+
stats.files_total = len(files)
|
|
6895
|
+
_p_disc.set_count(len(files))
|
|
6804
6896
|
|
|
6805
6897
|
# Scope the cache to the CURRENT root set (issue #108), but only from
|
|
6806
6898
|
# positive filesystem evidence (#485). A recognized new root still
|
|
@@ -7067,8 +7159,7 @@ def sync_codex_cache(
|
|
|
7067
7159
|
st = jp.stat()
|
|
7068
7160
|
except OSError as exc:
|
|
7069
7161
|
eprint(f"[codex-cache] stat failed for {jp}: {exc}")
|
|
7070
|
-
|
|
7071
|
-
stats.files_failed += 1 # §5.1 I/O decline → call dirty
|
|
7162
|
+
stats.files_failed += 1
|
|
7072
7163
|
continue
|
|
7073
7164
|
|
|
7074
7165
|
size = st.st_size
|
|
@@ -7483,8 +7574,7 @@ def sync_codex_cache(
|
|
|
7483
7574
|
except OSError as exc:
|
|
7484
7575
|
eprint(f"[codex-cache] could not read {jp}: {exc}")
|
|
7485
7576
|
anchor_resolver.discard_uncommitted_file()
|
|
7486
|
-
|
|
7487
|
-
stats.files_failed += 1 # §5.1 I/O decline → call dirty
|
|
7577
|
+
stats.files_failed += 1
|
|
7488
7578
|
continue
|
|
7489
7579
|
|
|
7490
7580
|
# Pull terminal session_id/model from the iterator's tracker.
|
|
@@ -7860,8 +7950,9 @@ def sync_codex_cache(
|
|
|
7860
7950
|
# entries the triggers just wrote. Best-effort, like the adoption below:
|
|
7861
7951
|
# the resolution is fully re-derivable on the next sync.
|
|
7862
7952
|
try:
|
|
7863
|
-
|
|
7864
|
-
|
|
7953
|
+
with _perf.phase("accounting"):
|
|
7954
|
+
resolved_models = _cctally_db_sib.backfill_codex_quota_observed_model(
|
|
7955
|
+
conn)
|
|
7865
7956
|
if resolved_models:
|
|
7866
7957
|
_bump_codex_physical_mutation_seq(conn)
|
|
7867
7958
|
conn.commit()
|
|
@@ -7869,6 +7960,7 @@ def sync_codex_cache(
|
|
|
7869
7960
|
conn.rollback()
|
|
7870
7961
|
if _cctally_db_sib._is_sqlite_corruption_error(exc):
|
|
7871
7962
|
raise
|
|
7963
|
+
stats.maintenance_failed = True
|
|
7872
7964
|
eprint("[cache-sync] could not resolve Codex quota model "
|
|
7873
7965
|
f"attribution: {exc}")
|
|
7874
7966
|
# Window-scoped spend adoption (spec
|
|
@@ -7880,8 +7972,9 @@ def sync_codex_cache(
|
|
|
7880
7972
|
# the stamp is fully re-derivable, so the next sync (or the migration)
|
|
7881
7973
|
# repeats it.
|
|
7882
7974
|
try:
|
|
7883
|
-
|
|
7884
|
-
|
|
7975
|
+
with _perf.phase("accounting"):
|
|
7976
|
+
adopted = apply_codex_window_spend_adoption(
|
|
7977
|
+
conn, touched=None if rebuild else adoption_spans)
|
|
7885
7978
|
conn.commit()
|
|
7886
7979
|
# Terse, and silent on zero: a rebuild re-derives every row and so
|
|
7887
7980
|
# legitimately re-stamps the same population each time, which would
|
|
@@ -7895,6 +7988,7 @@ def sync_codex_cache(
|
|
|
7895
7988
|
# Classified family corruption belongs to the shared recovery
|
|
7896
7989
|
# boundary, never to a best-effort local except.
|
|
7897
7990
|
raise
|
|
7991
|
+
stats.maintenance_failed = True
|
|
7898
7992
|
eprint(f"[cache-sync] could not adopt Codex window spend: {exc}")
|
|
7899
7993
|
# #500 §7.1: the STANDING half of operator attribution. The condition it
|
|
7900
7994
|
# repairs is created by ordinary ingest, not only by an operator command
|
|
@@ -7907,7 +8001,8 @@ def sync_codex_cache(
|
|
|
7907
8001
|
# rows it restores are one committed generation. Costs a store with no
|
|
7908
8002
|
# attribution records one indexed read; best-effort, like the pass above.
|
|
7909
8003
|
try:
|
|
7910
|
-
|
|
8004
|
+
with _perf.phase("accounting"):
|
|
8005
|
+
restored, readopted = reconcile_codex_window_attribution_spend(conn)
|
|
7911
8006
|
conn.commit()
|
|
7912
8007
|
if restored:
|
|
7913
8008
|
eprint(f"[cache-sync] restored {restored} Codex row(s) whose "
|
|
@@ -7917,6 +8012,7 @@ def sync_codex_cache(
|
|
|
7917
8012
|
conn.rollback()
|
|
7918
8013
|
if _cctally_db_sib._is_sqlite_corruption_error(exc):
|
|
7919
8014
|
raise
|
|
8015
|
+
stats.maintenance_failed = True
|
|
7920
8016
|
eprint("[cache-sync] could not reconcile Codex window "
|
|
7921
8017
|
f"attribution spend: {exc}")
|
|
7922
8018
|
# #582: keep only a generous mutation-sequence tail. A dashboard that
|
|
@@ -7997,6 +8093,24 @@ def sync_codex_cache(
|
|
|
7997
8093
|
deferred_cert_sigs = dict(certificate[1])
|
|
7998
8094
|
else:
|
|
7999
8095
|
project_after_unlock = True
|
|
8096
|
+
stats.full_walk_complete = bool(
|
|
8097
|
+
not targeted
|
|
8098
|
+
and walk_complete
|
|
8099
|
+
and stats.files_failed == 0
|
|
8100
|
+
and stats.files_deferred_torn == 0
|
|
8101
|
+
and not stats.prune_refused
|
|
8102
|
+
and not stats.maintenance_failed
|
|
8103
|
+
and (
|
|
8104
|
+
stats.files_processed + stats.files_skipped_unchanged
|
|
8105
|
+
== stats.files_total
|
|
8106
|
+
)
|
|
8107
|
+
)
|
|
8108
|
+
if stats.full_walk_complete:
|
|
8109
|
+
conn.execute(
|
|
8110
|
+
"INSERT OR IGNORE INTO cache_meta(key,value) VALUES(?, '1')",
|
|
8111
|
+
(_ingest_frontier.CODEX_FULL_WALK_COMPLETE_KEY,),
|
|
8112
|
+
)
|
|
8113
|
+
conn.commit()
|
|
8000
8114
|
finally:
|
|
8001
8115
|
release_cache_writer_flocks(held_writer_flocks)
|
|
8002
8116
|
|
|
@@ -8012,15 +8126,17 @@ def sync_codex_cache(
|
|
|
8012
8126
|
from _cctally_quota import _stats_projection_signatures_match
|
|
8013
8127
|
stats_conn = _cctally_core.open_db()
|
|
8014
8128
|
try:
|
|
8015
|
-
|
|
8016
|
-
|
|
8017
|
-
|
|
8018
|
-
|
|
8129
|
+
with _perf.phase("projector"):
|
|
8130
|
+
if not _stats_projection_signatures_match(
|
|
8131
|
+
stats_conn, deferred_cert_roots, deferred_cert_sigs or {}
|
|
8132
|
+
):
|
|
8133
|
+
project_after_unlock = True
|
|
8019
8134
|
finally:
|
|
8020
8135
|
stats_conn.close()
|
|
8021
8136
|
if project_after_unlock and quota_reconcile == "auto":
|
|
8022
8137
|
from _cctally_quota import reconcile_codex_quota_projection
|
|
8023
|
-
|
|
8138
|
+
with _perf.phase("projector"):
|
|
8139
|
+
reconcile_codex_quota_projection()
|
|
8024
8140
|
return stats
|
|
8025
8141
|
|
|
8026
8142
|
|
|
@@ -9295,11 +9411,12 @@ def _run_cache_plan_with_recovery(
|
|
|
9295
9411
|
):
|
|
9296
9412
|
active.close()
|
|
9297
9413
|
raise
|
|
9298
|
-
|
|
9299
|
-
|
|
9300
|
-
|
|
9301
|
-
|
|
9302
|
-
|
|
9414
|
+
with _perf.phase("recovery"):
|
|
9415
|
+
if not _recover_corrupt_cache(
|
|
9416
|
+
exc, origin=origin, active_conn=active,
|
|
9417
|
+
):
|
|
9418
|
+
raise
|
|
9419
|
+
active = open_cache_db()
|
|
9303
9420
|
_cache_storm_test_pause("cache_repair_recreated")
|
|
9304
9421
|
recovered = True
|
|
9305
9422
|
except BaseException:
|
|
@@ -9889,11 +10006,14 @@ def scope_conversations_db_to_account(
|
|
|
9889
10006
|
project_key TEXT,
|
|
9890
10007
|
project_label TEXT,
|
|
9891
10008
|
models_json TEXT,
|
|
9892
|
-
title TEXT
|
|
10009
|
+
title TEXT,
|
|
10010
|
+
render_revision INTEGER NOT NULL DEFAULT 0
|
|
9893
10011
|
);
|
|
9894
10012
|
"""
|
|
9895
10013
|
)
|
|
9896
|
-
|
|
10014
|
+
# These are read-scope TEMP projections. They must neither consume nor
|
|
10015
|
+
# advance the durable render frontier used by writer recomputes.
|
|
10016
|
+
_recompute_conversation_sessions(conn, advance_render_revision=False)
|
|
9897
10017
|
codex_keys = {
|
|
9898
10018
|
row[0]
|
|
9899
10019
|
for row in conn.execute(
|
|
@@ -9920,7 +10040,9 @@ def scope_conversations_db_to_account(
|
|
|
9920
10040
|
safe_project_attribution[conversation_key] = (
|
|
9921
10041
|
_codex_conversation_project_attribution(*thread)
|
|
9922
10042
|
)
|
|
9923
|
-
_recompute_codex_rollups(
|
|
10043
|
+
_recompute_codex_rollups(
|
|
10044
|
+
conn, codex_keys, advance_render_revision=False,
|
|
10045
|
+
)
|
|
9924
10046
|
# Project identity is safe conversation-level enrichment: it is already
|
|
9925
10047
|
# visible on the unqualified rail and contains only an opaque key plus the
|
|
9926
10048
|
# derived display label. Preserve those two fields for conversations that
|
package/bin/_cctally_config.py
CHANGED
|
@@ -300,6 +300,7 @@ ALLOWED_CONFIG_KEYS = (
|
|
|
300
300
|
"display.tz",
|
|
301
301
|
"alerts.enabled",
|
|
302
302
|
"alerts.projected_enabled",
|
|
303
|
+
"alerts.rate_change_enabled",
|
|
303
304
|
"alerts.notifier",
|
|
304
305
|
"alerts.command_template",
|
|
305
306
|
"alerts.quota",
|
|
@@ -1006,6 +1007,16 @@ def _config_known_value(config: dict, key: str) -> "object":
|
|
|
1006
1007
|
return bool(_get_alerts_config(config)["projected_enabled"])
|
|
1007
1008
|
except c._AlertsConfigError:
|
|
1008
1009
|
return False
|
|
1010
|
+
if key == "alerts.rate_change_enabled":
|
|
1011
|
+
# #661 S2 section 6.2's PUSH gate. Recording a metering-rate change is
|
|
1012
|
+
# unconditional; this key gates only the notification. Same posture as
|
|
1013
|
+
# alerts.projected_enabled — validated boolean, default False, and a
|
|
1014
|
+
# corrupt alerts block surfaces the default rather than erroring out of
|
|
1015
|
+
# a plain `config get`.
|
|
1016
|
+
try:
|
|
1017
|
+
return bool(_get_alerts_config(config)["rate_change_enabled"])
|
|
1018
|
+
except c._AlertsConfigError:
|
|
1019
|
+
return False
|
|
1009
1020
|
if key == "alerts.notifier":
|
|
1010
1021
|
# Validated dispatch backend (defaults to 'auto' when unset). A corrupt
|
|
1011
1022
|
# alerts block surfaces the default — mirrors alerts.enabled.
|
|
@@ -1468,6 +1479,53 @@ def _cmd_config_set(args: argparse.Namespace) -> int:
|
|
|
1468
1479
|
f"alerts.projected_enabled={'true' if normalized else 'false'}"
|
|
1469
1480
|
)
|
|
1470
1481
|
return 0
|
|
1482
|
+
if key == "alerts.rate_change_enabled":
|
|
1483
|
+
# #661 S2 section 6.2. Identical posture to alerts.projected_enabled
|
|
1484
|
+
# above, including the re-message: _normalize_alerts_enabled_value
|
|
1485
|
+
# hardcodes "alerts.enabled" in its ValueError text, so the branch
|
|
1486
|
+
# names the key the user actually typed.
|
|
1487
|
+
try:
|
|
1488
|
+
normalized = c._normalize_alerts_enabled_value(raw)
|
|
1489
|
+
except ValueError:
|
|
1490
|
+
print(
|
|
1491
|
+
f"cctally: invalid boolean value for "
|
|
1492
|
+
f"alerts.rate_change_enabled: "
|
|
1493
|
+
f"{raw!r} (expected true|false|yes|no|1|0|on|off)",
|
|
1494
|
+
file=sys.stderr,
|
|
1495
|
+
)
|
|
1496
|
+
return 2
|
|
1497
|
+
with config_writer_lock():
|
|
1498
|
+
config = _load_config_unlocked()
|
|
1499
|
+
existing_alerts = config.get("alerts")
|
|
1500
|
+
if existing_alerts is not None and not isinstance(
|
|
1501
|
+
existing_alerts, dict
|
|
1502
|
+
):
|
|
1503
|
+
print(
|
|
1504
|
+
"cctally: alerts config error: alerts must be an object",
|
|
1505
|
+
file=sys.stderr,
|
|
1506
|
+
)
|
|
1507
|
+
return 2
|
|
1508
|
+
alerts_block = dict(existing_alerts or {})
|
|
1509
|
+
alerts_block["rate_change_enabled"] = normalized
|
|
1510
|
+
try:
|
|
1511
|
+
_get_alerts_config({**config, "alerts": alerts_block})
|
|
1512
|
+
except _AlertsConfigError as exc:
|
|
1513
|
+
print(f"cctally: alerts config error: {exc}", file=sys.stderr)
|
|
1514
|
+
return 2
|
|
1515
|
+
config["alerts"] = alerts_block
|
|
1516
|
+
save_config(config)
|
|
1517
|
+
if getattr(args, "emit_json", False):
|
|
1518
|
+
print(
|
|
1519
|
+
json.dumps(
|
|
1520
|
+
{"alerts": {"rate_change_enabled": normalized}}, indent=2
|
|
1521
|
+
)
|
|
1522
|
+
)
|
|
1523
|
+
else:
|
|
1524
|
+
print(
|
|
1525
|
+
f"alerts.rate_change_enabled="
|
|
1526
|
+
f"{'true' if normalized else 'false'}"
|
|
1527
|
+
)
|
|
1528
|
+
return 0
|
|
1471
1529
|
if key == "alerts.notifier":
|
|
1472
1530
|
# Dispatch backend (Phase B). Plain string; the enum constraint is
|
|
1473
1531
|
# enforced by the pre-persist _get_alerts_config validation (so we never
|
|
@@ -2296,6 +2354,7 @@ def _cmd_config_unset(args: argparse.Namespace) -> int:
|
|
|
2296
2354
|
if key in (
|
|
2297
2355
|
"alerts.enabled",
|
|
2298
2356
|
"alerts.projected_enabled",
|
|
2357
|
+
"alerts.rate_change_enabled",
|
|
2299
2358
|
"alerts.notifier",
|
|
2300
2359
|
"alerts.command_template",
|
|
2301
2360
|
"alerts.quota",
|
|
@@ -2305,8 +2364,9 @@ def _cmd_config_unset(args: argparse.Namespace) -> int:
|
|
|
2305
2364
|
# self-deadlock per the gotcha in CLAUDE.md). Unsetting just the
|
|
2306
2365
|
# named key preserves any user-customized threshold lists
|
|
2307
2366
|
# (`weekly_thresholds`, `five_hour_thresholds`) and the sibling
|
|
2308
|
-
# enabled/projected_enabled/notifier/
|
|
2309
|
-
# enabled/projected_enabled/
|
|
2367
|
+
# enabled/projected_enabled/rate_change_enabled/notifier/
|
|
2368
|
+
# command_template keys. For enabled/projected_enabled/
|
|
2369
|
+
# rate_change_enabled/notifier the read-time validator
|
|
2310
2370
|
# (`_get_alerts_config`) re-applies the canonical default
|
|
2311
2371
|
# (`False` / `"auto"`) for the missing key on next get. NOT so for
|
|
2312
2372
|
# command_template when notifier == "command": the cross-field
|