cctally 1.89.2 → 1.90.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/bin/_cctally_cache_report.py +10 -5
- package/bin/_cctally_core.py +22 -4
- package/bin/_cctally_dashboard.py +158 -1
- package/bin/_cctally_dashboard_cache_report.py +56 -46
- package/bin/_cctally_dashboard_envelope.py +8 -0
- package/bin/_cctally_dashboard_share.py +2 -0
- package/bin/_cctally_dashboard_sources.py +82 -37
- package/bin/_cctally_db.py +27 -0
- package/bin/_cctally_parser.py +20 -0
- package/bin/_cctally_record.py +8 -38
- package/bin/_cctally_statusline.py +21 -2
- package/bin/_cctally_store.py +200 -0
- package/bin/_cctally_tui.py +33 -9
- package/bin/_lib_cache_report.py +199 -70
- package/bin/_lib_cache_report_wire.py +184 -0
- package/bin/cctally +14 -2
- package/dashboard/static/assets/{index-Ub8vwz1M.css → index-Bar8-S1i.css} +1 -1
- package/dashboard/static/assets/index-CRogVlEC.js +92 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +2 -1
- package/dashboard/static/assets/index-BgoYXdus.js +0 -92
|
@@ -761,14 +761,15 @@ def resolve_dashboard_source_semantics(
|
|
|
761
761
|
quota_alerts = c._get_quota_alerts_config(raw_config)
|
|
762
762
|
raw_cache_report = raw_config.get("cache_report")
|
|
763
763
|
raw_cache_threshold = (
|
|
764
|
-
raw_cache_report.get("anomaly_threshold_pp"
|
|
765
|
-
if isinstance(raw_cache_report, Mapping) else
|
|
766
|
-
)
|
|
767
|
-
cache_threshold = (
|
|
768
|
-
int(raw_cache_threshold)
|
|
769
|
-
if isinstance(raw_cache_threshold, int) and not isinstance(raw_cache_threshold, bool)
|
|
770
|
-
and 1 <= raw_cache_threshold <= 100 else 15
|
|
764
|
+
raw_cache_report.get("anomaly_threshold_pp")
|
|
765
|
+
if isinstance(raw_cache_report, Mapping) else None
|
|
771
766
|
)
|
|
767
|
+
# #443 S3 F17: one definition of what the setting means, shared with
|
|
768
|
+
# the Claude read path and the persistence gate. An absent key
|
|
769
|
+
# reaches the resolver as None and defaults there, not here.
|
|
770
|
+
cache_threshold = c._load_sibling(
|
|
771
|
+
"_lib_cache_report"
|
|
772
|
+
).resolve_cache_report_threshold(raw_cache_threshold)
|
|
772
773
|
raw_codex_budget = budget_config.get("codex")
|
|
773
774
|
codex_budget = (
|
|
774
775
|
MappingProxyType(dict(raw_codex_budget))
|
|
@@ -1043,6 +1044,8 @@ def _codex_cache_report_wire(
|
|
|
1043
1044
|
"""
|
|
1044
1045
|
c = sys.modules["cctally"]
|
|
1045
1046
|
crk = c._load_sibling("_lib_cache_report")
|
|
1047
|
+
# #443 S2 F18 — the one serializer shared with the Claude producer.
|
|
1048
|
+
wire = c._load_sibling("_lib_cache_report_wire")
|
|
1046
1049
|
display_tz = ZoneInfo(display_tz_name) if display_tz_name else None
|
|
1047
1050
|
cutoff = now_utc - dt.timedelta(days=window_days)
|
|
1048
1051
|
|
|
@@ -1107,24 +1110,35 @@ def _codex_cache_report_wire(
|
|
|
1107
1110
|
},
|
|
1108
1111
|
))
|
|
1109
1112
|
|
|
1110
|
-
|
|
1113
|
+
# One current day per invocation (#443 S3 F23): the focal day and the
|
|
1114
|
+
# entry filter below both resolve through the SAME zone the kernel
|
|
1115
|
+
# buckets by. ``display_tz or UTC`` diverged from host-local bucketing
|
|
1116
|
+
# on every non-UTC host, which published a fabricated spotlight and let
|
|
1117
|
+
# the breakdowns draw from a different entry population than the days.
|
|
1118
|
+
bucket_tz = crk._resolve_bucket_tz(display_tz)
|
|
1119
|
+
today_iso = now_utc.astimezone(bucket_tz).strftime("%Y-%m-%d")
|
|
1111
1120
|
if not wrapped:
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1121
|
+
# An empty store measured nothing, so ``observed`` is False and
|
|
1122
|
+
# every applicable predicate is unevaluated. The client
|
|
1123
|
+
# short-circuits on ``is_empty`` before reading either, so this
|
|
1124
|
+
# costs nothing and keeps the empty and populated returns one shape.
|
|
1125
|
+
return wire.build_cache_report_wire(
|
|
1126
|
+
provider="codex", window_days=window_days,
|
|
1127
|
+
anomaly_threshold_pp=anomaly_threshold_pp,
|
|
1128
|
+
anomaly_window_days=window_days,
|
|
1129
|
+
today={
|
|
1117
1130
|
"date": today_iso, "cache_hit_percent": 0.0,
|
|
1118
1131
|
"baseline_median_percent": None, "delta_pp": None,
|
|
1119
1132
|
"net_usd": 0.0, "saved_usd": 0.0, "wasted_usd": 0.0,
|
|
1120
1133
|
"anomaly_triggered": False, "anomaly_reasons": (),
|
|
1121
1134
|
"baseline_daily_row_count": 0,
|
|
1135
|
+
"anomaly_unevaluated": wire.CODEX_PREDICATES, "observed": False,
|
|
1122
1136
|
},
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1137
|
+
days=(), by_project=(), by_model=(),
|
|
1138
|
+
seven_day_net_usd=0.0, seven_day_anomaly_count=0,
|
|
1139
|
+
fourteen_day_counterfactual_usd=0.0,
|
|
1140
|
+
fourteen_day_efficiency_ratio=0.0, is_empty=True,
|
|
1141
|
+
)
|
|
1128
1142
|
|
|
1129
1143
|
result = crk._build_cache_report(
|
|
1130
1144
|
wrapped,
|
|
@@ -1137,7 +1151,14 @@ def _codex_cache_report_wire(
|
|
|
1137
1151
|
cost_calculator=lambda _model, _usage, _mode, cost: float(cost or 0.0),
|
|
1138
1152
|
)
|
|
1139
1153
|
raw_rows = sorted(result.rows, key=lambda row: row.date or "", reverse=True)
|
|
1140
|
-
|
|
1154
|
+
today_row = next((row for row in raw_rows if row.date == today_iso), None)
|
|
1155
|
+
# #443 F13/F14 — both charts label their rightmost element "Today"
|
|
1156
|
+
# positionally, so an idle day left the newest REAL row wearing that
|
|
1157
|
+
# label while the spotlight published a fabricated 0%. The Claude
|
|
1158
|
+
# builder solves this with a synthetic row for exactly this reason;
|
|
1159
|
+
# Codex now does the same. Slice AFTER inserting, as Claude does.
|
|
1160
|
+
synthetic_today = today_row is None
|
|
1161
|
+
day_dicts = [{
|
|
1141
1162
|
"date": row.date or "",
|
|
1142
1163
|
"cache_hit_percent": row.cache_hit_percent,
|
|
1143
1164
|
"input_tokens": row.input_tokens,
|
|
@@ -1149,15 +1170,28 @@ def _codex_cache_report_wire(
|
|
|
1149
1170
|
"net_usd": row.net_usd,
|
|
1150
1171
|
"anomaly_triggered": row.anomaly_triggered,
|
|
1151
1172
|
"anomaly_reasons": tuple(row.anomaly_reasons),
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1173
|
+
"anomaly_unevaluated": tuple(getattr(row, "anomaly_unevaluated", ())),
|
|
1174
|
+
"observed": True,
|
|
1175
|
+
} for row in raw_rows]
|
|
1176
|
+
if synthetic_today:
|
|
1177
|
+
day_dicts.insert(0, {
|
|
1178
|
+
"date": today_iso, "cache_hit_percent": 0.0,
|
|
1179
|
+
"input_tokens": 0, "output_tokens": 0,
|
|
1180
|
+
"cache_creation_tokens": 0, "cache_read_tokens": 0,
|
|
1181
|
+
"saved_usd": 0.0, "wasted_usd": 0.0, "net_usd": 0.0,
|
|
1182
|
+
"anomaly_triggered": False, "anomaly_reasons": (),
|
|
1183
|
+
"anomaly_unevaluated": wire.CODEX_PREDICATES, "observed": False,
|
|
1184
|
+
})
|
|
1185
|
+
days = tuple(day_dicts[:window_days])
|
|
1186
|
+
# #443 S3 F22: the rows the median was actually taken over, not every
|
|
1187
|
+
# non-today row. The old count was unbounded by the baseline window.
|
|
1188
|
+
baseline_count = result.today_baseline_sample_count
|
|
1155
1189
|
baseline = result.today_baseline_median
|
|
1156
1190
|
today_hit = today_row.cache_hit_percent if today_row else 0.0
|
|
1157
1191
|
kept_dates = {row["date"] for row in days}
|
|
1158
1192
|
kept_entries = [
|
|
1159
1193
|
entry for entry in wrapped
|
|
1160
|
-
if entry.timestamp.astimezone(
|
|
1194
|
+
if entry.timestamp.astimezone(bucket_tz).strftime("%Y-%m-%d") in kept_dates
|
|
1161
1195
|
]
|
|
1162
1196
|
by_project = crk._aggregate_cache_breakdown(
|
|
1163
1197
|
kept_entries, key_fn=lambda entry: entry.project_path,
|
|
@@ -1171,11 +1205,12 @@ def _codex_cache_report_wire(
|
|
|
1171
1205
|
saved_total = stable_sum(float(row["saved_usd"]) for row in days)
|
|
1172
1206
|
wasted_total = stable_sum(float(row["wasted_usd"]) for row in days)
|
|
1173
1207
|
efficiency_denom = saved_total + abs(wasted_total)
|
|
1174
|
-
return
|
|
1175
|
-
"
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1208
|
+
return wire.build_cache_report_wire(
|
|
1209
|
+
provider="codex",
|
|
1210
|
+
window_days=window_days,
|
|
1211
|
+
anomaly_threshold_pp=anomaly_threshold_pp,
|
|
1212
|
+
anomaly_window_days=window_days,
|
|
1213
|
+
today={
|
|
1179
1214
|
"date": today_iso,
|
|
1180
1215
|
"cache_hit_percent": today_hit,
|
|
1181
1216
|
"baseline_median_percent": baseline,
|
|
@@ -1186,24 +1221,34 @@ def _codex_cache_report_wire(
|
|
|
1186
1221
|
"anomaly_triggered": today_row.anomaly_triggered if today_row else False,
|
|
1187
1222
|
"anomaly_reasons": tuple(today_row.anomaly_reasons) if today_row else (),
|
|
1188
1223
|
"baseline_daily_row_count": baseline_count,
|
|
1224
|
+
"anomaly_unevaluated": (
|
|
1225
|
+
tuple(getattr(today_row, "anomaly_unevaluated", ()))
|
|
1226
|
+
if today_row else wire.CODEX_PREDICATES
|
|
1227
|
+
),
|
|
1228
|
+
"observed": today_row is not None,
|
|
1189
1229
|
},
|
|
1190
|
-
|
|
1191
|
-
|
|
1230
|
+
days=days,
|
|
1231
|
+
by_project=tuple({
|
|
1192
1232
|
"key": row.key, "cache_hit_percent": row.cache_hit_percent,
|
|
1193
1233
|
"net_usd": row.net_usd,
|
|
1194
1234
|
} for row in by_project),
|
|
1195
|
-
|
|
1235
|
+
by_model=tuple({
|
|
1196
1236
|
"key": row.key, "cache_hit_percent": row.cache_hit_percent,
|
|
1197
1237
|
"net_usd": row.net_usd,
|
|
1198
1238
|
} for row in by_model),
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1239
|
+
seven_day_net_usd=stable_sum(float(row["net_usd"]) for row in seven),
|
|
1240
|
+
# Placeholder: on the Codex path the builder RECOMPUTES this from
|
|
1241
|
+
# the day blocks it has already filtered, because a count taken
|
|
1242
|
+
# here would still include verdicts whose only reason is about to
|
|
1243
|
+
# be dropped as inapplicable. Computing it twice would just invite
|
|
1244
|
+
# the two to drift.
|
|
1245
|
+
seven_day_anomaly_count=0,
|
|
1246
|
+
fourteen_day_counterfactual_usd=saved_total,
|
|
1247
|
+
fourteen_day_efficiency_ratio=(
|
|
1203
1248
|
saved_total / efficiency_denom if efficiency_denom > 1e-9 else 0.0
|
|
1204
1249
|
),
|
|
1205
|
-
|
|
1206
|
-
|
|
1250
|
+
is_empty=False,
|
|
1251
|
+
)
|
|
1207
1252
|
|
|
1208
1253
|
|
|
1209
1254
|
def _codex_conversation_metadata(
|
package/bin/_cctally_db.py
CHANGED
|
@@ -302,6 +302,33 @@ class StatsEpochMismatchError(sqlite3.DatabaseError):
|
|
|
302
302
|
DB failure; ``main()`` maps it to a staged exit 3."""
|
|
303
303
|
|
|
304
304
|
|
|
305
|
+
class StatsEpochRebuildDeferred(BaseException):
|
|
306
|
+
"""A readable wrong-epoch stats index is rebuilding out of process.
|
|
307
|
+
|
|
308
|
+
Ordinary callers must not read the schema-incompatible old index or pay
|
|
309
|
+
whole-journal replay latency inline. ``outcome`` records whether this
|
|
310
|
+
caller spawned the worker, observed an existing attempt, or could not
|
|
311
|
+
spawn it; ``main()`` maps every case to prompt retry guidance and exit 3.
|
|
312
|
+
This deliberately derives directly from ``BaseException``: reporting
|
|
313
|
+
kernels contain many broad ``Exception`` / ``sqlite3.DatabaseError``
|
|
314
|
+
fallbacks that turn missing optional data into ``n/a``. Swallowing this
|
|
315
|
+
control signal there would publish a misleading partial report. The CLI
|
|
316
|
+
boundary, dashboard, and statusline catch it explicitly; ``finally``
|
|
317
|
+
cleanup still runs normally.
|
|
318
|
+
"""
|
|
319
|
+
|
|
320
|
+
def __init__(self, outcome: str) -> None:
|
|
321
|
+
self.outcome = str(outcome)
|
|
322
|
+
message = (
|
|
323
|
+
"could not start the stats.db index epoch rebuild; retry this "
|
|
324
|
+
"command shortly"
|
|
325
|
+
if self.outcome == "failed"
|
|
326
|
+
else "stats.db index epoch rebuild is running in the background; "
|
|
327
|
+
"retry shortly"
|
|
328
|
+
)
|
|
329
|
+
super().__init__(message)
|
|
330
|
+
|
|
331
|
+
|
|
305
332
|
_SQLITE_CORRUPTION_MESSAGES = (
|
|
306
333
|
"database disk image is malformed",
|
|
307
334
|
"file is not a database",
|
package/bin/_cctally_parser.py
CHANGED
|
@@ -3357,6 +3357,25 @@ def _build_codex_quota_verify_parser(subparsers, name, *, help_text, xref=None):
|
|
|
3357
3357
|
)
|
|
3358
3358
|
qv.set_defaults(func=c.cmd_codex_quota_verify_internal)
|
|
3359
3359
|
|
|
3360
|
+
def _build_stats_epoch_rebuild_parser(subparsers, name, *, help_text, xref=None):
|
|
3361
|
+
"""Build the issue-#453 detached stats epoch worker parser."""
|
|
3362
|
+
c = _cctally()
|
|
3363
|
+
worker = subparsers.add_parser(
|
|
3364
|
+
name,
|
|
3365
|
+
help=help_text,
|
|
3366
|
+
formatter_class=CLIHelpFormatter,
|
|
3367
|
+
description=textwrap.dedent(
|
|
3368
|
+
"""\
|
|
3369
|
+
Internal subcommand: converge a readable post-legacy
|
|
3370
|
+
wrong-epoch stats index through the existing journal
|
|
3371
|
+
resolver. Spawned by ordinary stats-backed commands so
|
|
3372
|
+
whole-journal replay never lands on their blocking path.
|
|
3373
|
+
Always returns 0; failures are logged and remain retryable.
|
|
3374
|
+
"""
|
|
3375
|
+
),
|
|
3376
|
+
)
|
|
3377
|
+
worker.set_defaults(func=c.cmd_stats_epoch_rebuild_internal)
|
|
3378
|
+
|
|
3360
3379
|
def _build_codex_replay_drain_parser(subparsers, name, *, help_text, xref=None):
|
|
3361
3380
|
"""Build the `_codex-replay-drain` parser (public #5 §4)."""
|
|
3362
3381
|
c = _cctally()
|
|
@@ -3458,6 +3477,7 @@ _REGISTRATION = (
|
|
|
3458
3477
|
_Reg('_update-check', _build_update_check_parser, argparse.SUPPRESS, None, None),
|
|
3459
3478
|
_Reg('_telemetry-beat', _build_telemetry_beat_parser, argparse.SUPPRESS, None, None),
|
|
3460
3479
|
_Reg('_codex-quota-verify', _build_codex_quota_verify_parser, argparse.SUPPRESS, None, None),
|
|
3480
|
+
_Reg('_stats-epoch-rebuild', _build_stats_epoch_rebuild_parser, argparse.SUPPRESS, None, None),
|
|
3461
3481
|
_Reg('_codex-replay-drain', _build_codex_replay_drain_parser, argparse.SUPPRESS, None, None),
|
|
3462
3482
|
_Reg('repair-symlinks', _build_repair_symlinks_parser, argparse.SUPPRESS, None, None),
|
|
3463
3483
|
)
|
package/bin/_cctally_record.py
CHANGED
|
@@ -4766,47 +4766,18 @@ def _codex_lifecycle_roots():
|
|
|
4766
4766
|
|
|
4767
4767
|
|
|
4768
4768
|
def _stats_epoch_rebuild_pending() -> bool:
|
|
4769
|
-
"""
|
|
4770
|
-
|
|
4771
|
-
Side-effect-free: a raw read-only ``PRAGMA user_version``, the same probe
|
|
4772
|
-
``resolve_stats_epoch_mismatch`` re-checks under the maintenance lock.
|
|
4773
|
-
|
|
4774
|
-
Deliberately narrow. A MISSING stats.db is a fresh install, where building
|
|
4775
|
-
the index is cheap and skipping it would leave the hook with nothing to do
|
|
4776
|
-
forever. An UNREADABLE one belongs to the corruption auto-heal path, not
|
|
4777
|
-
here. A LEGACY index (``user_version <= LEGACY_STATS_HEAD``) takes the
|
|
4778
|
-
migration route rather than the epoch rebuild, and predates every epoch
|
|
4779
|
-
this decision is about. Only a readable, post-legacy, wrong-epoch index —
|
|
4780
|
-
exactly what an upgrade across a ``STATS_INDEX_EPOCH`` bump produces —
|
|
4781
|
-
answers True.
|
|
4782
|
-
"""
|
|
4783
|
-
path = _cctally_core.DB_PATH
|
|
4784
|
-
try:
|
|
4785
|
-
if not path.exists():
|
|
4786
|
-
return False
|
|
4787
|
-
except OSError:
|
|
4788
|
-
return False
|
|
4769
|
+
"""Delegate the side-effect-free epoch probe to the store boundary."""
|
|
4789
4770
|
try:
|
|
4790
4771
|
import _cctally_store
|
|
4791
|
-
|
|
4772
|
+
return _cctally_store.stats_epoch_rebuild_pending()
|
|
4792
4773
|
except Exception:
|
|
4793
4774
|
return False
|
|
4794
|
-
if version < 0 or version <= _cctally_core.LEGACY_STATS_HEAD:
|
|
4795
|
-
return False
|
|
4796
|
-
return version != _cctally_core.STATS_INDEX_EPOCH
|
|
4797
4775
|
|
|
4798
4776
|
|
|
4799
4777
|
def _defer_stats_epoch_rebuild() -> str:
|
|
4800
|
-
"""Hand a pending
|
|
4801
|
-
|
|
4802
|
-
|
|
4803
|
-
``force_full`` pass opens stats.db, which is what performs the rebuild, and
|
|
4804
|
-
the whole-history pass it then runs is exactly the one the freshly rebuilt
|
|
4805
|
-
index needs anyway. Sharing the worker also shares its attempt-stamped
|
|
4806
|
-
throttle, so a rebuild that keeps dying cannot spawn one worker per tick.
|
|
4807
|
-
"""
|
|
4808
|
-
from _cctally_quota import _defer_codex_quota_verification
|
|
4809
|
-
return _defer_codex_quota_verification()
|
|
4778
|
+
"""Hand a pending epoch rebuild to the dedicated store worker."""
|
|
4779
|
+
import _cctally_store
|
|
4780
|
+
return _cctally_store.defer_stats_epoch_rebuild()
|
|
4810
4781
|
|
|
4811
4782
|
|
|
4812
4783
|
def _cmd_hook_tick_codex(
|
|
@@ -4866,10 +4837,9 @@ def _cmd_hook_tick_codex(
|
|
|
4866
4837
|
# journal rebuild — against Codex's 30-second hook timeout. A killed rebuild
|
|
4867
4838
|
# commits nothing, so the next tick repeats it: a non-converging
|
|
4868
4839
|
# 30-second-per-turn loop, which is the reported defect delivered by the
|
|
4869
|
-
# fix. Hand it to the
|
|
4870
|
-
#
|
|
4871
|
-
#
|
|
4872
|
-
# the next Codex turn re-checks immediately; the spawn itself is throttled.
|
|
4840
|
+
# fix. Hand it to the dedicated store-owned epoch worker and acknowledge
|
|
4841
|
+
# this tick as a no-op. No lifecycle marker is stamped, so the next Codex
|
|
4842
|
+
# turn re-checks immediately; store admission suppresses duplicate workers.
|
|
4873
4843
|
if _stats_epoch_rebuild_pending():
|
|
4874
4844
|
_defer_stats_epoch_rebuild()
|
|
4875
4845
|
log_outcome(sync="deferred", result="noop")
|
|
@@ -769,7 +769,10 @@ def _read_db_projection_once() -> "_candidates.DbProjection":
|
|
|
769
769
|
def _read_db_projection_stable(*, attempts: int = 3) -> "_candidates.DbProjection":
|
|
770
770
|
for _ in range(attempts):
|
|
771
771
|
before = _db_file_fingerprint()
|
|
772
|
-
|
|
772
|
+
try:
|
|
773
|
+
projection = _read_db_projection_once()
|
|
774
|
+
except _cctally().StatsEpochRebuildDeferred:
|
|
775
|
+
return _candidates.DbProjection(None, None, db_files=before)
|
|
773
776
|
after = _db_file_fingerprint()
|
|
774
777
|
if before == after and after["main"] is not None:
|
|
775
778
|
return dataclasses.replace(projection, db_files=after)
|
|
@@ -1009,6 +1012,8 @@ def _authoritative_record_usage(
|
|
|
1009
1012
|
|
|
1010
1013
|
try:
|
|
1011
1014
|
rc = _cctally().cmd_record_usage(args)
|
|
1015
|
+
except _cctally().StatsEpochRebuildDeferred as exc:
|
|
1016
|
+
return _AuthoritativeRecordResult("record_failed", str(exc))
|
|
1012
1017
|
except Exception as exc:
|
|
1013
1018
|
return _AuthoritativeRecordResult("record_failed", str(exc))
|
|
1014
1019
|
if rc != 0:
|
|
@@ -1163,7 +1168,13 @@ def _statusline_reduce_and_publish() -> "_candidates.ReductionDecision | None":
|
|
|
1163
1168
|
# statusline's AUTHORITATIVE publication step (`_authoritative_record_usage`)
|
|
1164
1169
|
# keeps the default authoritative mode — it is already try/except-wrapped to
|
|
1165
1170
|
# "record_failed".
|
|
1166
|
-
|
|
1171
|
+
try:
|
|
1172
|
+
record_rc = _cctally().cmd_record_usage(
|
|
1173
|
+
args, ingest_mode="opportunistic"
|
|
1174
|
+
)
|
|
1175
|
+
except _cctally().StatsEpochRebuildDeferred:
|
|
1176
|
+
return decision
|
|
1177
|
+
if record_rc != 0:
|
|
1167
1178
|
return decision
|
|
1168
1179
|
after = _read_db_projection_stable()
|
|
1169
1180
|
if _projection_changed(projection, after):
|
|
@@ -1586,6 +1597,10 @@ def _build_statusline_injections(warn_once):
|
|
|
1586
1597
|
range_start - c.BLOCK_DURATION, now + c.BLOCK_DURATION,
|
|
1587
1598
|
)
|
|
1588
1599
|
)
|
|
1600
|
+
except c.StatsEpochRebuildDeferred:
|
|
1601
|
+
recorded_windows, block_start_overrides, canonical_intervals = (
|
|
1602
|
+
[], {}, {},
|
|
1603
|
+
)
|
|
1589
1604
|
except Exception:
|
|
1590
1605
|
recorded_windows, block_start_overrides, canonical_intervals = (
|
|
1591
1606
|
[], {}, {},
|
|
@@ -1622,6 +1637,8 @@ def _build_statusline_injections(warn_once):
|
|
|
1622
1637
|
_acct_params = () if _sl_account is None else (_sl_account,)
|
|
1623
1638
|
try:
|
|
1624
1639
|
conn = open_db()
|
|
1640
|
+
except c.StatsEpochRebuildDeferred:
|
|
1641
|
+
return (None, None)
|
|
1625
1642
|
except Exception:
|
|
1626
1643
|
return (None, None)
|
|
1627
1644
|
try:
|
|
@@ -1707,6 +1724,8 @@ def _build_statusline_injections(warn_once):
|
|
|
1707
1724
|
def _db_latest_rate_limits():
|
|
1708
1725
|
try:
|
|
1709
1726
|
conn = open_db()
|
|
1727
|
+
except _cctally().StatsEpochRebuildDeferred:
|
|
1728
|
+
return None
|
|
1710
1729
|
except Exception:
|
|
1711
1730
|
return None
|
|
1712
1731
|
try:
|
package/bin/_cctally_store.py
CHANGED
|
@@ -1460,6 +1460,185 @@ HEAL_HOOK = _stats_heal_hook
|
|
|
1460
1460
|
# a HARD ERROR (``StatsEpochMismatchError``) — never a silent rebuild-to-empty.
|
|
1461
1461
|
|
|
1462
1462
|
_EPOCH_MISMATCH_ACTIVE = False
|
|
1463
|
+
STATS_EPOCH_REBUILD_COMMAND = "_stats-epoch-rebuild"
|
|
1464
|
+
_STATS_EPOCH_REBUILD_RETRY_SECONDS = 60.0
|
|
1465
|
+
|
|
1466
|
+
|
|
1467
|
+
def _stats_epoch_rebuild_path(name: str) -> pathlib.Path:
|
|
1468
|
+
return pathlib.Path(_cctally_core.APP_DIR) / name
|
|
1469
|
+
|
|
1470
|
+
|
|
1471
|
+
def _stats_epoch_rebuild_marker_path() -> pathlib.Path:
|
|
1472
|
+
return _stats_epoch_rebuild_path("stats-epoch-rebuild.pending")
|
|
1473
|
+
|
|
1474
|
+
|
|
1475
|
+
def _stats_epoch_rebuild_admission_path() -> pathlib.Path:
|
|
1476
|
+
return _stats_epoch_rebuild_path("stats-epoch-rebuild.admission.lock")
|
|
1477
|
+
|
|
1478
|
+
|
|
1479
|
+
def _stats_epoch_rebuild_worker_path() -> pathlib.Path:
|
|
1480
|
+
return _stats_epoch_rebuild_path("stats-epoch-rebuild.worker.lock")
|
|
1481
|
+
|
|
1482
|
+
|
|
1483
|
+
def _stats_epoch_rebuild_log_path() -> pathlib.Path:
|
|
1484
|
+
return pathlib.Path(_cctally_core.LOG_DIR) / "stats-epoch-rebuild.log"
|
|
1485
|
+
|
|
1486
|
+
|
|
1487
|
+
def _unlink_stats_epoch_marker() -> None:
|
|
1488
|
+
try:
|
|
1489
|
+
_stats_epoch_rebuild_marker_path().unlink()
|
|
1490
|
+
except FileNotFoundError:
|
|
1491
|
+
pass
|
|
1492
|
+
|
|
1493
|
+
|
|
1494
|
+
def _stats_epoch_rebuild_worker_active() -> bool:
|
|
1495
|
+
"""Probe the worker flock without waiting or disturbing its owner."""
|
|
1496
|
+
try:
|
|
1497
|
+
fd = os.open(
|
|
1498
|
+
_stats_epoch_rebuild_worker_path(),
|
|
1499
|
+
os.O_WRONLY | os.O_CREAT,
|
|
1500
|
+
0o600,
|
|
1501
|
+
)
|
|
1502
|
+
except OSError:
|
|
1503
|
+
return False
|
|
1504
|
+
try:
|
|
1505
|
+
try:
|
|
1506
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1507
|
+
except BlockingIOError:
|
|
1508
|
+
return True
|
|
1509
|
+
except OSError:
|
|
1510
|
+
return False
|
|
1511
|
+
try:
|
|
1512
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
1513
|
+
except OSError:
|
|
1514
|
+
pass
|
|
1515
|
+
return False
|
|
1516
|
+
finally:
|
|
1517
|
+
os.close(fd)
|
|
1518
|
+
|
|
1519
|
+
|
|
1520
|
+
def _log_stats_epoch_rebuild(outcome: str, *, error: Exception | None = None) -> None:
|
|
1521
|
+
"""Append one path-safe worker result line.
|
|
1522
|
+
|
|
1523
|
+
Detached worker streams are `/dev/null`; this small log is its only
|
|
1524
|
+
diagnostic. Error detail is deliberately structural (class plus numeric
|
|
1525
|
+
SQLite/OS code), never free-form exception text that may carry private
|
|
1526
|
+
paths or `key=value` fragments.
|
|
1527
|
+
"""
|
|
1528
|
+
try:
|
|
1529
|
+
log_path = _stats_epoch_rebuild_log_path()
|
|
1530
|
+
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1531
|
+
detail = ""
|
|
1532
|
+
if error is not None:
|
|
1533
|
+
code = getattr(error, "sqlite_errorcode", None)
|
|
1534
|
+
if code is None:
|
|
1535
|
+
code = getattr(error, "errno", None)
|
|
1536
|
+
detail = f" error={type(error).__name__}"
|
|
1537
|
+
if code is not None:
|
|
1538
|
+
detail += f" code={int(code)}"
|
|
1539
|
+
line = (
|
|
1540
|
+
f"{_cctally_core.now_utc_iso()} worker=stats-epoch-rebuild "
|
|
1541
|
+
f"result={outcome}{detail}\n"
|
|
1542
|
+
).encode("utf-8")
|
|
1543
|
+
fd = os.open(log_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600)
|
|
1544
|
+
try:
|
|
1545
|
+
os.write(fd, line)
|
|
1546
|
+
finally:
|
|
1547
|
+
os.close(fd)
|
|
1548
|
+
except Exception:
|
|
1549
|
+
pass
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
def defer_stats_epoch_rebuild() -> str:
|
|
1553
|
+
"""Schedule one retryable detached epoch worker without blocking callers."""
|
|
1554
|
+
try:
|
|
1555
|
+
pathlib.Path(_cctally_core.APP_DIR).mkdir(parents=True, exist_ok=True)
|
|
1556
|
+
admission_fd = os.open(
|
|
1557
|
+
_stats_epoch_rebuild_admission_path(),
|
|
1558
|
+
os.O_WRONLY | os.O_CREAT,
|
|
1559
|
+
0o600,
|
|
1560
|
+
)
|
|
1561
|
+
except OSError:
|
|
1562
|
+
return "failed"
|
|
1563
|
+
try:
|
|
1564
|
+
try:
|
|
1565
|
+
fcntl.flock(admission_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1566
|
+
except OSError:
|
|
1567
|
+
return "pending"
|
|
1568
|
+
marker = _stats_epoch_rebuild_marker_path()
|
|
1569
|
+
try:
|
|
1570
|
+
age = time.time() - marker.stat().st_mtime
|
|
1571
|
+
except FileNotFoundError:
|
|
1572
|
+
age = None
|
|
1573
|
+
except OSError:
|
|
1574
|
+
return "failed"
|
|
1575
|
+
if age is not None and age < _STATS_EPOCH_REBUILD_RETRY_SECONDS:
|
|
1576
|
+
return "pending"
|
|
1577
|
+
if _stats_epoch_rebuild_worker_active():
|
|
1578
|
+
# A representative replay outlives the marker retry interval.
|
|
1579
|
+
# Refresh the admission stamp instead of launching a process that
|
|
1580
|
+
# can only lose the worker flock and exit.
|
|
1581
|
+
try:
|
|
1582
|
+
os.utime(marker, None)
|
|
1583
|
+
except OSError:
|
|
1584
|
+
pass
|
|
1585
|
+
return "pending"
|
|
1586
|
+
try:
|
|
1587
|
+
marker_fd = os.open(marker, os.O_WRONLY | os.O_CREAT, 0o600)
|
|
1588
|
+
os.close(marker_fd)
|
|
1589
|
+
os.utime(marker, None)
|
|
1590
|
+
except OSError:
|
|
1591
|
+
return "failed"
|
|
1592
|
+
from _cctally_update import _spawn_detached
|
|
1593
|
+
if _spawn_detached(STATS_EPOCH_REBUILD_COMMAND):
|
|
1594
|
+
return "spawned"
|
|
1595
|
+
_unlink_stats_epoch_marker()
|
|
1596
|
+
return "failed"
|
|
1597
|
+
finally:
|
|
1598
|
+
try:
|
|
1599
|
+
fcntl.flock(admission_fd, fcntl.LOCK_UN)
|
|
1600
|
+
except OSError:
|
|
1601
|
+
pass
|
|
1602
|
+
os.close(admission_fd)
|
|
1603
|
+
|
|
1604
|
+
|
|
1605
|
+
def cmd_stats_epoch_rebuild_internal(args) -> int:
|
|
1606
|
+
"""Hidden detached worker: converge a pending stats epoch exactly once."""
|
|
1607
|
+
del args
|
|
1608
|
+
try:
|
|
1609
|
+
pathlib.Path(_cctally_core.APP_DIR).mkdir(parents=True, exist_ok=True)
|
|
1610
|
+
worker_fd = os.open(
|
|
1611
|
+
_stats_epoch_rebuild_worker_path(),
|
|
1612
|
+
os.O_WRONLY | os.O_CREAT,
|
|
1613
|
+
0o600,
|
|
1614
|
+
)
|
|
1615
|
+
except OSError as exc:
|
|
1616
|
+
_log_stats_epoch_rebuild("error", error=exc)
|
|
1617
|
+
return 0
|
|
1618
|
+
try:
|
|
1619
|
+
try:
|
|
1620
|
+
fcntl.flock(worker_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1621
|
+
except OSError:
|
|
1622
|
+
return 0
|
|
1623
|
+
if not stats_epoch_rebuild_pending():
|
|
1624
|
+
_unlink_stats_epoch_marker()
|
|
1625
|
+
_log_stats_epoch_rebuild("current")
|
|
1626
|
+
return 0
|
|
1627
|
+
try:
|
|
1628
|
+
conn = resolve_stats_epoch_mismatch()
|
|
1629
|
+
conn.close()
|
|
1630
|
+
except Exception as exc:
|
|
1631
|
+
_log_stats_epoch_rebuild("error", error=exc)
|
|
1632
|
+
return 0
|
|
1633
|
+
_unlink_stats_epoch_marker()
|
|
1634
|
+
_log_stats_epoch_rebuild("success")
|
|
1635
|
+
return 0
|
|
1636
|
+
finally:
|
|
1637
|
+
try:
|
|
1638
|
+
fcntl.flock(worker_fd, fcntl.LOCK_UN)
|
|
1639
|
+
except OSError:
|
|
1640
|
+
pass
|
|
1641
|
+
os.close(worker_fd)
|
|
1463
1642
|
|
|
1464
1643
|
|
|
1465
1644
|
def _raw_user_version(path) -> int:
|
|
@@ -1475,6 +1654,27 @@ def _raw_user_version(path) -> int:
|
|
|
1475
1654
|
return -1
|
|
1476
1655
|
|
|
1477
1656
|
|
|
1657
|
+
def stats_epoch_rebuild_pending(path=None) -> bool:
|
|
1658
|
+
"""Whether an ordinary live open would require whole-journal replay.
|
|
1659
|
+
|
|
1660
|
+
Missing indexes are cheap fresh installs. Unreadable indexes belong to the
|
|
1661
|
+
corruption classifier, and legacy indexes take the one-time cutover path.
|
|
1662
|
+
Only a readable post-legacy index at a non-current epoch is deferrable.
|
|
1663
|
+
"""
|
|
1664
|
+
candidate = pathlib.Path(
|
|
1665
|
+
_cctally_core.DB_PATH if path is None else path
|
|
1666
|
+
)
|
|
1667
|
+
try:
|
|
1668
|
+
if not candidate.exists():
|
|
1669
|
+
return False
|
|
1670
|
+
except OSError:
|
|
1671
|
+
return False
|
|
1672
|
+
version = _raw_user_version(candidate)
|
|
1673
|
+
if version < 0 or version <= _cctally_core.LEGACY_STATS_HEAD:
|
|
1674
|
+
return False
|
|
1675
|
+
return version != _cctally_core.STATS_INDEX_EPOCH
|
|
1676
|
+
|
|
1677
|
+
|
|
1478
1678
|
def resolve_stats_epoch_mismatch():
|
|
1479
1679
|
"""Resolve a version-mismatched stats.db by journal rebuild (spec §7.1).
|
|
1480
1680
|
Called by ``open_db`` after ``conn.close()`` — returns a fresh steady-state
|
package/bin/_cctally_tui.py
CHANGED
|
@@ -3719,8 +3719,12 @@ def _tui_build_snapshot_once(
|
|
|
3719
3719
|
# Cache-report panel + modal envelope block (spec
|
|
3720
3720
|
# 2026-05-21-cache-report-panel-design.md §5.2). Per-tick build
|
|
3721
3721
|
# alongside the projects envelope. Threshold is read from
|
|
3722
|
-
# ``config.json:cache_report.anomaly_threshold_pp``
|
|
3723
|
-
#
|
|
3722
|
+
# ``config.json:cache_report.anomaly_threshold_pp`` and resolved
|
|
3723
|
+
# by ``_lib_cache_report.resolve_cache_report_threshold`` — the
|
|
3724
|
+
# one definition shared with the Codex read path and the
|
|
3725
|
+
# persistence gate (#443 S3 F17), strict and silent (anything
|
|
3726
|
+
# that is not an in-range int becomes the default 15);
|
|
3727
|
+
# ``anomaly_window_days`` is hardcoded at 14 in v1.
|
|
3724
3728
|
# display_tz inherits the same resolved zone as every other
|
|
3725
3729
|
# panel so today-bucketing matches the envelope's ``display``
|
|
3726
3730
|
# block. Errors record on ``last_sync_error``; ``None`` lands
|
|
@@ -3730,13 +3734,11 @@ def _tui_build_snapshot_once(
|
|
|
3730
3734
|
with _perf.phase("build.cache_report"):
|
|
3731
3735
|
try:
|
|
3732
3736
|
cfg_cr = load_config().get("cache_report") or {}
|
|
3733
|
-
|
|
3734
|
-
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
|
|
3738
|
-
if threshold_pp < 1 or threshold_pp > 100:
|
|
3739
|
-
threshold_pp = 15
|
|
3737
|
+
threshold_pp = _cctally()._load_sibling(
|
|
3738
|
+
"_lib_cache_report"
|
|
3739
|
+
).resolve_cache_report_threshold(
|
|
3740
|
+
cfg_cr.get("anomaly_threshold_pp")
|
|
3741
|
+
)
|
|
3740
3742
|
_dash_mod = sys.modules["_cctally_dashboard"]
|
|
3741
3743
|
_bcr = _dash_mod.build_cache_report_snapshot
|
|
3742
3744
|
cache_report_block = _bcr(
|
|
@@ -6609,6 +6611,28 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
6609
6611
|
snap = dataclasses.replace(snap, last_sync_at=None, hydrating=False)
|
|
6610
6612
|
ref.set(snap)
|
|
6611
6613
|
hub.publish(snap)
|
|
6614
|
+
except _cctally().StatsEpochRebuildDeferred as exc:
|
|
6615
|
+
# #453: the first periodic tick runs before HTTP bind. Preserve the
|
|
6616
|
+
# initial hydrating/degraded frame while the dedicated replay owns
|
|
6617
|
+
# stats maintenance; a generic crash frame would clear the latch
|
|
6618
|
+
# before any client could observe it. The loop retries normally on
|
|
6619
|
+
# its next cadence and publishes a full frame after convergence.
|
|
6620
|
+
prev = ref.get()
|
|
6621
|
+
pending = dataclasses.replace(
|
|
6622
|
+
prev,
|
|
6623
|
+
last_sync_error=f"stats-open: {exc}",
|
|
6624
|
+
sync_failures=(
|
|
6625
|
+
SyncFailureAttribution(
|
|
6626
|
+
leg="stats-open",
|
|
6627
|
+
database="stats",
|
|
6628
|
+
corruption=False,
|
|
6629
|
+
),
|
|
6630
|
+
),
|
|
6631
|
+
generated_at=dt.datetime.now(dt.timezone.utc),
|
|
6632
|
+
hydrating=True,
|
|
6633
|
+
)
|
|
6634
|
+
ref.set(pending)
|
|
6635
|
+
hub.publish(pending)
|
|
6612
6636
|
except Exception as exc:
|
|
6613
6637
|
prev = ref.get()
|
|
6614
6638
|
crashed = dataclasses.replace(
|