cctally 1.99.0 → 1.100.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/bin/_cctally_dashboard.py +1370 -190
- package/bin/_cctally_dashboard_envelope.py +98 -4
- package/bin/_cctally_dashboard_perf.py +433 -0
- package/bin/_cctally_dashboard_sources.py +450 -56
- package/bin/_cctally_db.py +30 -14
- package/bin/_cctally_doctor.py +1331 -1148
- package/bin/_cctally_parser.py +86 -1
- package/bin/_cctally_quota.py +40 -21
- package/bin/_cctally_record.py +37 -2
- package/bin/_cctally_refresh.py +60 -10
- package/bin/_cctally_statusline.py +53 -7
- package/bin/_cctally_tui.py +335 -30
- package/bin/_cctally_update.py +28 -22
- package/bin/_lib_dashboard_sources.py +26 -7
- package/bin/_lib_doctor.py +37 -0
- package/bin/_lib_jsonl.py +4 -2
- package/bin/_lib_perf.py +132 -3
- package/bin/_lib_source_analytics.py +2 -2
- package/bin/_lib_tick_stats.py +538 -0
- package/bin/cctally +19 -7
- package/dashboard/static/assets/index-B5YfQEtn.css +1 -0
- package/dashboard/static/assets/index-Bt59nMMO.js +97 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +3 -1
- package/dashboard/static/assets/index-Bcbm-DNP.js +0 -97
- package/dashboard/static/assets/index-hJP4wlIO.css +0 -1
package/bin/_cctally_tui.py
CHANGED
|
@@ -228,6 +228,7 @@ from _lib_fmt import stable_sum
|
|
|
228
228
|
# shared no-op singleton), so the _tui_build_snapshot seam wraps below cost
|
|
229
229
|
# nothing on the default path.
|
|
230
230
|
import _lib_perf as _perf
|
|
231
|
+
import _lib_tick_stats as _tick_stats
|
|
231
232
|
import _lib_log
|
|
232
233
|
|
|
233
234
|
import importlib.util as _ilu
|
|
@@ -1064,6 +1065,12 @@ class SyncFailureAttribution:
|
|
|
1064
1065
|
leg: str
|
|
1065
1066
|
database: str
|
|
1066
1067
|
corruption: bool
|
|
1068
|
+
# #583 S2 §7. True only when the exception's PRIMARY SQLite code is
|
|
1069
|
+
# SQLITE_BUSY or SQLITE_LOCKED. Defaulted so every existing construction
|
|
1070
|
+
# site is unchanged and no current input reaches the new classifier
|
|
1071
|
+
# branch; `_tui_capture_sync_failure` is the one normal site that supplies
|
|
1072
|
+
# the real value, and it already receives the exception.
|
|
1073
|
+
sqlite_busy: bool = False
|
|
1067
1074
|
|
|
1068
1075
|
|
|
1069
1076
|
class _StatsSnapshotCorruption(Exception):
|
|
@@ -1120,10 +1127,24 @@ def _tui_capture_sync_failure(
|
|
|
1120
1127
|
attributed_database, corruption = _tui_attribute_corruption(
|
|
1121
1128
|
conn, exc, database=database
|
|
1122
1129
|
)
|
|
1130
|
+
# #583 S2 §7. Mask the primary code out of the extended one: SQLite
|
|
1131
|
+
# reports extended codes such as SQLITE_BUSY_SNAPSHOT (517), which this
|
|
1132
|
+
# repository already contends with in its multi-writer cache and
|
|
1133
|
+
# conversations locking, and an unmasked comparison would miss the most
|
|
1134
|
+
# likely case. Numeric only — a string-only hop carrying "database is
|
|
1135
|
+
# locked" must not set the flag, because Preserve 10 forbids widening raw
|
|
1136
|
+
# text matching. `_is_sqlite_corruption_error` is the precedent for
|
|
1137
|
+
# reading the code and is untouched.
|
|
1138
|
+
code = getattr(exc, "sqlite_errorcode", None)
|
|
1139
|
+
sqlite_busy = bool(
|
|
1140
|
+
isinstance(code, int)
|
|
1141
|
+
and (code & 0xFF) in (sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED)
|
|
1142
|
+
)
|
|
1123
1143
|
failure = SyncFailureAttribution(
|
|
1124
1144
|
leg=leg,
|
|
1125
1145
|
database=attributed_database,
|
|
1126
1146
|
corruption=corruption,
|
|
1147
|
+
sqlite_busy=sqlite_busy,
|
|
1127
1148
|
)
|
|
1128
1149
|
if (
|
|
1129
1150
|
failure.database == "stats"
|
|
@@ -1299,6 +1320,15 @@ class DataSnapshot:
|
|
|
1299
1320
|
# Placed LAST with a default so positional fixture constructors keep
|
|
1300
1321
|
# working.
|
|
1301
1322
|
hydrating: bool = False
|
|
1323
|
+
# ---- #583 S2: queue / activity state ----
|
|
1324
|
+
# Owned by ``_SnapshotRef`` and merged in at ``set()`` / mutation time.
|
|
1325
|
+
# Builders never populate it: A2 and the final publish both replace the
|
|
1326
|
+
# snapshot wholesale, so a request accepted mid-build would have its
|
|
1327
|
+
# counter overwritten by the older object the builder had assembled.
|
|
1328
|
+
# ``None`` means "no activity known yet"; the envelope renders that as an
|
|
1329
|
+
# idle object. Trailing default so positional fixture constructors keep
|
|
1330
|
+
# working; appears in NO ``--json``/CLI surface.
|
|
1331
|
+
sync_activity: dict | None = None
|
|
1302
1332
|
# ---- #300: change-signal for the dashboard's lazy detail fetchers ----
|
|
1303
1333
|
# A compact, deterministic string derived from the whole DB dispatch
|
|
1304
1334
|
# signature (``_snapshot_data_version(dispatch_sig)``): it changes iff ANY
|
|
@@ -3209,11 +3239,16 @@ def _tui_claude_budget_domain(
|
|
|
3209
3239
|
window_start_at=start_at,
|
|
3210
3240
|
window_end_at=end_at,
|
|
3211
3241
|
target_usd=target,
|
|
3212
|
-
|
|
3242
|
+
# stable_sum, not sum: both figures are published on the dashboard
|
|
3243
|
+
# wire and byte-compared by the dashboard goldens, and the built-in
|
|
3244
|
+
# sum() switched to Neumaier compensated summation for floats in
|
|
3245
|
+
# CPython 3.12, so the same spend renders 49.20424485 on 3.12+ and
|
|
3246
|
+
# 49.204244850000016 on 3.11.
|
|
3247
|
+
spent_usd=stable_sum(
|
|
3213
3248
|
cost for timestamp, cost in events
|
|
3214
3249
|
if start_at <= timestamp < now_utc
|
|
3215
3250
|
),
|
|
3216
|
-
recent_24h_usd=
|
|
3251
|
+
recent_24h_usd=stable_sum(
|
|
3217
3252
|
cost for timestamp, cost in events
|
|
3218
3253
|
if recent_start <= timestamp < now_utc
|
|
3219
3254
|
),
|
|
@@ -3259,6 +3294,28 @@ def _tui_claude_data_with_budget(
|
|
|
3259
3294
|
return merged
|
|
3260
3295
|
|
|
3261
3296
|
|
|
3297
|
+
def _tui_note_codex_regime(value: str) -> None:
|
|
3298
|
+
"""Stamp the REALISED Codex source-leg decision on the open tick (§1.5).
|
|
3299
|
+
|
|
3300
|
+
Read from what the leg actually did, not from ``CodexIngestStats.
|
|
3301
|
+
rows_changed``. ``_write_codex_file_batch`` can leave ``rows_changed == 0``
|
|
3302
|
+
while unconditionally advancing ``codex_physical_mutation_seq`` after a
|
|
3303
|
+
quota, thread, root, cursor or metadata write; that sequence reaches
|
|
3304
|
+
``codex_version`` through ``compute_signature``, ``reuse_coherent_source_
|
|
3305
|
+
state`` requires exact version equality, and the mismatch therefore drives
|
|
3306
|
+
a genuinely expensive Codex rebuild that ``rows_changed`` would stamp as
|
|
3307
|
+
idle. The repository already reached this conclusion for the #313 F4
|
|
3308
|
+
reconcile gate, in two comments in ``bin/_cctally_cache.py``.
|
|
3309
|
+
|
|
3310
|
+
Aggregated over the refresh by ``TickContext.set_codex_regime``: several
|
|
3311
|
+
builds can run inside one refresh and disagree, and last-write
|
|
3312
|
+
classification would move an expensive tick into the idle population.
|
|
3313
|
+
"""
|
|
3314
|
+
tick = _tick_stats.current()
|
|
3315
|
+
if tick is not None:
|
|
3316
|
+
tick.set_codex_regime(value)
|
|
3317
|
+
|
|
3318
|
+
|
|
3262
3319
|
def _tui_build_source_bundle(
|
|
3263
3320
|
*,
|
|
3264
3321
|
stats_conn,
|
|
@@ -3285,6 +3342,27 @@ def _tui_build_source_bundle(
|
|
|
3285
3342
|
c = _cctally()
|
|
3286
3343
|
cache_conn = c.open_cache_db()
|
|
3287
3344
|
cache_read_tx = False
|
|
3345
|
+
pin_started_ns: "int | None" = None
|
|
3346
|
+
|
|
3347
|
+
def _record_cache_pin() -> None:
|
|
3348
|
+
"""Stamp the elapsed hold exactly once, on whichever path ends it.
|
|
3349
|
+
|
|
3350
|
+
Called immediately before BOTH rollbacks — the normal one after the
|
|
3351
|
+
bundle is composed, and the `finally` one that runs when the build
|
|
3352
|
+
raises. A build that crashed still held the pin for however long it
|
|
3353
|
+
ran, and omitting that would bias the published figure toward the
|
|
3354
|
+
cheap ticks. `pin_started_ns` is cleared here so the second call on
|
|
3355
|
+
the exception path, where both rollbacks are reachable, is a no-op.
|
|
3356
|
+
"""
|
|
3357
|
+
nonlocal pin_started_ns
|
|
3358
|
+
if pin_started_ns is None:
|
|
3359
|
+
return
|
|
3360
|
+
elapsed = time.monotonic_ns() - pin_started_ns
|
|
3361
|
+
pin_started_ns = None
|
|
3362
|
+
tick = _tick_stats.current()
|
|
3363
|
+
if tick is not None:
|
|
3364
|
+
tick.mark_cache_pin(elapsed)
|
|
3365
|
+
|
|
3288
3366
|
try:
|
|
3289
3367
|
# Keep cache.db on one stable snapshot, but leave stats.db in
|
|
3290
3368
|
# statement-scoped autocommit. A dashboard source build can spend
|
|
@@ -3297,6 +3375,12 @@ def _tui_build_source_bundle(
|
|
|
3297
3375
|
if not cache_conn.in_transaction:
|
|
3298
3376
|
cache_conn.execute("BEGIN")
|
|
3299
3377
|
cache_read_tx = True
|
|
3378
|
+
# #583 S5 §2.4 / acceptance criterion 16: the hold is stamped at
|
|
3379
|
+
# the BEGIN and ROLLBACK boundaries themselves. This function's
|
|
3380
|
+
# cumulative duration also counts the work before BEGIN and after
|
|
3381
|
+
# ROLLBACK, so it is an upper bound on the hold rather than the
|
|
3382
|
+
# hold, and no document may quote it as one.
|
|
3383
|
+
pin_started_ns = time.monotonic_ns()
|
|
3300
3384
|
if common_range_start is None:
|
|
3301
3385
|
# Resolved through the SAME helper the callers use, with no daily
|
|
3302
3386
|
# panel. A bare `now_utc - 30 days` here is a microsecond-precise
|
|
@@ -3666,6 +3750,11 @@ def _tui_build_source_bundle(
|
|
|
3666
3750
|
# inherits it.
|
|
3667
3751
|
if codex is not None and aggregate_scope_failed(codex):
|
|
3668
3752
|
codex = None
|
|
3753
|
+
# #583 S1 §1.5: the realised decision, read at the `codex is None`
|
|
3754
|
+
# predicate this branch has just settled. Stamped only here, so a
|
|
3755
|
+
# tick whose Codex leg degraded on ingest failure or contention
|
|
3756
|
+
# stays `not_observed` — no build reached the decision at all.
|
|
3757
|
+
_tui_note_codex_regime("active" if codex is None else "idle")
|
|
3669
3758
|
if codex is None:
|
|
3670
3759
|
try:
|
|
3671
3760
|
codex = build_codex_source_state(
|
|
@@ -3682,6 +3771,15 @@ def _tui_build_source_bundle(
|
|
|
3682
3771
|
codex_quota_actual_thresholds=semantics.codex_quota_actual_thresholds,
|
|
3683
3772
|
codex_quota_projected_thresholds=semantics.codex_quota_projected_thresholds,
|
|
3684
3773
|
cache_report_anomaly_threshold_pp=semantics.cache_report_anomaly_threshold_pp,
|
|
3774
|
+
# #583 S5: the three digests this function already
|
|
3775
|
+
# computed above, handed to the quota memo's reuse
|
|
3776
|
+
# identity instead of being derived a second time per
|
|
3777
|
+
# memo call. Same values, same order, same connection,
|
|
3778
|
+
# and taken under the `assert_projection_readable` gate
|
|
3779
|
+
# already run at the top of this build.
|
|
3780
|
+
stats_identity=(
|
|
3781
|
+
stats_digest, accounts_digest, claude_digest,
|
|
3782
|
+
),
|
|
3685
3783
|
),
|
|
3686
3784
|
data_version=codex_version,
|
|
3687
3785
|
)
|
|
@@ -3745,7 +3843,23 @@ def _tui_build_source_bundle(
|
|
|
3745
3843
|
# whose statement-scoped autocommit reads may have mixed generations.
|
|
3746
3844
|
# Rejecting ordinary cache advancement starves publication whenever
|
|
3747
3845
|
# active Claude/Codex sessions append faster than this build completes.
|
|
3846
|
+
#
|
|
3847
|
+
# ONE EXCEPTION since #583 S5, stated here because a narrowing of this
|
|
3848
|
+
# transaction has to account for it. Every cache read this build ISSUES
|
|
3849
|
+
# runs on the pinned connection and sees the frozen generation, but the
|
|
3850
|
+
# Codex quota memo `_CODEX_QUOTA_OBSERVATION_CACHE` now survives across
|
|
3851
|
+
# builds, so a five-hour correlation read can be answered from a value
|
|
3852
|
+
# another caller loaded -- the share render or the cycle-detail route,
|
|
3853
|
+
# on an unpinned connection at a different generation. That value is
|
|
3854
|
+
# served only while its key is unmoved, and the key covers the ledger
|
|
3855
|
+
# sequence, the attribution revision, the database path and the three
|
|
3856
|
+
# digests taken above, so the evidence behind it provably has not
|
|
3857
|
+
# changed. The full argument is at `_cached_codex_quota_observations`
|
|
3858
|
+
# in `bin/_cctally_dashboard_sources.py`. The dashboard's own bounded
|
|
3859
|
+
# quota read is NOT in that class: it passes `memoize=False` and is
|
|
3860
|
+
# always physically issued on the pinned connection.
|
|
3748
3861
|
if cache_read_tx:
|
|
3862
|
+
_record_cache_pin()
|
|
3749
3863
|
cache_conn.rollback()
|
|
3750
3864
|
cache_read_tx = False
|
|
3751
3865
|
post_stats_digest = codex_stats_digest(stats_conn)
|
|
@@ -3773,6 +3887,7 @@ def _tui_build_source_bundle(
|
|
|
3773
3887
|
raise RuntimeError("source read generation moved during build")
|
|
3774
3888
|
return bundle
|
|
3775
3889
|
finally:
|
|
3890
|
+
_record_cache_pin()
|
|
3776
3891
|
if cache_read_tx:
|
|
3777
3892
|
cache_conn.rollback()
|
|
3778
3893
|
cache_conn.close()
|
|
@@ -3869,6 +3984,26 @@ def _tui_common_source_range_start(
|
|
|
3869
3984
|
return start
|
|
3870
3985
|
|
|
3871
3986
|
|
|
3987
|
+
def _tui_publish_final(tick, hub, snap, *, publication="final",
|
|
3988
|
+
monotonic_ns=None, utcnow=None):
|
|
3989
|
+
"""Publish a tick's closing frame, then close its record (#583 S1 §1.2).
|
|
3990
|
+
|
|
3991
|
+
The order is the contract and it is asserted by spec §7.3: the record is
|
|
3992
|
+
written AFTER ``hub.publish`` returns, so a reader that sees a ring entry
|
|
3993
|
+
knows the frame reached the hub rather than merely having been built. The
|
|
3994
|
+
two clocks are injectable for that gate; production passes neither.
|
|
3995
|
+
|
|
3996
|
+
``tick`` may be None so a caller outside a tick boundary still publishes.
|
|
3997
|
+
"""
|
|
3998
|
+
hub.publish(snap)
|
|
3999
|
+
if tick is None:
|
|
4000
|
+
return
|
|
4001
|
+
monotonic_ns = monotonic_ns or time.monotonic_ns
|
|
4002
|
+
utcnow = utcnow or (lambda: dt.datetime.now(dt.timezone.utc))
|
|
4003
|
+
tick.set_publication(publication)
|
|
4004
|
+
tick.finish(published_ns=monotonic_ns(), published_at=utcnow().isoformat())
|
|
4005
|
+
|
|
4006
|
+
|
|
3872
4007
|
def _tui_build_snapshot(
|
|
3873
4008
|
*,
|
|
3874
4009
|
now_utc: dt.datetime | None = None,
|
|
@@ -3877,7 +4012,60 @@ def _tui_build_snapshot(
|
|
|
3877
4012
|
precompute_envelope: bool = False,
|
|
3878
4013
|
runtime_bind: "str | None" = None,
|
|
3879
4014
|
) -> DataSnapshot:
|
|
3880
|
-
"""Build once, then perform at most one post-query stats heal/reopen.
|
|
4015
|
+
"""Build once, then perform at most one post-query stats heal/reopen.
|
|
4016
|
+
|
|
4017
|
+
#583 S1 §1.2/§1.3: opens a STANDALONE tick context only when no dashboard
|
|
4018
|
+
tick is already open on this thread, so a build made outside a refresh is
|
|
4019
|
+
recorded while an A2 partial build nested inside a live refresh is not
|
|
4020
|
+
double-counted as a second tick. THREE callers reach it that way — ``tui
|
|
4021
|
+
--render-once``, ``cctally-snapshot-measure``, and the dashboard's own
|
|
4022
|
+
pre-bind seed on the ``--no-sync`` branch of
|
|
4023
|
+
``_dashboard_initial_snapshot_once``. The spec names only the first two and
|
|
4024
|
+
says the A1 seed is outside the tick boundary because "it bypasses
|
|
4025
|
+
``_tui_build_snapshot``"; that is true of the ingesting branch and false of
|
|
4026
|
+
the ``--no-sync`` one, which calls straight through here. Recording it is
|
|
4027
|
+
right — it is a real build with a real cost — so the surface names the
|
|
4028
|
+
class rather than enumerating callers. The
|
|
4029
|
+
builder span is installed here as well as at the dashboard's own ``_build``
|
|
4030
|
+
wrapper, because those two standalone callers never reach that wrapper and
|
|
4031
|
+
would otherwise carry a total duration with no split.
|
|
4032
|
+
"""
|
|
4033
|
+
if _tick_stats.current() is not None:
|
|
4034
|
+
return _tui_build_snapshot_impl(
|
|
4035
|
+
now_utc=now_utc, skip_sync=skip_sync,
|
|
4036
|
+
display_tz_pref_override=display_tz_pref_override,
|
|
4037
|
+
precompute_envelope=precompute_envelope,
|
|
4038
|
+
runtime_bind=runtime_bind,
|
|
4039
|
+
)
|
|
4040
|
+
tick = _tick_stats.begin_tick(standalone=True)
|
|
4041
|
+
try:
|
|
4042
|
+
with tick.build_span():
|
|
4043
|
+
snap = _tui_build_snapshot_impl(
|
|
4044
|
+
now_utc=now_utc, skip_sync=skip_sync,
|
|
4045
|
+
display_tz_pref_override=display_tz_pref_override,
|
|
4046
|
+
precompute_envelope=precompute_envelope,
|
|
4047
|
+
runtime_bind=runtime_bind,
|
|
4048
|
+
)
|
|
4049
|
+
except BaseException:
|
|
4050
|
+
tick.mark_degraded()
|
|
4051
|
+
raise
|
|
4052
|
+
finally:
|
|
4053
|
+
tick.finish(
|
|
4054
|
+
published_ns=time.monotonic_ns(),
|
|
4055
|
+
published_at=dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
4056
|
+
)
|
|
4057
|
+
return snap
|
|
4058
|
+
|
|
4059
|
+
|
|
4060
|
+
def _tui_build_snapshot_impl(
|
|
4061
|
+
*,
|
|
4062
|
+
now_utc: dt.datetime | None = None,
|
|
4063
|
+
skip_sync: bool = False,
|
|
4064
|
+
display_tz_pref_override: "str | None" = None,
|
|
4065
|
+
precompute_envelope: bool = False,
|
|
4066
|
+
runtime_bind: "str | None" = None,
|
|
4067
|
+
) -> DataSnapshot:
|
|
4068
|
+
"""The build-and-heal body. See ``_tui_build_snapshot`` for the boundary."""
|
|
3881
4069
|
|
|
3882
4070
|
try:
|
|
3883
4071
|
return _tui_build_snapshot_once(
|
|
@@ -4035,9 +4223,19 @@ def _tui_build_snapshot_once(
|
|
|
4035
4223
|
claude_ingest_failed = False
|
|
4036
4224
|
codex_ingest_contended = False
|
|
4037
4225
|
codex_ingest_failed = False
|
|
4226
|
+
_tick = _tick_stats.current()
|
|
4038
4227
|
with _perf.phase("sync") as _p_sync:
|
|
4039
4228
|
_p_sync.set_meta(ingest=do_ingest)
|
|
4040
4229
|
if do_ingest:
|
|
4230
|
+
# #583 S1 §1.3: the internal-sync ingest span, for the direct
|
|
4231
|
+
# build path (`tui --render-once`, `cctally-snapshot-measure`).
|
|
4232
|
+
# Bracketed rather than a `with` block so the body below is not
|
|
4233
|
+
# reindented; a leaked span is closed by `TickContext.finish`.
|
|
4234
|
+
_ingest_span = (
|
|
4235
|
+
_tick.ingest_span() if _tick is not None else None
|
|
4236
|
+
)
|
|
4237
|
+
if _ingest_span is not None:
|
|
4238
|
+
_ingest_span.__enter__()
|
|
4041
4239
|
try:
|
|
4042
4240
|
cache_conn = _cctally().open_cache_db()
|
|
4043
4241
|
cache_mod = _cctally()._load_sibling("_cctally_cache")
|
|
@@ -4107,6 +4305,8 @@ def _tui_build_snapshot_once(
|
|
|
4107
4305
|
if precompute_envelope:
|
|
4108
4306
|
codex_ingest_failed = True
|
|
4109
4307
|
capture_failure("sync-cache-open", "cache", exc)
|
|
4308
|
+
if _ingest_span is not None:
|
|
4309
|
+
_ingest_span.__exit__(None, None, None)
|
|
4110
4310
|
# Force pure reads for every view builder below, independent of the
|
|
4111
4311
|
# caller's flag: the single ingest above is the only glob per tick.
|
|
4112
4312
|
skip_sync = True
|
|
@@ -4172,6 +4372,12 @@ def _tui_build_snapshot_once(
|
|
|
4172
4372
|
try:
|
|
4173
4373
|
_sc = _cctally()._load_sibling("_lib_snapshot_cache")
|
|
4174
4374
|
prior_key, prior_snap = _sc.dispatch_state()
|
|
4375
|
+
# #583 S1 §1.1: cold versus warm. A build with no prior
|
|
4376
|
+
# dispatch memo has nothing to reuse, so it pays the whole
|
|
4377
|
+
# source construction; the flag is sticky across the refresh,
|
|
4378
|
+
# because one cold build makes the tick a cold tick.
|
|
4379
|
+
if _tick is not None:
|
|
4380
|
+
_tick.set_cold(prior_snap is None)
|
|
4175
4381
|
if prior_snap is not None:
|
|
4176
4382
|
prior_source_bundle = getattr(prior_snap, "source_bundle", None)
|
|
4177
4383
|
except Exception as exc:
|
|
@@ -4242,6 +4448,8 @@ def _tui_build_snapshot_once(
|
|
|
4242
4448
|
)
|
|
4243
4449
|
assert _sc is not None
|
|
4244
4450
|
_sc.store_dispatch_state(dispatch_key, idle_snap)
|
|
4451
|
+
if _tick is not None:
|
|
4452
|
+
_tick.set_dispatch("idle") # §1.4: the reuse branch
|
|
4245
4453
|
_p_snapshot.__exit__(None, None, None)
|
|
4246
4454
|
if _perf.enabled():
|
|
4247
4455
|
_perf.stash_last(
|
|
@@ -4260,7 +4468,8 @@ def _tui_build_snapshot_once(
|
|
|
4260
4468
|
# `use_weekref_cost_cache=True`; any failure leaves the flag OFF
|
|
4261
4469
|
# (safe direct-compute fallback, byte-identical output).
|
|
4262
4470
|
try:
|
|
4263
|
-
|
|
4471
|
+
with _perf.phase("reconcile.cache_open"):
|
|
4472
|
+
_rc_cache_conn = _cctally().open_cache_db()
|
|
4264
4473
|
try:
|
|
4265
4474
|
with _perf.phase("reconcile.weekref") as _pr:
|
|
4266
4475
|
_pr.set_meta(hit=False)
|
|
@@ -4858,6 +5067,11 @@ def _tui_build_snapshot_once(
|
|
|
4858
5067
|
# into the process-global slot for the loopback /api/debug/backend
|
|
4859
5068
|
# endpoint. Whole-dict atomic assignment; never mutated after. No-op
|
|
4860
5069
|
# when tracing is off.
|
|
5070
|
+
if _tick is not None:
|
|
5071
|
+
# §1.4: the full branch. `full` outranks a later `idle` inside the
|
|
5072
|
+
# same refresh, because a refresh containing one full build cost
|
|
5073
|
+
# what a full build costs.
|
|
5074
|
+
_tick.set_dispatch("full")
|
|
4861
5075
|
_p_snapshot.__exit__(None, None, None)
|
|
4862
5076
|
if _perf.enabled():
|
|
4863
5077
|
_perf.stash_last(
|
|
@@ -4893,8 +5107,11 @@ def _tui_precompute_doctor_payload(
|
|
|
4893
5107
|
def _compute(now: dt.datetime, bind: "str | None") -> dict:
|
|
4894
5108
|
_ld = c._load_sibling("_lib_doctor")
|
|
4895
5109
|
try:
|
|
4896
|
-
|
|
4897
|
-
|
|
5110
|
+
with _perf.phase("doctor.gather"):
|
|
5111
|
+
_doc_state = c.doctor_gather_state(
|
|
5112
|
+
now_utc=now, runtime_bind=bind)
|
|
5113
|
+
with _perf.phase("doctor.checks"):
|
|
5114
|
+
_doc_report = _ld.run_checks(_doc_state)
|
|
4898
5115
|
return {
|
|
4899
5116
|
"severity": _doc_report.overall_severity,
|
|
4900
5117
|
"counts": dict(_doc_report.counts),
|
|
@@ -5193,7 +5410,13 @@ def _tui_build_idle_snapshot(prior, *, now_utc, precompute_envelope,
|
|
|
5193
5410
|
return dataclasses.replace(
|
|
5194
5411
|
prior,
|
|
5195
5412
|
generated_at=now_utc,
|
|
5196
|
-
last_sync_at
|
|
5413
|
+
# #583 S2 §6.1: `last_sync_at` means "last SUCCESSFUL validation". A
|
|
5414
|
+
# clean idle tick IS one — it re-verified through four independent
|
|
5415
|
+
# gates that nothing changed, so the reused rows are genuinely
|
|
5416
|
+
# current. A tick that recorded an error retains the prior stamp
|
|
5417
|
+
# rather than reporting itself as a fresh success.
|
|
5418
|
+
last_sync_at=(time.monotonic() if not errors
|
|
5419
|
+
else getattr(prior, "last_sync_at", None)),
|
|
5197
5420
|
last_sync_error=("; ".join(errors) if errors else None),
|
|
5198
5421
|
sync_failures=tuple(idle_failures),
|
|
5199
5422
|
doctor_payload=doctor_payload,
|
|
@@ -5248,7 +5471,9 @@ def _tui_stats_retry_degraded_snapshot(
|
|
|
5248
5471
|
errors.append(f"doctor-precompute: {doctor_exc}")
|
|
5249
5472
|
return dataclasses.replace(
|
|
5250
5473
|
_tui_empty_snapshot(now_utc),
|
|
5251
|
-
|
|
5474
|
+
# #583 S2 §6.1: this produced no successful snapshot, and it builds
|
|
5475
|
+
# from the empty snapshot, so there is no earlier success to preserve.
|
|
5476
|
+
last_sync_at=None,
|
|
5252
5477
|
last_sync_error="; ".join(errors),
|
|
5253
5478
|
sync_failures=(
|
|
5254
5479
|
SyncFailureAttribution(
|
|
@@ -7485,7 +7710,7 @@ class _A2ThrottleClock:
|
|
|
7485
7710
|
def _make_a2_progress_cb(*, ref, hub, build_partial, throttle, monotonic,
|
|
7486
7711
|
perf=_perf):
|
|
7487
7712
|
"""Build the A2 throttled ``sync_cache`` progress callback (extracted so the
|
|
7488
|
-
throttle +
|
|
7713
|
+
throttle + isolation logic is unit-testable with injected deps).
|
|
7489
7714
|
|
|
7490
7715
|
On proceed it builds a partial via ``build_partial()`` (a fresh, complete
|
|
7491
7716
|
``skip_sync=True`` snapshot over the current committed cache — NOT nested in
|
|
@@ -7494,19 +7719,28 @@ def _make_a2_progress_cb(*, ref, hub, build_partial, throttle, monotonic,
|
|
|
7494
7719
|
only the publish carries the latch, so the memo's retained object stays
|
|
7495
7720
|
clean). The dispatch-memo write itself happens inside ``build_partial()`` /
|
|
7496
7721
|
``_tui_build_snapshot`` (its snapshot-cache reconcile step), NOT in this cb.
|
|
7497
|
-
|
|
7498
|
-
|
|
7499
|
-
|
|
7500
|
-
|
|
7722
|
+
|
|
7723
|
+
#583 S1 §2.1: this used to return early whenever perf tracing was active,
|
|
7724
|
+
which meant arming the trace silently switched progressive fill off. The
|
|
7725
|
+
hazard that justified the suppression is real but narrower than a blanket
|
|
7726
|
+
skip — the partial build's unconditional ``_perf.reset_thread()`` rebinds
|
|
7727
|
+
``_tls.stack`` while the enclosing ``sync_cache`` still holds its ``walk``
|
|
7728
|
+
phase, whose ``Phase._stack`` is the original list, so the outer phase
|
|
7729
|
+
would later close into a detached or fragmented root. Isolating the partial
|
|
7730
|
+
build's thread state addresses exactly that, and the publication now
|
|
7731
|
+
happens whether or not tracing is on.
|
|
7501
7732
|
"""
|
|
7502
7733
|
def cb(_stats) -> None:
|
|
7503
|
-
if perf.enabled():
|
|
7504
|
-
return
|
|
7505
7734
|
now = monotonic()
|
|
7506
7735
|
if not throttle.should_fire(now):
|
|
7507
7736
|
return
|
|
7508
|
-
|
|
7509
|
-
|
|
7737
|
+
with perf.isolated_thread_state():
|
|
7738
|
+
snap = build_partial()
|
|
7739
|
+
# #583 S2 §6.2 publication point 3: publish what the reference now
|
|
7740
|
+
# HOLDS, not the local build. A request accepted while `build_partial`
|
|
7741
|
+
# ran is on the reference and absent from `snap`, so publishing `snap`
|
|
7742
|
+
# would erase its counter and the client would never settle.
|
|
7743
|
+
snap = ref.set(snap)
|
|
7510
7744
|
hub.publish(dataclasses.replace(snap, hydrating=True))
|
|
7511
7745
|
throttle.mark_done(monotonic())
|
|
7512
7746
|
return cb
|
|
@@ -7547,11 +7781,24 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7547
7781
|
# re-export AND ``monkeypatch.setitem(ns, "_tui_build_snapshot", spy)``
|
|
7548
7782
|
# in tests propagate into this closure body (a bare-name lookup would
|
|
7549
7783
|
# resolve in this sibling's __dict__ and miss the cctally-side patch).
|
|
7550
|
-
|
|
7551
|
-
|
|
7552
|
-
|
|
7553
|
-
|
|
7554
|
-
|
|
7784
|
+
#
|
|
7785
|
+
# #583 S1 §1.3: one of the TWO builder-span sites. This wrapper is
|
|
7786
|
+
# dashboard-local, so `tui --render-once` and `cctally-snapshot-measure`
|
|
7787
|
+
# never reach it and take the span at `_tui_build_snapshot`'s standalone
|
|
7788
|
+
# boundary instead. The span subtracts any ingest nested inside it.
|
|
7789
|
+
tick = _tick_stats.current()
|
|
7790
|
+
if tick is None:
|
|
7791
|
+
return sys.modules["cctally"]._tui_build_snapshot(
|
|
7792
|
+
now_utc=pinned_now, skip_sync=skip_sync,
|
|
7793
|
+
display_tz_pref_override=display_tz_pref_override,
|
|
7794
|
+
precompute_envelope=True, runtime_bind=runtime_bind,
|
|
7795
|
+
)
|
|
7796
|
+
with tick.build_span():
|
|
7797
|
+
return sys.modules["cctally"]._tui_build_snapshot(
|
|
7798
|
+
now_utc=pinned_now, skip_sync=skip_sync,
|
|
7799
|
+
display_tz_pref_override=display_tz_pref_override,
|
|
7800
|
+
precompute_envelope=True, runtime_bind=runtime_bind,
|
|
7801
|
+
)
|
|
7555
7802
|
|
|
7556
7803
|
def _locked(skip_sync: bool) -> None:
|
|
7557
7804
|
# #279 S5 F6.3 (gate P1-1): arm the snapshot-cache owner-thread tripwire
|
|
@@ -7560,11 +7807,19 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7560
7807
|
# on-call, so ownership transfers to the current rebuilder; the guards in
|
|
7561
7808
|
# _lib_snapshot_cache then catch a lock-bypassing foreign-thread mutation.
|
|
7562
7809
|
_cctally()._load_sibling("_lib_snapshot_cache").mark_owner_thread()
|
|
7810
|
+
# #583 S1 §1.2: the tick opens here and closes after the final publish,
|
|
7811
|
+
# so the progressive, final, deferred and crash publication paths are
|
|
7812
|
+
# all inside it. The A1 pre-bind seed is deliberately outside, because
|
|
7813
|
+
# it bypasses `_tui_build_snapshot` and writes no dispatch state.
|
|
7814
|
+
tick = _tick_stats.begin_tick()
|
|
7563
7815
|
try:
|
|
7564
7816
|
if not skip_sync:
|
|
7565
7817
|
# ── Decoupled ingest + build (§2.1) ─────────────────────────
|
|
7566
7818
|
import time as _time
|
|
7567
7819
|
sync_error = None
|
|
7820
|
+
# #583 S2 §6.1: the last SUCCESSFUL validation, read before
|
|
7821
|
+
# A2's partial republishes can overwrite the held snapshot.
|
|
7822
|
+
prior_sync_at = ref.get().last_sync_at
|
|
7568
7823
|
start = _time.monotonic()
|
|
7569
7824
|
throttle = _A2ThrottleClock(_A2_PARTIAL_THROTTLE_S, start=start)
|
|
7570
7825
|
cb = _make_a2_progress_cb(
|
|
@@ -7574,6 +7829,18 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7574
7829
|
)
|
|
7575
7830
|
cache_conn = _cctally().open_cache_db()
|
|
7576
7831
|
cache_mod = _cctally()._load_sibling("_cctally_cache")
|
|
7832
|
+
# #583 S1 §1.3: the ingest span, opened through the
|
|
7833
|
+
# contextmanager protocol rather than a `with` block so the
|
|
7834
|
+
# long try/except/finally below is not reindented. The A2
|
|
7835
|
+
# progress callback runs `build_partial()` SYNCHRONOUSLY inside
|
|
7836
|
+
# this region, so the builder spans it opens nest here and
|
|
7837
|
+
# their time is subtracted — without that, every progress build
|
|
7838
|
+
# is counted once as ingest and again as builder and
|
|
7839
|
+
# `ingest_ns + builder_ns` can exceed `duration_ns`. A span
|
|
7840
|
+
# left open by an escaping exception is closed by
|
|
7841
|
+
# `TickContext.finish`, so no try/finally is needed to bound it.
|
|
7842
|
+
_ingest = tick.ingest_span()
|
|
7843
|
+
_ingest.__enter__()
|
|
7577
7844
|
try:
|
|
7578
7845
|
# Under CCTALLY_PERF_TRACE the phase tree this standalone
|
|
7579
7846
|
# sync_cache builds is intentionally NOT surfaced in the live
|
|
@@ -7603,6 +7870,13 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7603
7870
|
sync_error = f"sync-cache: {exc}"
|
|
7604
7871
|
finally:
|
|
7605
7872
|
cache_conn.close()
|
|
7873
|
+
_ingest.__exit__(None, None, None)
|
|
7874
|
+
# #583 S1 §2.2: the ONE place a pending trace-arm request is
|
|
7875
|
+
# consumed. After the cache connection closes and immediately
|
|
7876
|
+
# before the authoritative build, so a request landing
|
|
7877
|
+
# mid-ingest cannot split one ingest across two tracing states,
|
|
7878
|
+
# and A2 partial builds never consume it.
|
|
7879
|
+
_perf.apply_pending()
|
|
7606
7880
|
snap = _build(skip_sync=True) # final: hydrating=False (default)
|
|
7607
7881
|
if sync_error is not None:
|
|
7608
7882
|
# Thread the standalone sync error into last_sync_error, sync
|
|
@@ -7612,10 +7886,27 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7612
7886
|
merged = (sync_error if not existing
|
|
7613
7887
|
else f"{sync_error}; {existing}")
|
|
7614
7888
|
snap = dataclasses.replace(snap, last_sync_error=merged)
|
|
7615
|
-
|
|
7616
|
-
|
|
7889
|
+
# #583 S2 §6.1: `last_sync_at` means "last SUCCESSFUL
|
|
7890
|
+
# validation", and `_tui_build_snapshot` stamps it
|
|
7891
|
+
# unconditionally. A build that recorded any failure carries
|
|
7892
|
+
# the prior successful value forward instead of reporting
|
|
7893
|
+
# itself as freshly synced.
|
|
7894
|
+
if snap.last_sync_error:
|
|
7895
|
+
snap = dataclasses.replace(snap, last_sync_at=prior_sync_at)
|
|
7896
|
+
# #583 S2 §6.2 publication point 4: publish what the reference
|
|
7897
|
+
# holds, so a request accepted during the build is not erased.
|
|
7898
|
+
# `set_final` also clears `rebuilding` in the same acquisition,
|
|
7899
|
+
# so this ONE frame reports the rebuild finished; the loop's
|
|
7900
|
+
# trailing `mark_rebuilding(False)` then finds no transition and
|
|
7901
|
+
# publishes nothing. That is what keeps an automatic tick at two
|
|
7902
|
+
# frames instead of three, PLUS one per A2 progress publish that
|
|
7903
|
+
# cleared the throttle above — several on a cold first-run
|
|
7904
|
+
# ingest, none on a warm one.
|
|
7905
|
+
snap = ref.set_final(snap)
|
|
7906
|
+
_tui_publish_final(tick, hub, snap)
|
|
7617
7907
|
return
|
|
7618
7908
|
# ── skip_sync=True: single-build path (POST /api/settings) ──────
|
|
7909
|
+
_perf.apply_pending() # §2.2: the no-ingest build's arm boundary
|
|
7619
7910
|
snap = _build(skip_sync=True)
|
|
7620
7911
|
# Mirror the startup override: suppress the monotonic sync stamp so
|
|
7621
7912
|
# the envelope keeps emitting sync_age_s=None and the client keeps
|
|
@@ -7624,8 +7915,10 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7624
7915
|
# from _tui_build_snapshot, restated here since this is a replace()
|
|
7625
7916
|
# clone site).
|
|
7626
7917
|
snap = dataclasses.replace(snap, last_sync_at=None, hydrating=False)
|
|
7627
|
-
|
|
7628
|
-
|
|
7918
|
+
# Terminal for this branch too, for the same reason: POST
|
|
7919
|
+
# /api/settings brackets its rebuild with the same flag pair.
|
|
7920
|
+
snap = ref.set_final(snap)
|
|
7921
|
+
_tui_publish_final(tick, hub, snap)
|
|
7629
7922
|
except _cctally().StatsRebuildDeferred as exc:
|
|
7630
7923
|
# #453: the first periodic tick runs before HTTP bind. Preserve the
|
|
7631
7924
|
# initial hydrating/degraded frame while the dedicated replay owns
|
|
@@ -7651,8 +7944,9 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7651
7944
|
generated_at=dt.datetime.now(dt.timezone.utc),
|
|
7652
7945
|
hydrating=True,
|
|
7653
7946
|
)
|
|
7654
|
-
ref.set(pending)
|
|
7655
|
-
|
|
7947
|
+
pending = ref.set(pending) # #583 S2 §6.2: publish what is held
|
|
7948
|
+
tick.mark_degraded()
|
|
7949
|
+
_tui_publish_final(tick, hub, pending, publication="degraded")
|
|
7656
7950
|
except Exception as exc:
|
|
7657
7951
|
prev = ref.get()
|
|
7658
7952
|
crashed = dataclasses.replace(
|
|
@@ -7668,8 +7962,19 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7668
7962
|
# seed/partial, so the client doesn't stay stuck in skeletons.
|
|
7669
7963
|
hydrating=False,
|
|
7670
7964
|
)
|
|
7671
|
-
ref.set(crashed)
|
|
7672
|
-
|
|
7965
|
+
crashed = ref.set(crashed) # #583 S2 §6.2: publish what is held
|
|
7966
|
+
tick.mark_degraded()
|
|
7967
|
+
_tui_publish_final(tick, hub, crashed, publication="degraded")
|
|
7968
|
+
finally:
|
|
7969
|
+
# A tick always closes. Every publication path above finishes it,
|
|
7970
|
+
# and `finish` is idempotent, so this only catches an escape none
|
|
7971
|
+
# of them handled — a BaseException such as KeyboardInterrupt.
|
|
7972
|
+
if not tick.finished:
|
|
7973
|
+
tick.mark_degraded()
|
|
7974
|
+
tick.finish(
|
|
7975
|
+
published_ns=time.monotonic_ns(),
|
|
7976
|
+
published_at=dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
7977
|
+
)
|
|
7673
7978
|
return _locked
|
|
7674
7979
|
|
|
7675
7980
|
|