cctally 1.99.1 → 1.100.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/bin/_cctally_dashboard.py +1370 -190
- package/bin/_cctally_dashboard_envelope.py +98 -4
- package/bin/_cctally_dashboard_perf.py +433 -0
- package/bin/_cctally_dashboard_sources.py +441 -54
- package/bin/_cctally_db.py +30 -14
- package/bin/_cctally_doctor.py +1331 -1148
- package/bin/_cctally_parser.py +70 -1
- package/bin/_cctally_quota.py +40 -21
- package/bin/_cctally_record.py +37 -2
- package/bin/_cctally_refresh.py +60 -10
- package/bin/_cctally_statusline.py +53 -7
- package/bin/_cctally_tui.py +328 -28
- package/bin/_cctally_update.py +28 -22
- package/bin/_lib_dashboard_sources.py +26 -7
- package/bin/_lib_doctor.py +37 -0
- package/bin/_lib_jsonl.py +4 -2
- package/bin/_lib_perf.py +132 -3
- package/bin/_lib_source_analytics.py +2 -2
- package/bin/_lib_tick_stats.py +538 -0
- package/bin/cctally +19 -7
- package/dashboard/static/assets/index-B5YfQEtn.css +1 -0
- package/dashboard/static/assets/index-Bt59nMMO.js +97 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +3 -1
- package/dashboard/static/assets/index-C5NBB2w9.js +0 -97
- package/dashboard/static/assets/index-hJP4wlIO.css +0 -1
package/bin/_cctally_tui.py
CHANGED
|
@@ -228,6 +228,7 @@ from _lib_fmt import stable_sum
|
|
|
228
228
|
# shared no-op singleton), so the _tui_build_snapshot seam wraps below cost
|
|
229
229
|
# nothing on the default path.
|
|
230
230
|
import _lib_perf as _perf
|
|
231
|
+
import _lib_tick_stats as _tick_stats
|
|
231
232
|
import _lib_log
|
|
232
233
|
|
|
233
234
|
import importlib.util as _ilu
|
|
@@ -1064,6 +1065,12 @@ class SyncFailureAttribution:
|
|
|
1064
1065
|
leg: str
|
|
1065
1066
|
database: str
|
|
1066
1067
|
corruption: bool
|
|
1068
|
+
# #583 S2 §7. True only when the exception's PRIMARY SQLite code is
|
|
1069
|
+
# SQLITE_BUSY or SQLITE_LOCKED. Defaulted so every existing construction
|
|
1070
|
+
# site is unchanged and no current input reaches the new classifier
|
|
1071
|
+
# branch; `_tui_capture_sync_failure` is the one normal site that supplies
|
|
1072
|
+
# the real value, and it already receives the exception.
|
|
1073
|
+
sqlite_busy: bool = False
|
|
1067
1074
|
|
|
1068
1075
|
|
|
1069
1076
|
class _StatsSnapshotCorruption(Exception):
|
|
@@ -1120,10 +1127,24 @@ def _tui_capture_sync_failure(
|
|
|
1120
1127
|
attributed_database, corruption = _tui_attribute_corruption(
|
|
1121
1128
|
conn, exc, database=database
|
|
1122
1129
|
)
|
|
1130
|
+
# #583 S2 §7. Mask the primary code out of the extended one: SQLite
|
|
1131
|
+
# reports extended codes such as SQLITE_BUSY_SNAPSHOT (517), which this
|
|
1132
|
+
# repository already contends with in its multi-writer cache and
|
|
1133
|
+
# conversations locking, and an unmasked comparison would miss the most
|
|
1134
|
+
# likely case. Numeric only — a string-only hop carrying "database is
|
|
1135
|
+
# locked" must not set the flag, because Preserve 10 forbids widening raw
|
|
1136
|
+
# text matching. `_is_sqlite_corruption_error` is the precedent for
|
|
1137
|
+
# reading the code and is untouched.
|
|
1138
|
+
code = getattr(exc, "sqlite_errorcode", None)
|
|
1139
|
+
sqlite_busy = bool(
|
|
1140
|
+
isinstance(code, int)
|
|
1141
|
+
and (code & 0xFF) in (sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED)
|
|
1142
|
+
)
|
|
1123
1143
|
failure = SyncFailureAttribution(
|
|
1124
1144
|
leg=leg,
|
|
1125
1145
|
database=attributed_database,
|
|
1126
1146
|
corruption=corruption,
|
|
1147
|
+
sqlite_busy=sqlite_busy,
|
|
1127
1148
|
)
|
|
1128
1149
|
if (
|
|
1129
1150
|
failure.database == "stats"
|
|
@@ -1299,6 +1320,15 @@ class DataSnapshot:
|
|
|
1299
1320
|
# Placed LAST with a default so positional fixture constructors keep
|
|
1300
1321
|
# working.
|
|
1301
1322
|
hydrating: bool = False
|
|
1323
|
+
# ---- #583 S2: queue / activity state ----
|
|
1324
|
+
# Owned by ``_SnapshotRef`` and merged in at ``set()`` / mutation time.
|
|
1325
|
+
# Builders never populate it: A2 and the final publish both replace the
|
|
1326
|
+
# snapshot wholesale, so a request accepted mid-build would have its
|
|
1327
|
+
# counter overwritten by the older object the builder had assembled.
|
|
1328
|
+
# ``None`` means "no activity known yet"; the envelope renders that as an
|
|
1329
|
+
# idle object. Trailing default so positional fixture constructors keep
|
|
1330
|
+
# working; appears in NO ``--json``/CLI surface.
|
|
1331
|
+
sync_activity: dict | None = None
|
|
1302
1332
|
# ---- #300: change-signal for the dashboard's lazy detail fetchers ----
|
|
1303
1333
|
# A compact, deterministic string derived from the whole DB dispatch
|
|
1304
1334
|
# signature (``_snapshot_data_version(dispatch_sig)``): it changes iff ANY
|
|
@@ -3264,6 +3294,28 @@ def _tui_claude_data_with_budget(
|
|
|
3264
3294
|
return merged
|
|
3265
3295
|
|
|
3266
3296
|
|
|
3297
|
+
def _tui_note_codex_regime(value: str) -> None:
|
|
3298
|
+
"""Stamp the REALISED Codex source-leg decision on the open tick (§1.5).
|
|
3299
|
+
|
|
3300
|
+
Read from what the leg actually did, not from ``CodexIngestStats.
|
|
3301
|
+
rows_changed``. ``_write_codex_file_batch`` can leave ``rows_changed == 0``
|
|
3302
|
+
while unconditionally advancing ``codex_physical_mutation_seq`` after a
|
|
3303
|
+
quota, thread, root, cursor or metadata write; that sequence reaches
|
|
3304
|
+
``codex_version`` through ``compute_signature``, ``reuse_coherent_source_
|
|
3305
|
+
state`` requires exact version equality, and the mismatch therefore drives
|
|
3306
|
+
a genuinely expensive Codex rebuild that ``rows_changed`` would stamp as
|
|
3307
|
+
idle. The repository already reached this conclusion for the #313 F4
|
|
3308
|
+
reconcile gate, in two comments in ``bin/_cctally_cache.py``.
|
|
3309
|
+
|
|
3310
|
+
Aggregated over the refresh by ``TickContext.set_codex_regime``: several
|
|
3311
|
+
builds can run inside one refresh and disagree, and last-write
|
|
3312
|
+
classification would move an expensive tick into the idle population.
|
|
3313
|
+
"""
|
|
3314
|
+
tick = _tick_stats.current()
|
|
3315
|
+
if tick is not None:
|
|
3316
|
+
tick.set_codex_regime(value)
|
|
3317
|
+
|
|
3318
|
+
|
|
3267
3319
|
def _tui_build_source_bundle(
|
|
3268
3320
|
*,
|
|
3269
3321
|
stats_conn,
|
|
@@ -3290,6 +3342,27 @@ def _tui_build_source_bundle(
|
|
|
3290
3342
|
c = _cctally()
|
|
3291
3343
|
cache_conn = c.open_cache_db()
|
|
3292
3344
|
cache_read_tx = False
|
|
3345
|
+
pin_started_ns: "int | None" = None
|
|
3346
|
+
|
|
3347
|
+
def _record_cache_pin() -> None:
|
|
3348
|
+
"""Stamp the elapsed hold exactly once, on whichever path ends it.
|
|
3349
|
+
|
|
3350
|
+
Called immediately before BOTH rollbacks — the normal one after the
|
|
3351
|
+
bundle is composed, and the `finally` one that runs when the build
|
|
3352
|
+
raises. A build that crashed still held the pin for however long it
|
|
3353
|
+
ran, and omitting that would bias the published figure toward the
|
|
3354
|
+
cheap ticks. `pin_started_ns` is cleared here so the second call on
|
|
3355
|
+
the exception path, where both rollbacks are reachable, is a no-op.
|
|
3356
|
+
"""
|
|
3357
|
+
nonlocal pin_started_ns
|
|
3358
|
+
if pin_started_ns is None:
|
|
3359
|
+
return
|
|
3360
|
+
elapsed = time.monotonic_ns() - pin_started_ns
|
|
3361
|
+
pin_started_ns = None
|
|
3362
|
+
tick = _tick_stats.current()
|
|
3363
|
+
if tick is not None:
|
|
3364
|
+
tick.mark_cache_pin(elapsed)
|
|
3365
|
+
|
|
3293
3366
|
try:
|
|
3294
3367
|
# Keep cache.db on one stable snapshot, but leave stats.db in
|
|
3295
3368
|
# statement-scoped autocommit. A dashboard source build can spend
|
|
@@ -3302,6 +3375,12 @@ def _tui_build_source_bundle(
|
|
|
3302
3375
|
if not cache_conn.in_transaction:
|
|
3303
3376
|
cache_conn.execute("BEGIN")
|
|
3304
3377
|
cache_read_tx = True
|
|
3378
|
+
# #583 S5 §2.4 / acceptance criterion 16: the hold is stamped at
|
|
3379
|
+
# the BEGIN and ROLLBACK boundaries themselves. This function's
|
|
3380
|
+
# cumulative duration also counts the work before BEGIN and after
|
|
3381
|
+
# ROLLBACK, so it is an upper bound on the hold rather than the
|
|
3382
|
+
# hold, and no document may quote it as one.
|
|
3383
|
+
pin_started_ns = time.monotonic_ns()
|
|
3305
3384
|
if common_range_start is None:
|
|
3306
3385
|
# Resolved through the SAME helper the callers use, with no daily
|
|
3307
3386
|
# panel. A bare `now_utc - 30 days` here is a microsecond-precise
|
|
@@ -3671,6 +3750,11 @@ def _tui_build_source_bundle(
|
|
|
3671
3750
|
# inherits it.
|
|
3672
3751
|
if codex is not None and aggregate_scope_failed(codex):
|
|
3673
3752
|
codex = None
|
|
3753
|
+
# #583 S1 §1.5: the realised decision, read at the `codex is None`
|
|
3754
|
+
# predicate this branch has just settled. Stamped only here, so a
|
|
3755
|
+
# tick whose Codex leg degraded on ingest failure or contention
|
|
3756
|
+
# stays `not_observed` — no build reached the decision at all.
|
|
3757
|
+
_tui_note_codex_regime("active" if codex is None else "idle")
|
|
3674
3758
|
if codex is None:
|
|
3675
3759
|
try:
|
|
3676
3760
|
codex = build_codex_source_state(
|
|
@@ -3687,6 +3771,15 @@ def _tui_build_source_bundle(
|
|
|
3687
3771
|
codex_quota_actual_thresholds=semantics.codex_quota_actual_thresholds,
|
|
3688
3772
|
codex_quota_projected_thresholds=semantics.codex_quota_projected_thresholds,
|
|
3689
3773
|
cache_report_anomaly_threshold_pp=semantics.cache_report_anomaly_threshold_pp,
|
|
3774
|
+
# #583 S5: the three digests this function already
|
|
3775
|
+
# computed above, handed to the quota memo's reuse
|
|
3776
|
+
# identity instead of being derived a second time per
|
|
3777
|
+
# memo call. Same values, same order, same connection,
|
|
3778
|
+
# and taken under the `assert_projection_readable` gate
|
|
3779
|
+
# already run at the top of this build.
|
|
3780
|
+
stats_identity=(
|
|
3781
|
+
stats_digest, accounts_digest, claude_digest,
|
|
3782
|
+
),
|
|
3690
3783
|
),
|
|
3691
3784
|
data_version=codex_version,
|
|
3692
3785
|
)
|
|
@@ -3750,7 +3843,23 @@ def _tui_build_source_bundle(
|
|
|
3750
3843
|
# whose statement-scoped autocommit reads may have mixed generations.
|
|
3751
3844
|
# Rejecting ordinary cache advancement starves publication whenever
|
|
3752
3845
|
# active Claude/Codex sessions append faster than this build completes.
|
|
3846
|
+
#
|
|
3847
|
+
# ONE EXCEPTION since #583 S5, stated here because a narrowing of this
|
|
3848
|
+
# transaction has to account for it. Every cache read this build ISSUES
|
|
3849
|
+
# runs on the pinned connection and sees the frozen generation, but the
|
|
3850
|
+
# Codex quota memo `_CODEX_QUOTA_OBSERVATION_CACHE` now survives across
|
|
3851
|
+
# builds, so a five-hour correlation read can be answered from a value
|
|
3852
|
+
# another caller loaded -- the share render or the cycle-detail route,
|
|
3853
|
+
# on an unpinned connection at a different generation. That value is
|
|
3854
|
+
# served only while its key is unmoved, and the key covers the ledger
|
|
3855
|
+
# sequence, the attribution revision, the database path and the three
|
|
3856
|
+
# digests taken above, so the evidence behind it provably has not
|
|
3857
|
+
# changed. The full argument is at `_cached_codex_quota_observations`
|
|
3858
|
+
# in `bin/_cctally_dashboard_sources.py`. The dashboard's own bounded
|
|
3859
|
+
# quota read is NOT in that class: it passes `memoize=False` and is
|
|
3860
|
+
# always physically issued on the pinned connection.
|
|
3753
3861
|
if cache_read_tx:
|
|
3862
|
+
_record_cache_pin()
|
|
3754
3863
|
cache_conn.rollback()
|
|
3755
3864
|
cache_read_tx = False
|
|
3756
3865
|
post_stats_digest = codex_stats_digest(stats_conn)
|
|
@@ -3778,6 +3887,7 @@ def _tui_build_source_bundle(
|
|
|
3778
3887
|
raise RuntimeError("source read generation moved during build")
|
|
3779
3888
|
return bundle
|
|
3780
3889
|
finally:
|
|
3890
|
+
_record_cache_pin()
|
|
3781
3891
|
if cache_read_tx:
|
|
3782
3892
|
cache_conn.rollback()
|
|
3783
3893
|
cache_conn.close()
|
|
@@ -3874,6 +3984,26 @@ def _tui_common_source_range_start(
|
|
|
3874
3984
|
return start
|
|
3875
3985
|
|
|
3876
3986
|
|
|
3987
|
+
def _tui_publish_final(tick, hub, snap, *, publication="final",
|
|
3988
|
+
monotonic_ns=None, utcnow=None):
|
|
3989
|
+
"""Publish a tick's closing frame, then close its record (#583 S1 §1.2).
|
|
3990
|
+
|
|
3991
|
+
The order is the contract and it is asserted by spec §7.3: the record is
|
|
3992
|
+
written AFTER ``hub.publish`` returns, so a reader that sees a ring entry
|
|
3993
|
+
knows the frame reached the hub rather than merely having been built. The
|
|
3994
|
+
two clocks are injectable for that gate; production passes neither.
|
|
3995
|
+
|
|
3996
|
+
``tick`` may be None so a caller outside a tick boundary still publishes.
|
|
3997
|
+
"""
|
|
3998
|
+
hub.publish(snap)
|
|
3999
|
+
if tick is None:
|
|
4000
|
+
return
|
|
4001
|
+
monotonic_ns = monotonic_ns or time.monotonic_ns
|
|
4002
|
+
utcnow = utcnow or (lambda: dt.datetime.now(dt.timezone.utc))
|
|
4003
|
+
tick.set_publication(publication)
|
|
4004
|
+
tick.finish(published_ns=monotonic_ns(), published_at=utcnow().isoformat())
|
|
4005
|
+
|
|
4006
|
+
|
|
3877
4007
|
def _tui_build_snapshot(
|
|
3878
4008
|
*,
|
|
3879
4009
|
now_utc: dt.datetime | None = None,
|
|
@@ -3882,7 +4012,60 @@ def _tui_build_snapshot(
|
|
|
3882
4012
|
precompute_envelope: bool = False,
|
|
3883
4013
|
runtime_bind: "str | None" = None,
|
|
3884
4014
|
) -> DataSnapshot:
|
|
3885
|
-
"""Build once, then perform at most one post-query stats heal/reopen.
|
|
4015
|
+
"""Build once, then perform at most one post-query stats heal/reopen.
|
|
4016
|
+
|
|
4017
|
+
#583 S1 §1.2/§1.3: opens a STANDALONE tick context only when no dashboard
|
|
4018
|
+
tick is already open on this thread, so a build made outside a refresh is
|
|
4019
|
+
recorded while an A2 partial build nested inside a live refresh is not
|
|
4020
|
+
double-counted as a second tick. THREE callers reach it that way — ``tui
|
|
4021
|
+
--render-once``, ``cctally-snapshot-measure``, and the dashboard's own
|
|
4022
|
+
pre-bind seed on the ``--no-sync`` branch of
|
|
4023
|
+
``_dashboard_initial_snapshot_once``. The spec names only the first two and
|
|
4024
|
+
says the A1 seed is outside the tick boundary because "it bypasses
|
|
4025
|
+
``_tui_build_snapshot``"; that is true of the ingesting branch and false of
|
|
4026
|
+
the ``--no-sync`` one, which calls straight through here. Recording it is
|
|
4027
|
+
right — it is a real build with a real cost — so the surface names the
|
|
4028
|
+
class rather than enumerating callers. The
|
|
4029
|
+
builder span is installed here as well as at the dashboard's own ``_build``
|
|
4030
|
+
wrapper, because those two standalone callers never reach that wrapper and
|
|
4031
|
+
would otherwise carry a total duration with no split.
|
|
4032
|
+
"""
|
|
4033
|
+
if _tick_stats.current() is not None:
|
|
4034
|
+
return _tui_build_snapshot_impl(
|
|
4035
|
+
now_utc=now_utc, skip_sync=skip_sync,
|
|
4036
|
+
display_tz_pref_override=display_tz_pref_override,
|
|
4037
|
+
precompute_envelope=precompute_envelope,
|
|
4038
|
+
runtime_bind=runtime_bind,
|
|
4039
|
+
)
|
|
4040
|
+
tick = _tick_stats.begin_tick(standalone=True)
|
|
4041
|
+
try:
|
|
4042
|
+
with tick.build_span():
|
|
4043
|
+
snap = _tui_build_snapshot_impl(
|
|
4044
|
+
now_utc=now_utc, skip_sync=skip_sync,
|
|
4045
|
+
display_tz_pref_override=display_tz_pref_override,
|
|
4046
|
+
precompute_envelope=precompute_envelope,
|
|
4047
|
+
runtime_bind=runtime_bind,
|
|
4048
|
+
)
|
|
4049
|
+
except BaseException:
|
|
4050
|
+
tick.mark_degraded()
|
|
4051
|
+
raise
|
|
4052
|
+
finally:
|
|
4053
|
+
tick.finish(
|
|
4054
|
+
published_ns=time.monotonic_ns(),
|
|
4055
|
+
published_at=dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
4056
|
+
)
|
|
4057
|
+
return snap
|
|
4058
|
+
|
|
4059
|
+
|
|
4060
|
+
def _tui_build_snapshot_impl(
|
|
4061
|
+
*,
|
|
4062
|
+
now_utc: dt.datetime | None = None,
|
|
4063
|
+
skip_sync: bool = False,
|
|
4064
|
+
display_tz_pref_override: "str | None" = None,
|
|
4065
|
+
precompute_envelope: bool = False,
|
|
4066
|
+
runtime_bind: "str | None" = None,
|
|
4067
|
+
) -> DataSnapshot:
|
|
4068
|
+
"""The build-and-heal body. See ``_tui_build_snapshot`` for the boundary."""
|
|
3886
4069
|
|
|
3887
4070
|
try:
|
|
3888
4071
|
return _tui_build_snapshot_once(
|
|
@@ -4040,9 +4223,19 @@ def _tui_build_snapshot_once(
|
|
|
4040
4223
|
claude_ingest_failed = False
|
|
4041
4224
|
codex_ingest_contended = False
|
|
4042
4225
|
codex_ingest_failed = False
|
|
4226
|
+
_tick = _tick_stats.current()
|
|
4043
4227
|
with _perf.phase("sync") as _p_sync:
|
|
4044
4228
|
_p_sync.set_meta(ingest=do_ingest)
|
|
4045
4229
|
if do_ingest:
|
|
4230
|
+
# #583 S1 §1.3: the internal-sync ingest span, for the direct
|
|
4231
|
+
# build path (`tui --render-once`, `cctally-snapshot-measure`).
|
|
4232
|
+
# Bracketed rather than a `with` block so the body below is not
|
|
4233
|
+
# reindented; a leaked span is closed by `TickContext.finish`.
|
|
4234
|
+
_ingest_span = (
|
|
4235
|
+
_tick.ingest_span() if _tick is not None else None
|
|
4236
|
+
)
|
|
4237
|
+
if _ingest_span is not None:
|
|
4238
|
+
_ingest_span.__enter__()
|
|
4046
4239
|
try:
|
|
4047
4240
|
cache_conn = _cctally().open_cache_db()
|
|
4048
4241
|
cache_mod = _cctally()._load_sibling("_cctally_cache")
|
|
@@ -4112,6 +4305,8 @@ def _tui_build_snapshot_once(
|
|
|
4112
4305
|
if precompute_envelope:
|
|
4113
4306
|
codex_ingest_failed = True
|
|
4114
4307
|
capture_failure("sync-cache-open", "cache", exc)
|
|
4308
|
+
if _ingest_span is not None:
|
|
4309
|
+
_ingest_span.__exit__(None, None, None)
|
|
4115
4310
|
# Force pure reads for every view builder below, independent of the
|
|
4116
4311
|
# caller's flag: the single ingest above is the only glob per tick.
|
|
4117
4312
|
skip_sync = True
|
|
@@ -4177,6 +4372,12 @@ def _tui_build_snapshot_once(
|
|
|
4177
4372
|
try:
|
|
4178
4373
|
_sc = _cctally()._load_sibling("_lib_snapshot_cache")
|
|
4179
4374
|
prior_key, prior_snap = _sc.dispatch_state()
|
|
4375
|
+
# #583 S1 §1.1: cold versus warm. A build with no prior
|
|
4376
|
+
# dispatch memo has nothing to reuse, so it pays the whole
|
|
4377
|
+
# source construction; the flag is sticky across the refresh,
|
|
4378
|
+
# because one cold build makes the tick a cold tick.
|
|
4379
|
+
if _tick is not None:
|
|
4380
|
+
_tick.set_cold(prior_snap is None)
|
|
4180
4381
|
if prior_snap is not None:
|
|
4181
4382
|
prior_source_bundle = getattr(prior_snap, "source_bundle", None)
|
|
4182
4383
|
except Exception as exc:
|
|
@@ -4247,6 +4448,8 @@ def _tui_build_snapshot_once(
|
|
|
4247
4448
|
)
|
|
4248
4449
|
assert _sc is not None
|
|
4249
4450
|
_sc.store_dispatch_state(dispatch_key, idle_snap)
|
|
4451
|
+
if _tick is not None:
|
|
4452
|
+
_tick.set_dispatch("idle") # §1.4: the reuse branch
|
|
4250
4453
|
_p_snapshot.__exit__(None, None, None)
|
|
4251
4454
|
if _perf.enabled():
|
|
4252
4455
|
_perf.stash_last(
|
|
@@ -4265,7 +4468,8 @@ def _tui_build_snapshot_once(
|
|
|
4265
4468
|
# `use_weekref_cost_cache=True`; any failure leaves the flag OFF
|
|
4266
4469
|
# (safe direct-compute fallback, byte-identical output).
|
|
4267
4470
|
try:
|
|
4268
|
-
|
|
4471
|
+
with _perf.phase("reconcile.cache_open"):
|
|
4472
|
+
_rc_cache_conn = _cctally().open_cache_db()
|
|
4269
4473
|
try:
|
|
4270
4474
|
with _perf.phase("reconcile.weekref") as _pr:
|
|
4271
4475
|
_pr.set_meta(hit=False)
|
|
@@ -4863,6 +5067,11 @@ def _tui_build_snapshot_once(
|
|
|
4863
5067
|
# into the process-global slot for the loopback /api/debug/backend
|
|
4864
5068
|
# endpoint. Whole-dict atomic assignment; never mutated after. No-op
|
|
4865
5069
|
# when tracing is off.
|
|
5070
|
+
if _tick is not None:
|
|
5071
|
+
# §1.4: the full branch. `full` outranks a later `idle` inside the
|
|
5072
|
+
# same refresh, because a refresh containing one full build cost
|
|
5073
|
+
# what a full build costs.
|
|
5074
|
+
_tick.set_dispatch("full")
|
|
4866
5075
|
_p_snapshot.__exit__(None, None, None)
|
|
4867
5076
|
if _perf.enabled():
|
|
4868
5077
|
_perf.stash_last(
|
|
@@ -4898,8 +5107,11 @@ def _tui_precompute_doctor_payload(
|
|
|
4898
5107
|
def _compute(now: dt.datetime, bind: "str | None") -> dict:
|
|
4899
5108
|
_ld = c._load_sibling("_lib_doctor")
|
|
4900
5109
|
try:
|
|
4901
|
-
|
|
4902
|
-
|
|
5110
|
+
with _perf.phase("doctor.gather"):
|
|
5111
|
+
_doc_state = c.doctor_gather_state(
|
|
5112
|
+
now_utc=now, runtime_bind=bind)
|
|
5113
|
+
with _perf.phase("doctor.checks"):
|
|
5114
|
+
_doc_report = _ld.run_checks(_doc_state)
|
|
4903
5115
|
return {
|
|
4904
5116
|
"severity": _doc_report.overall_severity,
|
|
4905
5117
|
"counts": dict(_doc_report.counts),
|
|
@@ -5198,7 +5410,13 @@ def _tui_build_idle_snapshot(prior, *, now_utc, precompute_envelope,
|
|
|
5198
5410
|
return dataclasses.replace(
|
|
5199
5411
|
prior,
|
|
5200
5412
|
generated_at=now_utc,
|
|
5201
|
-
last_sync_at
|
|
5413
|
+
# #583 S2 §6.1: `last_sync_at` means "last SUCCESSFUL validation". A
|
|
5414
|
+
# clean idle tick IS one — it re-verified through four independent
|
|
5415
|
+
# gates that nothing changed, so the reused rows are genuinely
|
|
5416
|
+
# current. A tick that recorded an error retains the prior stamp
|
|
5417
|
+
# rather than reporting itself as a fresh success.
|
|
5418
|
+
last_sync_at=(time.monotonic() if not errors
|
|
5419
|
+
else getattr(prior, "last_sync_at", None)),
|
|
5202
5420
|
last_sync_error=("; ".join(errors) if errors else None),
|
|
5203
5421
|
sync_failures=tuple(idle_failures),
|
|
5204
5422
|
doctor_payload=doctor_payload,
|
|
@@ -5253,7 +5471,9 @@ def _tui_stats_retry_degraded_snapshot(
|
|
|
5253
5471
|
errors.append(f"doctor-precompute: {doctor_exc}")
|
|
5254
5472
|
return dataclasses.replace(
|
|
5255
5473
|
_tui_empty_snapshot(now_utc),
|
|
5256
|
-
|
|
5474
|
+
# #583 S2 §6.1: this produced no successful snapshot, and it builds
|
|
5475
|
+
# from the empty snapshot, so there is no earlier success to preserve.
|
|
5476
|
+
last_sync_at=None,
|
|
5257
5477
|
last_sync_error="; ".join(errors),
|
|
5258
5478
|
sync_failures=(
|
|
5259
5479
|
SyncFailureAttribution(
|
|
@@ -7490,7 +7710,7 @@ class _A2ThrottleClock:
|
|
|
7490
7710
|
def _make_a2_progress_cb(*, ref, hub, build_partial, throttle, monotonic,
|
|
7491
7711
|
perf=_perf):
|
|
7492
7712
|
"""Build the A2 throttled ``sync_cache`` progress callback (extracted so the
|
|
7493
|
-
throttle +
|
|
7713
|
+
throttle + isolation logic is unit-testable with injected deps).
|
|
7494
7714
|
|
|
7495
7715
|
On proceed it builds a partial via ``build_partial()`` (a fresh, complete
|
|
7496
7716
|
``skip_sync=True`` snapshot over the current committed cache — NOT nested in
|
|
@@ -7499,19 +7719,28 @@ def _make_a2_progress_cb(*, ref, hub, build_partial, throttle, monotonic,
|
|
|
7499
7719
|
only the publish carries the latch, so the memo's retained object stays
|
|
7500
7720
|
clean). The dispatch-memo write itself happens inside ``build_partial()`` /
|
|
7501
7721
|
``_tui_build_snapshot`` (its snapshot-cache reconcile step), NOT in this cb.
|
|
7502
|
-
|
|
7503
|
-
|
|
7504
|
-
|
|
7505
|
-
|
|
7722
|
+
|
|
7723
|
+
#583 S1 §2.1: this used to return early whenever perf tracing was active,
|
|
7724
|
+
which meant arming the trace silently switched progressive fill off. The
|
|
7725
|
+
hazard that justified the suppression is real but narrower than a blanket
|
|
7726
|
+
skip — the partial build's unconditional ``_perf.reset_thread()`` rebinds
|
|
7727
|
+
``_tls.stack`` while the enclosing ``sync_cache`` still holds its ``walk``
|
|
7728
|
+
phase, whose ``Phase._stack`` is the original list, so the outer phase
|
|
7729
|
+
would later close into a detached or fragmented root. Isolating the partial
|
|
7730
|
+
build's thread state addresses exactly that, and the publication now
|
|
7731
|
+
happens whether or not tracing is on.
|
|
7506
7732
|
"""
|
|
7507
7733
|
def cb(_stats) -> None:
|
|
7508
|
-
if perf.enabled():
|
|
7509
|
-
return
|
|
7510
7734
|
now = monotonic()
|
|
7511
7735
|
if not throttle.should_fire(now):
|
|
7512
7736
|
return
|
|
7513
|
-
|
|
7514
|
-
|
|
7737
|
+
with perf.isolated_thread_state():
|
|
7738
|
+
snap = build_partial()
|
|
7739
|
+
# #583 S2 §6.2 publication point 3: publish what the reference now
|
|
7740
|
+
# HOLDS, not the local build. A request accepted while `build_partial`
|
|
7741
|
+
# ran is on the reference and absent from `snap`, so publishing `snap`
|
|
7742
|
+
# would erase its counter and the client would never settle.
|
|
7743
|
+
snap = ref.set(snap)
|
|
7515
7744
|
hub.publish(dataclasses.replace(snap, hydrating=True))
|
|
7516
7745
|
throttle.mark_done(monotonic())
|
|
7517
7746
|
return cb
|
|
@@ -7552,11 +7781,24 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7552
7781
|
# re-export AND ``monkeypatch.setitem(ns, "_tui_build_snapshot", spy)``
|
|
7553
7782
|
# in tests propagate into this closure body (a bare-name lookup would
|
|
7554
7783
|
# resolve in this sibling's __dict__ and miss the cctally-side patch).
|
|
7555
|
-
|
|
7556
|
-
|
|
7557
|
-
|
|
7558
|
-
|
|
7559
|
-
|
|
7784
|
+
#
|
|
7785
|
+
# #583 S1 §1.3: one of the TWO builder-span sites. This wrapper is
|
|
7786
|
+
# dashboard-local, so `tui --render-once` and `cctally-snapshot-measure`
|
|
7787
|
+
# never reach it and take the span at `_tui_build_snapshot`'s standalone
|
|
7788
|
+
# boundary instead. The span subtracts any ingest nested inside it.
|
|
7789
|
+
tick = _tick_stats.current()
|
|
7790
|
+
if tick is None:
|
|
7791
|
+
return sys.modules["cctally"]._tui_build_snapshot(
|
|
7792
|
+
now_utc=pinned_now, skip_sync=skip_sync,
|
|
7793
|
+
display_tz_pref_override=display_tz_pref_override,
|
|
7794
|
+
precompute_envelope=True, runtime_bind=runtime_bind,
|
|
7795
|
+
)
|
|
7796
|
+
with tick.build_span():
|
|
7797
|
+
return sys.modules["cctally"]._tui_build_snapshot(
|
|
7798
|
+
now_utc=pinned_now, skip_sync=skip_sync,
|
|
7799
|
+
display_tz_pref_override=display_tz_pref_override,
|
|
7800
|
+
precompute_envelope=True, runtime_bind=runtime_bind,
|
|
7801
|
+
)
|
|
7560
7802
|
|
|
7561
7803
|
def _locked(skip_sync: bool) -> None:
|
|
7562
7804
|
# #279 S5 F6.3 (gate P1-1): arm the snapshot-cache owner-thread tripwire
|
|
@@ -7565,11 +7807,19 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7565
7807
|
# on-call, so ownership transfers to the current rebuilder; the guards in
|
|
7566
7808
|
# _lib_snapshot_cache then catch a lock-bypassing foreign-thread mutation.
|
|
7567
7809
|
_cctally()._load_sibling("_lib_snapshot_cache").mark_owner_thread()
|
|
7810
|
+
# #583 S1 §1.2: the tick opens here and closes after the final publish,
|
|
7811
|
+
# so the progressive, final, deferred and crash publication paths are
|
|
7812
|
+
# all inside it. The A1 pre-bind seed is deliberately outside, because
|
|
7813
|
+
# it bypasses `_tui_build_snapshot` and writes no dispatch state.
|
|
7814
|
+
tick = _tick_stats.begin_tick()
|
|
7568
7815
|
try:
|
|
7569
7816
|
if not skip_sync:
|
|
7570
7817
|
# ── Decoupled ingest + build (§2.1) ─────────────────────────
|
|
7571
7818
|
import time as _time
|
|
7572
7819
|
sync_error = None
|
|
7820
|
+
# #583 S2 §6.1: the last SUCCESSFUL validation, read before
|
|
7821
|
+
# A2's partial republishes can overwrite the held snapshot.
|
|
7822
|
+
prior_sync_at = ref.get().last_sync_at
|
|
7573
7823
|
start = _time.monotonic()
|
|
7574
7824
|
throttle = _A2ThrottleClock(_A2_PARTIAL_THROTTLE_S, start=start)
|
|
7575
7825
|
cb = _make_a2_progress_cb(
|
|
@@ -7579,6 +7829,18 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7579
7829
|
)
|
|
7580
7830
|
cache_conn = _cctally().open_cache_db()
|
|
7581
7831
|
cache_mod = _cctally()._load_sibling("_cctally_cache")
|
|
7832
|
+
# #583 S1 §1.3: the ingest span, opened through the
|
|
7833
|
+
# contextmanager protocol rather than a `with` block so the
|
|
7834
|
+
# long try/except/finally below is not reindented. The A2
|
|
7835
|
+
# progress callback runs `build_partial()` SYNCHRONOUSLY inside
|
|
7836
|
+
# this region, so the builder spans it opens nest here and
|
|
7837
|
+
# their time is subtracted — without that, every progress build
|
|
7838
|
+
# is counted once as ingest and again as builder and
|
|
7839
|
+
# `ingest_ns + builder_ns` can exceed `duration_ns`. A span
|
|
7840
|
+
# left open by an escaping exception is closed by
|
|
7841
|
+
# `TickContext.finish`, so no try/finally is needed to bound it.
|
|
7842
|
+
_ingest = tick.ingest_span()
|
|
7843
|
+
_ingest.__enter__()
|
|
7582
7844
|
try:
|
|
7583
7845
|
# Under CCTALLY_PERF_TRACE the phase tree this standalone
|
|
7584
7846
|
# sync_cache builds is intentionally NOT surfaced in the live
|
|
@@ -7608,6 +7870,13 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7608
7870
|
sync_error = f"sync-cache: {exc}"
|
|
7609
7871
|
finally:
|
|
7610
7872
|
cache_conn.close()
|
|
7873
|
+
_ingest.__exit__(None, None, None)
|
|
7874
|
+
# #583 S1 §2.2: the ONE place a pending trace-arm request is
|
|
7875
|
+
# consumed. After the cache connection closes and immediately
|
|
7876
|
+
# before the authoritative build, so a request landing
|
|
7877
|
+
# mid-ingest cannot split one ingest across two tracing states,
|
|
7878
|
+
# and A2 partial builds never consume it.
|
|
7879
|
+
_perf.apply_pending()
|
|
7611
7880
|
snap = _build(skip_sync=True) # final: hydrating=False (default)
|
|
7612
7881
|
if sync_error is not None:
|
|
7613
7882
|
# Thread the standalone sync error into last_sync_error, sync
|
|
@@ -7617,10 +7886,27 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7617
7886
|
merged = (sync_error if not existing
|
|
7618
7887
|
else f"{sync_error}; {existing}")
|
|
7619
7888
|
snap = dataclasses.replace(snap, last_sync_error=merged)
|
|
7620
|
-
|
|
7621
|
-
|
|
7889
|
+
# #583 S2 §6.1: `last_sync_at` means "last SUCCESSFUL
|
|
7890
|
+
# validation", and `_tui_build_snapshot` stamps it
|
|
7891
|
+
# unconditionally. A build that recorded any failure carries
|
|
7892
|
+
# the prior successful value forward instead of reporting
|
|
7893
|
+
# itself as freshly synced.
|
|
7894
|
+
if snap.last_sync_error:
|
|
7895
|
+
snap = dataclasses.replace(snap, last_sync_at=prior_sync_at)
|
|
7896
|
+
# #583 S2 §6.2 publication point 4: publish what the reference
|
|
7897
|
+
# holds, so a request accepted during the build is not erased.
|
|
7898
|
+
# `set_final` also clears `rebuilding` in the same acquisition,
|
|
7899
|
+
# so this ONE frame reports the rebuild finished; the loop's
|
|
7900
|
+
# trailing `mark_rebuilding(False)` then finds no transition and
|
|
7901
|
+
# publishes nothing. That is what keeps an automatic tick at two
|
|
7902
|
+
# frames instead of three, PLUS one per A2 progress publish that
|
|
7903
|
+
# cleared the throttle above — several on a cold first-run
|
|
7904
|
+
# ingest, none on a warm one.
|
|
7905
|
+
snap = ref.set_final(snap)
|
|
7906
|
+
_tui_publish_final(tick, hub, snap)
|
|
7622
7907
|
return
|
|
7623
7908
|
# ── skip_sync=True: single-build path (POST /api/settings) ──────
|
|
7909
|
+
_perf.apply_pending() # §2.2: the no-ingest build's arm boundary
|
|
7624
7910
|
snap = _build(skip_sync=True)
|
|
7625
7911
|
# Mirror the startup override: suppress the monotonic sync stamp so
|
|
7626
7912
|
# the envelope keeps emitting sync_age_s=None and the client keeps
|
|
@@ -7629,8 +7915,10 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7629
7915
|
# from _tui_build_snapshot, restated here since this is a replace()
|
|
7630
7916
|
# clone site).
|
|
7631
7917
|
snap = dataclasses.replace(snap, last_sync_at=None, hydrating=False)
|
|
7632
|
-
|
|
7633
|
-
|
|
7918
|
+
# Terminal for this branch too, for the same reason: POST
|
|
7919
|
+
# /api/settings brackets its rebuild with the same flag pair.
|
|
7920
|
+
snap = ref.set_final(snap)
|
|
7921
|
+
_tui_publish_final(tick, hub, snap)
|
|
7634
7922
|
except _cctally().StatsRebuildDeferred as exc:
|
|
7635
7923
|
# #453: the first periodic tick runs before HTTP bind. Preserve the
|
|
7636
7924
|
# initial hydrating/degraded frame while the dedicated replay owns
|
|
@@ -7656,8 +7944,9 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7656
7944
|
generated_at=dt.datetime.now(dt.timezone.utc),
|
|
7657
7945
|
hydrating=True,
|
|
7658
7946
|
)
|
|
7659
|
-
ref.set(pending)
|
|
7660
|
-
|
|
7947
|
+
pending = ref.set(pending) # #583 S2 §6.2: publish what is held
|
|
7948
|
+
tick.mark_degraded()
|
|
7949
|
+
_tui_publish_final(tick, hub, pending, publication="degraded")
|
|
7661
7950
|
except Exception as exc:
|
|
7662
7951
|
prev = ref.get()
|
|
7663
7952
|
crashed = dataclasses.replace(
|
|
@@ -7673,8 +7962,19 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
7673
7962
|
# seed/partial, so the client doesn't stay stuck in skeletons.
|
|
7674
7963
|
hydrating=False,
|
|
7675
7964
|
)
|
|
7676
|
-
ref.set(crashed)
|
|
7677
|
-
|
|
7965
|
+
crashed = ref.set(crashed) # #583 S2 §6.2: publish what is held
|
|
7966
|
+
tick.mark_degraded()
|
|
7967
|
+
_tui_publish_final(tick, hub, crashed, publication="degraded")
|
|
7968
|
+
finally:
|
|
7969
|
+
# A tick always closes. Every publication path above finishes it,
|
|
7970
|
+
# and `finish` is idempotent, so this only catches an escape none
|
|
7971
|
+
# of them handled — a BaseException such as KeyboardInterrupt.
|
|
7972
|
+
if not tick.finished:
|
|
7973
|
+
tick.mark_degraded()
|
|
7974
|
+
tick.finish(
|
|
7975
|
+
published_ns=time.monotonic_ns(),
|
|
7976
|
+
published_at=dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
7977
|
+
)
|
|
7678
7978
|
return _locked
|
|
7679
7979
|
|
|
7680
7980
|
|