cctally 1.108.0 → 1.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +6 -6
- package/bin/_cctally_account.py +4 -0
- package/bin/_cctally_alerts.py +6 -2
- package/bin/_cctally_cache.py +1652 -166
- package/bin/_cctally_config.py +3 -2
- package/bin/_cctally_core.py +594 -33
- package/bin/_cctally_dashboard.py +1585 -201
- package/bin/_cctally_dashboard_conversation.py +494 -48
- package/bin/_cctally_dashboard_envelope.py +167 -11
- package/bin/_cctally_dashboard_share.py +154 -36
- package/bin/_cctally_dashboard_sources.py +2615 -422
- package/bin/_cctally_db.py +678 -5
- package/bin/_cctally_diagnosis_sources.py +238 -26
- package/bin/_cctally_doctor.py +99 -18
- package/bin/_cctally_five_hour.py +1295 -40
- package/bin/_cctally_forecast.py +124 -28
- package/bin/_cctally_journal.py +408 -38
- package/bin/_cctally_milestone_history.py +150 -14
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_percent_breakdown.py +6 -1
- package/bin/_cctally_project.py +48 -76
- package/bin/_cctally_quota.py +39 -7
- package/bin/_cctally_quota_model.py +17 -2
- package/bin/_cctally_record.py +1647 -443
- package/bin/_cctally_rederive.py +899 -30
- package/bin/_cctally_refresh.py +2 -1
- package/bin/_cctally_share.py +24 -5
- package/bin/_cctally_source_analytics.py +1035 -78
- package/bin/_cctally_statusline.py +83 -19
- package/bin/_cctally_store.py +61 -4
- package/bin/_cctally_transcript.py +27 -2
- package/bin/_cctally_tui.py +264 -76
- package/bin/_cctally_weekrefs.py +28 -5
- package/bin/_lib_blocks.py +127 -20
- package/bin/_lib_cache_report.py +20 -5
- package/bin/_lib_codex_conversation_query.py +297 -64
- package/bin/_lib_codex_hooks.py +102 -0
- package/bin/_lib_codex_metadata.py +207 -0
- package/bin/_lib_conversation.py +119 -0
- package/bin/_lib_conversation_dispatch.py +31 -1
- package/bin/_lib_conversation_query.py +315 -51
- package/bin/_lib_conversation_retention.py +413 -25
- package/bin/_lib_dashboard_sources.py +267 -7
- package/bin/_lib_diagnosis.py +10 -0
- package/bin/_lib_diff_kernel.py +12 -1
- package/bin/_lib_doctor.py +230 -8
- package/bin/_lib_ingest_frontier.py +1366 -49
- package/bin/_lib_journal.py +19 -2
- package/bin/_lib_merge_gate.py +307 -0
- package/bin/_lib_pricing.py +588 -29
- package/bin/_lib_record.py +130 -12
- package/bin/_lib_rederive.py +903 -2
- package/bin/_lib_render.py +9 -3
- package/bin/_lib_retained_size.py +27 -2
- package/bin/_lib_segment_summary.py +66 -8
- package/bin/_lib_share_templates.py +28 -18
- package/bin/_lib_snapshot_cache.py +115 -1
- package/bin/_lib_source_retry.py +320 -0
- package/bin/_lib_statusline_candidates.py +400 -36
- package/bin/_lib_tick_stats.py +119 -0
- package/bin/_lib_view_models.py +256 -76
- package/bin/cctally +69 -0
- package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
- package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
- package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
- package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
- package/dashboard/static/assets/index-COhxe3vU.css +1 -0
- package/dashboard/static/assets/index-DdU62sov.js +13 -0
- package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
- package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +4 -1
- package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
- package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
- package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
- package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
- package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
- package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
- package/dashboard/static/assets/index-CIWsbux3.js +0 -13
- package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
- package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
package/bin/_cctally_journal.py
CHANGED
|
@@ -887,6 +887,12 @@ class SegmentElisionPlan:
|
|
|
887
887
|
certificate_covers=self._covers(name, summary),
|
|
888
888
|
resolution_seen=self.resolution_seen,
|
|
889
889
|
stat_identity=(stat_result.st_dev, stat_result.st_ino),
|
|
890
|
+
# #834 S2 (#827). Lazy on purpose: the predicate evaluates this ONLY
|
|
891
|
+
# on a device-only identity change, so the common path still reads
|
|
892
|
+
# no segment bytes. Hashing here on every segment would read exactly
|
|
893
|
+
# what elision exists to skip.
|
|
894
|
+
prefix_proof=lambda: _complete_prefix_digest(
|
|
895
|
+
name, int(summary.complete_line_covered_offset)),
|
|
890
896
|
)
|
|
891
897
|
if not ok:
|
|
892
898
|
self.scanned += 1
|
|
@@ -1040,6 +1046,34 @@ def _refill_elided_quota_raw(quota_raw, gaps, *, failures=None):
|
|
|
1040
1046
|
return out, complete
|
|
1041
1047
|
|
|
1042
1048
|
|
|
1049
|
+
def _complete_prefix_digest(name, covered_offset) -> str:
|
|
1050
|
+
"""Hex `sha256` of segment ``name``'s bytes in `[0, covered_offset)`.
|
|
1051
|
+
|
|
1052
|
+
#834 S2 (#827). The byte-identity proof `summary_is_elidable` requires
|
|
1053
|
+
before it reuses a summary across a DEVICE-ONLY identity change. Read
|
|
1054
|
+
straight from the file rather than from the pass's own accumulator, because
|
|
1055
|
+
the question is what the file holds NOW, not what some earlier pass folded.
|
|
1056
|
+
|
|
1057
|
+
Raises `OSError` when the segment cannot be read, and the predicate treats
|
|
1058
|
+
that as no proof and refuses — an unreadable prefix costs a re-read of one
|
|
1059
|
+
segment, while accepting it would skip retained data silently.
|
|
1060
|
+
"""
|
|
1061
|
+
digest = hashlib.sha256()
|
|
1062
|
+
remaining = int(covered_offset)
|
|
1063
|
+
with _open_segment_for_read(_cctally_core.JOURNAL_DIR / name) as fh:
|
|
1064
|
+
while remaining > 0:
|
|
1065
|
+
chunk = fh.read(min(_SEGMENT_READ_CHUNK, remaining))
|
|
1066
|
+
if not chunk:
|
|
1067
|
+
break
|
|
1068
|
+
remaining -= len(chunk)
|
|
1069
|
+
digest.update(chunk)
|
|
1070
|
+
if remaining:
|
|
1071
|
+
# A short read is not a proof either: the file no longer holds the
|
|
1072
|
+
# prefix the summary describes.
|
|
1073
|
+
raise OSError(f"segment {name} is shorter than its summarized prefix")
|
|
1074
|
+
return digest.hexdigest()
|
|
1075
|
+
|
|
1076
|
+
|
|
1043
1077
|
class _SegmentSummaryCollector:
|
|
1044
1078
|
"""Per-segment traversal facts, accumulated as the pass streams.
|
|
1045
1079
|
|
|
@@ -1056,7 +1090,7 @@ class _SegmentSummaryCollector:
|
|
|
1056
1090
|
|
|
1057
1091
|
__slots__ = ("summaries", "last_seen", "_name", "_lo", "_stat",
|
|
1058
1092
|
"_lines", "_bytes", "_decodes", "_malformed", "_retained",
|
|
1059
|
-
"_line_end")
|
|
1093
|
+
"_line_end", "_prefix_digest")
|
|
1060
1094
|
|
|
1061
1095
|
def __init__(self) -> None:
|
|
1062
1096
|
self.summaries: dict = {}
|
|
@@ -1074,6 +1108,13 @@ class _SegmentSummaryCollector:
|
|
|
1074
1108
|
self._malformed = 0
|
|
1075
1109
|
self._retained = 0
|
|
1076
1110
|
self._line_end = 0
|
|
1111
|
+
# #834 S2 (#827): the running digest of the COMPLETE prefix this summary
|
|
1112
|
+
# will describe. Accumulated from the bytes the pass is already reading,
|
|
1113
|
+
# so recording it costs one `sha256.update` per line and no extra I/O.
|
|
1114
|
+
# It is what settles a device-only identity change: `st_dev` is assigned
|
|
1115
|
+
# at mount time, so a remount presents an unchanged file at a new device
|
|
1116
|
+
# number, and an inode number is unique only within a device.
|
|
1117
|
+
self._prefix_digest = hashlib.sha256()
|
|
1077
1118
|
|
|
1078
1119
|
def begin(self, name, lo, hi, stat_result, running) -> None:
|
|
1079
1120
|
"""Open ``name``, closing whatever segment was open before it.
|
|
@@ -1095,6 +1136,12 @@ class _SegmentSummaryCollector:
|
|
|
1095
1136
|
self._lines += 1
|
|
1096
1137
|
self._bytes += len(raw) + 1
|
|
1097
1138
|
self._line_end = int(end_offset)
|
|
1139
|
+
# `_iter_segment_lines` yields each complete line WITHOUT its newline and
|
|
1140
|
+
# splits on exactly one `\n`, and this writer only summarizes segments
|
|
1141
|
+
# opened at offset 0, so line-plus-newline reproduces the file's bytes in
|
|
1142
|
+
# `[0, self._line_end)` verbatim.
|
|
1143
|
+
self._prefix_digest.update(raw)
|
|
1144
|
+
self._prefix_digest.update(b"\n")
|
|
1098
1145
|
|
|
1099
1146
|
def decoded(self, retained: bool) -> None:
|
|
1100
1147
|
"""One decoded record: one traversal decode AND one `decoded` element.
|
|
@@ -1141,6 +1188,7 @@ class _SegmentSummaryCollector:
|
|
|
1141
1188
|
malformed=self._malformed,
|
|
1142
1189
|
quota_only=self._retained == 0,
|
|
1143
1190
|
decoded_entry_count=self._decodes,
|
|
1191
|
+
complete_prefix_sha256=self._prefix_digest.hexdigest(),
|
|
1144
1192
|
last_seen_stamped=(
|
|
1145
1193
|
{} if partial is None else dict(partial.stamped)),
|
|
1146
1194
|
last_seen_legacy_claude_at=(
|
|
@@ -1367,6 +1415,9 @@ class IngestContext:
|
|
|
1367
1415
|
# writes while still exercising the same SQLite derivation/fold code.
|
|
1368
1416
|
event_sink: "list | None" = None
|
|
1369
1417
|
projection_writes: bool = True
|
|
1418
|
+
# Operator-reviewed scratch replay only. Normal ingest leaves this empty.
|
|
1419
|
+
held_weekly_observation_ids: frozenset[str] = field(default_factory=frozenset)
|
|
1420
|
+
reviewed_weekly_basis_by_account: dict = field(default_factory=dict)
|
|
1370
1421
|
# #374 write boundary: emissions WITHHELD this cycle because they would have
|
|
1371
1422
|
# violated the same-revision rule. Each entry is a `DroppedConflict`; the row
|
|
1372
1423
|
# was converged from the already-journaled effective event instead.
|
|
@@ -3367,29 +3418,161 @@ def _now_iso() -> str:
|
|
|
3367
3418
|
)
|
|
3368
3419
|
|
|
3369
3420
|
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
|
|
3373
|
-
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3379
|
-
|
|
3380
|
-
|
|
3381
|
-
|
|
3382
|
-
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
3387
|
-
|
|
3388
|
-
|
|
3389
|
-
|
|
3390
|
-
|
|
3391
|
-
|
|
3392
|
-
|
|
3421
|
+
#: #769 S11 (#824) — the axis dispositions and the snapshot action returned by
|
|
3422
|
+
#: ``_usage_snapshot_fold_decision``. Spelled as module-level constants because
|
|
3423
|
+
#: the pipeline, the replay path and the tests all compare against them by name;
|
|
3424
|
+
#: a bare string literal at a call site is how two of them silently collapse.
|
|
3425
|
+
#:
|
|
3426
|
+
#: WEEKLY axis. ``WEEKLY_OBSERVED`` — the incoming weekly percent is at or above
|
|
3427
|
+
#: the reset-aware in-window maximum, so it IS the week's evidence.
|
|
3428
|
+
#: ``WEEKLY_HELD_CLAMP`` — it is strictly below that maximum, so the weekly axis
|
|
3429
|
+
#: is held at the maximum and the incoming value is recorded only as the raw
|
|
3430
|
+
#: reading. On reviewed scratch replay, an exact observation id can also hold
|
|
3431
|
+
#: a reading above the basis; either way it is not fresh weekly evidence.
|
|
3432
|
+
WEEKLY_OBSERVED = "observed"
|
|
3433
|
+
WEEKLY_HELD_CLAMP = "held_clamp"
|
|
3434
|
+
|
|
3435
|
+
#: FIVE-HOUR axis. ``FIVE_HOUR_MISSING`` — the observation carries no five-hour
|
|
3436
|
+
#: percent or no canonical window key, so there is no five-hour evidence at all.
|
|
3437
|
+
#: ``FIVE_HOUR_AS_OBSERVED`` — the reading is at or above the in-window maximum.
|
|
3438
|
+
#: ``FIVE_HOUR_CLAMPED`` — it is below, so the effective value is the maximum.
|
|
3439
|
+
#: The five-hour clamp adjusts the value UP and NEVER gates the row, which is
|
|
3440
|
+
#: the rule the pre-#824 early return on the weekly clamp used to skip past.
|
|
3441
|
+
FIVE_HOUR_MISSING = "missing"
|
|
3442
|
+
FIVE_HOUR_AS_OBSERVED = "as_observed"
|
|
3443
|
+
FIVE_HOUR_CLAMPED = "clamped"
|
|
3444
|
+
|
|
3445
|
+
#: The materialized outcome. ``SNAPSHOT_WRITE_OBSERVED`` — write an ordinary
|
|
3446
|
+
#: row. ``SNAPSHOT_WRITE_HELD_5H`` — write a row whose weekly value and boundary
|
|
3447
|
+
#: are carried forward from the latest non-held basis and whose five-hour
|
|
3448
|
+
#: fields, capture time and source are the tick's own
|
|
3449
|
+
#: (``weekly_observation_held = 1``). ``SNAPSHOT_SKIP_NO_CHANGE`` — neither axis
|
|
3450
|
+
#: carries evidence the stored row does not already have.
|
|
3451
|
+
SNAPSHOT_WRITE_OBSERVED = "write_observed"
|
|
3452
|
+
SNAPSHOT_WRITE_HELD_5H = "write_held_5h"
|
|
3453
|
+
SNAPSHOT_SKIP_NO_CHANGE = "skip_no_change"
|
|
3454
|
+
|
|
3455
|
+
|
|
3456
|
+
@dataclass(frozen=True)
|
|
3457
|
+
class WeeklyBasis:
|
|
3458
|
+
"""The latest NON-held ``weekly_usage_snapshots`` row for this week and
|
|
3459
|
+
account — the row a held row copies its weekly value and boundary from.
|
|
3460
|
+
|
|
3461
|
+
Selected with ``weekly_observation_held = 0`` so a held row never becomes
|
|
3462
|
+
the basis for the next one. Compounding the carry that way would let a
|
|
3463
|
+
boundary drift arbitrarily far from any observation that produced it, which
|
|
3464
|
+
is the lineage half of review finding 7.
|
|
3465
|
+
"""
|
|
3466
|
+
|
|
3467
|
+
snapshot_id: int
|
|
3468
|
+
captured_at_utc: str
|
|
3469
|
+
weekly_percent: float
|
|
3470
|
+
week_start_date: "str | None"
|
|
3471
|
+
week_end_date: "str | None"
|
|
3472
|
+
week_start_at: "str | None"
|
|
3473
|
+
week_end_at: "str | None"
|
|
3474
|
+
|
|
3475
|
+
|
|
3476
|
+
def _latest_weekly_basis(conn, account_key, *, week_start_date=None):
|
|
3477
|
+
"""Return accepted weekly evidence before a later axis may delete rows."""
|
|
3478
|
+
sql = (
|
|
3479
|
+
"SELECT id, captured_at_utc, weekly_percent, week_start_date, "
|
|
3480
|
+
" week_end_date, week_start_at, week_end_at "
|
|
3481
|
+
"FROM weekly_usage_snapshots "
|
|
3482
|
+
"WHERE account_key = ? AND weekly_observation_held = 0 "
|
|
3483
|
+
)
|
|
3484
|
+
args = (account_key,)
|
|
3485
|
+
if week_start_date is not None:
|
|
3486
|
+
sql += "AND week_start_date = ? "
|
|
3487
|
+
args += (week_start_date,)
|
|
3488
|
+
row = conn.execute(
|
|
3489
|
+
sql + "ORDER BY captured_at_utc DESC, id DESC LIMIT 1", args,
|
|
3490
|
+
).fetchone()
|
|
3491
|
+
if row is None:
|
|
3492
|
+
return None
|
|
3493
|
+
return WeeklyBasis(
|
|
3494
|
+
snapshot_id=int(row[0]), captured_at_utc=str(row[1]),
|
|
3495
|
+
weekly_percent=float(row[2]), week_start_date=row[3],
|
|
3496
|
+
week_end_date=row[4], week_start_at=row[5], week_end_at=row[6],
|
|
3497
|
+
)
|
|
3498
|
+
|
|
3499
|
+
|
|
3500
|
+
@dataclass(frozen=True)
|
|
3501
|
+
class WeeklyFold:
|
|
3502
|
+
"""The weekly axis of one fold decision."""
|
|
3503
|
+
|
|
3504
|
+
raw_pct: float
|
|
3505
|
+
effective_pct: float
|
|
3506
|
+
disposition: str
|
|
3507
|
+
basis: "WeeklyBasis | None"
|
|
3508
|
+
|
|
3509
|
+
|
|
3510
|
+
@dataclass(frozen=True)
|
|
3511
|
+
class FiveHourFold:
|
|
3512
|
+
"""The five-hour axis of one fold decision.
|
|
3513
|
+
|
|
3514
|
+
``rollover_heal`` is deliberately ORTHOGONAL to the snapshot action: the
|
|
3515
|
+
incoming observation can name a new physical window that no
|
|
3516
|
+
``five_hour_blocks`` row anchors yet while the snapshot itself is skipped,
|
|
3517
|
+
and the block-only heal is owed in exactly that case. Folding it into the
|
|
3518
|
+
action would lose the case where healing is required and no row is written.
|
|
3519
|
+
"""
|
|
3520
|
+
|
|
3521
|
+
raw_pct: "float | None"
|
|
3522
|
+
effective_pct: "float | None"
|
|
3523
|
+
disposition: str
|
|
3524
|
+
window_key: "int | None"
|
|
3525
|
+
rollover_heal: bool
|
|
3526
|
+
|
|
3527
|
+
|
|
3528
|
+
@dataclass(frozen=True)
|
|
3529
|
+
class UsageSnapshotFoldResult:
|
|
3530
|
+
"""One Claude rate-limit observation, folded per axis (#769 S11, #824).
|
|
3531
|
+
|
|
3532
|
+
It replaces the ``(skip, adjusted_five_hour, reason)`` tuple, whose single
|
|
3533
|
+
boolean could not express the case this session exists to fix: a tick whose
|
|
3534
|
+
weekly percent clamps while its five-hour reading genuinely grew. The tuple
|
|
3535
|
+
also carried a ``deduped`` meaning inside ``reason``; ``snapshot_action``
|
|
3536
|
+
now describes the final materialized pair directly, so that distinction no
|
|
3537
|
+
longer has to be reconstructed from a string at every call site.
|
|
3538
|
+
"""
|
|
3539
|
+
|
|
3540
|
+
weekly: WeeklyFold
|
|
3541
|
+
five_hour: FiveHourFold
|
|
3542
|
+
snapshot_action: str
|
|
3543
|
+
|
|
3544
|
+
|
|
3545
|
+
def _usage_snapshot_fold_decision(
|
|
3546
|
+
conn, payload, *, force_weekly_hold=False, reviewed_weekly_basis=None,
|
|
3547
|
+
) -> UsageSnapshotFoldResult:
|
|
3548
|
+
"""The apply-time accept/hold/skip decision for a Claude rate-limit obs —
|
|
3549
|
+
the predicate ported from `cmd_record_usage`'s insert guard
|
|
3550
|
+
(bin/_cctally_record.py), made at fold time (spec §4.5 / §5.3).
|
|
3551
|
+
|
|
3552
|
+
Returns a :class:`UsageSnapshotFoldResult`. The two axes are evaluated
|
|
3553
|
+
INDEPENDENTLY, which is the #824 change:
|
|
3554
|
+
|
|
3555
|
+
- the weekly leg takes the reset-aware 7d HWM clamp (`_reset_aware_floor`
|
|
3556
|
+
+ reset-aware MAX + `hwm_clamp_applies`). A clamp no longer RETURNS; it
|
|
3557
|
+
sets `weekly.disposition = WEEKLY_HELD_CLAMP` and carries the maximum as
|
|
3558
|
+
`weekly.effective_pct`;
|
|
3559
|
+
- the 5h leg adjusts `five_hour.effective_pct` UP to the in-window MAX and
|
|
3560
|
+
never gates, exactly as before — but it now runs on the clamped path
|
|
3561
|
+
too, where the old early return meant the tick's five-hour reading was
|
|
3562
|
+
returned RAW and then discarded entirely by the pipeline;
|
|
3563
|
+
- `snapshot_action` then selects the materialized outcome from both.
|
|
3564
|
+
|
|
3565
|
+
The clamp/hold distinction is what `_pipeline_claude_usage` gates the weekly
|
|
3566
|
+
milestone on, and the reason it must survive: a dedup skip AGREES with the
|
|
3567
|
+
stored row, so re-running the derivation chokepoints against it heals a
|
|
3568
|
+
killed tick; a hold CONTRADICTS it, so deriving a weekly milestone from the
|
|
3569
|
+
carried value fabricates a crossing (the 2026-09-01 incident, a fabricated
|
|
3570
|
+
13% milestone, and milestones are forward-only within an epoch).
|
|
3571
|
+
|
|
3572
|
+
Every value returned is derived from the same `conn` state and payload, so
|
|
3573
|
+
the whole result is exactly as replay-deterministic as the boolean it
|
|
3574
|
+
replaces.
|
|
3575
|
+
"""
|
|
3393
3576
|
week_start_date = payload["week_start_date"]
|
|
3394
3577
|
week_start_at = payload.get("week_start_at")
|
|
3395
3578
|
week_end_at = payload.get("week_end_at")
|
|
@@ -3403,22 +3586,56 @@ def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object, str]:
|
|
|
3403
3586
|
# single-account install where every row shares one key).
|
|
3404
3587
|
account_key = payload.get("account_key") or _lib_accounts.UNATTRIBUTED
|
|
3405
3588
|
|
|
3589
|
+
# ── Weekly axis ────────────────────────────────────────────────────────
|
|
3406
3590
|
clamp_floor_iso = _cctally_core._reset_aware_floor(
|
|
3407
3591
|
conn, week_start_date, week_start_at, week_end_at,
|
|
3408
3592
|
account_key=account_key,
|
|
3409
3593
|
) or "1970-01-01T00:00:00Z"
|
|
3594
|
+
# `weekly_observation_held = 0` (#769 S11): a held row's weekly value is a
|
|
3595
|
+
# copy of an OLDER row's, so counting it here would let pre-credit evidence
|
|
3596
|
+
# re-enter a post-credit maximum and clamp a genuine reading. The MAX is a
|
|
3597
|
+
# weekly read, and every weekly read excludes held rows.
|
|
3410
3598
|
max_row = conn.execute(
|
|
3411
3599
|
"SELECT MAX(weekly_percent) FROM weekly_usage_snapshots "
|
|
3412
3600
|
"WHERE week_start_date = ? AND account_key = ? "
|
|
3601
|
+
" AND weekly_observation_held = 0 "
|
|
3413
3602
|
" AND unixepoch(captured_at_utc) >= unixepoch(?)",
|
|
3414
3603
|
(week_start_date, account_key, clamp_floor_iso),
|
|
3415
3604
|
).fetchone()
|
|
3416
3605
|
max_v = max_row[0] if max_row else None
|
|
3417
|
-
|
|
3418
|
-
|
|
3606
|
+
weekly_held = force_weekly_hold or _lib_record.hwm_clamp_applies(
|
|
3607
|
+
weekly_percent, max_v)
|
|
3608
|
+
|
|
3609
|
+
basis = None
|
|
3610
|
+
if weekly_held:
|
|
3611
|
+
# A reviewed hold may carry an old raw boundary. Its basis is selected
|
|
3612
|
+
# before reset/credit detection because five-hour credit can delete
|
|
3613
|
+
# that very row while preserving the held observation's 5h effect.
|
|
3614
|
+
basis = (
|
|
3615
|
+
reviewed_weekly_basis if force_weekly_hold
|
|
3616
|
+
else _latest_weekly_basis(
|
|
3617
|
+
conn, account_key, week_start_date=week_start_date)
|
|
3618
|
+
)
|
|
3619
|
+
weekly_effective = (
|
|
3620
|
+
basis.weekly_percent if force_weekly_hold and basis is not None
|
|
3621
|
+
else float(max_v) if weekly_held and max_v is not None
|
|
3622
|
+
else weekly_percent
|
|
3623
|
+
)
|
|
3624
|
+
|
|
3625
|
+
weekly = WeeklyFold(
|
|
3626
|
+
raw_pct=weekly_percent,
|
|
3627
|
+
effective_pct=weekly_effective,
|
|
3628
|
+
disposition=WEEKLY_HELD_CLAMP if weekly_held else WEEKLY_OBSERVED,
|
|
3629
|
+
basis=basis,
|
|
3630
|
+
)
|
|
3419
3631
|
|
|
3632
|
+
# ── Five-hour axis ─────────────────────────────────────────────────────
|
|
3633
|
+
# Runs unconditionally. Before #824 the weekly clamp returned above this
|
|
3634
|
+
# block, so a weekly-clamped tick reported its five-hour reading RAW.
|
|
3420
3635
|
adjusted_5h = five_hour_percent
|
|
3636
|
+
five_hour_disposition = FIVE_HOUR_MISSING
|
|
3421
3637
|
if five_hour_percent is not None and five_hour_window_key is not None:
|
|
3638
|
+
five_hour_disposition = FIVE_HOUR_AS_OBSERVED
|
|
3422
3639
|
max_5h_row = conn.execute(
|
|
3423
3640
|
"SELECT MAX(five_hour_percent) FROM weekly_usage_snapshots "
|
|
3424
3641
|
"WHERE five_hour_window_key = ? AND account_key = ? "
|
|
@@ -3433,20 +3650,76 @@ def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object, str]:
|
|
|
3433
3650
|
max_5h = max_5h_row[0] if max_5h_row else None
|
|
3434
3651
|
if _lib_record.hwm_clamp_applies(float(five_hour_percent), max_5h):
|
|
3435
3652
|
adjusted_5h = float(max_5h)
|
|
3436
|
-
|
|
3653
|
+
five_hour_disposition = FIVE_HOUR_CLAMPED
|
|
3654
|
+
else:
|
|
3655
|
+
adjusted_5h = float(five_hour_percent)
|
|
3656
|
+
|
|
3657
|
+
# ── What the stored state already knows ────────────────────────────────
|
|
3658
|
+
# HELD-INCLUSIVE, deliberately. This is a five-hour read as much as a
|
|
3659
|
+
# weekly one: a held row carries the latest five-hour value, so excluding
|
|
3660
|
+
# it would make the next held tick look like new five-hour evidence and
|
|
3661
|
+
# write a duplicate row every tick. Its weekly value is by construction a
|
|
3662
|
+
# copy of the basis's, so including it changes no weekly comparison.
|
|
3663
|
+
last_week_start_date = (
|
|
3664
|
+
basis.week_start_date if force_weekly_hold and basis is not None
|
|
3665
|
+
else week_start_date
|
|
3666
|
+
)
|
|
3437
3667
|
last = conn.execute(
|
|
3438
|
-
"SELECT weekly_percent, five_hour_percent
|
|
3668
|
+
"SELECT weekly_percent, five_hour_percent, five_hour_window_key "
|
|
3669
|
+
"FROM weekly_usage_snapshots "
|
|
3439
3670
|
"WHERE week_start_date = ? AND account_key = ? "
|
|
3440
3671
|
"ORDER BY captured_at_utc DESC, id DESC LIMIT 1",
|
|
3441
|
-
(
|
|
3672
|
+
(last_week_start_date, account_key),
|
|
3442
3673
|
).fetchone()
|
|
3443
|
-
if last is not None
|
|
3444
|
-
|
|
3445
|
-
|
|
3446
|
-
|
|
3674
|
+
last_5h = last[1] if last is not None else None
|
|
3675
|
+
last_window_key = last[2] if last is not None else None
|
|
3676
|
+
|
|
3677
|
+
five_hour_unchanged = adjusted_5h is None or (
|
|
3678
|
+
last_5h is not None and float(last_5h) == float(adjusted_5h)
|
|
3679
|
+
)
|
|
3680
|
+
|
|
3681
|
+
# ── The window-rollover heal, orthogonal to the action ─────────────────
|
|
3682
|
+
rollover_heal = False
|
|
3683
|
+
if adjusted_5h is not None and five_hour_window_key is not None:
|
|
3684
|
+
if last_window_key is None or int(last_window_key) != int(
|
|
3685
|
+
five_hour_window_key
|
|
3447
3686
|
):
|
|
3448
|
-
|
|
3449
|
-
|
|
3687
|
+
rollover_heal = conn.execute(
|
|
3688
|
+
"SELECT 1 FROM five_hour_blocks WHERE five_hour_window_key = ? "
|
|
3689
|
+
" AND account_key = ? LIMIT 1",
|
|
3690
|
+
(int(five_hour_window_key), account_key),
|
|
3691
|
+
).fetchone() is None
|
|
3692
|
+
|
|
3693
|
+
five_hour = FiveHourFold(
|
|
3694
|
+
raw_pct=None if five_hour_percent is None else float(five_hour_percent),
|
|
3695
|
+
effective_pct=adjusted_5h,
|
|
3696
|
+
disposition=five_hour_disposition,
|
|
3697
|
+
window_key=None if five_hour_window_key is None
|
|
3698
|
+
else int(five_hour_window_key),
|
|
3699
|
+
rollover_heal=rollover_heal,
|
|
3700
|
+
)
|
|
3701
|
+
|
|
3702
|
+
# ── The materialized outcome ───────────────────────────────────────────
|
|
3703
|
+
if weekly_held:
|
|
3704
|
+
# The weekly axis carries nothing new. An ordinary clamp writes only
|
|
3705
|
+
# for five-hour evidence, including on an explicit reviewed hold.
|
|
3706
|
+
# A held row also needs an accepted basis boundary to carry forward.
|
|
3707
|
+
if five_hour_unchanged or basis is None:
|
|
3708
|
+
action = SNAPSHOT_SKIP_NO_CHANGE
|
|
3709
|
+
else:
|
|
3710
|
+
action = SNAPSHOT_WRITE_HELD_5H
|
|
3711
|
+
else:
|
|
3712
|
+
weekly_unchanged = (
|
|
3713
|
+
last is not None and float(last[0]) == weekly_percent
|
|
3714
|
+
)
|
|
3715
|
+
action = (
|
|
3716
|
+
SNAPSHOT_SKIP_NO_CHANGE
|
|
3717
|
+
if weekly_unchanged and five_hour_unchanged
|
|
3718
|
+
else SNAPSHOT_WRITE_OBSERVED
|
|
3719
|
+
)
|
|
3720
|
+
|
|
3721
|
+
return UsageSnapshotFoldResult(
|
|
3722
|
+
weekly=weekly, five_hour=five_hour, snapshot_action=action)
|
|
3450
3723
|
|
|
3451
3724
|
|
|
3452
3725
|
# NOTE (rev 3): the direct obs -> weekly_usage_snapshots fold is GONE. That
|
|
@@ -3666,6 +3939,7 @@ _EVT_KIND_PROVIDER = {
|
|
|
3666
3939
|
# planner raises `RederiveConflict` on every run.
|
|
3667
3940
|
_ACCOUNTS_MACHINERY_KINDS = frozenset(
|
|
3668
3941
|
("account_observe", "account_label", "accounts_cutover",
|
|
3942
|
+
"claude_weekly_observation_decision",
|
|
3669
3943
|
"codex_file_account",
|
|
3670
3944
|
"codex_window_attribution", "codex_window_attribution_retract"))
|
|
3671
3945
|
|
|
@@ -7486,6 +7760,16 @@ _REBUILD_MILESTONE_ORDER = 60
|
|
|
7486
7760
|
|
|
7487
7761
|
# Journal-covered families counted in the RebuildResult report (+ the two
|
|
7488
7762
|
# re-materialized quota projection families, useful for the operator command).
|
|
7763
|
+
#
|
|
7764
|
+
# Two stats tables are deliberately ABSENT, and the absence is a decision
|
|
7765
|
+
# rather than an oversight: `weekly_reset_debounce_state` (#750 S3) and
|
|
7766
|
+
# `five_hour_credit_confirmation_state` (#769 S2). Both hold disposable
|
|
7767
|
+
# operational state that the journal does not describe, so a rebuild
|
|
7768
|
+
# reproduces none of their rows and a count over them would report zero on
|
|
7769
|
+
# every rebuild while implying the rebuild had checked something. Membership
|
|
7770
|
+
# here is "this family is journal truth the rebuild re-derives", which neither
|
|
7771
|
+
# is. `tests/test_epoch_1013_reset_origin.py` and
|
|
7772
|
+
# `tests/test_epoch_1014_source_local_confirmation.py` pin both exclusions.
|
|
7489
7773
|
_REBUILD_COUNT_TABLES = (
|
|
7490
7774
|
"weekly_usage_snapshots", "weekly_cost_snapshots", "week_reset_events",
|
|
7491
7775
|
"five_hour_reset_events", "five_hour_blocks", "five_hour_block_models",
|
|
@@ -7659,6 +7943,7 @@ _REBUILD_REQUIRED_TABLES = frozenset(
|
|
|
7659
7943
|
"five_hour_block_models",
|
|
7660
7944
|
"five_hour_block_projects",
|
|
7661
7945
|
"five_hour_blocks",
|
|
7946
|
+
"five_hour_credit_confirmation_state",
|
|
7662
7947
|
"five_hour_milestones",
|
|
7663
7948
|
"five_hour_reset_events",
|
|
7664
7949
|
"journal_cursor",
|
|
@@ -7735,7 +8020,7 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
|
|
|
7735
8020
|
# omitted column, constraint, partial predicate, or index definition. An epoch
|
|
7736
8021
|
# schema change must update this contract alongside STATS_INDEX_EPOCH.
|
|
7737
8022
|
_REBUILD_SCHEMA_FINGERPRINT = (
|
|
7738
|
-
"
|
|
8023
|
+
"b8f18315bce871bd9692ba1db73cf7e2bda716b360616fc65149ce209248906d"
|
|
7739
8024
|
)
|
|
7740
8025
|
|
|
7741
8026
|
|
|
@@ -8573,8 +8858,9 @@ def _open_publication_connection(destination) -> sqlite3.Connection:
|
|
|
8573
8858
|
|
|
8574
8859
|
conn = _cctally_store.stats_open_guarded(
|
|
8575
8860
|
pathlib.Path(destination),
|
|
8576
|
-
|
|
8577
|
-
|
|
8861
|
+
# #778: forward the opener's `cached_statements=0`.
|
|
8862
|
+
connect=lambda path, **kw: sqlite3.connect(
|
|
8863
|
+
pathlib.Path(path).resolve().as_uri(), uri=True, **kw
|
|
8578
8864
|
),
|
|
8579
8865
|
recover_interruptions=False,
|
|
8580
8866
|
)
|
|
@@ -10378,6 +10664,84 @@ def _observe_selector_desynchronization(path) -> "dict | None":
|
|
|
10378
10664
|
}
|
|
10379
10665
|
|
|
10380
10666
|
|
|
10667
|
+
def _rematerialize_five_hour_high_water(destination) -> None:
|
|
10668
|
+
"""Rewrite the `hwm-5h` projection file from a freshly published LIVE stats
|
|
10669
|
+
index (#769 S11, #824).
|
|
10670
|
+
|
|
10671
|
+
`hwm-7d` deliberately has no such pass, and the reason is recorded above:
|
|
10672
|
+
the SQL 7d high-water clamp re-establishes the floor on the next status-line
|
|
10673
|
+
tick, so a stale or absent file self-heals. The five-hour axis needs one
|
|
10674
|
+
now, because a weekly-clamped tick's five-hour evidence lives only in a
|
|
10675
|
+
`weekly_observation_held` row: `db rebuild` reconstructs that row from the
|
|
10676
|
+
journal without rerunning the live pipeline, and `db rederive`'s scratch
|
|
10677
|
+
derivation runs with projection writes disabled, so neither path would
|
|
10678
|
+
otherwise leave `hwm-5h` describing the index it just published — and the
|
|
10679
|
+
two would disagree with each other about it.
|
|
10680
|
+
|
|
10681
|
+
The read is HELD-INCLUSIVE, which is the whole point: the held row is where
|
|
10682
|
+
the evidence is. The write is unconditional rather than monotonic, because
|
|
10683
|
+
the published index IS the truth here — a rebuild holds maintenance
|
|
10684
|
+
exclusive and the bounded ingest lock, so no concurrent tick can have
|
|
10685
|
+
written a value this pass does not already account for, and a five-hour
|
|
10686
|
+
credit legitimately lowers the value.
|
|
10687
|
+
|
|
10688
|
+
The MAX is bounded on the window's latest `effective_reset_at_utc` and
|
|
10689
|
+
scoped to one account, mirroring the fold's own five-hour clamp. An
|
|
10690
|
+
unfloored MAX is not the effective value: an in-place five-hour credit
|
|
10691
|
+
retires the rows captured before it WITHOUT deleting them (the stale-replica
|
|
10692
|
+
DELETE bands only at or after the credit instant), so the pre-credit peak
|
|
10693
|
+
survives in the index and an unfloored read republishes it — re-inflating
|
|
10694
|
+
the status line's no-regression floor past a level the meter has already
|
|
10695
|
+
moved past, which is the exact outcome the credit's own force-write exists
|
|
10696
|
+
to prevent. The account comes from the same latest row that selects the
|
|
10697
|
+
window, because a reset generation is meaningful only inside the account
|
|
10698
|
+
that was issued it; `hwm-5h` is nonetheless a single global file, so which
|
|
10699
|
+
account owns it across a multi-account install is #832's question, not one
|
|
10700
|
+
this pass can settle.
|
|
10701
|
+
|
|
10702
|
+
`hwm-7d` is never read or written here, so no held weekly value can reach
|
|
10703
|
+
the file the status line renders the weekly axis from.
|
|
10704
|
+
|
|
10705
|
+
Best-effort and silent, matching every other projection-file writer: a
|
|
10706
|
+
projection is re-derivable by definition and must never fail a publication.
|
|
10707
|
+
"""
|
|
10708
|
+
try:
|
|
10709
|
+
conn = sqlite3.connect(f"file:{destination}?mode=ro", uri=True)
|
|
10710
|
+
except sqlite3.Error:
|
|
10711
|
+
return
|
|
10712
|
+
try:
|
|
10713
|
+
latest = conn.execute(
|
|
10714
|
+
"SELECT five_hour_window_key, account_key FROM weekly_usage_snapshots "
|
|
10715
|
+
"WHERE five_hour_window_key IS NOT NULL "
|
|
10716
|
+
"ORDER BY captured_at_utc DESC, id DESC LIMIT 1"
|
|
10717
|
+
).fetchone()
|
|
10718
|
+
if latest is None or latest[0] is None:
|
|
10719
|
+
return
|
|
10720
|
+
window_key = int(latest[0])
|
|
10721
|
+
account_key = latest[1] or _lib_accounts.UNATTRIBUTED
|
|
10722
|
+
row = conn.execute(
|
|
10723
|
+
"SELECT MAX(five_hour_percent) FROM weekly_usage_snapshots "
|
|
10724
|
+
"WHERE five_hour_window_key = ? AND account_key = ? "
|
|
10725
|
+
" AND unixepoch(captured_at_utc) >= unixepoch(COALESCE("
|
|
10726
|
+
" (SELECT effective_reset_at_utc FROM five_hour_reset_events "
|
|
10727
|
+
" WHERE five_hour_window_key = ? AND account_key = ? "
|
|
10728
|
+
" ORDER BY id DESC LIMIT 1),"
|
|
10729
|
+
" '1970-01-01T00:00:00Z'))",
|
|
10730
|
+
(window_key, account_key, window_key, account_key),
|
|
10731
|
+
).fetchone()
|
|
10732
|
+
except sqlite3.Error:
|
|
10733
|
+
return
|
|
10734
|
+
finally:
|
|
10735
|
+
conn.close()
|
|
10736
|
+
if row is None or row[0] is None:
|
|
10737
|
+
return
|
|
10738
|
+
try:
|
|
10739
|
+
(_cctally_core.APP_DIR / "hwm-5h").write_text(
|
|
10740
|
+
f"{window_key} {float(row[0])}\n")
|
|
10741
|
+
except OSError:
|
|
10742
|
+
pass
|
|
10743
|
+
|
|
10744
|
+
|
|
10381
10745
|
def rebuild_stats_index(
|
|
10382
10746
|
*,
|
|
10383
10747
|
context: RebuildContext,
|
|
@@ -11026,6 +11390,12 @@ def _rebuild_stats_index_locked(
|
|
|
11026
11390
|
phase_seconds["publication"] = round(
|
|
11027
11391
|
time.monotonic() - publication_started, 6)
|
|
11028
11392
|
|
|
11393
|
+
# #769 S11 (#824). LIVE destination only. A `target_path` build is a scratch
|
|
11394
|
+
# or preview index that nothing renders from, and mutating a projection file
|
|
11395
|
+
# on its behalf would let a preview change what the status line shows.
|
|
11396
|
+
if target_path is None:
|
|
11397
|
+
_rematerialize_five_hour_high_water(dest)
|
|
11398
|
+
|
|
11029
11399
|
return RebuildResult(
|
|
11030
11400
|
rows_by_table=rows_by_table, malformed=malformed,
|
|
11031
11401
|
duration_s=time.monotonic() - start, segments_read=segments_read,
|