cctally 1.110.0 → 1.112.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +5 -6
- package/bin/_cctally_cache.py +14 -7
- package/bin/_cctally_core.py +12 -1
- package/bin/_cctally_dashboard.py +60 -154
- package/bin/_cctally_dashboard_envelope.py +7 -6
- package/bin/_cctally_dashboard_sources.py +90 -55
- package/bin/_cctally_doctor.py +73 -27
- package/bin/_cctally_forecast.py +2 -1
- package/bin/_cctally_journal.py +147 -6
- package/bin/_cctally_milestones.py +29 -3
- package/bin/_cctally_quota.py +10 -1
- package/bin/_cctally_record.py +136 -4
- package/bin/_cctally_rederive.py +311 -8
- package/bin/_cctally_source_analytics.py +20 -8
- package/bin/_cctally_tui.py +546 -45
- package/bin/_cctally_update.py +420 -14
- package/bin/_lib_cache_report.py +7 -10
- package/bin/_lib_codex_conversation_query.py +40 -4
- package/bin/_lib_codex_metadata.py +57 -11
- package/bin/_lib_cost_provenance.py +303 -0
- package/bin/_lib_dashboard_sources.py +263 -28
- package/bin/_lib_doctor.py +2 -2
- package/bin/_lib_ingest_frontier.py +137 -38
- package/bin/_lib_pricing.py +24 -263
- package/bin/_lib_rederive.py +704 -36
- package/bin/_lib_snapshot_cache.py +332 -9
- package/bin/cctally +18 -5
- package/dashboard/static/assets/{ConversationsView-BB9vUiL6.js → ConversationsView-CEZ-BUyi.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-DFY2iPDf.js → DoctorModal-CE1oH_Zy.js} +1 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-DAv39uSo.js → ProjectsDrillPanel-B3DZoCeD.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-DSAWo2Fp.js → SourceDetailModal-CHp9EgLP.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-R_9dOVnh.js → UpdateModal-5iKST6jo.js} +1 -1
- package/dashboard/static/assets/{index-DdU62sov.js → index-Cm0VTxJC.js} +2 -2
- package/dashboard/static/assets/index-Dt4tqf43.css +1 -0
- package/dashboard/static/assets/{outlineNavigation-BOCLcRa6.js → outlineNavigation-DkXGL8J6.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +2 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +0 -1
- package/dashboard/static/assets/index-COhxe3vU.css +0 -1
|
@@ -55,15 +55,17 @@ from _lib_dashboard_sources import (
|
|
|
55
55
|
SourceDashboardState,
|
|
56
56
|
SourceDashboardWarning,
|
|
57
57
|
assess_codex_projection_coherence,
|
|
58
|
+
alerted_at_order_key,
|
|
59
|
+
alerted_at_order_key_sql,
|
|
58
60
|
build_metadata_health,
|
|
59
61
|
canonical_alerted_at,
|
|
60
|
-
canonical_alerted_at_sql,
|
|
61
62
|
claude_stats_digest,
|
|
62
63
|
combined_accounting_version,
|
|
63
64
|
codex_stats_digest,
|
|
64
65
|
dashboard_resource_key,
|
|
65
66
|
)
|
|
66
67
|
from _lib_quota import (
|
|
68
|
+
FUTURE_CLOCK_SKEW_SECONDS,
|
|
67
69
|
QuotaWindowIdentity,
|
|
68
70
|
build_blocks,
|
|
69
71
|
build_history,
|
|
@@ -627,26 +629,12 @@ def _codex_quota_reuse_identity(
|
|
|
627
629
|
"stats_identity must be the three-digest tuple or None")
|
|
628
630
|
elif not isinstance(stats_conn, sqlite3.Connection):
|
|
629
631
|
return None
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
"SELECT 1 FROM sqlite_master WHERE type='table' "
|
|
637
|
-
"AND name='quota_window_change_log'"
|
|
638
|
-
).fetchone()
|
|
639
|
-
revision_row = cache_conn.execute(
|
|
640
|
-
"SELECT value FROM cache_meta "
|
|
641
|
-
"WHERE key='codex_window_attribution_revision'"
|
|
642
|
-
).fetchone()
|
|
643
|
-
db_path = next(
|
|
644
|
-
str(row[2]) for row in cache_conn.execute("PRAGMA database_list")
|
|
645
|
-
if str(row[1]) == "main"
|
|
646
|
-
)
|
|
647
|
-
except (sqlite3.Error, StopIteration):
|
|
648
|
-
return None
|
|
649
|
-
if table is None:
|
|
632
|
+
# #857: the cache.db legs are read through the snapshot-cache kernel, which
|
|
633
|
+
# states each leg's query once and shares it with the doctor's quota probe
|
|
634
|
+
# and the dashboard's quota-dependency identity.
|
|
635
|
+
db_path = _lib_snapshot_cache.codex_main_database_path(cache_conn)
|
|
636
|
+
legs = _lib_snapshot_cache.codex_quota_observation_legs(cache_conn)
|
|
637
|
+
if db_path is None or legs is None:
|
|
650
638
|
return None
|
|
651
639
|
if stats_identity is None:
|
|
652
640
|
try:
|
|
@@ -660,12 +648,7 @@ def _codex_quota_reuse_identity(
|
|
|
660
648
|
return None
|
|
661
649
|
except sqlite3.Error:
|
|
662
650
|
return None
|
|
663
|
-
return (
|
|
664
|
-
db_path,
|
|
665
|
-
0 if seq_row is None else int(seq_row[0]),
|
|
666
|
-
"" if revision_row is None else str(revision_row[0]),
|
|
667
|
-
*stats_identity,
|
|
668
|
-
)
|
|
651
|
+
return (db_path, *legs, *stats_identity)
|
|
669
652
|
|
|
670
653
|
|
|
671
654
|
def _cached_codex_quota_observations(**kwargs) -> tuple[object, ...]:
|
|
@@ -1190,51 +1173,82 @@ def _codex_next_decision_at(
|
|
|
1190
1173
|
crossing. That is one rebuild per deadline (a handful per weekly cycle), not
|
|
1191
1174
|
one per tick.
|
|
1192
1175
|
|
|
1193
|
-
The deadline is the ``min`` of
|
|
1194
|
-
at or before ``now_utc
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1176
|
+
The deadline is the ``min`` of every candidate below, dropping any
|
|
1177
|
+
candidate at or before ``now_utc``. #857 made the set COMPLETE, because a
|
|
1178
|
+
settled cycle-unavailable generation now idles until this instant: a
|
|
1179
|
+
transition with no candidate is never picked up at all, and one whose
|
|
1180
|
+
candidate fires early is lost too — the rebuild at the early instant drops
|
|
1181
|
+
the candidate (it is no longer after ``now``) while the transition it stood
|
|
1182
|
+
for is still ahead. Each candidate is therefore the FIRST instant its
|
|
1183
|
+
transition is observable, under the resolver's own comparison:
|
|
1184
|
+
|
|
1185
|
+
1. every weekly baseline's ``canonical_resets_at`` — selected or not, since
|
|
1186
|
+
an UNSELECTED boundary expiring is what resolves a ``conflicting``
|
|
1187
|
+
account (the resolver drops ``resets_at <= now``, so the reset instant
|
|
1188
|
+
itself is the first expired one). Every selected cycle's ``resets_at`` is
|
|
1189
|
+
one of these; they stay listed explicitly for the #341 multi-account case
|
|
1190
|
+
where account A expires while B stays live;
|
|
1191
|
+
2. every weekly baseline's nominal start plus one microsecond: the resolver
|
|
1192
|
+
rejects ``start >= now`` (#599), so a window captured ahead of its start
|
|
1193
|
+
becomes eligible strictly after it, on frozen evidence;
|
|
1194
|
+
3. for every weekly history with a baseline that has not reset: the
|
|
1195
|
+
fresh -> stale flip of its latest physical capture, which
|
|
1196
|
+
``quota_freshness`` takes when the whole-second age EXCEEDS
|
|
1197
|
+
``stale_after_seconds`` — ``capture + stale_after + 1s``, not
|
|
1198
|
+
``capture + stale_after`` — and the future -> fresh flip, when the
|
|
1199
|
+
truncated age stops being below ``-FUTURE_CLOCK_SKEW_SECONDS`` —
|
|
1200
|
+
strictly after ``capture - (skew + 1)s``. This is a deliberate SUPERSET
|
|
1201
|
+
of the §3.2 ranking participants, because an extra candidate can only
|
|
1202
|
+
pull the deadline EARLIER and costs one rebuild at an instant where the
|
|
1203
|
+
verdict could have changed;
|
|
1204
|
+
4. the ``captured_at`` of any future-dated weekly observation (baseline
|
|
1205
|
+
eligibility, ``captured_at <= now``). Recorded BEFORE the model-scoped
|
|
1206
|
+
skip: the pool label is read from the baseline and ``limit_name`` is not
|
|
1207
|
+
part of history identity, so a future capture can move a history out of
|
|
1208
|
+
a model-scoped pool.
|
|
1206
1209
|
|
|
1207
1210
|
Returns ``None`` when nothing can flip. Server-only: it rides ``clock_data``
|
|
1208
1211
|
and never reaches the public source envelope.
|
|
1209
1212
|
"""
|
|
1213
|
+
one_microsecond = dt.timedelta(microseconds=1)
|
|
1210
1214
|
candidates: list[dt.datetime] = []
|
|
1211
1215
|
for cycle in cycles:
|
|
1212
1216
|
candidates.append(cycle.resets_at.astimezone(UTC))
|
|
1213
1217
|
for history in build_history(tuple(observations)):
|
|
1214
1218
|
if history.identity.window_minutes != 10_080:
|
|
1215
1219
|
continue
|
|
1220
|
+
# 4. A capture ahead of ``now`` is not baseline-eligible yet, so this
|
|
1221
|
+
# runs even for a history with no live baseline at all.
|
|
1222
|
+
for observation in (
|
|
1223
|
+
*history.observations, *history.physical_observations,
|
|
1224
|
+
):
|
|
1225
|
+
captured_at = observation.captured_at.astimezone(UTC)
|
|
1226
|
+
if captured_at > now_utc:
|
|
1227
|
+
candidates.append(captured_at)
|
|
1216
1228
|
baseline = select_baseline(history.observations, now_utc)
|
|
1217
1229
|
if codex_history_is_model_scoped(history, baseline=baseline):
|
|
1218
1230
|
continue
|
|
1231
|
+
if baseline is None:
|
|
1232
|
+
continue
|
|
1219
1233
|
# #428: the anchor, so the decision deadline describes the same window
|
|
1220
1234
|
# the hero publishes rather than a jitter sibling up to 600s away.
|
|
1221
|
-
|
|
1235
|
+
resets_at = baseline.canonical_resets_at.astimezone(UTC)
|
|
1236
|
+
window = dt.timedelta(minutes=history.identity.window_minutes)
|
|
1237
|
+
candidates.append(resets_at) # 1.
|
|
1238
|
+
candidates.append(resets_at - window + one_microsecond) # 2.
|
|
1239
|
+
if resets_at > now_utc: # 3.
|
|
1222
1240
|
latest = latest_physical_observation(history.physical_observations)
|
|
1223
1241
|
if latest is not None:
|
|
1242
|
+
latest_at = latest.captured_at.astimezone(UTC)
|
|
1243
|
+
candidates.append(latest_at + dt.timedelta(
|
|
1244
|
+
seconds=stale_after_seconds(
|
|
1245
|
+
history.identity.window_minutes) + 1,
|
|
1246
|
+
))
|
|
1224
1247
|
candidates.append(
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
)
|
|
1248
|
+
latest_at
|
|
1249
|
+
- dt.timedelta(seconds=FUTURE_CLOCK_SKEW_SECONDS + 1)
|
|
1250
|
+
+ one_microsecond
|
|
1229
1251
|
)
|
|
1230
|
-
# A capture ahead of ``now`` is not baseline-eligible yet, so this runs
|
|
1231
|
-
# even for a history with no live baseline at all.
|
|
1232
|
-
for observation in (
|
|
1233
|
-
*history.observations, *history.physical_observations,
|
|
1234
|
-
):
|
|
1235
|
-
captured_at = observation.captured_at.astimezone(UTC)
|
|
1236
|
-
if captured_at > now_utc:
|
|
1237
|
-
candidates.append(captured_at)
|
|
1238
1252
|
live = [candidate for candidate in candidates if candidate > now_utc]
|
|
1239
1253
|
return min(live) if live else None
|
|
1240
1254
|
|
|
@@ -4447,7 +4461,8 @@ def _alerts_wire(
|
|
|
4447
4461
|
# row that fired most recently but crossed longest ago was dropped at the
|
|
4448
4462
|
# LIMIT and never reached the panel at all. `created_at` stays as an
|
|
4449
4463
|
# equal-valued compatibility alias for a client reading a pre-v7 envelope.
|
|
4450
|
-
|
|
4464
|
+
# #869 F14: the full instant, so a same-second pair keeps its newer row.
|
|
4465
|
+
canon = alerted_at_order_key_sql()
|
|
4451
4466
|
|
|
4452
4467
|
def _instants(raw: object) -> dict[str, str]:
|
|
4453
4468
|
value = canonical_alerted_at(raw)
|
|
@@ -4506,7 +4521,7 @@ def _alerts_wire(
|
|
|
4506
4521
|
return ()
|
|
4507
4522
|
return tuple(sorted(
|
|
4508
4523
|
rows,
|
|
4509
|
-
key=lambda item:
|
|
4524
|
+
key=lambda item: alerted_at_order_key(item["alerted_at"]),
|
|
4510
4525
|
reverse=True,
|
|
4511
4526
|
)[:SOURCE_HISTORY_LIMIT])
|
|
4512
4527
|
|
|
@@ -7441,6 +7456,13 @@ class _CodexAccountingCapture:
|
|
|
7441
7456
|
#: safety came from the compare-and-swap rather than from the signature,
|
|
7442
7457
|
#: and carrying the value makes the signature answer for itself.
|
|
7443
7458
|
population_signature: tuple | None = None
|
|
7459
|
+
#: #857: the accounting provenance token THIS capture's accounting result
|
|
7460
|
+
#: returned — carried for the same reason as the signature above, because
|
|
7461
|
+
#: overlay publication or a concurrent capture can move the module state
|
|
7462
|
+
#: before the fold stamps it. ``None`` when the capture bypassed the
|
|
7463
|
+
#: qualified accounting read: such a generation names no population, and
|
|
7464
|
+
#: borrowing the process's current token would name one it never read.
|
|
7465
|
+
provenance_token: int | None = None
|
|
7444
7466
|
|
|
7445
7467
|
|
|
7446
7468
|
#: The detail routes' horizon: one year ending at the generation instant. The
|
|
@@ -7532,6 +7554,8 @@ def _capture_codex_accounting(
|
|
|
7532
7554
|
changed_old: tuple[object, ...] = ()
|
|
7533
7555
|
changed_new: tuple[object, ...] = ()
|
|
7534
7556
|
population_signature: tuple | None = None
|
|
7557
|
+
# #857: set only by a qualified read that returned; every bypass keeps it.
|
|
7558
|
+
provenance_token: int | None = None
|
|
7535
7559
|
|
|
7536
7560
|
def fallback_entries() -> tuple[object, ...]:
|
|
7537
7561
|
nonlocal metadata_read_failed, failed_legs
|
|
@@ -7600,6 +7624,7 @@ def _capture_codex_accounting(
|
|
|
7600
7624
|
changed_old = cached_accounting.changed_old
|
|
7601
7625
|
changed_new = cached_accounting.changed_new
|
|
7602
7626
|
population_signature = cached_accounting.population_signature
|
|
7627
|
+
provenance_token = cached_accounting.provenance_token
|
|
7603
7628
|
entries: tuple[object, ...] = qualified_entries
|
|
7604
7629
|
except QualifiedMetadataUnavailable as exc:
|
|
7605
7630
|
_lib_log.get_logger("dashboard").warning(
|
|
@@ -7644,6 +7669,8 @@ def _capture_codex_accounting(
|
|
|
7644
7669
|
failed_legs=failed_legs,
|
|
7645
7670
|
population_signature=(
|
|
7646
7671
|
None if metadata_incomplete else population_signature),
|
|
7672
|
+
provenance_token=(
|
|
7673
|
+
None if metadata_incomplete else provenance_token),
|
|
7647
7674
|
)
|
|
7648
7675
|
|
|
7649
7676
|
|
|
@@ -9253,6 +9280,14 @@ def _build_codex_source_state(
|
|
|
9253
9280
|
"codex_next_decision_at": _codex_next_decision_at(
|
|
9254
9281
|
quota_observations, cycles_all, context.now_utc,
|
|
9255
9282
|
),
|
|
9283
|
+
# #857: the accounting provenance token of the population THIS
|
|
9284
|
+
# build's capture read (`None` when it bypassed the qualified
|
|
9285
|
+
# read). A retained generation is admitted only while this equals
|
|
9286
|
+
# the token of the population the process last consumed, so a
|
|
9287
|
+
# build that consumed accounting and then failed to publish leaves
|
|
9288
|
+
# every older generation refused, whatever the failure site.
|
|
9289
|
+
_lib_snapshot_cache.CODEX_ACCOUNTING_PROVENANCE_KEY: (
|
|
9290
|
+
accounting_capture.provenance_token),
|
|
9256
9291
|
},
|
|
9257
9292
|
private_session_labels=private_session_labels,
|
|
9258
9293
|
# #819: the SAME reading that gated decoration above. Attaching it here
|
package/bin/_cctally_doctor.py
CHANGED
|
@@ -1032,31 +1032,17 @@ def _codex_quota_observation_signal() -> "tuple | None":
|
|
|
1032
1032
|
except sqlite3.Error:
|
|
1033
1033
|
return None
|
|
1034
1034
|
try:
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
).
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
seq_row = conn.execute(
|
|
1042
|
-
"SELECT seq FROM sqlite_sequence "
|
|
1043
|
-
"WHERE name='quota_window_change_log'"
|
|
1044
|
-
).fetchone()
|
|
1045
|
-
revision_row = conn.execute(
|
|
1046
|
-
"SELECT value FROM cache_meta "
|
|
1047
|
-
"WHERE key='codex_window_attribution_revision'"
|
|
1048
|
-
).fetchone()
|
|
1049
|
-
except sqlite3.Error:
|
|
1050
|
-
return None
|
|
1035
|
+
# #857: the two ledger legs are stated once, in the snapshot-cache
|
|
1036
|
+
# kernel, and shared with the dashboard's Codex quota-dependency
|
|
1037
|
+
# identity rather than restated here.
|
|
1038
|
+
legs = _cctally()._load_sibling(
|
|
1039
|
+
"_lib_snapshot_cache"
|
|
1040
|
+
).codex_quota_observation_legs(conn)
|
|
1051
1041
|
finally:
|
|
1052
1042
|
conn.close()
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
stat.st_ino,
|
|
1057
|
-
0 if seq_row is None else int(seq_row[0]),
|
|
1058
|
-
"" if revision_row is None else str(revision_row[0]),
|
|
1059
|
-
)
|
|
1043
|
+
if legs is None:
|
|
1044
|
+
return None
|
|
1045
|
+
return (str(path), stat.st_dev, stat.st_ino, *legs)
|
|
1060
1046
|
|
|
1061
1047
|
|
|
1062
1048
|
def _load_codex_quota_observations_for_doctor(*, force_cold: bool = False):
|
|
@@ -1079,6 +1065,67 @@ def _load_codex_quota_observations_for_doctor(*, force_cold: bool = False):
|
|
|
1079
1065
|
return loaded
|
|
1080
1066
|
|
|
1081
1067
|
|
|
1068
|
+
# The all-history project-metadata partition is the other large immutable input
|
|
1069
|
+
# in a warm dashboard Doctor gather. On the production-shaped #857 copy it
|
|
1070
|
+
# cost 422-432 ms per call while returning the same six counts every 30-second
|
|
1071
|
+
# Doctor refresh. Retain only that raw partition, never a Doctor verdict or a
|
|
1072
|
+
# clock-derived field, under the cache identity plus the existing physical
|
|
1073
|
+
# mutation sequence. Codex ingest advances that sequence in the same
|
|
1074
|
+
# transaction as token, thread, root, cursor, and metadata writes, which are
|
|
1075
|
+
# exactly the three-table inputs of ``load_codex_project_metadata_health``.
|
|
1076
|
+
# Replacement is covered separately by device/inode, and an absent or malformed
|
|
1077
|
+
# signal always takes the cold path. The path is the one the passed connection
|
|
1078
|
+
# opened, never the global ``CACHE_DB_PATH``: the sequence is read from that
|
|
1079
|
+
# connection, so the file identity must describe the same file (#857).
|
|
1080
|
+
_CODEX_PROJECT_METADATA_MEMO: "dict[tuple, object]" = {}
|
|
1081
|
+
|
|
1082
|
+
|
|
1083
|
+
def _codex_project_metadata_signal(
|
|
1084
|
+
cache_conn: sqlite3.Connection,
|
|
1085
|
+
) -> "tuple | None":
|
|
1086
|
+
try:
|
|
1087
|
+
path = next(
|
|
1088
|
+
str(row[2]) for row in cache_conn.execute("PRAGMA database_list")
|
|
1089
|
+
if len(row) > 2 and str(row[1]) == "main" and row[2]
|
|
1090
|
+
)
|
|
1091
|
+
stat = os.stat(path)
|
|
1092
|
+
row = cache_conn.execute(
|
|
1093
|
+
"SELECT value FROM cache_meta "
|
|
1094
|
+
"WHERE key='codex_physical_mutation_seq'"
|
|
1095
|
+
).fetchone()
|
|
1096
|
+
if row is None:
|
|
1097
|
+
return None
|
|
1098
|
+
sequence = int(row[0])
|
|
1099
|
+
if sequence < 0:
|
|
1100
|
+
return None
|
|
1101
|
+
except (OSError, StopIteration, sqlite3.Error, TypeError, ValueError):
|
|
1102
|
+
return None
|
|
1103
|
+
return (path, stat.st_dev, stat.st_ino, sequence)
|
|
1104
|
+
|
|
1105
|
+
|
|
1106
|
+
def _load_codex_project_metadata_health_for_doctor(
|
|
1107
|
+
cache_conn: sqlite3.Connection,
|
|
1108
|
+
*,
|
|
1109
|
+
force_cold: bool = False,
|
|
1110
|
+
):
|
|
1111
|
+
signal = (
|
|
1112
|
+
None if force_cold
|
|
1113
|
+
else _codex_project_metadata_signal(cache_conn)
|
|
1114
|
+
)
|
|
1115
|
+
if signal is not None:
|
|
1116
|
+
cached = _CODEX_PROJECT_METADATA_MEMO.get(signal)
|
|
1117
|
+
if cached is not None:
|
|
1118
|
+
return cached
|
|
1119
|
+
analytics = _cctally()._load_sibling("_cctally_source_analytics")
|
|
1120
|
+
loaded = analytics.load_codex_project_metadata_health(
|
|
1121
|
+
cache_conn=cache_conn,
|
|
1122
|
+
)
|
|
1123
|
+
if signal is not None:
|
|
1124
|
+
_CODEX_PROJECT_METADATA_MEMO.clear()
|
|
1125
|
+
_CODEX_PROJECT_METADATA_MEMO[signal] = loaded
|
|
1126
|
+
return loaded
|
|
1127
|
+
|
|
1128
|
+
|
|
1082
1129
|
_QUOTA_NO_CHANGE: dict = {
|
|
1083
1130
|
"active": False, "assessed": True, "effective_from": None,
|
|
1084
1131
|
"previous_units_per_point": None, "new_units_per_point": None,
|
|
@@ -1835,10 +1882,9 @@ def _doctor_gather_state_impl(
|
|
|
1835
1882
|
# connection. A failed probe is health evidence, not an
|
|
1836
1883
|
# empty corpus: the kernel renders it as a distinct FAIL.
|
|
1837
1884
|
try:
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
cache_conn=conn,
|
|
1885
|
+
health = _load_codex_project_metadata_health_for_doctor(
|
|
1886
|
+
conn,
|
|
1887
|
+
force_cold=_force_cold_inputs,
|
|
1842
1888
|
)
|
|
1843
1889
|
codex_project_metadata_health = {
|
|
1844
1890
|
"total_rows": health.total_rows,
|
package/bin/_cctally_forecast.py
CHANGED
|
@@ -3173,7 +3173,8 @@ def cmd_budget(args: argparse.Namespace) -> int:
|
|
|
3173
3173
|
|
|
3174
3174
|
def _codex_backstop_leg(ictx):
|
|
3175
3175
|
c.maybe_record_codex_budget_milestone(
|
|
3176
|
-
{}, conn=ictx.conn, alert_sink=ictx.pending_alerts
|
|
3176
|
+
{}, conn=ictx.conn, alert_sink=ictx.pending_alerts,
|
|
3177
|
+
journal_ctx=ictx)
|
|
3177
3178
|
if projected_on:
|
|
3178
3179
|
c.maybe_record_projected_alert(
|
|
3179
3180
|
{}, only_metrics={"codex_budget_usd"},
|
package/bin/_cctally_journal.py
CHANGED
|
@@ -1409,12 +1409,19 @@ class IngestContext:
|
|
|
1409
1409
|
# (reset INSERT OR IGNORE rowcount == 1), so a crash-replayed reset never
|
|
1410
1410
|
# re-suppresses with a divergent list.
|
|
1411
1411
|
suppression_map: dict = field(default_factory=dict)
|
|
1412
|
+
# Indexed by (event kind, local row id) until harvest persists the marker.
|
|
1413
|
+
pricing_provenance: dict = field(default_factory=dict)
|
|
1412
1414
|
# Task B rederive seam. Normal ingest leaves both defaults unchanged.
|
|
1413
1415
|
# A scratch planner supplies an in-memory sink so derived events are captured
|
|
1414
1416
|
# instead of appended to the durable journal, and disables projection-file
|
|
1415
1417
|
# writes while still exercising the same SQLite derivation/fold code.
|
|
1416
1418
|
event_sink: "list | None" = None
|
|
1417
1419
|
projection_writes: bool = True
|
|
1420
|
+
# #869 `db rederive` scratch replay only. Its keys name the (account_key,
|
|
1421
|
+
# window key) blocks the planner must prove; each value becomes the
|
|
1422
|
+
# retained entries behind the totals that block last applied. Live ingest
|
|
1423
|
+
# leaves it None and keeps no entries.
|
|
1424
|
+
block_source_evidence: "dict | None" = None
|
|
1418
1425
|
# Operator-reviewed scratch replay only. Normal ingest leaves this empty.
|
|
1419
1426
|
held_weekly_observation_ids: frozenset[str] = field(default_factory=frozenset)
|
|
1420
1427
|
reviewed_weekly_basis_by_account: dict = field(default_factory=dict)
|
|
@@ -4127,7 +4134,10 @@ def _apply_generic_evt(conn, evt):
|
|
|
4127
4134
|
return None
|
|
4128
4135
|
cols = {"journal_id": evt["id"]}
|
|
4129
4136
|
for key, value in payload.items():
|
|
4130
|
-
|
|
4137
|
+
# `journal_identity_version` names how the event id was minted (#869
|
|
4138
|
+
# alert-latch re-fire incarnations: bm2/pjm2/pbm2); it is journal
|
|
4139
|
+
# metadata, never a column.
|
|
4140
|
+
if key in ("kind", "_pricing", "journal_identity_version"):
|
|
4131
4141
|
continue
|
|
4132
4142
|
if key in spec.fk_refs:
|
|
4133
4143
|
column, ref_table = spec.fk_refs[key]
|
|
@@ -4493,7 +4503,7 @@ def _apply_block_close(conn, evt):
|
|
|
4493
4503
|
parent = {"journal_id": evt["id"]}
|
|
4494
4504
|
children = {}
|
|
4495
4505
|
for key, value in payload.items():
|
|
4496
|
-
if key
|
|
4506
|
+
if key in ("kind", "_pricing"):
|
|
4497
4507
|
continue
|
|
4498
4508
|
if key in _BLOCK_CHILD_KEYS:
|
|
4499
4509
|
children[key] = value or []
|
|
@@ -4552,7 +4562,7 @@ def _apply_reset_with_suppression(conn, evt):
|
|
|
4552
4562
|
return None
|
|
4553
4563
|
cols = {"journal_id": evt["id"]}
|
|
4554
4564
|
for key, value in payload.items():
|
|
4555
|
-
if key in ("kind", "suppression", "suppression_table"):
|
|
4565
|
+
if key in ("kind", "suppression", "suppression_table", "_pricing"):
|
|
4556
4566
|
continue
|
|
4557
4567
|
if key in spec.fk_refs:
|
|
4558
4568
|
column, ref_table = spec.fk_refs[key]
|
|
@@ -4834,6 +4844,59 @@ def _build_harvest_evt(ctx, spec, row):
|
|
|
4834
4844
|
{k: cr[k] for k in cr.keys() if k not in ("id", "block_id")}
|
|
4835
4845
|
for cr in child_rows
|
|
4836
4846
|
]
|
|
4847
|
+
if spec.kind == "five_hour_block_close":
|
|
4848
|
+
raw_pricing = row["pricing_provenance_json"]
|
|
4849
|
+
if raw_pricing is not None:
|
|
4850
|
+
try:
|
|
4851
|
+
pricing = json.loads(raw_pricing)
|
|
4852
|
+
except (TypeError, ValueError) as exc:
|
|
4853
|
+
raise JournalError(
|
|
4854
|
+
"five_hour_block_close has malformed computation-time "
|
|
4855
|
+
"pricing provenance"
|
|
4856
|
+
) from exc
|
|
4857
|
+
if (
|
|
4858
|
+
not isinstance(pricing, dict)
|
|
4859
|
+
or pricing.get("version") != 1
|
|
4860
|
+
or not isinstance(pricing.get("pricingDate"), str)
|
|
4861
|
+
or not pricing["pricingDate"]
|
|
4862
|
+
or not isinstance(pricing.get("unpricedModels"), list)
|
|
4863
|
+
or any(not isinstance(model, str) or not model
|
|
4864
|
+
for model in pricing["unpricedModels"])
|
|
4865
|
+
or pricing["unpricedModels"] != sorted(set(
|
|
4866
|
+
pricing["unpricedModels"]
|
|
4867
|
+
))
|
|
4868
|
+
or (
|
|
4869
|
+
"uncertainModels" in pricing
|
|
4870
|
+
and (
|
|
4871
|
+
not isinstance(pricing["uncertainModels"], list)
|
|
4872
|
+
or any(not isinstance(model, str) or not model
|
|
4873
|
+
for model in pricing["uncertainModels"])
|
|
4874
|
+
or pricing["uncertainModels"] != sorted(set(
|
|
4875
|
+
pricing["uncertainModels"]
|
|
4876
|
+
))
|
|
4877
|
+
)
|
|
4878
|
+
)
|
|
4879
|
+
):
|
|
4880
|
+
raise JournalError(
|
|
4881
|
+
"five_hour_block_close has incomplete computation-time "
|
|
4882
|
+
"pricing provenance"
|
|
4883
|
+
)
|
|
4884
|
+
positive_models = {
|
|
4885
|
+
child["model"] for child in payload.get("_models", [])
|
|
4886
|
+
if sum(int(child.get(key) or 0) for key in (
|
|
4887
|
+
"input_tokens", "output_tokens", "cache_create_tokens",
|
|
4888
|
+
"cache_read_tokens",
|
|
4889
|
+
)) > 0
|
|
4890
|
+
}
|
|
4891
|
+
if not (
|
|
4892
|
+
set(pricing["unpricedModels"])
|
|
4893
|
+
| set(pricing.get("uncertainModels") or [])
|
|
4894
|
+
) <= positive_models:
|
|
4895
|
+
raise JournalError(
|
|
4896
|
+
"five_hour_block_close pricing provenance does not match "
|
|
4897
|
+
"its computed model population"
|
|
4898
|
+
)
|
|
4899
|
+
payload["_pricing"] = pricing
|
|
4837
4900
|
# #750 S3 §1.1: ONE producer for the id parts and for the suppression-map
|
|
4838
4901
|
# key. A family with an `id_parts_fn` (today only `week_reset_events`,
|
|
4839
4902
|
# whose identity is dual-shaped) supplies both from that callable; every
|
|
@@ -4851,11 +4914,85 @@ def _build_harvest_evt(ctx, spec, row):
|
|
|
4851
4914
|
supp = ctx.suppression_map.get(supp_key)
|
|
4852
4915
|
if supp:
|
|
4853
4916
|
payload["suppression"] = list(supp)
|
|
4917
|
+
provenance = ctx.pricing_provenance.get((spec.kind, int(row["id"])))
|
|
4918
|
+
if provenance is not None:
|
|
4919
|
+
payload["_pricing"] = provenance
|
|
4854
4920
|
eid = _lib_journal.evt_id(spec.id_prefix, *parts)
|
|
4921
|
+
refire_prefix = _LATCH_REFIRE_ID_PREFIXES.get(spec.kind)
|
|
4922
|
+
if (
|
|
4923
|
+
refire_prefix is not None
|
|
4924
|
+
and ctx.event_sink is None
|
|
4925
|
+
and _latch_event_was_retired(conn, eid)
|
|
4926
|
+
):
|
|
4927
|
+
# #869 F6/V1: a correction retired this natural key's earlier crossing
|
|
4928
|
+
# with a higher-revision tombstone, so a revision-0 event under the
|
|
4929
|
+
# same id can never become effective. A later crossing is a new
|
|
4930
|
+
# incarnation: its id adds the crossing instant, while the table's
|
|
4931
|
+
# natural key still deduplicates it. Every other crossing, and every
|
|
4932
|
+
# already-journaled id, keeps the legacy spelling.
|
|
4933
|
+
eid = _lib_journal.evt_id(
|
|
4934
|
+
refire_prefix, *parts, row["crossed_at_utc"])
|
|
4935
|
+
payload["journal_identity_version"] = 2
|
|
4855
4936
|
at = row[spec.at_column] if spec.at_column else _now_iso()
|
|
4856
4937
|
return _lib_journal.make_evt(kind=spec.kind, id=eid, at=at, payload=payload)
|
|
4857
4938
|
|
|
4858
4939
|
|
|
4940
|
+
#: #869 F6/V1. The id prefix, per alert-latch kind, of a crossing that re-fires
|
|
4941
|
+
#: after a correction tombstoned the natural key's earlier crossing (the legacy
|
|
4942
|
+
#: prefix + identity version 2). `db rederive` retires these latches, so each
|
|
4943
|
+
#: one needs a spelling a later live crossing can journal under.
|
|
4944
|
+
#:
|
|
4945
|
+
#: Coupling: the live writers (`_claude_latch_recreates_retired_crossing` in
|
|
4946
|
+
#: `_cctally_record`) stamp a recreated CLAUDE latch without notifying, on the
|
|
4947
|
+
#: premise that every Claude-latch tombstone is the claude-usage family's
|
|
4948
|
+
#: stale-config retirement, never a verdict that the crossing was false. A
|
|
4949
|
+
#: future rule that retires a Claude latch as a false crossing must record
|
|
4950
|
+
#: that distinction durably, or the genuine later crossing is silently
|
|
4951
|
+
#: suppressed.
|
|
4952
|
+
_LATCH_REFIRE_ID_PREFIXES = {
|
|
4953
|
+
"budget": "bm2",
|
|
4954
|
+
"projected": "pjm2",
|
|
4955
|
+
"project_budget": "pbm2",
|
|
4956
|
+
}
|
|
4957
|
+
|
|
4958
|
+
|
|
4959
|
+
def _latch_event_was_retired(conn, legacy_event_id) -> bool:
|
|
4960
|
+
"""True iff the effective selection for this natural-key id is a tombstone.
|
|
4961
|
+
|
|
4962
|
+
Read from the live effective metadata the rebuild published, never
|
|
4963
|
+
recomputed. A missing row or an active event keeps the legacy id.
|
|
4964
|
+
"""
|
|
4965
|
+
prior = _metadata_row(conn, legacy_event_id)
|
|
4966
|
+
return prior is not None and str(prior[1]) == "tombstone"
|
|
4967
|
+
|
|
4968
|
+
|
|
4969
|
+
def latch_crossing_was_retired(conn, kind, rowid) -> bool:
|
|
4970
|
+
"""True iff alert-latch row ``rowid`` re-creates a retired crossing (#869).
|
|
4971
|
+
|
|
4972
|
+
``kind`` is a latch family with a re-fire incarnation (``budget``,
|
|
4973
|
+
``projected``, ``project_budget``). The row's legacy natural-key event id
|
|
4974
|
+
is minted exactly as harvest mints it, and the answer is the live effective
|
|
4975
|
+
metadata's: a tombstone means a completed correction batch retired an
|
|
4976
|
+
earlier crossing of this natural key, so harvest will journal this row
|
|
4977
|
+
under the kind's version-2 prefix. A live writer calls this in the same
|
|
4978
|
+
transaction as its insert, before harvest runs, so both decisions read one
|
|
4979
|
+
durable fact.
|
|
4980
|
+
"""
|
|
4981
|
+
if kind not in _LATCH_REFIRE_ID_PREFIXES:
|
|
4982
|
+
raise ValueError(f"{kind!r} has no re-fire incarnation")
|
|
4983
|
+
[spec] = [item for item in _HARVEST_SPECS if item.kind == kind]
|
|
4984
|
+
if spec.fk_refs or spec.id_parts_fn is not None:
|
|
4985
|
+
raise ValueError(f"{kind!r} ids are not plain natural-key columns")
|
|
4986
|
+
row = conn.execute(
|
|
4987
|
+
f"SELECT {', '.join(spec.id_parts)} FROM {spec.table} WHERE id = ?",
|
|
4988
|
+
(int(rowid),),
|
|
4989
|
+
).fetchone()
|
|
4990
|
+
if row is None:
|
|
4991
|
+
return False
|
|
4992
|
+
return _latch_event_was_retired(
|
|
4993
|
+
conn, _lib_journal.evt_id(spec.id_prefix, *tuple(row)))
|
|
4994
|
+
|
|
4995
|
+
|
|
4859
4996
|
def _emit_harvest_row(ctx, spec, row):
|
|
4860
4997
|
"""Journal and stamp one already-selected natural-keyed row."""
|
|
4861
4998
|
conn = ctx.conn
|
|
@@ -5334,7 +5471,10 @@ def _run_config_reconcile(ctx, reconcile_config) -> None:
|
|
|
5334
5471
|
if fn is None:
|
|
5335
5472
|
continue
|
|
5336
5473
|
try:
|
|
5337
|
-
|
|
5474
|
+
if axis == "codex_budget":
|
|
5475
|
+
fn(validated_budget, conn=ctx.conn, journal_ctx=ctx)
|
|
5476
|
+
else:
|
|
5477
|
+
fn(validated_budget, conn=ctx.conn)
|
|
5338
5478
|
except Exception as exc: # best-effort; never break the cycle over a reconcile
|
|
5339
5479
|
print(f"[budget-reconcile] {name} failed: {exc}", file=sys.stderr)
|
|
5340
5480
|
# Per-project reconcile takes `touched_projects` as its 2nd positional
|
|
@@ -6086,7 +6226,8 @@ def _effective_event_for_convergence(conn, event_id) -> dict:
|
|
|
6086
6226
|
|
|
6087
6227
|
# Effect keys that ride an evt payload but are NOT target-table columns.
|
|
6088
6228
|
_EVT_EFFECT_KEYS = frozenset(
|
|
6089
|
-
{"kind", "suppression", "suppression_table", "floor_suppression",
|
|
6229
|
+
{"kind", "suppression", "suppression_table", "floor_suppression",
|
|
6230
|
+
"hwm_floor", "_pricing", "journal_identity_version"}
|
|
6090
6231
|
)
|
|
6091
6232
|
|
|
6092
6233
|
|
|
@@ -8020,7 +8161,7 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
|
|
|
8020
8161
|
# omitted column, constraint, partial predicate, or index definition. An epoch
|
|
8021
8162
|
# schema change must update this contract alongside STATS_INDEX_EPOCH.
|
|
8022
8163
|
_REBUILD_SCHEMA_FINGERPRINT = (
|
|
8023
|
-
"
|
|
8164
|
+
"e0f2225f00843952b4887fa8afb046edacd3da0642d0f402ebaa5219a6e30697"
|
|
8024
8165
|
)
|
|
8025
8166
|
|
|
8026
8167
|
|