cctally 1.113.0 → 1.114.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +4 -6
- package/bin/_cctally_cache.py +1683 -485
- package/bin/_cctally_core.py +223 -1
- package/bin/_cctally_dashboard.py +96 -18
- package/bin/_cctally_dashboard_perf.py +128 -9
- package/bin/_cctally_dashboard_sources.py +267 -204
- package/bin/_cctally_db.py +861 -18
- package/bin/_cctally_diagnosis_sources.py +22 -4
- package/bin/_cctally_doctor.py +136 -18
- package/bin/_cctally_five_hour.py +2 -2
- package/bin/_cctally_forecast.py +27 -6
- package/bin/_cctally_journal.py +67 -5
- package/bin/_cctally_milestone_history.py +82 -9
- package/bin/_cctally_percent_breakdown.py +3 -2
- package/bin/_cctally_quota.py +201 -33
- package/bin/_cctally_record.py +14 -0
- package/bin/_cctally_rederive.py +23 -9
- package/bin/_cctally_source_analytics.py +60 -15
- package/bin/_cctally_statusline.py +17 -0
- package/bin/_cctally_store.py +80 -5
- package/bin/_cctally_tui.py +4 -3
- package/bin/_cctally_weekrefs.py +128 -21
- package/bin/_lib_cache_report.py +35 -6
- package/bin/_lib_codex_conversation.py +32 -0
- package/bin/_lib_codex_conversation_query.py +164 -30
- package/bin/_lib_conversation_query.py +95 -56
- package/bin/_lib_conversation_retention.py +1605 -640
- package/bin/_lib_cost_provenance.py +53 -2
- package/bin/_lib_doctor.py +263 -69
- package/bin/_lib_fmt.py +11 -0
- package/bin/_lib_merge_gate.py +469 -51
- package/bin/_lib_perf.py +27 -1
- package/bin/_lib_pricing.py +193 -8
- package/bin/_lib_pricing_check.py +20 -4
- package/bin/_lib_reclaim_planner.py +792 -0
- package/bin/_lib_rederive.py +225 -77
- package/bin/_lib_snapshot_cache.py +21 -0
- package/bin/_lib_sqlite_close.py +81 -0
- package/bin/_lib_subscription_weeks.py +100 -17
- package/bin/_lib_tick_stats.py +217 -17
- package/bin/_lib_wal_checkpoint.py +1077 -0
- package/bin/_lib_write_budget.py +483 -0
- package/bin/_lib_write_io.py +703 -0
- package/bin/cctally +8 -0
- package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
- package/dashboard/static/assets/index-BKITk1CT.css +1 -0
- package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +6 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
- package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
|
@@ -1984,10 +1984,12 @@ _CODEX_THREADS_BY_NATIVE_SQL = """
|
|
|
1984
1984
|
# reading `cache_create_tokens` off `session_entries` carries the split
|
|
1985
1985
|
# column, because a missed site does not raise and does not move a golden — it
|
|
1986
1986
|
# silently under-prices — and this map is one keystroke away from feeding a
|
|
1987
|
-
# cost fold.
|
|
1987
|
+
# cost fold. `input_tokens` (#929) is appended LAST so the existing row
|
|
1988
|
+
# indexes stay valid: a flagged turn's wasted estimate selects its pricing card
|
|
1989
|
+
# from the request's real prompt, input + creation + read.
|
|
1988
1990
|
_CLAUDE_TURN_TOKENS_SQL = (
|
|
1989
1991
|
"SELECT msg_id, req_id, cache_create_tokens, cache_read_tokens, "
|
|
1990
|
-
" cache_create_1h_tokens, speed "
|
|
1992
|
+
" cache_create_1h_tokens, speed, input_tokens "
|
|
1991
1993
|
"FROM session_entries WHERE "
|
|
1992
1994
|
)
|
|
1993
1995
|
|
|
@@ -2148,7 +2150,8 @@ def _placeholders(count: int) -> str:
|
|
|
2148
2150
|
def _claude_turn_token_map(conn: sqlite3.Connection,
|
|
2149
2151
|
keys: Sequence[tuple], account_key: str | None
|
|
2150
2152
|
) -> dict[tuple, dict]:
|
|
2151
|
-
"""`{(msg_id, req_id): {"cache_creation", "cache_read", "
|
|
2153
|
+
"""`{(msg_id, req_id): {"cache_creation", "cache_read", "cache_1h",
|
|
2154
|
+
"speed", "input"}}`.
|
|
2152
2155
|
|
|
2153
2156
|
The cache-churn walk begins BEFORE the window, and those turns are state:
|
|
2154
2157
|
their tokens move the running maximum and they contribute no support,
|
|
@@ -2171,6 +2174,7 @@ def _claude_turn_token_map(conn: sqlite3.Connection,
|
|
|
2171
2174
|
"cache_read": int(row[3] or 0),
|
|
2172
2175
|
"cache_1h": row[4],
|
|
2173
2176
|
"speed": row[5],
|
|
2177
|
+
"input": int(row[6] or 0),
|
|
2174
2178
|
}
|
|
2175
2179
|
return usage
|
|
2176
2180
|
|
|
@@ -2432,8 +2436,13 @@ def _evaluate_cache_churn(scope: DiagnosisScope, bundle: StoreBundle,
|
|
|
2432
2436
|
continue
|
|
2433
2437
|
flagged_keys.add(key)
|
|
2434
2438
|
flagged_sessions.add(session)
|
|
2439
|
+
# #929: the flagged request's real prompt selects the card.
|
|
2440
|
+
turn = usage.get(key) or {}
|
|
2435
2441
|
wasted.append(query._cache_failure_wasted_usd(
|
|
2436
|
-
model, lost, speed=speed
|
|
2442
|
+
model, lost, speed=speed,
|
|
2443
|
+
prompt_tokens=(turn.get("input", 0)
|
|
2444
|
+
+ turn.get("cache_creation", 0)
|
|
2445
|
+
+ turn.get("cache_read", 0))))
|
|
2437
2446
|
|
|
2438
2447
|
known = set(all_sessions)
|
|
2439
2448
|
seeded = set(streams)
|
|
@@ -4412,6 +4421,15 @@ def _start_forked_provider(args) -> tuple[int, int]:
|
|
|
4412
4421
|
with os.fdopen(write_fd, "wb") as stream:
|
|
4413
4422
|
pickle.dump(payload, stream, protocol=5)
|
|
4414
4423
|
finally:
|
|
4424
|
+
# #901 W9 (Q14): `os._exit` skips `atexit`; a provider task that
|
|
4425
|
+
# synced a store hands its frames to the checkpoint policy's
|
|
4426
|
+
# finalizer. With nothing armed it does nothing.
|
|
4427
|
+
try:
|
|
4428
|
+
import _lib_wal_checkpoint
|
|
4429
|
+
|
|
4430
|
+
_lib_wal_checkpoint.finalize()
|
|
4431
|
+
except BaseException: # noqa: BLE001
|
|
4432
|
+
pass
|
|
4415
4433
|
os._exit(0)
|
|
4416
4434
|
os.close(write_fd)
|
|
4417
4435
|
return pid, read_fd
|
package/bin/_cctally_doctor.py
CHANGED
|
@@ -470,6 +470,78 @@ def _gather_writer_guard_log(now_utc: dt.datetime):
|
|
|
470
470
|
}
|
|
471
471
|
|
|
472
472
|
|
|
473
|
+
#: Bounded per-instance probe of a running dashboard's write telemetry.
|
|
474
|
+
DASHBOARD_WRITE_IO_TIMEOUT_SECONDS = 2.0
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def _fetch_dashboard_write_io(descriptor: dict, *, timeout_s: float) -> dict:
|
|
478
|
+
"""One loopback probe of a discovered dashboard (#901 §5.6).
|
|
479
|
+
|
|
480
|
+
Reuses `dashboard-perf`'s client — IP-literal loopback target, an
|
|
481
|
+
`Origin` matching the `Host` — and sends NO token, so an instance that
|
|
482
|
+
requires one answers 401 and is reported `authentication_required`: no
|
|
483
|
+
authentication bypass exists. The instance id in the answer must equal
|
|
484
|
+
the descriptor's, or the probe is an `identity_mismatch`."""
|
|
485
|
+
import _lib_write_io as wio
|
|
486
|
+
|
|
487
|
+
perf = _cctally()._cctally_dashboard_perf
|
|
488
|
+
base = {"instance_id": descriptor["instanceId"],
|
|
489
|
+
"port": descriptor["port"], "pid": descriptor["pid"]}
|
|
490
|
+
host = wio.loopback_target(descriptor.get("host"))
|
|
491
|
+
if host is None:
|
|
492
|
+
return {**base, "probe": "failed", "reason": "not_loopback_reachable",
|
|
493
|
+
"write_io": None}
|
|
494
|
+
try:
|
|
495
|
+
payload = perf._request(host, descriptor["port"], perf.WRITE_IO_PATH,
|
|
496
|
+
token=None, timeout=timeout_s)
|
|
497
|
+
except perf.DashboardPerfError as exc:
|
|
498
|
+
return {**base, "probe": "failed", "reason": exc.reason,
|
|
499
|
+
"write_io": None}
|
|
500
|
+
write_io = payload.get("writeIo") if isinstance(payload, dict) else None
|
|
501
|
+
if not isinstance(write_io, dict):
|
|
502
|
+
return {**base, "probe": "failed", "reason": "malformed_response",
|
|
503
|
+
"write_io": None}
|
|
504
|
+
if write_io.get("instanceId") != descriptor["instanceId"]:
|
|
505
|
+
return {**base, "probe": "failed", "reason": "identity_mismatch",
|
|
506
|
+
"write_io": None}
|
|
507
|
+
return {**base, "probe": "ok", "reason": None, "write_io": write_io}
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def _gather_dashboard_disk_writes(
|
|
511
|
+
*, fetch=None, timeout_s: float = DASHBOARD_WRITE_IO_TIMEOUT_SECONDS,
|
|
512
|
+
discover: bool = True,
|
|
513
|
+
) -> dict:
|
|
514
|
+
"""Running dashboards' write telemetry for `performance.dashboard_disk_
|
|
515
|
+
writes` (#901 §5.6).
|
|
516
|
+
|
|
517
|
+
In the dashboard process the local instance's own snapshot is read
|
|
518
|
+
directly — no self-HTTP and no recursive gather. Anywhere else (the CLI,
|
|
519
|
+
the TUI) instances are discovered through their owner-readable
|
|
520
|
+
descriptors; a stale one (dead or reused pid) is ignored and never
|
|
521
|
+
deleted, because doctor is read-only. Discovery runs only for the
|
|
522
|
+
`cctally doctor` CLI (PR-11); with ``discover`` false nothing is probed and
|
|
523
|
+
the mode is ``not_probed``."""
|
|
524
|
+
import _lib_write_io as wio
|
|
525
|
+
|
|
526
|
+
local = wio.local_instance()
|
|
527
|
+
if local is not None:
|
|
528
|
+
return {"mode": "in_process", "instances": [{
|
|
529
|
+
"instance_id": local["instanceId"], "port": local["port"],
|
|
530
|
+
"pid": local["pid"], "probe": "ok", "reason": None,
|
|
531
|
+
"write_io": wio.write_io_payload(),
|
|
532
|
+
}]}
|
|
533
|
+
if not discover:
|
|
534
|
+
return {"mode": "not_probed", "instances": []}
|
|
535
|
+
probe = fetch or _fetch_dashboard_write_io
|
|
536
|
+
instances = [
|
|
537
|
+
probe(descriptor, timeout_s=timeout_s)
|
|
538
|
+
for descriptor in wio.read_instance_descriptors(_cctally_core.APP_DIR)
|
|
539
|
+
if wio.descriptor_is_live(descriptor)
|
|
540
|
+
]
|
|
541
|
+
instances.sort(key=lambda entry: (entry["port"], entry["instance_id"]))
|
|
542
|
+
return {"mode": "discovery", "instances": instances}
|
|
543
|
+
|
|
544
|
+
|
|
473
545
|
def _gather_statusline_pipeline(c, *, now_utc: dt.datetime) -> dict:
|
|
474
546
|
"""Read the #318 statusline pipeline without creating or pruning files."""
|
|
475
547
|
now_epoch = int(now_utc.timestamp())
|
|
@@ -874,6 +946,26 @@ def _gather_codex_window_attribution_state() -> "dict | None":
|
|
|
874
946
|
conn.close()
|
|
875
947
|
|
|
876
948
|
|
|
949
|
+
def _journal_protocol_violation_rows(conn: sqlite3.Connection) -> list:
|
|
950
|
+
"""Every stored protocol violation, parsed, in (batch, kind, fingerprint)
|
|
951
|
+
order: the doctor's shallow journal leg.
|
|
952
|
+
|
|
953
|
+
#901 Amendment 19 T1 (spec I1): the doctor holds every violation anyway,
|
|
954
|
+
so they are ordered here, in Python, where ``ORDER BY batch_id, kind,
|
|
955
|
+
fingerprint`` left a partial sorter over each batch's rows — and one batch
|
|
956
|
+
is not a bound. ``fingerprint`` is the primary key, so the order is total
|
|
957
|
+
and identical (text compared as UTF-8 bytes, SQLite's BINARY collation).
|
|
958
|
+
"""
|
|
959
|
+
rows = conn.execute(
|
|
960
|
+
"SELECT batch_id, kind, fingerprint, violation_json "
|
|
961
|
+
"FROM journal_protocol_violations"
|
|
962
|
+
).fetchall()
|
|
963
|
+
rows.sort(key=lambda row: tuple(
|
|
964
|
+
(value is not None, str(value or "").encode("utf-8"))
|
|
965
|
+
for value in row[:3]))
|
|
966
|
+
return [json.loads(str(row[3])) for row in rows]
|
|
967
|
+
|
|
968
|
+
|
|
877
969
|
def _gather_accounts_state(now_utc: "dt.datetime") -> dict:
|
|
878
970
|
"""Best-effort account-attribution state for the doctor `accounts.*` legs
|
|
879
971
|
(#341). Never raises: identity + registry reads are read-only and each guard
|
|
@@ -936,14 +1028,21 @@ def _gather_accounts_state(now_utc: "dt.datetime") -> dict:
|
|
|
936
1028
|
cutoff = (now_utc - dt.timedelta(days=7)).astimezone(
|
|
937
1029
|
dt.timezone.utc).isoformat()
|
|
938
1030
|
try:
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
1031
|
+
# #901 Amendment 19 T1 (spec I1): counted in Python over the
|
|
1032
|
+
# streamed rows, one total per class, where `GROUP BY
|
|
1033
|
+
# account_key` sorted the trailing week's every snapshot through a
|
|
1034
|
+
# temp b-tree — and a week of activity is not a bound.
|
|
1035
|
+
attributed = unattributed = 0
|
|
1036
|
+
for (account_key,) in conn.execute(
|
|
1037
|
+
"SELECT account_key FROM weekly_usage_snapshots "
|
|
1038
|
+
"WHERE captured_at_utc >= ?", (cutoff,)
|
|
1039
|
+
):
|
|
943
1040
|
if account_key and account_key != "unattributed":
|
|
944
|
-
|
|
1041
|
+
attributed += 1
|
|
945
1042
|
else:
|
|
946
|
-
|
|
1043
|
+
unattributed += 1
|
|
1044
|
+
state["recent_attributed"] += attributed
|
|
1045
|
+
state["recent_unattributed"] += unattributed
|
|
947
1046
|
except sqlite3.DatabaseError:
|
|
948
1047
|
pass
|
|
949
1048
|
finally:
|
|
@@ -961,9 +1060,13 @@ def _gather_accounts_state(now_utc: "dt.datetime") -> dict:
|
|
|
961
1060
|
# contains no ingest and is not a whole tick; a whole tick is ingest plus
|
|
962
1061
|
# builder, and the builder is 70.5% of one measured live. Against a whole tick
|
|
963
1062
|
# the same probe is therefore 14%. It is one all-history
|
|
964
|
-
# `load_codex_quota_observations` call
|
|
965
|
-
# (PARTITION BY ...)` across every retained row
|
|
966
|
-
#
|
|
1063
|
+
# `load_codex_quota_observations(latest_per_identity=True)` call. Its SQL used
|
|
1064
|
+
# to run a `MAX(...) OVER (PARTITION BY ...)` across every retained row
|
|
1065
|
+
# (277,207 on that store; 565K and ~1.1 GB of temp-file writes per call on the
|
|
1066
|
+
# #901 live store); since #901 it reads the trigger-maintained
|
|
1067
|
+
# `codex_quota_partition_latest` summary plus one indexed seek per raw
|
|
1068
|
+
# partition. The memo below therefore remains a CPU optimisation and is no
|
|
1069
|
+
# longer load-bearing for disk writes.
|
|
967
1070
|
#
|
|
968
1071
|
# Only the ROWS are retained, and only while their evidence is unmoved. Every
|
|
969
1072
|
# `now`-derived interpretation over them is recomputed on every gather:
|
|
@@ -1249,6 +1352,7 @@ def doctor_gather_state(
|
|
|
1249
1352
|
runtime_bind: "str | None" = None,
|
|
1250
1353
|
deep: bool = False,
|
|
1251
1354
|
force_cold_inputs: bool = False,
|
|
1355
|
+
discover_dashboards: bool = False,
|
|
1252
1356
|
):
|
|
1253
1357
|
"""Gather doctor state while excluding cache-family replacement.
|
|
1254
1358
|
|
|
@@ -1262,6 +1366,11 @@ def doctor_gather_state(
|
|
|
1262
1366
|
caller can obtain a genuinely fresh gather at a chosen instant. It exists
|
|
1263
1367
|
for the equality gate that compares a reusing gather against a fresh one at
|
|
1264
1368
|
an identical ``now_utc``; production callers leave it false.
|
|
1369
|
+
|
|
1370
|
+
``discover_dashboards`` (PR-11) lets the gather probe other running
|
|
1371
|
+
dashboards for their disk writes. Only the `cctally doctor` CLI passes it:
|
|
1372
|
+
the TUI and every snapshot build gather on each rebuild, where serial
|
|
1373
|
+
2-second probes of every live descriptor do not belong.
|
|
1265
1374
|
"""
|
|
1266
1375
|
cache_lock = None
|
|
1267
1376
|
cache_probe_allowed = not _cctally_core.CACHE_DB_PATH.exists()
|
|
@@ -1317,6 +1426,7 @@ def doctor_gather_state(
|
|
|
1317
1426
|
_cache_probe_allowed=cache_probe_allowed,
|
|
1318
1427
|
_cache_repair_marker=cache_repair_marker,
|
|
1319
1428
|
_force_cold_inputs=force_cold_inputs,
|
|
1429
|
+
_discover_dashboards=discover_dashboards,
|
|
1320
1430
|
)
|
|
1321
1431
|
finally:
|
|
1322
1432
|
if cache_lock is not None:
|
|
@@ -1334,6 +1444,7 @@ def _doctor_gather_state_impl(
|
|
|
1334
1444
|
_cache_probe_allowed: bool,
|
|
1335
1445
|
_cache_repair_marker: dict,
|
|
1336
1446
|
_force_cold_inputs: bool = False,
|
|
1447
|
+
_discover_dashboards: bool = False,
|
|
1337
1448
|
):
|
|
1338
1449
|
"""I/O chokepoint for `cctally doctor` (spec §7.2).
|
|
1339
1450
|
|
|
@@ -1727,6 +1838,7 @@ def _doctor_gather_state_impl(
|
|
|
1727
1838
|
conv_rollup_sync_in_progress = False
|
|
1728
1839
|
conversations_db_page_count = None
|
|
1729
1840
|
conversations_db_freelist_count = None
|
|
1841
|
+
conversations_db_page_size = None
|
|
1730
1842
|
conversations_reclaim_pending = None
|
|
1731
1843
|
codex_prune_refusals: list[dict] = []
|
|
1732
1844
|
conversation_rollup_pricing_refusal: "dict | None" = None
|
|
@@ -1751,6 +1863,9 @@ def _doctor_gather_state_impl(
|
|
|
1751
1863
|
row = conn.execute("PRAGMA freelist_count").fetchone()
|
|
1752
1864
|
if row and row[0] is not None:
|
|
1753
1865
|
conversations_db_freelist_count = int(row[0])
|
|
1866
|
+
row = conn.execute("PRAGMA page_size").fetchone()
|
|
1867
|
+
if row and row[0] is not None:
|
|
1868
|
+
conversations_db_page_size = int(row[0])
|
|
1754
1869
|
except sqlite3.Error:
|
|
1755
1870
|
pass
|
|
1756
1871
|
try:
|
|
@@ -2407,14 +2522,8 @@ def _doctor_gather_state_impl(
|
|
|
2407
2522
|
if _cctally_core.DB_PATH.exists():
|
|
2408
2523
|
pc = _stats_ro_guarded()
|
|
2409
2524
|
try:
|
|
2410
|
-
protocol_rows =
|
|
2411
|
-
|
|
2412
|
-
for row in pc.execute(
|
|
2413
|
-
"SELECT violation_json "
|
|
2414
|
-
"FROM journal_protocol_violations "
|
|
2415
|
-
"ORDER BY batch_id, kind, fingerprint"
|
|
2416
|
-
)
|
|
2417
|
-
]
|
|
2525
|
+
protocol_rows = (
|
|
2526
|
+
_journal_protocol_violation_rows(pc))
|
|
2418
2527
|
journal_protocol_violations = [
|
|
2419
2528
|
item for item in protocol_rows
|
|
2420
2529
|
if not item.get("auditId")
|
|
@@ -2696,6 +2805,13 @@ def _doctor_gather_state_impl(
|
|
|
2696
2805
|
_gather_codex_window_attribution_state()
|
|
2697
2806
|
if _cache_probe_allowed else None)
|
|
2698
2807
|
|
|
2808
|
+
# #901 §5.6: running dashboards' disk writes. Read-only; degrades.
|
|
2809
|
+
try:
|
|
2810
|
+
with _lib_perf.phase("doctor.dashboard_disk_writes"):
|
|
2811
|
+
dashboard_disk_writes = _gather_dashboard_disk_writes(
|
|
2812
|
+
discover=_discover_dashboards)
|
|
2813
|
+
except Exception: # noqa: BLE001 — a probe failure must not fail doctor
|
|
2814
|
+
dashboard_disk_writes = {"mode": "error", "instances": []}
|
|
2699
2815
|
return _lib_doctor.DoctorState(
|
|
2700
2816
|
symlink_state=symlink_state,
|
|
2701
2817
|
path_includes_local_bin=path_includes,
|
|
@@ -2792,6 +2908,8 @@ def _doctor_gather_state_impl(
|
|
|
2792
2908
|
cache_db_freelist_count=cache_db_freelist_count,
|
|
2793
2909
|
conversations_db_page_count=conversations_db_page_count,
|
|
2794
2910
|
conversations_db_freelist_count=conversations_db_freelist_count,
|
|
2911
|
+
conversations_db_page_size=conversations_db_page_size,
|
|
2912
|
+
dashboard_disk_writes=dashboard_disk_writes,
|
|
2795
2913
|
conversations_reclaim_pending=conversations_reclaim_pending,
|
|
2796
2914
|
codex_quota_windows=codex_quota_windows,
|
|
2797
2915
|
codex_hook_roots=codex_hook_roots,
|
|
@@ -2846,7 +2964,7 @@ def cmd_doctor(args: argparse.Namespace) -> int:
|
|
|
2846
2964
|
# #279 S2 F5b: deep=True runs PRAGMA quick_check(1) — CLI-only (the
|
|
2847
2965
|
# dashboard/TUI gather callers stay deep=False so their per-rebuild
|
|
2848
2966
|
# gather never pays the multi-second cost on a large cache.db).
|
|
2849
|
-
state = c.doctor_gather_state(deep=True)
|
|
2967
|
+
state = c.doctor_gather_state(deep=True, discover_dashboards=True)
|
|
2850
2968
|
report = _lib_doctor.run_checks(state)
|
|
2851
2969
|
if getattr(args, "json", False):
|
|
2852
2970
|
print(encode_dashboard_json(
|
|
@@ -2689,10 +2689,10 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
|
|
|
2689
2689
|
])
|
|
2690
2690
|
continue
|
|
2691
2691
|
m = ev
|
|
2692
|
-
cum =
|
|
2692
|
+
cum = _c._fmt_usd_accounting(m['blockCostUSD'], 6)
|
|
2693
2693
|
marg = (
|
|
2694
2694
|
"n/a" if m["marginalCostUSD"] is None
|
|
2695
|
-
else
|
|
2695
|
+
else _c._fmt_usd_accounting(m['marginalCostUSD'], 6)
|
|
2696
2696
|
)
|
|
2697
2697
|
# #834 S1 (#836): the EFFECTIVE weekly value, never the raw one. A
|
|
2698
2698
|
# weekly-clamped tick stored a reading no reader ever saw, and this
|
package/bin/_cctally_forecast.py
CHANGED
|
@@ -153,6 +153,25 @@ def _shape_week_samples(rows, *, include_held: bool,
|
|
|
153
153
|
return out
|
|
154
154
|
|
|
155
155
|
|
|
156
|
+
def _week_rows_by_capture(cursor, *, by_id: bool) -> list:
|
|
157
|
+
"""One week's sample rows in capture order: ``captured_at_utc`` ascending,
|
|
158
|
+
then ``id`` when ``by_id`` (the row's trailing column, which this strips).
|
|
159
|
+
|
|
160
|
+
#901 Amendment 19 T1 (spec I1): the reader returns every row of the week
|
|
161
|
+
anyway, so ordering them here costs no structure beyond its own result,
|
|
162
|
+
where ``ORDER BY captured_at_utc`` sorted the whole week through a temp
|
|
163
|
+
b-tree — and one subscription week is not a bound. ``list.sort`` is stable
|
|
164
|
+
over the same scan SQLite's in-memory sorter consumed, so rows tied on one
|
|
165
|
+
capture instant keep the order they had.
|
|
166
|
+
"""
|
|
167
|
+
rows = cursor.fetchall()
|
|
168
|
+
if by_id:
|
|
169
|
+
rows.sort(key=lambda row: (row[0], row[-1]))
|
|
170
|
+
return [row[:-1] for row in rows]
|
|
171
|
+
rows.sort(key=lambda row: row[0])
|
|
172
|
+
return rows
|
|
173
|
+
|
|
174
|
+
|
|
156
175
|
def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime,
|
|
157
176
|
*, account_key: "str | None" = None,
|
|
158
177
|
include_held: bool = False,
|
|
@@ -264,6 +283,8 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
|
|
|
264
283
|
we_date = dt.date.fromisoformat(drow[1])
|
|
265
284
|
week_start_at = dt.datetime.combine(ws_date, dt.time(0, 0), local_tz).astimezone(dt.timezone.utc)
|
|
266
285
|
week_end_at = dt.datetime.combine(we_date + dt.timedelta(days=1), dt.time(0, 0), local_tz).astimezone(dt.timezone.utc)
|
|
286
|
+
# One date: `idx_usage_week_time` (week_start_date, captured_at_utc
|
|
287
|
+
# DESC, id DESC) serves this order outright, with no sorter.
|
|
267
288
|
rows = conn.execute(
|
|
268
289
|
"SELECT captured_at_utc, weekly_percent, five_hour_percent, "
|
|
269
290
|
" weekly_observation_held" + _owner_column + " "
|
|
@@ -293,16 +314,16 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
|
|
|
293
314
|
matching_texts.append(r[0])
|
|
294
315
|
chosen_date = chosen[2]
|
|
295
316
|
placeholders = ",".join("?" * len(matching_texts))
|
|
296
|
-
rows = conn.execute(
|
|
317
|
+
rows = _week_rows_by_capture(conn.execute(
|
|
297
318
|
f"SELECT captured_at_utc, weekly_percent, five_hour_percent, "
|
|
298
|
-
f" weekly_observation_held{_owner_column}
|
|
319
|
+
f" weekly_observation_held{_owner_column}"
|
|
320
|
+
f"{', id' if include_account else ''} "
|
|
299
321
|
f"FROM weekly_usage_snapshots "
|
|
300
322
|
f"WHERE (week_start_at IN ({placeholders}) "
|
|
301
323
|
f" OR (week_start_at IS NULL AND week_start_date = ?))"
|
|
302
|
-
f"{_acct_pred}{_held_pred}
|
|
303
|
-
f"ORDER BY captured_at_utc{_sample_order}",
|
|
324
|
+
f"{_acct_pred}{_held_pred}",
|
|
304
325
|
tuple(matching_texts) + (chosen_date,) + _acct_p,
|
|
305
|
-
)
|
|
326
|
+
), by_id=include_account)
|
|
306
327
|
samples = _shape_week_samples(rows, include_held=include_held,
|
|
307
328
|
include_account=include_account)
|
|
308
329
|
samples = [s for s in samples if s[0] <= now_utc]
|
|
@@ -2518,7 +2539,7 @@ def cmd_report(args: argparse.Namespace) -> int:
|
|
|
2518
2539
|
m_rows: list[list[str]] = []
|
|
2519
2540
|
for idx, m in enumerate(milestone_rows, start=1):
|
|
2520
2541
|
pct = f"{int(m['percent_threshold'])}%"
|
|
2521
|
-
cum =
|
|
2542
|
+
cum = c._fmt_usd_accounting(float(m['cumulative_cost_usd']), 6)
|
|
2522
2543
|
marg = c.observation_gap_marginal_cell(
|
|
2523
2544
|
m["marginal_cost_usd"],
|
|
2524
2545
|
withheld=(idx - 1) in gap_disclosure.withheld_indexes,
|
package/bin/_cctally_journal.py
CHANGED
|
@@ -2083,13 +2083,18 @@ _FILE_ACCOUNT_INSERT = (
|
|
|
2083
2083
|
# to reach this statement for a record the map insert dropped, so the two guards
|
|
2084
2084
|
# are belt-and-suspenders over disjoint failure modes: this one covers any
|
|
2085
2085
|
# constraint the incarnation table might gain that the map table lacks.
|
|
2086
|
+
#
|
|
2087
|
+
# #901 §5.3c (W8): the MAX-set writes only when the incarnation advances, like
|
|
2088
|
+
# its ingest twin `_cctally_cache.set_codex_file_incarnation`, so replaying a
|
|
2089
|
+
# decision the store already holds issues no write.
|
|
2086
2090
|
_FILE_INCARNATION_INSERT = (
|
|
2087
2091
|
"INSERT OR IGNORE INTO codex_file_incarnations "
|
|
2088
2092
|
"(file_identity, incarnation, updated_at_utc) "
|
|
2089
2093
|
"VALUES (?,?,?) "
|
|
2090
2094
|
"ON CONFLICT(file_identity) DO UPDATE SET "
|
|
2091
|
-
" incarnation =
|
|
2092
|
-
" updated_at_utc = excluded.updated_at_utc"
|
|
2095
|
+
" incarnation = excluded.incarnation, "
|
|
2096
|
+
" updated_at_utc = excluded.updated_at_utc "
|
|
2097
|
+
"WHERE excluded.incarnation > codex_file_incarnations.incarnation"
|
|
2093
2098
|
)
|
|
2094
2099
|
|
|
2095
2100
|
|
|
@@ -2496,6 +2501,14 @@ _QUOTA_SNAPSHOT_UPSERT_CLAUSE = (
|
|
|
2496
2501
|
)
|
|
2497
2502
|
|
|
2498
2503
|
_QUOTA_SNAPSHOT_UPSERT = _QUOTA_SNAPSHOT_INSERT + _QUOTA_SNAPSHOT_UPSERT_CLAUSE
|
|
2504
|
+
|
|
2505
|
+
# The natural key of `_QUOTA_SNAPSHOT_UPSERT_CLAUSE`, read before a replay
|
|
2506
|
+
# inserts (#901 W11): `(source, source_path, line_offset, logical_limit_key)`.
|
|
2507
|
+
_QUOTA_SNAPSHOT_STORED_ACCOUNT = (
|
|
2508
|
+
"SELECT account_key FROM quota_window_snapshots "
|
|
2509
|
+
"WHERE source = ? AND source_path = ? AND line_offset = ? "
|
|
2510
|
+
"AND logical_limit_key = ?"
|
|
2511
|
+
)
|
|
2499
2512
|
_QUOTA_SNAPSHOT_UPSERT_LEGACY = (
|
|
2500
2513
|
_QUOTA_SNAPSHOT_INSERT_LEGACY + _QUOTA_SNAPSHOT_UPSERT_CLAUSE
|
|
2501
2514
|
)
|
|
@@ -2695,8 +2708,20 @@ def _apply_quota_records(
|
|
|
2695
2708
|
row_values = _quota_snapshot_values(rec, anchor)
|
|
2696
2709
|
if not has_anchor:
|
|
2697
2710
|
row_values = row_values[:-1]
|
|
2711
|
+
# #901 §5.3c (W11, Q14): an INSERT into this AUTOINCREMENT table
|
|
2712
|
+
# rewrites `sqlite_sequence` even when every row it carries is
|
|
2713
|
+
# rejected, so replaying observations the walk already materialized
|
|
2714
|
+
# wrote a page per ingest cycle. The stored row is read first and an
|
|
2715
|
+
# insert that would change nothing is not issued; a genuine
|
|
2716
|
+
# correction (a decision now governing a row stamped otherwise) still
|
|
2717
|
+
# runs. AUTOINCREMENT and the sequence values are untouched.
|
|
2718
|
+
stored = cache.execute(
|
|
2719
|
+
_QUOTA_SNAPSHOT_STORED_ACCOUNT,
|
|
2720
|
+
(row_values[0], row_values[2], row_values[3], row_values[6]),
|
|
2721
|
+
).fetchone()
|
|
2698
2722
|
if not covered:
|
|
2699
|
-
|
|
2723
|
+
if stored is None:
|
|
2724
|
+
cache.execute(insert_sql, row_values)
|
|
2700
2725
|
continue
|
|
2701
2726
|
observed = rec.get("account")
|
|
2702
2727
|
payload = rec.get("payload") or {}
|
|
@@ -2719,6 +2744,8 @@ def _apply_quota_records(
|
|
|
2719
2744
|
)
|
|
2720
2745
|
values = list(row_values)
|
|
2721
2746
|
values[16] = decided
|
|
2747
|
+
if stored is not None and stored[0] == decided:
|
|
2748
|
+
continue # the upsert's guarded DO UPDATE would change nothing
|
|
2722
2749
|
cache.execute(upsert_sql, tuple(values))
|
|
2723
2750
|
|
|
2724
2751
|
|
|
@@ -3259,7 +3286,12 @@ def _cache_applier(decoded, *, cursor=None, covered_to=None,
|
|
|
3259
3286
|
return stop_idx
|
|
3260
3287
|
try:
|
|
3261
3288
|
import _cctally_cache
|
|
3289
|
+
import _cctally_store
|
|
3262
3290
|
cache.execute("PRAGMA busy_timeout=15000")
|
|
3291
|
+
# #901 §4.7 (Q11): a raw cache writer, so it holds its statement
|
|
3292
|
+
# journals in memory like the store openers' writers do. Only the
|
|
3293
|
+
# temp store; locks and transaction order are unchanged.
|
|
3294
|
+
_cctally_store.apply_writer_temp_store(cache)
|
|
3263
3295
|
# Read and check the predecessor BEFORE the transaction opens
|
|
3264
3296
|
# (spec §4.3, as corrected). `prior_is_extendable` checks contiguity
|
|
3265
3297
|
# against this cycle's STARTING cursor — a predecessor applied
|
|
@@ -8117,6 +8149,8 @@ _REBUILD_REQUIRED_TABLES = frozenset(
|
|
|
8117
8149
|
)
|
|
8118
8150
|
_REBUILD_REQUIRED_INDEXES = frozenset(
|
|
8119
8151
|
{
|
|
8152
|
+
"idx_budget_milestones_alert_digest",
|
|
8153
|
+
"idx_budget_milestones_codex_digest",
|
|
8120
8154
|
"idx_budget_milestones_journal_id",
|
|
8121
8155
|
"idx_budget_milestones_journal_id_null",
|
|
8122
8156
|
"idx_cost_week_start_at_time",
|
|
@@ -8128,6 +8162,7 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
|
|
|
8128
8162
|
"idx_five_hour_blocks_block_start",
|
|
8129
8163
|
"idx_five_hour_blocks_journal_id",
|
|
8130
8164
|
"idx_five_hour_blocks_journal_id_null",
|
|
8165
|
+
"idx_five_hour_milestones_alert_digest",
|
|
8131
8166
|
"idx_five_hour_milestones_block",
|
|
8132
8167
|
"idx_five_hour_milestones_journal_id",
|
|
8133
8168
|
"idx_five_hour_milestones_journal_id_null",
|
|
@@ -8135,17 +8170,36 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
|
|
|
8135
8170
|
"idx_five_hour_reset_events_journal_id_null",
|
|
8136
8171
|
"idx_journal_protocol_violations_batch",
|
|
8137
8172
|
"idx_meter_rate_change_events_key",
|
|
8173
|
+
"idx_percent_milestones_alert_digest",
|
|
8138
8174
|
"idx_percent_milestones_journal_id",
|
|
8139
8175
|
"idx_percent_milestones_journal_id_null",
|
|
8176
|
+
"idx_percent_milestones_week_date",
|
|
8177
|
+
"idx_project_budget_milestones_alert_digest",
|
|
8140
8178
|
"idx_project_budget_milestones_journal_id",
|
|
8141
8179
|
"idx_project_budget_milestones_journal_id_null",
|
|
8180
|
+
"idx_projected_milestones_alert_digest",
|
|
8181
|
+
"idx_projected_milestones_codex_digest",
|
|
8142
8182
|
"idx_projected_milestones_journal_id",
|
|
8143
8183
|
"idx_projected_milestones_journal_id_null",
|
|
8144
8184
|
"idx_quota_blocks_active",
|
|
8185
|
+
"idx_quota_blocks_digest",
|
|
8186
|
+
"idx_quota_blocks_root_group_pairs",
|
|
8187
|
+
"idx_quota_blocks_weekly_reset_order",
|
|
8188
|
+
"idx_quota_blocks_weekly_single_root_order",
|
|
8145
8189
|
"idx_quota_milestones_active",
|
|
8190
|
+
"idx_quota_milestones_digest",
|
|
8191
|
+
"idx_quota_milestones_root",
|
|
8192
|
+
"idx_quota_projection_state_digest",
|
|
8146
8193
|
"idx_quota_threshold_events_active",
|
|
8194
|
+
"idx_quota_threshold_events_digest",
|
|
8195
|
+
"idx_quota_threshold_events_root",
|
|
8196
|
+
"idx_usage_subscription_anchor_order",
|
|
8197
|
+
"idx_usage_subscription_anchor_pick",
|
|
8198
|
+
"idx_usage_week_boundary_group",
|
|
8199
|
+
"idx_usage_week_date_group",
|
|
8147
8200
|
"idx_usage_week_start_at_time",
|
|
8148
8201
|
"idx_usage_week_time",
|
|
8202
|
+
"idx_week_reset_events_effective_order",
|
|
8149
8203
|
"idx_week_reset_events_journal_id",
|
|
8150
8204
|
"idx_week_reset_events_journal_id_null",
|
|
8151
8205
|
"idx_week_reset_events_legacy_tuple",
|
|
@@ -8161,7 +8215,7 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
|
|
|
8161
8215
|
# omitted column, constraint, partial predicate, or index definition. An epoch
|
|
8162
8216
|
# schema change must update this contract alongside STATS_INDEX_EPOCH.
|
|
8163
8217
|
_REBUILD_SCHEMA_FINGERPRINT = (
|
|
8164
|
-
"
|
|
8218
|
+
"2ef6dab067703e612360245f9fbe4522290046dc6d8d7b009798a384cb5c27b8"
|
|
8165
8219
|
)
|
|
8166
8220
|
|
|
8167
8221
|
|
|
@@ -9006,7 +9060,11 @@ def _open_publication_connection(destination) -> sqlite3.Connection:
|
|
|
9006
9060
|
recover_interruptions=False,
|
|
9007
9061
|
)
|
|
9008
9062
|
try:
|
|
9009
|
-
|
|
9063
|
+
# #901 §4.7 does not enumerate this writer, so it keeps SQLite's
|
|
9064
|
+
# default temp store: the publication rewrites whole relations inside
|
|
9065
|
+
# one transaction, and an in-memory statement journal for that would
|
|
9066
|
+
# be proportional to the index.
|
|
9067
|
+
_cctally_store.apply_policy(conn, "stats", memory_temp_store=False)
|
|
9010
9068
|
except BaseException:
|
|
9011
9069
|
try:
|
|
9012
9070
|
conn.close()
|
|
@@ -10527,6 +10585,10 @@ def _run_bounded_recovery(
|
|
|
10527
10585
|
try:
|
|
10528
10586
|
cache = sqlite3.connect(str(cache_path), timeout=15.0)
|
|
10529
10587
|
cache.execute("PRAGMA busy_timeout=15000")
|
|
10588
|
+
# #901 §4.7 (Q11): statement journals in memory, before the
|
|
10589
|
+
# chunk's transaction; nothing else about the chunk changes.
|
|
10590
|
+
import _cctally_store
|
|
10591
|
+
_cctally_store.apply_writer_temp_store(cache)
|
|
10530
10592
|
cache.execute("BEGIN IMMEDIATE")
|
|
10531
10593
|
if chunk_index > 0:
|
|
10532
10594
|
seq, digest = _recovery_state(cache)
|