cctally 1.113.0 → 1.114.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/README.md +4 -6
  3. package/bin/_cctally_cache.py +1683 -485
  4. package/bin/_cctally_core.py +223 -1
  5. package/bin/_cctally_dashboard.py +96 -18
  6. package/bin/_cctally_dashboard_perf.py +128 -9
  7. package/bin/_cctally_dashboard_sources.py +267 -204
  8. package/bin/_cctally_db.py +861 -18
  9. package/bin/_cctally_diagnosis_sources.py +22 -4
  10. package/bin/_cctally_doctor.py +136 -18
  11. package/bin/_cctally_five_hour.py +2 -2
  12. package/bin/_cctally_forecast.py +27 -6
  13. package/bin/_cctally_journal.py +67 -5
  14. package/bin/_cctally_milestone_history.py +82 -9
  15. package/bin/_cctally_percent_breakdown.py +3 -2
  16. package/bin/_cctally_quota.py +201 -33
  17. package/bin/_cctally_record.py +14 -0
  18. package/bin/_cctally_rederive.py +23 -9
  19. package/bin/_cctally_source_analytics.py +60 -15
  20. package/bin/_cctally_statusline.py +17 -0
  21. package/bin/_cctally_store.py +80 -5
  22. package/bin/_cctally_tui.py +4 -3
  23. package/bin/_cctally_weekrefs.py +128 -21
  24. package/bin/_lib_cache_report.py +35 -6
  25. package/bin/_lib_codex_conversation.py +32 -0
  26. package/bin/_lib_codex_conversation_query.py +164 -30
  27. package/bin/_lib_conversation_query.py +95 -56
  28. package/bin/_lib_conversation_retention.py +1605 -640
  29. package/bin/_lib_cost_provenance.py +53 -2
  30. package/bin/_lib_doctor.py +263 -69
  31. package/bin/_lib_fmt.py +11 -0
  32. package/bin/_lib_merge_gate.py +469 -51
  33. package/bin/_lib_perf.py +27 -1
  34. package/bin/_lib_pricing.py +193 -8
  35. package/bin/_lib_pricing_check.py +20 -4
  36. package/bin/_lib_reclaim_planner.py +792 -0
  37. package/bin/_lib_rederive.py +225 -77
  38. package/bin/_lib_snapshot_cache.py +21 -0
  39. package/bin/_lib_sqlite_close.py +81 -0
  40. package/bin/_lib_subscription_weeks.py +100 -17
  41. package/bin/_lib_tick_stats.py +217 -17
  42. package/bin/_lib_wal_checkpoint.py +1077 -0
  43. package/bin/_lib_write_budget.py +483 -0
  44. package/bin/_lib_write_io.py +703 -0
  45. package/bin/cctally +8 -0
  46. package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
  47. package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
  48. package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
  49. package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
  50. package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
  51. package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
  52. package/dashboard/static/assets/index-BKITk1CT.css +1 -0
  53. package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
  54. package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
  55. package/dashboard/static/dashboard.html +2 -2
  56. package/package.json +6 -1
  57. package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
  58. package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
@@ -1984,10 +1984,12 @@ _CODEX_THREADS_BY_NATIVE_SQL = """
1984
1984
  # reading `cache_create_tokens` off `session_entries` carries the split
1985
1985
  # column, because a missed site does not raise and does not move a golden — it
1986
1986
  # silently under-prices — and this map is one keystroke away from feeding a
1987
- # cost fold.
1987
+ # cost fold. `input_tokens` (#929) is appended LAST so the existing row
1988
+ # indexes stay valid: a flagged turn's wasted estimate selects its pricing card
1989
+ # from the request's real prompt, input + creation + read.
1988
1990
  _CLAUDE_TURN_TOKENS_SQL = (
1989
1991
  "SELECT msg_id, req_id, cache_create_tokens, cache_read_tokens, "
1990
- " cache_create_1h_tokens, speed "
1992
+ " cache_create_1h_tokens, speed, input_tokens "
1991
1993
  "FROM session_entries WHERE "
1992
1994
  )
1993
1995
 
@@ -2148,7 +2150,8 @@ def _placeholders(count: int) -> str:
2148
2150
  def _claude_turn_token_map(conn: sqlite3.Connection,
2149
2151
  keys: Sequence[tuple], account_key: str | None
2150
2152
  ) -> dict[tuple, dict]:
2151
- """`{(msg_id, req_id): {"cache_creation", "cache_read", "speed"}}`.
2153
+ """`{(msg_id, req_id): {"cache_creation", "cache_read", "cache_1h",
2154
+ "speed", "input"}}`.
2152
2155
 
2153
2156
  The cache-churn walk begins BEFORE the window, and those turns are state:
2154
2157
  their tokens move the running maximum and they contribute no support,
@@ -2171,6 +2174,7 @@ def _claude_turn_token_map(conn: sqlite3.Connection,
2171
2174
  "cache_read": int(row[3] or 0),
2172
2175
  "cache_1h": row[4],
2173
2176
  "speed": row[5],
2177
+ "input": int(row[6] or 0),
2174
2178
  }
2175
2179
  return usage
2176
2180
 
@@ -2432,8 +2436,13 @@ def _evaluate_cache_churn(scope: DiagnosisScope, bundle: StoreBundle,
2432
2436
  continue
2433
2437
  flagged_keys.add(key)
2434
2438
  flagged_sessions.add(session)
2439
+ # #929: the flagged request's real prompt selects the card.
2440
+ turn = usage.get(key) or {}
2435
2441
  wasted.append(query._cache_failure_wasted_usd(
2436
- model, lost, speed=speed))
2442
+ model, lost, speed=speed,
2443
+ prompt_tokens=(turn.get("input", 0)
2444
+ + turn.get("cache_creation", 0)
2445
+ + turn.get("cache_read", 0))))
2437
2446
 
2438
2447
  known = set(all_sessions)
2439
2448
  seeded = set(streams)
@@ -4412,6 +4421,15 @@ def _start_forked_provider(args) -> tuple[int, int]:
4412
4421
  with os.fdopen(write_fd, "wb") as stream:
4413
4422
  pickle.dump(payload, stream, protocol=5)
4414
4423
  finally:
4424
+ # #901 W9 (Q14): `os._exit` skips `atexit`; a provider task that
4425
+ # synced a store hands its frames to the checkpoint policy's
4426
+ # finalizer. With nothing armed it does nothing.
4427
+ try:
4428
+ import _lib_wal_checkpoint
4429
+
4430
+ _lib_wal_checkpoint.finalize()
4431
+ except BaseException: # noqa: BLE001
4432
+ pass
4415
4433
  os._exit(0)
4416
4434
  os.close(write_fd)
4417
4435
  return pid, read_fd
@@ -470,6 +470,78 @@ def _gather_writer_guard_log(now_utc: dt.datetime):
470
470
  }
471
471
 
472
472
 
473
+ #: Bounded per-instance probe of a running dashboard's write telemetry.
474
+ DASHBOARD_WRITE_IO_TIMEOUT_SECONDS = 2.0
475
+
476
+
477
+ def _fetch_dashboard_write_io(descriptor: dict, *, timeout_s: float) -> dict:
478
+ """One loopback probe of a discovered dashboard (#901 §5.6).
479
+
480
+ Reuses `dashboard-perf`'s client — IP-literal loopback target, an
481
+ `Origin` matching the `Host` — and sends NO token, so an instance that
482
+ requires one answers 401 and is reported `authentication_required`: no
483
+ authentication bypass exists. The instance id in the answer must equal
484
+ the descriptor's, or the probe is an `identity_mismatch`."""
485
+ import _lib_write_io as wio
486
+
487
+ perf = _cctally()._cctally_dashboard_perf
488
+ base = {"instance_id": descriptor["instanceId"],
489
+ "port": descriptor["port"], "pid": descriptor["pid"]}
490
+ host = wio.loopback_target(descriptor.get("host"))
491
+ if host is None:
492
+ return {**base, "probe": "failed", "reason": "not_loopback_reachable",
493
+ "write_io": None}
494
+ try:
495
+ payload = perf._request(host, descriptor["port"], perf.WRITE_IO_PATH,
496
+ token=None, timeout=timeout_s)
497
+ except perf.DashboardPerfError as exc:
498
+ return {**base, "probe": "failed", "reason": exc.reason,
499
+ "write_io": None}
500
+ write_io = payload.get("writeIo") if isinstance(payload, dict) else None
501
+ if not isinstance(write_io, dict):
502
+ return {**base, "probe": "failed", "reason": "malformed_response",
503
+ "write_io": None}
504
+ if write_io.get("instanceId") != descriptor["instanceId"]:
505
+ return {**base, "probe": "failed", "reason": "identity_mismatch",
506
+ "write_io": None}
507
+ return {**base, "probe": "ok", "reason": None, "write_io": write_io}
508
+
509
+
510
+ def _gather_dashboard_disk_writes(
511
+ *, fetch=None, timeout_s: float = DASHBOARD_WRITE_IO_TIMEOUT_SECONDS,
512
+ discover: bool = True,
513
+ ) -> dict:
514
+ """Running dashboards' write telemetry for `performance.dashboard_disk_
515
+ writes` (#901 §5.6).
516
+
517
+ In the dashboard process the local instance's own snapshot is read
518
+ directly — no self-HTTP and no recursive gather. Anywhere else (the CLI,
519
+ the TUI) instances are discovered through their owner-readable
520
+ descriptors; a stale one (dead or reused pid) is ignored and never
521
+ deleted, because doctor is read-only. Discovery runs only for the
522
+ `cctally doctor` CLI (PR-11); with ``discover`` false nothing is probed and
523
+ the mode is ``not_probed``."""
524
+ import _lib_write_io as wio
525
+
526
+ local = wio.local_instance()
527
+ if local is not None:
528
+ return {"mode": "in_process", "instances": [{
529
+ "instance_id": local["instanceId"], "port": local["port"],
530
+ "pid": local["pid"], "probe": "ok", "reason": None,
531
+ "write_io": wio.write_io_payload(),
532
+ }]}
533
+ if not discover:
534
+ return {"mode": "not_probed", "instances": []}
535
+ probe = fetch or _fetch_dashboard_write_io
536
+ instances = [
537
+ probe(descriptor, timeout_s=timeout_s)
538
+ for descriptor in wio.read_instance_descriptors(_cctally_core.APP_DIR)
539
+ if wio.descriptor_is_live(descriptor)
540
+ ]
541
+ instances.sort(key=lambda entry: (entry["port"], entry["instance_id"]))
542
+ return {"mode": "discovery", "instances": instances}
543
+
544
+
473
545
  def _gather_statusline_pipeline(c, *, now_utc: dt.datetime) -> dict:
474
546
  """Read the #318 statusline pipeline without creating or pruning files."""
475
547
  now_epoch = int(now_utc.timestamp())
@@ -874,6 +946,26 @@ def _gather_codex_window_attribution_state() -> "dict | None":
874
946
  conn.close()
875
947
 
876
948
 
949
+ def _journal_protocol_violation_rows(conn: sqlite3.Connection) -> list:
950
+ """Every stored protocol violation, parsed, in (batch, kind, fingerprint)
951
+ order: the doctor's shallow journal leg.
952
+
953
+ #901 Amendment 19 T1 (spec I1): the doctor holds every violation anyway,
954
+ so they are ordered here, in Python, where ``ORDER BY batch_id, kind,
955
+ fingerprint`` left a partial sorter over each batch's rows — and one batch
956
+ is not a bound. ``fingerprint`` is the primary key, so the order is total
957
+ and identical (text compared as UTF-8 bytes, SQLite's BINARY collation).
958
+ """
959
+ rows = conn.execute(
960
+ "SELECT batch_id, kind, fingerprint, violation_json "
961
+ "FROM journal_protocol_violations"
962
+ ).fetchall()
963
+ rows.sort(key=lambda row: tuple(
964
+ (value is not None, str(value or "").encode("utf-8"))
965
+ for value in row[:3]))
966
+ return [json.loads(str(row[3])) for row in rows]
967
+
968
+
877
969
  def _gather_accounts_state(now_utc: "dt.datetime") -> dict:
878
970
  """Best-effort account-attribution state for the doctor `accounts.*` legs
879
971
  (#341). Never raises: identity + registry reads are read-only and each guard
@@ -936,14 +1028,21 @@ def _gather_accounts_state(now_utc: "dt.datetime") -> dict:
936
1028
  cutoff = (now_utc - dt.timedelta(days=7)).astimezone(
937
1029
  dt.timezone.utc).isoformat()
938
1030
  try:
939
- for account_key, cnt in conn.execute(
940
- "SELECT account_key, COUNT(*) FROM weekly_usage_snapshots "
941
- "WHERE captured_at_utc >= ? GROUP BY account_key", (cutoff,)
942
- ).fetchall():
1031
+ # #901 Amendment 19 T1 (spec I1): counted in Python over the
1032
+ # streamed rows, one total per class, where `GROUP BY
1033
+ # account_key` sorted the trailing week's every snapshot through a
1034
+ # temp b-tree — and a week of activity is not a bound.
1035
+ attributed = unattributed = 0
1036
+ for (account_key,) in conn.execute(
1037
+ "SELECT account_key FROM weekly_usage_snapshots "
1038
+ "WHERE captured_at_utc >= ?", (cutoff,)
1039
+ ):
943
1040
  if account_key and account_key != "unattributed":
944
- state["recent_attributed"] += int(cnt)
1041
+ attributed += 1
945
1042
  else:
946
- state["recent_unattributed"] += int(cnt)
1043
+ unattributed += 1
1044
+ state["recent_attributed"] += attributed
1045
+ state["recent_unattributed"] += unattributed
947
1046
  except sqlite3.DatabaseError:
948
1047
  pass
949
1048
  finally:
@@ -961,9 +1060,13 @@ def _gather_accounts_state(now_utc: "dt.datetime") -> dict:
961
1060
  # contains no ingest and is not a whole tick; a whole tick is ingest plus
962
1061
  # builder, and the builder is 70.5% of one measured live. Against a whole tick
963
1062
  # the same probe is therefore 14%. It is one all-history
964
- # `load_codex_quota_observations` call whose SQL runs a `MAX(...) OVER
965
- # (PARTITION BY ...)` across every retained row (277,207 on that store) to
966
- # return one row per identity (16).
1063
+ # `load_codex_quota_observations(latest_per_identity=True)` call. Its SQL used
1064
+ # to run a `MAX(...) OVER (PARTITION BY ...)` across every retained row
1065
+ # (277,207 on that store; 565K and ~1.1 GB of temp-file writes per call on the
1066
+ # #901 live store); since #901 it reads the trigger-maintained
1067
+ # `codex_quota_partition_latest` summary plus one indexed seek per raw
1068
+ # partition. The memo below therefore remains a CPU optimisation and is no
1069
+ # longer load-bearing for disk writes.
967
1070
  #
968
1071
  # Only the ROWS are retained, and only while their evidence is unmoved. Every
969
1072
  # `now`-derived interpretation over them is recomputed on every gather:
@@ -1249,6 +1352,7 @@ def doctor_gather_state(
1249
1352
  runtime_bind: "str | None" = None,
1250
1353
  deep: bool = False,
1251
1354
  force_cold_inputs: bool = False,
1355
+ discover_dashboards: bool = False,
1252
1356
  ):
1253
1357
  """Gather doctor state while excluding cache-family replacement.
1254
1358
 
@@ -1262,6 +1366,11 @@ def doctor_gather_state(
1262
1366
  caller can obtain a genuinely fresh gather at a chosen instant. It exists
1263
1367
  for the equality gate that compares a reusing gather against a fresh one at
1264
1368
  an identical ``now_utc``; production callers leave it false.
1369
+
1370
+ ``discover_dashboards`` (PR-11) lets the gather probe other running
1371
+ dashboards for their disk writes. Only the `cctally doctor` CLI passes it:
1372
+ the TUI and every snapshot build gather on each rebuild, where serial
1373
+ 2-second probes of every live descriptor do not belong.
1265
1374
  """
1266
1375
  cache_lock = None
1267
1376
  cache_probe_allowed = not _cctally_core.CACHE_DB_PATH.exists()
@@ -1317,6 +1426,7 @@ def doctor_gather_state(
1317
1426
  _cache_probe_allowed=cache_probe_allowed,
1318
1427
  _cache_repair_marker=cache_repair_marker,
1319
1428
  _force_cold_inputs=force_cold_inputs,
1429
+ _discover_dashboards=discover_dashboards,
1320
1430
  )
1321
1431
  finally:
1322
1432
  if cache_lock is not None:
@@ -1334,6 +1444,7 @@ def _doctor_gather_state_impl(
1334
1444
  _cache_probe_allowed: bool,
1335
1445
  _cache_repair_marker: dict,
1336
1446
  _force_cold_inputs: bool = False,
1447
+ _discover_dashboards: bool = False,
1337
1448
  ):
1338
1449
  """I/O chokepoint for `cctally doctor` (spec §7.2).
1339
1450
 
@@ -1727,6 +1838,7 @@ def _doctor_gather_state_impl(
1727
1838
  conv_rollup_sync_in_progress = False
1728
1839
  conversations_db_page_count = None
1729
1840
  conversations_db_freelist_count = None
1841
+ conversations_db_page_size = None
1730
1842
  conversations_reclaim_pending = None
1731
1843
  codex_prune_refusals: list[dict] = []
1732
1844
  conversation_rollup_pricing_refusal: "dict | None" = None
@@ -1751,6 +1863,9 @@ def _doctor_gather_state_impl(
1751
1863
  row = conn.execute("PRAGMA freelist_count").fetchone()
1752
1864
  if row and row[0] is not None:
1753
1865
  conversations_db_freelist_count = int(row[0])
1866
+ row = conn.execute("PRAGMA page_size").fetchone()
1867
+ if row and row[0] is not None:
1868
+ conversations_db_page_size = int(row[0])
1754
1869
  except sqlite3.Error:
1755
1870
  pass
1756
1871
  try:
@@ -2407,14 +2522,8 @@ def _doctor_gather_state_impl(
2407
2522
  if _cctally_core.DB_PATH.exists():
2408
2523
  pc = _stats_ro_guarded()
2409
2524
  try:
2410
- protocol_rows = [
2411
- json.loads(str(row[0]))
2412
- for row in pc.execute(
2413
- "SELECT violation_json "
2414
- "FROM journal_protocol_violations "
2415
- "ORDER BY batch_id, kind, fingerprint"
2416
- )
2417
- ]
2525
+ protocol_rows = (
2526
+ _journal_protocol_violation_rows(pc))
2418
2527
  journal_protocol_violations = [
2419
2528
  item for item in protocol_rows
2420
2529
  if not item.get("auditId")
@@ -2696,6 +2805,13 @@ def _doctor_gather_state_impl(
2696
2805
  _gather_codex_window_attribution_state()
2697
2806
  if _cache_probe_allowed else None)
2698
2807
 
2808
+ # #901 §5.6: running dashboards' disk writes. Read-only; degrades.
2809
+ try:
2810
+ with _lib_perf.phase("doctor.dashboard_disk_writes"):
2811
+ dashboard_disk_writes = _gather_dashboard_disk_writes(
2812
+ discover=_discover_dashboards)
2813
+ except Exception: # noqa: BLE001 — a probe failure must not fail doctor
2814
+ dashboard_disk_writes = {"mode": "error", "instances": []}
2699
2815
  return _lib_doctor.DoctorState(
2700
2816
  symlink_state=symlink_state,
2701
2817
  path_includes_local_bin=path_includes,
@@ -2792,6 +2908,8 @@ def _doctor_gather_state_impl(
2792
2908
  cache_db_freelist_count=cache_db_freelist_count,
2793
2909
  conversations_db_page_count=conversations_db_page_count,
2794
2910
  conversations_db_freelist_count=conversations_db_freelist_count,
2911
+ conversations_db_page_size=conversations_db_page_size,
2912
+ dashboard_disk_writes=dashboard_disk_writes,
2795
2913
  conversations_reclaim_pending=conversations_reclaim_pending,
2796
2914
  codex_quota_windows=codex_quota_windows,
2797
2915
  codex_hook_roots=codex_hook_roots,
@@ -2846,7 +2964,7 @@ def cmd_doctor(args: argparse.Namespace) -> int:
2846
2964
  # #279 S2 F5b: deep=True runs PRAGMA quick_check(1) — CLI-only (the
2847
2965
  # dashboard/TUI gather callers stay deep=False so their per-rebuild
2848
2966
  # gather never pays the multi-second cost on a large cache.db).
2849
- state = c.doctor_gather_state(deep=True)
2967
+ state = c.doctor_gather_state(deep=True, discover_dashboards=True)
2850
2968
  report = _lib_doctor.run_checks(state)
2851
2969
  if getattr(args, "json", False):
2852
2970
  print(encode_dashboard_json(
@@ -2689,10 +2689,10 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
2689
2689
  ])
2690
2690
  continue
2691
2691
  m = ev
2692
- cum = f"${m['blockCostUSD']:.6f}"
2692
+ cum = _c._fmt_usd_accounting(m['blockCostUSD'], 6)
2693
2693
  marg = (
2694
2694
  "n/a" if m["marginalCostUSD"] is None
2695
- else f"${m['marginalCostUSD']:.6f}"
2695
+ else _c._fmt_usd_accounting(m['marginalCostUSD'], 6)
2696
2696
  )
2697
2697
  # #834 S1 (#836): the EFFECTIVE weekly value, never the raw one. A
2698
2698
  # weekly-clamped tick stored a reading no reader ever saw, and this
@@ -153,6 +153,25 @@ def _shape_week_samples(rows, *, include_held: bool,
153
153
  return out
154
154
 
155
155
 
156
+ def _week_rows_by_capture(cursor, *, by_id: bool) -> list:
157
+ """One week's sample rows in capture order: ``captured_at_utc`` ascending,
158
+ then ``id`` when ``by_id`` (the row's trailing column, which this strips).
159
+
160
+ #901 Amendment 19 T1 (spec I1): the reader returns every row of the week
161
+ anyway, so ordering them here costs no structure beyond its own result,
162
+ where ``ORDER BY captured_at_utc`` sorted the whole week through a temp
163
+ b-tree — and one subscription week is not a bound. ``list.sort`` is stable
164
+ over the same scan SQLite's in-memory sorter consumed, so rows tied on one
165
+ capture instant keep the order they had.
166
+ """
167
+ rows = cursor.fetchall()
168
+ if by_id:
169
+ rows.sort(key=lambda row: (row[0], row[-1]))
170
+ return [row[:-1] for row in rows]
171
+ rows.sort(key=lambda row: row[0])
172
+ return rows
173
+
174
+
156
175
  def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime,
157
176
  *, account_key: "str | None" = None,
158
177
  include_held: bool = False,
@@ -264,6 +283,8 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
264
283
  we_date = dt.date.fromisoformat(drow[1])
265
284
  week_start_at = dt.datetime.combine(ws_date, dt.time(0, 0), local_tz).astimezone(dt.timezone.utc)
266
285
  week_end_at = dt.datetime.combine(we_date + dt.timedelta(days=1), dt.time(0, 0), local_tz).astimezone(dt.timezone.utc)
286
+ # One date: `idx_usage_week_time` (week_start_date, captured_at_utc
287
+ # DESC, id DESC) serves this order outright, with no sorter.
267
288
  rows = conn.execute(
268
289
  "SELECT captured_at_utc, weekly_percent, five_hour_percent, "
269
290
  " weekly_observation_held" + _owner_column + " "
@@ -293,16 +314,16 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
293
314
  matching_texts.append(r[0])
294
315
  chosen_date = chosen[2]
295
316
  placeholders = ",".join("?" * len(matching_texts))
296
- rows = conn.execute(
317
+ rows = _week_rows_by_capture(conn.execute(
297
318
  f"SELECT captured_at_utc, weekly_percent, five_hour_percent, "
298
- f" weekly_observation_held{_owner_column} "
319
+ f" weekly_observation_held{_owner_column}"
320
+ f"{', id' if include_account else ''} "
299
321
  f"FROM weekly_usage_snapshots "
300
322
  f"WHERE (week_start_at IN ({placeholders}) "
301
323
  f" OR (week_start_at IS NULL AND week_start_date = ?))"
302
- f"{_acct_pred}{_held_pred} "
303
- f"ORDER BY captured_at_utc{_sample_order}",
324
+ f"{_acct_pred}{_held_pred}",
304
325
  tuple(matching_texts) + (chosen_date,) + _acct_p,
305
- ).fetchall()
326
+ ), by_id=include_account)
306
327
  samples = _shape_week_samples(rows, include_held=include_held,
307
328
  include_account=include_account)
308
329
  samples = [s for s in samples if s[0] <= now_utc]
@@ -2518,7 +2539,7 @@ def cmd_report(args: argparse.Namespace) -> int:
2518
2539
  m_rows: list[list[str]] = []
2519
2540
  for idx, m in enumerate(milestone_rows, start=1):
2520
2541
  pct = f"{int(m['percent_threshold'])}%"
2521
- cum = f"${float(m['cumulative_cost_usd']):.6f}"
2542
+ cum = c._fmt_usd_accounting(float(m['cumulative_cost_usd']), 6)
2522
2543
  marg = c.observation_gap_marginal_cell(
2523
2544
  m["marginal_cost_usd"],
2524
2545
  withheld=(idx - 1) in gap_disclosure.withheld_indexes,
@@ -2083,13 +2083,18 @@ _FILE_ACCOUNT_INSERT = (
2083
2083
  # to reach this statement for a record the map insert dropped, so the two guards
2084
2084
  # are belt-and-suspenders over disjoint failure modes: this one covers any
2085
2085
  # constraint the incarnation table might gain that the map table lacks.
2086
+ #
2087
+ # #901 §5.3c (W8): the MAX-set writes only when the incarnation advances, like
2088
+ # its ingest twin `_cctally_cache.set_codex_file_incarnation`, so replaying a
2089
+ # decision the store already holds issues no write.
2086
2090
  _FILE_INCARNATION_INSERT = (
2087
2091
  "INSERT OR IGNORE INTO codex_file_incarnations "
2088
2092
  "(file_identity, incarnation, updated_at_utc) "
2089
2093
  "VALUES (?,?,?) "
2090
2094
  "ON CONFLICT(file_identity) DO UPDATE SET "
2091
- " incarnation = MAX(codex_file_incarnations.incarnation, excluded.incarnation), "
2092
- " updated_at_utc = excluded.updated_at_utc"
2095
+ " incarnation = excluded.incarnation, "
2096
+ " updated_at_utc = excluded.updated_at_utc "
2097
+ "WHERE excluded.incarnation > codex_file_incarnations.incarnation"
2093
2098
  )
2094
2099
 
2095
2100
 
@@ -2496,6 +2501,14 @@ _QUOTA_SNAPSHOT_UPSERT_CLAUSE = (
2496
2501
  )
2497
2502
 
2498
2503
  _QUOTA_SNAPSHOT_UPSERT = _QUOTA_SNAPSHOT_INSERT + _QUOTA_SNAPSHOT_UPSERT_CLAUSE
2504
+
2505
+ # The natural key of `_QUOTA_SNAPSHOT_UPSERT_CLAUSE`, read before a replay
2506
+ # inserts (#901 W11): `(source, source_path, line_offset, logical_limit_key)`.
2507
+ _QUOTA_SNAPSHOT_STORED_ACCOUNT = (
2508
+ "SELECT account_key FROM quota_window_snapshots "
2509
+ "WHERE source = ? AND source_path = ? AND line_offset = ? "
2510
+ "AND logical_limit_key = ?"
2511
+ )
2499
2512
  _QUOTA_SNAPSHOT_UPSERT_LEGACY = (
2500
2513
  _QUOTA_SNAPSHOT_INSERT_LEGACY + _QUOTA_SNAPSHOT_UPSERT_CLAUSE
2501
2514
  )
@@ -2695,8 +2708,20 @@ def _apply_quota_records(
2695
2708
  row_values = _quota_snapshot_values(rec, anchor)
2696
2709
  if not has_anchor:
2697
2710
  row_values = row_values[:-1]
2711
+ # #901 §5.3c (W11, Q14): an INSERT into this AUTOINCREMENT table
2712
+ # rewrites `sqlite_sequence` even when every row it carries is
2713
+ # rejected, so replaying observations the walk already materialized
2714
+ # wrote a page per ingest cycle. The stored row is read first and an
2715
+ # insert that would change nothing is not issued; a genuine
2716
+ # correction (a decision now governing a row stamped otherwise) still
2717
+ # runs. AUTOINCREMENT and the sequence values are untouched.
2718
+ stored = cache.execute(
2719
+ _QUOTA_SNAPSHOT_STORED_ACCOUNT,
2720
+ (row_values[0], row_values[2], row_values[3], row_values[6]),
2721
+ ).fetchone()
2698
2722
  if not covered:
2699
- cache.execute(insert_sql, row_values)
2723
+ if stored is None:
2724
+ cache.execute(insert_sql, row_values)
2700
2725
  continue
2701
2726
  observed = rec.get("account")
2702
2727
  payload = rec.get("payload") or {}
@@ -2719,6 +2744,8 @@ def _apply_quota_records(
2719
2744
  )
2720
2745
  values = list(row_values)
2721
2746
  values[16] = decided
2747
+ if stored is not None and stored[0] == decided:
2748
+ continue # the upsert's guarded DO UPDATE would change nothing
2722
2749
  cache.execute(upsert_sql, tuple(values))
2723
2750
 
2724
2751
 
@@ -3259,7 +3286,12 @@ def _cache_applier(decoded, *, cursor=None, covered_to=None,
3259
3286
  return stop_idx
3260
3287
  try:
3261
3288
  import _cctally_cache
3289
+ import _cctally_store
3262
3290
  cache.execute("PRAGMA busy_timeout=15000")
3291
+ # #901 §4.7 (Q11): a raw cache writer, so it holds its statement
3292
+ # journals in memory like the store openers' writers do. Only the
3293
+ # temp store; locks and transaction order are unchanged.
3294
+ _cctally_store.apply_writer_temp_store(cache)
3263
3295
  # Read and check the predecessor BEFORE the transaction opens
3264
3296
  # (spec §4.3, as corrected). `prior_is_extendable` checks contiguity
3265
3297
  # against this cycle's STARTING cursor — a predecessor applied
@@ -8117,6 +8149,8 @@ _REBUILD_REQUIRED_TABLES = frozenset(
8117
8149
  )
8118
8150
  _REBUILD_REQUIRED_INDEXES = frozenset(
8119
8151
  {
8152
+ "idx_budget_milestones_alert_digest",
8153
+ "idx_budget_milestones_codex_digest",
8120
8154
  "idx_budget_milestones_journal_id",
8121
8155
  "idx_budget_milestones_journal_id_null",
8122
8156
  "idx_cost_week_start_at_time",
@@ -8128,6 +8162,7 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
8128
8162
  "idx_five_hour_blocks_block_start",
8129
8163
  "idx_five_hour_blocks_journal_id",
8130
8164
  "idx_five_hour_blocks_journal_id_null",
8165
+ "idx_five_hour_milestones_alert_digest",
8131
8166
  "idx_five_hour_milestones_block",
8132
8167
  "idx_five_hour_milestones_journal_id",
8133
8168
  "idx_five_hour_milestones_journal_id_null",
@@ -8135,17 +8170,36 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
8135
8170
  "idx_five_hour_reset_events_journal_id_null",
8136
8171
  "idx_journal_protocol_violations_batch",
8137
8172
  "idx_meter_rate_change_events_key",
8173
+ "idx_percent_milestones_alert_digest",
8138
8174
  "idx_percent_milestones_journal_id",
8139
8175
  "idx_percent_milestones_journal_id_null",
8176
+ "idx_percent_milestones_week_date",
8177
+ "idx_project_budget_milestones_alert_digest",
8140
8178
  "idx_project_budget_milestones_journal_id",
8141
8179
  "idx_project_budget_milestones_journal_id_null",
8180
+ "idx_projected_milestones_alert_digest",
8181
+ "idx_projected_milestones_codex_digest",
8142
8182
  "idx_projected_milestones_journal_id",
8143
8183
  "idx_projected_milestones_journal_id_null",
8144
8184
  "idx_quota_blocks_active",
8185
+ "idx_quota_blocks_digest",
8186
+ "idx_quota_blocks_root_group_pairs",
8187
+ "idx_quota_blocks_weekly_reset_order",
8188
+ "idx_quota_blocks_weekly_single_root_order",
8145
8189
  "idx_quota_milestones_active",
8190
+ "idx_quota_milestones_digest",
8191
+ "idx_quota_milestones_root",
8192
+ "idx_quota_projection_state_digest",
8146
8193
  "idx_quota_threshold_events_active",
8194
+ "idx_quota_threshold_events_digest",
8195
+ "idx_quota_threshold_events_root",
8196
+ "idx_usage_subscription_anchor_order",
8197
+ "idx_usage_subscription_anchor_pick",
8198
+ "idx_usage_week_boundary_group",
8199
+ "idx_usage_week_date_group",
8147
8200
  "idx_usage_week_start_at_time",
8148
8201
  "idx_usage_week_time",
8202
+ "idx_week_reset_events_effective_order",
8149
8203
  "idx_week_reset_events_journal_id",
8150
8204
  "idx_week_reset_events_journal_id_null",
8151
8205
  "idx_week_reset_events_legacy_tuple",
@@ -8161,7 +8215,7 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
8161
8215
  # omitted column, constraint, partial predicate, or index definition. An epoch
8162
8216
  # schema change must update this contract alongside STATS_INDEX_EPOCH.
8163
8217
  _REBUILD_SCHEMA_FINGERPRINT = (
8164
- "e0f2225f00843952b4887fa8afb046edacd3da0642d0f402ebaa5219a6e30697"
8218
+ "2ef6dab067703e612360245f9fbe4522290046dc6d8d7b009798a384cb5c27b8"
8165
8219
  )
8166
8220
 
8167
8221
 
@@ -9006,7 +9060,11 @@ def _open_publication_connection(destination) -> sqlite3.Connection:
9006
9060
  recover_interruptions=False,
9007
9061
  )
9008
9062
  try:
9009
- _cctally_store.apply_policy(conn, "stats")
9063
+ # #901 §4.7 does not enumerate this writer, so it keeps SQLite's
9064
+ # default temp store: the publication rewrites whole relations inside
9065
+ # one transaction, and an in-memory statement journal for that would
9066
+ # be proportional to the index.
9067
+ _cctally_store.apply_policy(conn, "stats", memory_temp_store=False)
9010
9068
  except BaseException:
9011
9069
  try:
9012
9070
  conn.close()
@@ -10527,6 +10585,10 @@ def _run_bounded_recovery(
10527
10585
  try:
10528
10586
  cache = sqlite3.connect(str(cache_path), timeout=15.0)
10529
10587
  cache.execute("PRAGMA busy_timeout=15000")
10588
+ # #901 §4.7 (Q11): statement journals in memory, before the
10589
+ # chunk's transaction; nothing else about the chunk changes.
10590
+ import _cctally_store
10591
+ _cctally_store.apply_writer_temp_store(cache)
10530
10592
  cache.execute("BEGIN IMMEDIATE")
10531
10593
  if chunk_index > 0:
10532
10594
  seq, digest = _recovery_state(cache)