cctally 1.109.0 → 1.111.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +52 -0
  2. package/README.md +6 -6
  3. package/bin/_cctally_cache.py +62 -13
  4. package/bin/_cctally_core.py +8 -1
  5. package/bin/_cctally_dashboard.py +422 -195
  6. package/bin/_cctally_dashboard_conversation.py +231 -40
  7. package/bin/_cctally_dashboard_envelope.py +48 -32
  8. package/bin/_cctally_dashboard_sources.py +503 -109
  9. package/bin/_cctally_db.py +44 -0
  10. package/bin/_cctally_diagnosis_sources.py +181 -15
  11. package/bin/_cctally_doctor.py +15 -3
  12. package/bin/_cctally_forecast.py +25 -12
  13. package/bin/_cctally_journal.py +59 -32
  14. package/bin/_cctally_milestone_history.py +65 -9
  15. package/bin/_cctally_parser.py +5 -0
  16. package/bin/_cctally_record.py +50 -12
  17. package/bin/_cctally_rederive.py +899 -30
  18. package/bin/_cctally_source_analytics.py +567 -133
  19. package/bin/_cctally_transcript.py +27 -2
  20. package/bin/_cctally_tui.py +39 -6
  21. package/bin/_cctally_update.py +420 -14
  22. package/bin/_lib_cache_report.py +7 -10
  23. package/bin/_lib_codex_conversation_query.py +286 -63
  24. package/bin/_lib_codex_metadata.py +207 -0
  25. package/bin/_lib_conversation_query.py +254 -37
  26. package/bin/_lib_doctor.py +16 -2
  27. package/bin/_lib_pricing.py +80 -276
  28. package/bin/_lib_rederive.py +903 -2
  29. package/bin/_lib_view_models.py +5 -0
  30. package/bin/cctally +18 -5
  31. package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
  32. package/dashboard/static/assets/{DoctorModal-BZGXOlc2.js → DoctorModal-DFY2iPDf.js} +1 -1
  33. package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
  34. package/dashboard/static/assets/{ProjectsDrillPanel-KbO_KWa7.js → ProjectsDrillPanel-DAv39uSo.js} +1 -1
  35. package/dashboard/static/assets/{SourceDetailModal-BrOPLVRL.js → SourceDetailModal-DSAWo2Fp.js} +1 -1
  36. package/dashboard/static/assets/{UpdateModal-CYgEFAi4.js → UpdateModal-R_9dOVnh.js} +3 -3
  37. package/dashboard/static/assets/index-COhxe3vU.css +1 -0
  38. package/dashboard/static/assets/index-DdU62sov.js +13 -0
  39. package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
  40. package/dashboard/static/assets/{useKeymap-DQx_A1MB.js → useKeymap-CW7Up0Ay.js} +1 -1
  41. package/dashboard/static/dashboard.html +3 -3
  42. package/package.json +2 -1
  43. package/dashboard/static/assets/ConversationsView-TO5bpIyc.js +0 -72
  44. package/dashboard/static/assets/ModalRoot-C_fvugTl.js +0 -1
  45. package/dashboard/static/assets/index-Cj_l7T7u.css +0 -1
  46. package/dashboard/static/assets/index-DLVwACUM.js +0 -13
  47. package/dashboard/static/assets/outlineNavigation-DXX-RjKl.js +0 -9
@@ -2083,6 +2083,8 @@ def _run_pending_migrations(
2083
2083
  db_label, db_version=cur_version, max_known=len(registry),
2084
2084
  )
2085
2085
  if cur_version == len(registry):
2086
+ if db_label == "conversations.db":
2087
+ _ensure_codex_session_meta_provenance_index(conn)
2086
2088
  # When the registry is currently empty (today's cache.db case),
2087
2089
  # still leave the schema_migrations table behind so a later
2088
2090
  # transition to len(registry) >= 1 can distinguish populated
@@ -2124,6 +2126,9 @@ def _run_pending_migrations(
2124
2126
  db_label, _first_pending_migration_name(conn, registry, cur_version)
2125
2127
  )
2126
2128
 
2129
+ if db_label == "conversations.db":
2130
+ _ensure_codex_session_meta_provenance_index(conn)
2131
+
2127
2132
  # Track whether schema_migrations existed before this open so we can
2128
2133
  # detect the fresh-install path. After bootstrap, even a "first time
2129
2134
  # opened with framework code" DB might have rows from the legacy
@@ -5475,6 +5480,45 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
5475
5480
  _apply_codex_conversation_source_identity(conn)
5476
5481
 
5477
5482
 
5483
+ def _ensure_codex_session_meta_provenance_index(
5484
+ conn: sqlite3.Connection,
5485
+ ) -> bool:
5486
+ """Ensure the account-scoped anonymization provenance index exists.
5487
+
5488
+ ``codex_conversation_events`` is an append-only physical log and can be
5489
+ much larger than the normalized transcript. The scoped anonymizer reads
5490
+ only ``session_meta`` payloads, so an index restricted to that record type
5491
+ makes the read proportional to the retained provenance rows rather than
5492
+ every event. This is an idempotent, data-free schema ensure run by the
5493
+ conversations dispatcher on both its fast and pending paths; keeping the
5494
+ helper here also gives the schema migration layer one stable delivery
5495
+ point for fresh and already-current stores.
5496
+
5497
+ A pre-conversations store may not have the event table yet. That is a
5498
+ valid no-op; the schema apply creates it before the dispatcher runs.
5499
+ Other SQLite errors propagate so a damaged store cannot silently fall back
5500
+ to a full physical-event scan.
5501
+ """
5502
+ if conn.execute(
5503
+ "SELECT 1 FROM sqlite_master "
5504
+ "WHERE type='table' AND name='codex_conversation_events'"
5505
+ ).fetchone() is None:
5506
+ return False
5507
+ if conn.execute(
5508
+ "SELECT 1 FROM sqlite_master "
5509
+ "WHERE type='index' AND name='idx_codex_events_session_meta_provenance'"
5510
+ ).fetchone() is not None:
5511
+ return True
5512
+ conn.execute(
5513
+ "CREATE INDEX IF NOT EXISTS idx_codex_events_session_meta_provenance "
5514
+ "ON codex_conversation_events("
5515
+ "source_root_key, account_key, conversation_key, payload_json) "
5516
+ "WHERE record_type='session_meta'"
5517
+ )
5518
+ conn.commit()
5519
+ return True
5520
+
5521
+
5478
5522
  #: `cache_meta` keys the stamp backfill uses. The cursor is the durable resume
5479
5523
  #: point (the last source path whose rows are fully stamped); the completion
5480
5524
  #: marker is what a rebuild's publication gate reads. Both are values, not
@@ -40,10 +40,12 @@ import json
40
40
  import os
41
41
  import pickle
42
42
  import re
43
+ import signal
43
44
  import shlex
44
45
  import sqlite3
45
46
  import sys
46
47
  import threading
48
+ import time
47
49
  from collections import defaultdict
48
50
  from dataclasses import dataclass, field
49
51
  from functools import cached_property
@@ -71,6 +73,14 @@ UTC = dt.timezone.utc
71
73
  # bounded wait rather than an immediate `database is locked`.
72
74
  READ_BUSY_TIMEOUT_MS = 4000
73
75
 
76
+ # A provider read can lose its component-local coherence check to the writer
77
+ # that is publishing the next generation. Retry the whole provider build a
78
+ # small number of times while a monotonic admission budget remains. The
79
+ # `_establish` retry remains deliberately narrower: it still rereads one
80
+ # component once and refuses a second divergence within that build.
81
+ DIAGNOSIS_RETRY_MAX_ATTEMPTS = 3
82
+ DIAGNOSIS_RETRY_DEADLINE_SECONDS = 1.0
83
+
74
84
  _STORE_PATHS = {
75
85
  "cache": "CACHE_DB_PATH",
76
86
  "stats": "DB_PATH",
@@ -1045,28 +1055,22 @@ def _compiled_codex_pricer(model: str, speed: str):
1045
1055
  reasoning_output_tokens: int) -> float:
1046
1056
  del reasoning_output_tokens
1047
1057
  non_cached_input = max(0, input_tokens - cached_input_tokens)
1058
+ long_context = input_tokens > threshold
1048
1059
  if non_cached_input <= 0 or not input_rate:
1049
1060
  input_cost = 0.0
1050
- elif non_cached_input > threshold and input_tier is not None:
1051
- input_cost = (threshold * input_rate
1052
- + (non_cached_input - threshold) * input_tier)
1053
1061
  else:
1054
- input_cost = non_cached_input * input_rate
1062
+ input_cost = non_cached_input * (
1063
+ input_tier if long_context and input_tier is not None else input_rate)
1055
1064
  if cached_input_tokens <= 0 or not cache_rate:
1056
1065
  cached_input_cost = 0.0
1057
- elif cached_input_tokens > threshold and cache_tier is not None:
1058
- cached_input_cost = (
1059
- threshold * cache_rate
1060
- + (cached_input_tokens - threshold) * cache_tier)
1061
1066
  else:
1062
- cached_input_cost = cached_input_tokens * cache_rate
1067
+ cached_input_cost = cached_input_tokens * (
1068
+ cache_tier if long_context and cache_tier is not None else cache_rate)
1063
1069
  if output_tokens <= 0 or not output_rate:
1064
1070
  output_cost = 0.0
1065
- elif output_tokens > threshold and output_tier is not None:
1066
- output_cost = (threshold * output_rate
1067
- + (output_tokens - threshold) * output_tier)
1068
1071
  else:
1069
- output_cost = output_tokens * output_rate
1072
+ output_cost = output_tokens * (
1073
+ output_tier if long_context and output_tier is not None else output_rate)
1070
1074
  base = input_cost + cached_input_cost + output_cost
1071
1075
  if speed == "fast":
1072
1076
  base *= multiplier
@@ -4345,11 +4349,43 @@ def build_provider_diagnosis(scope: DiagnosisScope, *,
4345
4349
  )
4346
4350
 
4347
4351
 
4352
+ def _build_provider_with_retry(scope: DiagnosisScope, *,
4353
+ transcripts_visible: bool
4354
+ ) -> kernel.ProviderResult:
4355
+ """Build one provider, admitting only bounded fresh retries.
4356
+
4357
+ A `generation_incoherent` result means the provider's read crossed two
4358
+ writes during one attempt. Restarting the provider build is safe because
4359
+ every attempt owns fresh read-only connections and facts. Other typed
4360
+ establishment failures and unexpected exceptions retain their existing
4361
+ handling at the caller; they are never converted into retries.
4362
+ """
4363
+ deadline = time.monotonic() + DIAGNOSIS_RETRY_DEADLINE_SECONDS
4364
+ for attempt in range(DIAGNOSIS_RETRY_MAX_ATTEMPTS):
4365
+ try:
4366
+ return build_provider_diagnosis(
4367
+ scope, transcripts_visible=transcripts_visible)
4368
+ except EstablishmentFailure as exc:
4369
+ if exc.code != EstablishmentError.GENERATION_INCOHERENT.value:
4370
+ raise
4371
+ if attempt + 1 >= DIAGNOSIS_RETRY_MAX_ATTEMPTS:
4372
+ raise
4373
+ remaining = deadline - time.monotonic()
4374
+ if remaining <= 0:
4375
+ raise
4376
+ # Give an in-flight writer a chance to finish before reopening
4377
+ # every store. Normal coherent reads pay no delay.
4378
+ time.sleep(min(0.1 * (attempt + 1), remaining))
4379
+ if time.monotonic() >= deadline:
4380
+ raise
4381
+ raise AssertionError("provider diagnosis retry loop did not return")
4382
+
4383
+
4348
4384
  def _build_provider_task(args) -> kernel.ProviderResult:
4349
4385
  """Pickle-safe provider build for the independent `all` branches."""
4350
4386
  scope, transcripts_visible = args
4351
4387
  try:
4352
- result = build_provider_diagnosis(
4388
+ result = _build_provider_with_retry(
4353
4389
  scope, transcripts_visible=transcripts_visible)
4354
4390
  except EstablishmentFailure as exc:
4355
4391
  if exc.code != EstablishmentError.STORE_UNAVAILABLE.value:
@@ -4419,6 +4455,136 @@ def _spawn_worker_init(cctally_path: str,
4419
4455
  setattr(_cctally_core, name, pathlib.Path(raw))
4420
4456
 
4421
4457
 
4458
+ _DIAGNOSIS_PROCESS_TERMINATE_GRACE_SECONDS = 0.5
4459
+
4460
+
4461
+ def _bounded_process_entry(result_path: str, task, args,
4462
+ initializer, initargs) -> None:
4463
+ """Run one pickle-safe task in its own killable process group."""
4464
+ if hasattr(os, "setsid"):
4465
+ try:
4466
+ os.setsid()
4467
+ except OSError:
4468
+ # The parent still terminates this process directly if the group
4469
+ # was not established. No descendants exist before this point.
4470
+ pass
4471
+ try:
4472
+ if initializer is not None:
4473
+ initializer(*initargs)
4474
+ payload = ("ok", task(*args))
4475
+ except EstablishmentFailure as exc:
4476
+ payload = ("establishment", exc.code, exc.message)
4477
+ except BaseException as exc:
4478
+ payload = ("unexpected", type(exc).__name__, str(exc))
4479
+ with open(result_path, "wb") as stream:
4480
+ pickle.dump(payload, stream, protocol=5)
4481
+
4482
+
4483
+ def _terminate_process_group(process) -> None:
4484
+ """Terminate and reap one bounded worker plus any provider descendants."""
4485
+ grouped = False
4486
+ if process.pid is not None and hasattr(os, "killpg"):
4487
+ try:
4488
+ os.killpg(process.pid, signal.SIGTERM)
4489
+ grouped = True
4490
+ except ProcessLookupError:
4491
+ pass
4492
+ except OSError:
4493
+ pass
4494
+ if process.is_alive() and not grouped:
4495
+ process.terminate()
4496
+ process.join(_DIAGNOSIS_PROCESS_TERMINATE_GRACE_SECONDS)
4497
+ if grouped and process.pid is not None:
4498
+ try:
4499
+ # The leader may have exited on SIGTERM while a provider child
4500
+ # ignored it. The process group remains addressable until every
4501
+ # descendant is gone, so always finish the group with SIGKILL.
4502
+ os.killpg(process.pid, signal.SIGKILL)
4503
+ except ProcessLookupError:
4504
+ pass
4505
+ except OSError:
4506
+ pass
4507
+ elif process.is_alive():
4508
+ process.kill()
4509
+ process.join(_DIAGNOSIS_PROCESS_TERMINATE_GRACE_SECONDS)
4510
+
4511
+
4512
+ def _run_process_with_deadline(task, args, *, timeout_seconds: float,
4513
+ initializer=None, initargs=()):
4514
+ """Run TASK in a one-shot process and forcibly retire it at the deadline."""
4515
+ import multiprocessing
4516
+ import tempfile
4517
+
4518
+ if timeout_seconds <= 0:
4519
+ raise TimeoutError("diagnosis process deadline expired")
4520
+ methods = multiprocessing.get_all_start_methods()
4521
+ method = "forkserver" if "forkserver" in methods else "spawn"
4522
+ context = multiprocessing.get_context(method)
4523
+ fd, result_path = tempfile.mkstemp(prefix="cctally-diagnosis-")
4524
+ os.close(fd)
4525
+ process = context.Process(
4526
+ target=_bounded_process_entry,
4527
+ args=(result_path, task, args, initializer, initargs),
4528
+ name="cctally-diagnosis-process",
4529
+ )
4530
+ try:
4531
+ process.start()
4532
+ process.join(timeout_seconds)
4533
+ if process.is_alive():
4534
+ _terminate_process_group(process)
4535
+ raise TimeoutError("diagnosis process deadline expired")
4536
+ if process.exitcode != 0:
4537
+ raise RuntimeError(
4538
+ f"diagnosis process exited {process.exitcode}")
4539
+ try:
4540
+ with open(result_path, "rb") as stream:
4541
+ payload = pickle.load(stream)
4542
+ except (EOFError, OSError, pickle.PickleError) as exc:
4543
+ raise RuntimeError("diagnosis process returned no result") from exc
4544
+ finally:
4545
+ if process.is_alive():
4546
+ _terminate_process_group(process)
4547
+ try:
4548
+ os.unlink(result_path)
4549
+ except FileNotFoundError:
4550
+ pass
4551
+ process.close()
4552
+ if payload[0] == "ok":
4553
+ return payload[1]
4554
+ if payload[0] == "establishment":
4555
+ raise EstablishmentFailure(payload[1], payload[2])
4556
+ raise RuntimeError(f"{payload[1]}: {payload[2]}")
4557
+
4558
+
4559
+ def _build_diagnosis_process_task(scope: DiagnosisScope,
4560
+ measured_at: dt.datetime | None,
4561
+ transcripts_visible: bool):
4562
+ return build_diagnosis(
4563
+ scope, measured_at=measured_at,
4564
+ transcripts_visible=transcripts_visible,
4565
+ )
4566
+
4567
+
4568
+ def build_diagnosis_bounded(scope: DiagnosisScope, *,
4569
+ measured_at: dt.datetime | None = None,
4570
+ transcripts_visible: bool,
4571
+ timeout_seconds: float) -> kernel.DiagnosisReport:
4572
+ """Build one dashboard diagnosis in a process retired at TIMEOUT_SECONDS."""
4573
+ c = _cctally()
4574
+ paths = {
4575
+ name: str(getattr(_cctally_core, name))
4576
+ for name in ("CACHE_DB_PATH", "DB_PATH",
4577
+ "CONVERSATIONS_DB_PATH", "CONFIG_PATH")
4578
+ }
4579
+ return _run_process_with_deadline(
4580
+ _build_diagnosis_process_task,
4581
+ (scope, measured_at, transcripts_visible),
4582
+ timeout_seconds=timeout_seconds,
4583
+ initializer=_spawn_worker_init,
4584
+ initargs=(str(c.__file__), paths),
4585
+ )
4586
+
4587
+
4422
4588
  def _build_isolated_provider(args):
4423
4589
  """Run one provider in a portable, one-shot safe process worker."""
4424
4590
  import concurrent.futures
@@ -4467,7 +4633,7 @@ def build_diagnosis(scope: DiagnosisScope,
4467
4633
  label=scope.label,
4468
4634
  )
4469
4635
  try:
4470
- return build_provider_diagnosis(
4636
+ return _build_provider_with_retry(
4471
4637
  provider_scope, transcripts_visible=transcripts_visible)
4472
4638
  except EstablishmentFailure as exc:
4473
4639
  # A store that cannot be OPENED withholds its provider rather than
@@ -1810,9 +1810,15 @@ def _doctor_gather_state_impl(
1810
1810
  if row:
1811
1811
  codex_entries_count = int(row[0]) if row[0] is not None else 0
1812
1812
  if row[1]:
1813
- codex_last_entry_at = parse_iso_datetime(
1814
- row[1], "codex_session_entries.timestamp_utc",
1815
- ).astimezone(dt.timezone.utc)
1813
+ try:
1814
+ codex_last_entry_at = parse_iso_datetime(
1815
+ row[1], "codex_session_entries.timestamp_utc",
1816
+ ).astimezone(dt.timezone.utc)
1817
+ except ValueError:
1818
+ # A malformed newest row cannot establish cache
1819
+ # freshness. Continue to the all-history health
1820
+ # partition so doctor can count the bad row.
1821
+ codex_last_entry_at = None
1816
1822
  try:
1817
1823
  row = conn.execute(
1818
1824
  "SELECT COUNT(*) FROM quota_window_snapshots "
@@ -1839,7 +1845,13 @@ def _doctor_gather_state_impl(
1839
1845
  "qualified_rows": health.qualified_rows,
1840
1846
  "missing_conversation_key_rows": health.missing_conversation_key_rows,
1841
1847
  "missing_thread_join_rows": health.missing_thread_join_rows,
1848
+ "undecodable_metadata_rows": (
1849
+ health.undecodable_metadata_rows),
1842
1850
  }
1851
+ if health.malformed_accounting_rows:
1852
+ codex_project_metadata_health[
1853
+ "malformed_accounting_rows"] = (
1854
+ health.malformed_accounting_rows)
1843
1855
  except Exception as exc:
1844
1856
  codex_project_metadata_error = type(exc).__name__
1845
1857
  except sqlite3.OperationalError as exc:
@@ -127,16 +127,16 @@ def _resolve_forecast_now(as_of: str | None) -> dt.datetime:
127
127
  return _command_as_of()
128
128
 
129
129
 
130
- def _shape_week_samples(rows, *, include_held: bool) -> list:
130
+ def _shape_week_samples(rows, *, include_held: bool,
131
+ include_account: bool = False) -> list:
131
132
  """Turn snapshot rows into the sample tuples `_fetch_current_week_snapshots`
132
133
  returns. Both of that function's row-fetch legs go through here so the two
133
134
  cannot drift on the tuple width.
134
135
 
135
136
  `rows` are `(captured_at_utc, weekly_percent, five_hour_percent,
136
- weekly_observation_held)`. The held flag is appended to the sample only
137
- when the caller asked for held rows, because a caller that excluded them
138
- has nothing to tell apart and the three-element shape is what every
139
- existing consumer unpacks.
137
+ weekly_observation_held[, account_key])`. The held flag is appended only
138
+ for a held-inclusive read, and the account only for an explicit identity
139
+ read. Existing consumers keep the historical three- or four-field shape.
140
140
  """
141
141
  out = []
142
142
  for row in rows:
@@ -147,13 +147,16 @@ def _shape_week_samples(rows, *, include_held: bool) -> list:
147
147
  )
148
148
  if include_held:
149
149
  sample = sample + (int(row[3] or 0),)
150
+ if include_account:
151
+ sample = sample + (row[4],)
150
152
  out.append(sample)
151
153
  return out
152
154
 
153
155
 
154
156
  def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime,
155
157
  *, account_key: "str | None" = None,
156
- include_held: bool = False):
158
+ include_held: bool = False,
159
+ include_account: bool = False):
157
160
  """Return (week_start_at, week_end_at, list[(captured_at, percent, five_hr)])
158
161
  for the subscription week containing `now_utc`, or None if no snapshot
159
162
  exists for the current week.
@@ -199,6 +202,10 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
199
202
  five-hour axis takes the newest sample, a caller reading the weekly
200
203
  axis takes the newest sample whose flag is 0, and neither can mistake
201
204
  a carried-forward weekly value for an observed one.
205
+
206
+ ``include_account`` appends the physical account key after the held flag
207
+ for the multi-account Current Week card only. The default wire-independent
208
+ sample shape and merged forecast/cost reads remain unchanged.
202
209
  """
203
210
  # #341: optional account scoping — same predicate on every leg so the whole
204
211
  # window resolution stays consistent (a real key or the `unattributed`
@@ -209,6 +216,10 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
209
216
  # `include_held=True` is a pure superset of the default read rather than a
210
217
  # read that can also resolve a different week.
211
218
  _held_pred = "" if include_held else " AND weekly_observation_held = 0"
219
+ # The TUI's multi-account card needs the physical owner of the sample it
220
+ # actually displays. Keep the historical sample shape for every other read.
221
+ _owner_column = ", account_key" if include_account else ""
222
+ _sample_order = " ASC, id ASC" if include_account else " ASC"
212
223
  candidates = conn.execute(
213
224
  "SELECT week_start_at, week_end_at, week_start_date, MAX(captured_at_utc) AS latest_cap "
214
225
  "FROM weekly_usage_snapshots "
@@ -255,13 +266,14 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
255
266
  week_end_at = dt.datetime.combine(we_date + dt.timedelta(days=1), dt.time(0, 0), local_tz).astimezone(dt.timezone.utc)
256
267
  rows = conn.execute(
257
268
  "SELECT captured_at_utc, weekly_percent, five_hour_percent, "
258
- " weekly_observation_held "
269
+ " weekly_observation_held" + _owner_column + " "
259
270
  "FROM weekly_usage_snapshots "
260
271
  "WHERE week_start_date = ?" + _acct_pred + _held_pred +
261
- " ORDER BY captured_at_utc ASC",
272
+ " ORDER BY captured_at_utc" + _sample_order,
262
273
  (drow[0],) + _acct_p,
263
274
  ).fetchall()
264
- samples = _shape_week_samples(rows, include_held=include_held)
275
+ samples = _shape_week_samples(rows, include_held=include_held,
276
+ include_account=include_account)
265
277
  samples = [s for s in samples if s[0] <= now_utc]
266
278
  return week_start_at, week_end_at, samples
267
279
  row = chosen
@@ -283,15 +295,16 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
283
295
  placeholders = ",".join("?" * len(matching_texts))
284
296
  rows = conn.execute(
285
297
  f"SELECT captured_at_utc, weekly_percent, five_hour_percent, "
286
- f" weekly_observation_held "
298
+ f" weekly_observation_held{_owner_column} "
287
299
  f"FROM weekly_usage_snapshots "
288
300
  f"WHERE (week_start_at IN ({placeholders}) "
289
301
  f" OR (week_start_at IS NULL AND week_start_date = ?))"
290
302
  f"{_acct_pred}{_held_pred} "
291
- f"ORDER BY captured_at_utc ASC",
303
+ f"ORDER BY captured_at_utc{_sample_order}",
292
304
  tuple(matching_texts) + (chosen_date,) + _acct_p,
293
305
  ).fetchall()
294
- samples = _shape_week_samples(rows, include_held=include_held)
306
+ samples = _shape_week_samples(rows, include_held=include_held,
307
+ include_account=include_account)
295
308
  samples = [s for s in samples if s[0] <= now_utc]
296
309
  return week_start_at, week_end_at, samples
297
310
 
@@ -1415,6 +1415,9 @@ class IngestContext:
1415
1415
  # writes while still exercising the same SQLite derivation/fold code.
1416
1416
  event_sink: "list | None" = None
1417
1417
  projection_writes: bool = True
1418
+ # Operator-reviewed scratch replay only. Normal ingest leaves this empty.
1419
+ held_weekly_observation_ids: frozenset[str] = field(default_factory=frozenset)
1420
+ reviewed_weekly_basis_by_account: dict = field(default_factory=dict)
1418
1421
  # #374 write boundary: emissions WITHHELD this cycle because they would have
1419
1422
  # violated the same-revision rule. Each entry is a `DroppedConflict`; the row
1420
1423
  # was converged from the already-journaled effective event instead.
@@ -3424,7 +3427,8 @@ def _now_iso() -> str:
3424
3427
  #: the reset-aware in-window maximum, so it IS the week's evidence.
3425
3428
  #: ``WEEKLY_HELD_CLAMP`` — it is strictly below that maximum, so the weekly axis
3426
3429
  #: is held at the maximum and the incoming value is recorded only as the raw
3427
- #: reading. A held weekly value is never fresh weekly evidence.
3430
+ #: reading. On reviewed scratch replay, an exact observation id can also hold
3431
+ #: a reading above the basis; either way it is not fresh weekly evidence.
3428
3432
  WEEKLY_OBSERVED = "observed"
3429
3433
  WEEKLY_HELD_CLAMP = "held_clamp"
3430
3434
 
@@ -3469,6 +3473,30 @@ class WeeklyBasis:
3469
3473
  week_end_at: "str | None"
3470
3474
 
3471
3475
 
3476
+ def _latest_weekly_basis(conn, account_key, *, week_start_date=None):
3477
+ """Return accepted weekly evidence before a later axis may delete rows."""
3478
+ sql = (
3479
+ "SELECT id, captured_at_utc, weekly_percent, week_start_date, "
3480
+ " week_end_date, week_start_at, week_end_at "
3481
+ "FROM weekly_usage_snapshots "
3482
+ "WHERE account_key = ? AND weekly_observation_held = 0 "
3483
+ )
3484
+ args = (account_key,)
3485
+ if week_start_date is not None:
3486
+ sql += "AND week_start_date = ? "
3487
+ args += (week_start_date,)
3488
+ row = conn.execute(
3489
+ sql + "ORDER BY captured_at_utc DESC, id DESC LIMIT 1", args,
3490
+ ).fetchone()
3491
+ if row is None:
3492
+ return None
3493
+ return WeeklyBasis(
3494
+ snapshot_id=int(row[0]), captured_at_utc=str(row[1]),
3495
+ weekly_percent=float(row[2]), week_start_date=row[3],
3496
+ week_end_date=row[4], week_start_at=row[5], week_end_at=row[6],
3497
+ )
3498
+
3499
+
3472
3500
  @dataclass(frozen=True)
3473
3501
  class WeeklyFold:
3474
3502
  """The weekly axis of one fold decision."""
@@ -3514,7 +3542,9 @@ class UsageSnapshotFoldResult:
3514
3542
  snapshot_action: str
3515
3543
 
3516
3544
 
3517
- def _usage_snapshot_fold_decision(conn, payload) -> UsageSnapshotFoldResult:
3545
+ def _usage_snapshot_fold_decision(
3546
+ conn, payload, *, force_weekly_hold=False, reviewed_weekly_basis=None,
3547
+ ) -> UsageSnapshotFoldResult:
3518
3548
  """The apply-time accept/hold/skip decision for a Claude rate-limit obs —
3519
3549
  the predicate ported from `cmd_record_usage`'s insert guard
3520
3550
  (bin/_cctally_record.py), made at fold time (spec §4.5 / §5.3).
@@ -3535,9 +3565,9 @@ def _usage_snapshot_fold_decision(conn, payload) -> UsageSnapshotFoldResult:
3535
3565
  The clamp/hold distinction is what `_pipeline_claude_usage` gates the weekly
3536
3566
  milestone on, and the reason it must survive: a dedup skip AGREES with the
3537
3567
  stored row, so re-running the derivation chokepoints against it heals a
3538
- killed tick; a clamp CONTRADICTS it, so deriving a weekly milestone from the
3539
- higher stored value fabricates a crossing (the 2026-09-01 incident, a
3540
- fabricated 13% milestone, and milestones are forward-only within an epoch).
3568
+ killed tick; a hold CONTRADICTS it, so deriving a weekly milestone from the
3569
+ carried value fabricates a crossing (the 2026-09-01 incident, a fabricated
3570
+ 13% milestone, and milestones are forward-only within an epoch).
3541
3571
 
3542
3572
  Every value returned is derived from the same `conn` state and payload, so
3543
3573
  the whole result is exactly as replay-deterministic as the boolean it
@@ -3573,30 +3603,24 @@ def _usage_snapshot_fold_decision(conn, payload) -> UsageSnapshotFoldResult:
3573
3603
  (week_start_date, account_key, clamp_floor_iso),
3574
3604
  ).fetchone()
3575
3605
  max_v = max_row[0] if max_row else None
3576
- weekly_held = _lib_record.hwm_clamp_applies(weekly_percent, max_v)
3577
- weekly_effective = float(max_v) if weekly_held else weekly_percent
3606
+ weekly_held = force_weekly_hold or _lib_record.hwm_clamp_applies(
3607
+ weekly_percent, max_v)
3578
3608
 
3579
3609
  basis = None
3580
3610
  if weekly_held:
3581
- basis_row = conn.execute(
3582
- "SELECT id, captured_at_utc, weekly_percent, week_start_date, "
3583
- " week_end_date, week_start_at, week_end_at "
3584
- "FROM weekly_usage_snapshots "
3585
- "WHERE week_start_date = ? AND account_key = ? "
3586
- " AND weekly_observation_held = 0 "
3587
- "ORDER BY captured_at_utc DESC, id DESC LIMIT 1",
3588
- (week_start_date, account_key),
3589
- ).fetchone()
3590
- if basis_row is not None:
3591
- basis = WeeklyBasis(
3592
- snapshot_id=int(basis_row[0]),
3593
- captured_at_utc=str(basis_row[1]),
3594
- weekly_percent=float(basis_row[2]),
3595
- week_start_date=basis_row[3],
3596
- week_end_date=basis_row[4],
3597
- week_start_at=basis_row[5],
3598
- week_end_at=basis_row[6],
3599
- )
3611
+ # A reviewed hold may carry an old raw boundary. Its basis is selected
3612
+ # before reset/credit detection because five-hour credit can delete
3613
+ # that very row while preserving the held observation's 5h effect.
3614
+ basis = (
3615
+ reviewed_weekly_basis if force_weekly_hold
3616
+ else _latest_weekly_basis(
3617
+ conn, account_key, week_start_date=week_start_date)
3618
+ )
3619
+ weekly_effective = (
3620
+ basis.weekly_percent if force_weekly_hold and basis is not None
3621
+ else float(max_v) if weekly_held and max_v is not None
3622
+ else weekly_percent
3623
+ )
3600
3624
 
3601
3625
  weekly = WeeklyFold(
3602
3626
  raw_pct=weekly_percent,
@@ -3636,12 +3660,16 @@ def _usage_snapshot_fold_decision(conn, payload) -> UsageSnapshotFoldResult:
3636
3660
  # it would make the next held tick look like new five-hour evidence and
3637
3661
  # write a duplicate row every tick. Its weekly value is by construction a
3638
3662
  # copy of the basis's, so including it changes no weekly comparison.
3663
+ last_week_start_date = (
3664
+ basis.week_start_date if force_weekly_hold and basis is not None
3665
+ else week_start_date
3666
+ )
3639
3667
  last = conn.execute(
3640
3668
  "SELECT weekly_percent, five_hour_percent, five_hour_window_key "
3641
3669
  "FROM weekly_usage_snapshots "
3642
3670
  "WHERE week_start_date = ? AND account_key = ? "
3643
3671
  "ORDER BY captured_at_utc DESC, id DESC LIMIT 1",
3644
- (week_start_date, account_key),
3672
+ (last_week_start_date, account_key),
3645
3673
  ).fetchone()
3646
3674
  last_5h = last[1] if last is not None else None
3647
3675
  last_window_key = last[2] if last is not None else None
@@ -3673,11 +3701,9 @@ def _usage_snapshot_fold_decision(conn, payload) -> UsageSnapshotFoldResult:
3673
3701
 
3674
3702
  # ── The materialized outcome ───────────────────────────────────────────
3675
3703
  if weekly_held:
3676
- # The weekly axis carries nothing new by definition — its value is the
3677
- # stored maximum. Only the five-hour axis can justify a row, and it can
3678
- # only justify a HELD one. A clamp with no basis cannot be materialized
3679
- # at all (no boundary to copy), so it degrades to a skip rather than
3680
- # writing a row with a fabricated week.
3704
+ # The weekly axis carries nothing new. An ordinary clamp writes only
3705
+ # for five-hour evidence, including on an explicit reviewed hold.
3706
+ # A held row also needs an accepted basis boundary to carry forward.
3681
3707
  if five_hour_unchanged or basis is None:
3682
3708
  action = SNAPSHOT_SKIP_NO_CHANGE
3683
3709
  else:
@@ -3913,6 +3939,7 @@ _EVT_KIND_PROVIDER = {
3913
3939
  # planner raises `RederiveConflict` on every run.
3914
3940
  _ACCOUNTS_MACHINERY_KINDS = frozenset(
3915
3941
  ("account_observe", "account_label", "accounts_cutover",
3942
+ "claude_weekly_observation_decision",
3916
3943
  "codex_file_account",
3917
3944
  "codex_window_attribution", "codex_window_attribution_retract"))
3918
3945