cctally 1.82.1 → 1.83.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/CHANGELOG.md +63 -0
  2. package/README.md +12 -5
  3. package/bin/_cctally_alerts.py +8 -1
  4. package/bin/_cctally_cache.py +912 -149
  5. package/bin/_cctally_config.py +43 -4
  6. package/bin/_cctally_core.py +933 -759
  7. package/bin/_cctally_dashboard.py +157 -47
  8. package/bin/_cctally_dashboard_cache_report.py +13 -6
  9. package/bin/_cctally_dashboard_conversation.py +1 -0
  10. package/bin/_cctally_dashboard_envelope.py +116 -8
  11. package/bin/_cctally_dashboard_share.py +50 -19
  12. package/bin/_cctally_dashboard_sources.py +223 -48
  13. package/bin/_cctally_db.py +605 -128
  14. package/bin/_cctally_doctor.py +417 -28
  15. package/bin/_cctally_five_hour.py +12 -5
  16. package/bin/_cctally_journal.py +2050 -156
  17. package/bin/_cctally_journal_repair.py +519 -0
  18. package/bin/_cctally_milestone_history.py +142 -56
  19. package/bin/_cctally_milestones.py +179 -111
  20. package/bin/_cctally_parser.py +42 -0
  21. package/bin/_cctally_project.py +24 -18
  22. package/bin/_cctally_quota.py +139 -25
  23. package/bin/_cctally_record.py +279 -108
  24. package/bin/_cctally_rederive.py +1052 -0
  25. package/bin/_cctally_reporting.py +58 -53
  26. package/bin/_cctally_setup.py +1 -0
  27. package/bin/_cctally_source_analytics.py +4 -1
  28. package/bin/_cctally_statusline.py +11 -11
  29. package/bin/_cctally_store.py +1039 -31
  30. package/bin/_cctally_sync_week.py +17 -8
  31. package/bin/_cctally_tui.py +350 -44
  32. package/bin/_cctally_update.py +133 -8
  33. package/bin/_cctally_weekrefs.py +14 -0
  34. package/bin/_lib_aggregators.py +10 -6
  35. package/bin/_lib_cache_report.py +101 -9
  36. package/bin/_lib_codex_pools.py +82 -0
  37. package/bin/_lib_conversation_query.py +81 -33
  38. package/bin/_lib_dashboard_sources.py +75 -0
  39. package/bin/_lib_diff_kernel.py +28 -15
  40. package/bin/_lib_doctor.py +342 -4
  41. package/bin/_lib_journal.py +924 -2
  42. package/bin/_lib_jsonl.py +43 -14
  43. package/bin/_lib_pricing.py +140 -21
  44. package/bin/_lib_rederive.py +395 -0
  45. package/bin/_lib_share.py +58 -2
  46. package/bin/cctally +56 -8
  47. package/dashboard/static/assets/{index-BKM43pxK.js → index-3bgCMVHb.js} +52 -52
  48. package/dashboard/static/assets/index-D27EIHEI.css +1 -0
  49. package/dashboard/static/dashboard.html +2 -2
  50. package/package.json +5 -1
  51. package/dashboard/static/assets/index-Dk1nplOz.css +0 -1
@@ -32,6 +32,7 @@ import os
32
32
  import pathlib
33
33
  import shutil
34
34
  import sqlite3
35
+ import subprocess
35
36
  import sys
36
37
 
37
38
  import _cctally_core
@@ -40,28 +41,210 @@ from _cctally_core import _now_utc, eprint, now_utc_iso, parse_iso_datetime
40
41
  from _lib_dashboard_json import encode_dashboard_json
41
42
 
42
43
 
44
+ #: Record types the deep journal conflict scan retains (#374). The selector
45
+ #: (`_lib_journal.resolve_effective_events`) reads only `evt`, `correction` and
46
+ #: `correction_batch`; `op` is kept because the rebuild-equivalent account
47
+ #: normalization is defined over evt/op records. Everything else — above all the
48
+ #: `obs` lines, ~97% of a real journal — is dropped as it is decoded, so the deep
49
+ #: gather's peak RSS tracks the decision history rather than the whole journal.
50
+ _CONFLICT_SCAN_RECORD_TYPES = frozenset(
51
+ {"evt", "correction", "correction_batch", "op"})
52
+
53
+ #: Doctor needs only recent evidence for this diagnostic. Bound both line count
54
+ #: and bytes so a corrupt single-line file cannot defeat the tail limit.
55
+ _GUARD_LOG_TAIL_LINES = 256
56
+ _GUARD_LOG_TAIL_BYTES = 64 * 1024
57
+
58
+
43
59
  def _cctally():
44
60
  """Resolve the current `cctally` module at call-time (spec §3.1)."""
45
61
  return sys.modules["cctally"]
46
62
 
47
63
 
64
+ def _stats_ro_guarded():
65
+ """A `mode=ro` stats connection that participates in the #386 opener protocol.
66
+
67
+ Read-only does not mean side-effect-free: measured on this platform, a
68
+ `mode=ro` connection to a WAL database with absent sidecars CREATES
69
+ `stats.db-shm` and `stats.db-wal`. Spec §3.1's third clause therefore
70
+ covers these diagnostics, and both callers already degrade on
71
+ `sqlite3.Error` (which `StatsDbMaintenanceError` is).
72
+ """
73
+ import _cctally_store
74
+
75
+ return _cctally_store.stats_open_guarded(
76
+ _cctally_core.DB_PATH,
77
+ connect=lambda p: sqlite3.connect(f"file:{p}?mode=ro", uri=True),
78
+ )
79
+
80
+
48
81
  def _journal_heal_incident(kind: str, name: str, now_utc: dt.datetime) -> dict:
49
82
  """One auto-heal artifact record for the doctor journal leg (§9). Parses the
50
- ``%Y%m%dT%H%M%SZ`` timestamp trailing the name (quarantine dir
51
- ``<db>.db-<ts>`` or forensics file ``<db>.db-corruption-forensics-<ts>.json``)
52
- into an age; a name that doesn't parse degrades to ``age_s=None``."""
83
+ legacy ``%Y%m%dT%H%M%SZ`` or collision-safe rebuild
84
+ ``%Y%m%dT%H%M%S_%f`` timestamp trailing the name (quarantine dir
85
+ ``<db>.db-<ts>`` or forensics file
86
+ ``<db>.db-corruption-forensics-<ts>.json``) into an age; a name that
87
+ doesn't parse degrades to ``age_s=None``."""
53
88
  base = name[:-5] if name.endswith(".json") else name
54
89
  ts = base.rsplit("-", 1)[-1]
55
90
  age_s = None
56
- try:
57
- parsed = dt.datetime.strptime(ts, "%Y%m%dT%H%M%SZ").replace(
58
- tzinfo=dt.timezone.utc)
59
- age_s = int((now_utc - parsed).total_seconds())
60
- except ValueError:
61
- pass
91
+ for timestamp_format in ("%Y%m%dT%H%M%SZ", "%Y%m%dT%H%M%S_%f"):
92
+ try:
93
+ parsed = dt.datetime.strptime(ts, timestamp_format).replace(
94
+ tzinfo=dt.timezone.utc)
95
+ age_s = int((now_utc - parsed).total_seconds())
96
+ break
97
+ except ValueError:
98
+ continue
62
99
  return {"kind": kind, "name": name, "age_s": age_s}
63
100
 
64
101
 
102
+ def _read_guard_log_tail(path) -> list[str]:
103
+ """Read at most the configured byte and line tail from one guard log."""
104
+ with path.open("rb") as fh:
105
+ fh.seek(0, os.SEEK_END)
106
+ size = fh.tell()
107
+ read_size = min(size, _GUARD_LOG_TAIL_BYTES)
108
+ fh.seek(size - read_size)
109
+ raw = fh.read(read_size)
110
+ if size > read_size:
111
+ # The byte window may start mid-line. Drop that fragment so every
112
+ # reported entry is one complete writer-guard record.
113
+ _, separator, raw = raw.partition(b"\n")
114
+ if not separator:
115
+ raw = b""
116
+ lines = [
117
+ line for line in raw.decode("utf-8", errors="replace").splitlines()
118
+ if line.strip()
119
+ ]
120
+ return lines[-_GUARD_LOG_TAIL_LINES:]
121
+
122
+
123
+ def _gather_backup_sync_state(
124
+ app_dir,
125
+ *,
126
+ platform_name: str | None = None,
127
+ home_dir=None,
128
+ probe_time_machine: bool = True,
129
+ which=shutil.which,
130
+ run=subprocess.run,
131
+ ) -> dict:
132
+ """Classify known macOS file-level backup/sync coverage without mutation."""
133
+ platform_name = (
134
+ platform_name
135
+ or os.environ.get("CCTALLY_DOCTOR_FIXTURE_PLATFORM")
136
+ or sys.platform
137
+ )
138
+ if platform_name != "darwin":
139
+ return {"status": "unsupported", "provider": None}
140
+
141
+ try:
142
+ app_path = pathlib.Path(app_dir).resolve(strict=False)
143
+ home = pathlib.Path(
144
+ home_dir if home_dir is not None else pathlib.Path.home()
145
+ ).resolve(strict=False)
146
+ except (OSError, RuntimeError):
147
+ return {"status": "unavailable", "provider": None}
148
+
149
+ def inside(root) -> bool:
150
+ try:
151
+ app_path.relative_to(pathlib.Path(root).resolve(strict=False))
152
+ return True
153
+ except (ValueError, OSError, RuntimeError):
154
+ return False
155
+
156
+ if inside(home / "Library/Mobile Documents/com~apple~CloudDocs"):
157
+ return {"status": "included", "provider": "iCloud Drive"}
158
+ if inside(home / "Dropbox"):
159
+ return {"status": "included", "provider": "Dropbox"}
160
+ cloud_storage = home / "Library/CloudStorage"
161
+ if inside(cloud_storage):
162
+ try:
163
+ relative = app_path.relative_to(cloud_storage.resolve(strict=False))
164
+ except (ValueError, OSError, RuntimeError):
165
+ relative = pathlib.Path()
166
+ if relative.parts and relative.parts[0].lower().startswith("dropbox"):
167
+ return {"status": "included", "provider": "Dropbox"}
168
+
169
+ # The dashboard gathers Doctor state on every envelope rebuild. Keep that
170
+ # hot path subprocess-free; the CLI's deep gather performs the bounded
171
+ # Time Machine probes below. Static cloud-root classification above is safe
172
+ # and cheap in either mode.
173
+ if not probe_time_machine:
174
+ return {"status": "unavailable", "provider": "Time Machine"}
175
+
176
+ try:
177
+ tmutil = which("tmutil")
178
+ except Exception:
179
+ tmutil = None
180
+ if not tmutil:
181
+ return {"status": "unavailable", "provider": "Time Machine"}
182
+ try:
183
+ destinations = run(
184
+ [tmutil, "destinationinfo"],
185
+ stdout=subprocess.PIPE,
186
+ stderr=subprocess.PIPE,
187
+ text=True,
188
+ timeout=2,
189
+ check=False,
190
+ )
191
+ except (OSError, subprocess.SubprocessError):
192
+ return {"status": "unavailable", "provider": "Time Machine"}
193
+ destination_text = f"{destinations.stdout}\n{destinations.stderr}".lower()
194
+ if destinations.returncode != 0:
195
+ status = (
196
+ "absent"
197
+ if "no destinations configured" in destination_text
198
+ else "unavailable"
199
+ )
200
+ return {"status": status, "provider": "Time Machine"}
201
+ if not destinations.stdout.strip():
202
+ return {"status": "unavailable", "provider": "Time Machine"}
203
+ try:
204
+ exclusion = run(
205
+ [tmutil, "isexcluded", str(app_path)],
206
+ stdout=subprocess.PIPE,
207
+ stderr=subprocess.PIPE,
208
+ text=True,
209
+ timeout=2,
210
+ check=False,
211
+ )
212
+ except (OSError, subprocess.SubprocessError):
213
+ return {"status": "unavailable", "provider": "Time Machine"}
214
+ if exclusion.returncode != 0:
215
+ return {"status": "unavailable", "provider": "Time Machine"}
216
+ output = exclusion.stdout.strip().lower()
217
+ if output.startswith("[excluded]"):
218
+ return {"status": "excluded", "provider": "Time Machine"}
219
+ if output.startswith("[included]"):
220
+ return {"status": "included", "provider": "Time Machine"}
221
+ return {"status": "unavailable", "provider": "Time Machine"}
222
+
223
+
224
+ def _gather_writer_guard_log(now_utc: dt.datetime):
225
+ """Gather the bounded writer-guard tail; absent/unreadable is normal."""
226
+ import _cctally_store as _store_mod_guard
227
+
228
+ guard_path = _store_mod_guard._guard_log_path()
229
+ if not guard_path.exists():
230
+ return None
231
+ lines = _read_guard_log_tail(guard_path)
232
+ newest_age = None
233
+ if lines:
234
+ stamp = lines[-1].split("\t", 1)[0]
235
+ try:
236
+ when = _cctally_core.parse_iso_datetime(stamp, "guard log")
237
+ newest_age = max(0, int((now_utc - when).total_seconds()))
238
+ except Exception:
239
+ newest_age = None
240
+ return {
241
+ "entries": len(lines),
242
+ "newest_age_s": newest_age,
243
+ "path": str(guard_path),
244
+ "sample": lines[-1] if lines else None,
245
+ }
246
+
247
+
65
248
  def _gather_statusline_pipeline(c, *, now_utc: dt.datetime) -> dict:
66
249
  """Read the #318 statusline pipeline without creating or pruning files."""
67
250
  now_epoch = int(now_utc.timestamp())
@@ -203,7 +386,10 @@ def _gather_accounts_state(now_utc: "dt.datetime") -> dict:
203
386
  if not _cctally_core.DB_PATH.exists():
204
387
  return state
205
388
  try:
206
- conn = sqlite3.connect(f"file:{_cctally_core.DB_PATH}?mode=ro", uri=True)
389
+ # #386: a `mode=ro` reader participates in the opener protocol too —
390
+ # measured, it CREATES `-shm`/`-wal` when they are absent. See
391
+ # `_cctally_store.stats_open_guarded`.
392
+ conn = _stats_ro_guarded()
207
393
  except sqlite3.Error:
208
394
  return state
209
395
  try:
@@ -314,13 +500,16 @@ def doctor_gather_state(
314
500
  cache_lock.close()
315
501
  cache_lock = None
316
502
 
317
- return _doctor_gather_state_impl(
318
- now_utc=now_utc,
319
- runtime_bind=runtime_bind,
320
- deep=deep,
321
- _cache_probe_allowed=cache_probe_allowed,
322
- _cache_repair_marker=cache_repair_marker,
323
- )
503
+ import _cctally_store
504
+
505
+ with _cctally_store.suppress_interrupted_stats_recovery():
506
+ return _doctor_gather_state_impl(
507
+ now_utc=now_utc,
508
+ runtime_bind=runtime_bind,
509
+ deep=deep,
510
+ _cache_probe_allowed=cache_probe_allowed,
511
+ _cache_repair_marker=cache_repair_marker,
512
+ )
324
513
  finally:
325
514
  if cache_lock is not None:
326
515
  try:
@@ -354,6 +543,11 @@ def _doctor_gather_state_impl(
354
543
  if now_utc is None:
355
544
  now_utc = _now_utc()
356
545
 
546
+ backup_sync_state = _gather_backup_sync_state(
547
+ _cctally_core.APP_DIR,
548
+ probe_time_machine=deep,
549
+ )
550
+
357
551
  # ── Install ──────────────────────────────────────────────────────
358
552
  # #279 S2 F5d: guard the only two unguarded statements in the
359
553
  # otherwise fail-soft gather — an exception here would kill the whole
@@ -449,9 +643,29 @@ def _doctor_gather_state_impl(
449
643
 
450
644
  # ── DB ───────────────────────────────────────────────────────────
451
645
  try:
452
- stats_db_status = c._db_status_for(_cctally_core.DB_PATH, c._STATS_MIGRATIONS, "stats.db")
646
+ import _cctally_store
647
+
648
+ interrupted = _cctally_store.stats_interrupted_rebuild_evidence(
649
+ _cctally_core.DB_PATH
650
+ )
651
+ if interrupted is not None and interrupted.get("live") is True:
652
+ stats_db_status = {
653
+ "path": str(_cctally_core.DB_PATH),
654
+ "user_version": 0,
655
+ "registry_size": len(c._STATS_MIGRATIONS),
656
+ "migrations": [],
657
+ }
658
+ else:
659
+ stats_db_status = c._db_status_for(
660
+ _cctally_core.DB_PATH,
661
+ c._STATS_MIGRATIONS,
662
+ "stats.db",
663
+ recover_interrupted_stats=False,
664
+ )
453
665
  if not _cctally_core.DB_PATH.exists():
454
666
  stats_db_status["_file_exists"] = False
667
+ if interrupted is not None:
668
+ stats_db_status["_interrupted_rebuild"] = interrupted
455
669
  except sqlite3.Error as exc:
456
670
  stats_db_status = {"path": str(_cctally_core.DB_PATH), "user_version": 0,
457
671
  "registry_size": len(c._STATS_MIGRATIONS),
@@ -494,7 +708,15 @@ def _doctor_gather_state_impl(
494
708
  credited_weeks: list[dict] | None = None
495
709
  try:
496
710
  if _cctally_core.DB_PATH.exists():
497
- conn = sqlite3.connect(str(_cctally_core.DB_PATH))
711
+ # #386 spec section 3.1, third clause: EVERY opener of the live stats
712
+ # family participates in the replacement protocol, read-only probes
713
+ # included. The open mode stays read-WRITE deliberately — switching a
714
+ # WAL DB whose `-shm` may be absent to `mode=ro` fails
715
+ # SQLITE_CANTOPEN, which is not corruption and has been misread as
716
+ # such on this project twice. Participation, not read-only-ness, is
717
+ # what the clause requires.
718
+ import _cctally_store as _store_mod
719
+ conn = _store_mod.stats_open_guarded(_cctally_core.DB_PATH)
498
720
  try:
499
721
  try:
500
722
  row = conn.execute(
@@ -876,7 +1098,15 @@ def _doctor_gather_state_impl(
876
1098
  _path.exists()
877
1099
  and (_label != "cache" or _cache_probe_allowed)
878
1100
  ):
879
- _conn = sqlite3.connect(str(_path))
1101
+ # #386: the stats leg holds a read-write handle for the whole
1102
+ # of a full quick_check — the longest-lived stats handle any
1103
+ # diagnostic takes — so it participates in the replacement
1104
+ # protocol. The cache leg keeps its own opener.
1105
+ if _label == "stats":
1106
+ import _cctally_store as _store_mod
1107
+ _conn = _store_mod.stats_open_guarded(_path)
1108
+ else:
1109
+ _conn = sqlite3.connect(str(_path))
880
1110
  try:
881
1111
  _row = _conn.execute(
882
1112
  "PRAGMA quick_check(1)").fetchone()
@@ -1068,11 +1298,16 @@ def _doctor_gather_state_impl(
1068
1298
  journal_present = False
1069
1299
  journal_appendable = None
1070
1300
  journal_segment_count = 0
1301
+ journal_has_bytes = False
1071
1302
  journal_malformed_count = None
1072
1303
  journal_torn_tail_count = None
1073
1304
  journal_cursor_lag_bytes = None
1074
1305
  journal_hw_segment = None
1075
1306
  journal_cursor_segment = None
1307
+ journal_conflicts = None
1308
+ journal_protocol_violations = None
1309
+ journal_protocol_acknowledged = None
1310
+ journal_protocol_error = None
1076
1311
  try:
1077
1312
  jdir = _cctally_core.JOURNAL_DIR
1078
1313
  journal_present = jdir.exists()
@@ -1087,6 +1322,35 @@ def _doctor_gather_state_impl(
1087
1322
  except Exception:
1088
1323
  segs = []
1089
1324
  journal_segment_count = len(segs)
1325
+ # #402: the disposable stats index persists the most recent complete
1326
+ # selector result. Shallow Dashboard/TUI gathers read that bounded
1327
+ # summary instead of rescanning a production-sized journal and
1328
+ # therefore cannot turn known taint into a false OK.
1329
+ try:
1330
+ if _cctally_core.DB_PATH.exists():
1331
+ pc = _stats_ro_guarded()
1332
+ try:
1333
+ protocol_rows = [
1334
+ json.loads(str(row[0]))
1335
+ for row in pc.execute(
1336
+ "SELECT violation_json "
1337
+ "FROM journal_protocol_violations "
1338
+ "ORDER BY batch_id, kind, fingerprint"
1339
+ )
1340
+ ]
1341
+ journal_protocol_violations = [
1342
+ item for item in protocol_rows
1343
+ if not item.get("auditId")
1344
+ ]
1345
+ journal_protocol_acknowledged = [
1346
+ item for item in protocol_rows
1347
+ if item.get("auditId")
1348
+ ]
1349
+ finally:
1350
+ pc.close()
1351
+ except (sqlite3.Error, ValueError, TypeError):
1352
+ journal_protocol_violations = None
1353
+ journal_protocol_acknowledged = None
1090
1354
  sizes: dict = {}
1091
1355
  for seg in segs:
1092
1356
  try:
@@ -1095,12 +1359,19 @@ def _doctor_gather_state_impl(
1095
1359
  sizes[seg] = 0
1096
1360
  if segs:
1097
1361
  journal_hw_segment = segs[-1]
1362
+ journal_has_bytes = _jr._has_retained_journal_bytes(
1363
+ sizes.values()
1364
+ )
1098
1365
  # deep-gated malformed / torn-tail scan (reads the whole journal;
1099
1366
  # the dashboard's per-rebuild gather stays deep=False so it never
1100
1367
  # pays this at the 10× envelope — mirrors the quick_check legs).
1101
1368
  if deep and segs:
1102
1369
  malformed = 0
1103
1370
  torn = 0
1371
+ decoded_records: list = []
1372
+ protocol_evidence = []
1373
+ prior_high_water = None
1374
+ cutover_value = None
1104
1375
  for seg in segs:
1105
1376
  try:
1106
1377
  data = (jdir / seg).read_bytes()
@@ -1113,24 +1384,124 @@ def _doctor_gather_state_impl(
1113
1384
  # every element except the last is a complete line; the last
1114
1385
  # is either "" (ended in \n) or the torn partial — not a
1115
1386
  # mid-file line, so it is never counted as malformed.
1387
+ offset = 0
1116
1388
  for raw in data.split(b"\n")[:-1]:
1117
- if raw and _jl.decode_line(raw) is None:
1389
+ if not raw:
1390
+ prior_high_water = (seg, offset + 1)
1391
+ offset += 1
1392
+ continue
1393
+ record = _jl.decode_line(raw)
1394
+ if record is None:
1118
1395
  malformed += 1
1396
+ prior_high_water = (
1397
+ seg,
1398
+ offset + len(raw) + 1,
1399
+ )
1400
+ offset += len(raw) + 1
1401
+ continue
1402
+ _jr._capture_protocol_prefix_evidence(
1403
+ record,
1404
+ prior_high_water,
1405
+ protocol_evidence,
1406
+ )
1407
+ # first cutover op wins, exactly as
1408
+ # `find_accounts_cutover_op` scans — captured here so the
1409
+ # conflict scan does not decode the whole journal twice.
1410
+ if (cutover_value is None
1411
+ and record.get("id") == _jr.CUTOVER_OP_ID):
1412
+ payload = record.get("payload")
1413
+ if isinstance(payload, dict):
1414
+ cutover_value = payload.get(
1415
+ "claude_legacy_account")
1416
+ # RETAIN ONLY what the selector consumes. `obs` lines are
1417
+ # ~97% of a real journal (984k of 1.02M) and
1418
+ # `resolve_effective_events` ignores them entirely —
1419
+ # keeping them cost 4.3 GB of peak RSS for an identical
1420
+ # result (#374 review).
1421
+ if record.get("t") in _CONFLICT_SCAN_RECORD_TYPES:
1422
+ decoded_records.append(record)
1423
+ prior_high_water = (
1424
+ seg,
1425
+ offset + len(raw) + 1,
1426
+ )
1427
+ offset += len(raw) + 1
1119
1428
  journal_malformed_count = malformed
1120
1429
  journal_torn_tail_count = torn
1430
+ # #374: same-revision quarantine, via the SHARED selector over
1431
+ # rebuild-equivalent input. Raw `(id, rev)` grouping would report
1432
+ # lower-revision groups a completed rev-1 batch legitimately
1433
+ # superseded, and false account conflicts that the rebuild's
1434
+ # `_normalize_legacy_account_stamp` resolves — so normalize
1435
+ # exactly as `rebuild_stats_index` does, then select.
1436
+ try:
1437
+ cutover_claude = (
1438
+ cutover_value if cutover_value is not None
1439
+ else _jr.resolve_cutover_claude_account()
1440
+ )
1441
+ for record in decoded_records:
1442
+ _jr._normalize_legacy_account_stamp(
1443
+ record, cutover_claude)
1444
+ selection = _jl.resolve_effective_events(
1445
+ decoded_records,
1446
+ protocol_prefix_evidence=protocol_evidence,
1447
+ )
1448
+ except _jl.JournalProtocolError as exc:
1449
+ # Out-of-scope malformed known record: selection did not
1450
+ # finish, so conflicts/tainted-batch results are unavailable.
1451
+ journal_protocol_error = str(exc)
1452
+ journal_conflicts = None
1453
+ journal_protocol_violations = None
1454
+ journal_protocol_acknowledged = None
1455
+ except Exception:
1456
+ journal_conflicts = None
1457
+ journal_protocol_violations = None
1458
+ journal_protocol_acknowledged = None
1459
+ else:
1460
+ journal_conflicts = [
1461
+ conflict.to_dict() for conflict in selection.conflicts
1462
+ ]
1463
+ journal_protocol_violations = [
1464
+ violation.to_dict()
1465
+ for violation in selection.protocol_violations
1466
+ ]
1467
+ journal_protocol_acknowledged = [
1468
+ violation.to_dict()
1469
+ for violation in (
1470
+ selection.acknowledged_protocol_violations
1471
+ )
1472
+ ]
1121
1473
  # ingest cursor lag: unconsumed bytes between the stats index cursor
1122
1474
  # and the journal high-water, in canonical (segment, offset) order.
1123
1475
  cursor = None
1124
1476
  try:
1125
1477
  if _cctally_core.DB_PATH.exists():
1126
- jc = sqlite3.connect(
1127
- f"file:{_cctally_core.DB_PATH}?mode=ro", uri=True)
1478
+ jc = _stats_ro_guarded() # #386 opener protocol
1128
1479
  try:
1129
- crow = jc.execute(
1130
- "SELECT segment, offset FROM journal_cursor "
1131
- "WHERE id = 1").fetchone()
1132
- if crow:
1133
- cursor = (crow[0], int(crow[1]))
1480
+ cursor_columns = {
1481
+ str(row[1])
1482
+ for row in jc.execute(
1483
+ "PRAGMA table_info(journal_cursor)"
1484
+ )
1485
+ }
1486
+ if {
1487
+ "applied_segment", "applied_offset"
1488
+ } <= cursor_columns:
1489
+ crow = jc.execute(
1490
+ "SELECT segment, offset, applied_segment, "
1491
+ "applied_offset FROM journal_cursor "
1492
+ "WHERE id = 1").fetchone()
1493
+ if (
1494
+ crow is not None
1495
+ and crow[2] is not None
1496
+ and crow[3] is not None
1497
+ ):
1498
+ cursor = (crow[2], int(crow[3]))
1499
+ else:
1500
+ legacy = jc.execute(
1501
+ "SELECT segment, offset FROM journal_cursor "
1502
+ "WHERE id = 1").fetchone()
1503
+ if legacy is not None:
1504
+ cursor = (legacy[0], int(legacy[1]))
1134
1505
  except sqlite3.OperationalError:
1135
1506
  pass # pre-cutover DB has no journal_cursor table
1136
1507
  finally:
@@ -1181,6 +1552,16 @@ def _doctor_gather_state_impl(
1181
1552
  d["age_s"] if d["age_s"] is not None else 0))
1182
1553
  journal_heal_incidents = _incidents
1183
1554
 
1555
+ # #386/#389 stats sole-writer guard log (spec §6.4). Read-only, fail-soft: an
1556
+ # absent log is the NORMAL state and must read as INFO, never as a gather
1557
+ # failure. Read only the bounded tail; rotation and cross-process throttling
1558
+ # bound the writer side independently.
1559
+ journal_writer_guard = None
1560
+ try:
1561
+ journal_writer_guard = _gather_writer_guard_log(now_utc)
1562
+ except (OSError, Exception):
1563
+ journal_writer_guard = None
1564
+
1184
1565
  cctally_version_tuple = _lib_changelog._read_latest_changelog_version()
1185
1566
  cctally_version = (
1186
1567
  cctally_version_tuple[0] if cctally_version_tuple else "unknown"
@@ -1251,6 +1632,11 @@ def _doctor_gather_state_impl(
1251
1632
  locks_held=locks_held,
1252
1633
  # #297: cache.db WAL size backstop (gathered outside the deep branch).
1253
1634
  cache_db_wal_bytes=cache_db_wal_bytes,
1635
+ # #374: quarantined same-revision groups + structural protocol violation.
1636
+ journal_conflicts=journal_conflicts,
1637
+ journal_protocol_violations=journal_protocol_violations,
1638
+ journal_protocol_acknowledged=journal_protocol_acknowledged,
1639
+ journal_protocol_error=journal_protocol_error,
1254
1640
  # #315: read-only cache free-page evidence for the reclaim hint.
1255
1641
  cache_db_page_count=cache_db_page_count,
1256
1642
  cache_db_freelist_count=cache_db_freelist_count,
@@ -1266,15 +1652,18 @@ def _doctor_gather_state_impl(
1266
1652
  journal_present=journal_present,
1267
1653
  journal_appendable=journal_appendable,
1268
1654
  journal_segment_count=journal_segment_count,
1655
+ journal_has_bytes=journal_has_bytes,
1269
1656
  journal_malformed_count=journal_malformed_count,
1270
1657
  journal_torn_tail_count=journal_torn_tail_count,
1271
1658
  journal_cursor_lag_bytes=journal_cursor_lag_bytes,
1272
1659
  journal_hw_segment=journal_hw_segment,
1273
1660
  journal_cursor_segment=journal_cursor_segment,
1274
1661
  journal_heal_incidents=journal_heal_incidents,
1662
+ journal_writer_guard=journal_writer_guard,
1275
1663
  # Multi-account attribution legs (#341).
1276
1664
  accounts_state=_gather_accounts_state(now_utc),
1277
1665
  cache_repair_marker=cache_repair_marker,
1666
+ backup_sync_state=backup_sync_state,
1278
1667
  )
1279
1668
 
1280
1669
 
@@ -1629,8 +1629,16 @@ def _backfill_five_hour_blocks(
1629
1629
  ).fetchone()
1630
1630
  crossed = 1 if cross_row is not None else 0
1631
1631
 
1632
- # is_closed: 1 if the canonical reset moment is already past.
1633
- is_closed = 1 if resets_dt < now_dt else 0
1632
+ # A rebuild's only-missing row is the trailing unjournaled
1633
+ # projection, never a close decision. Wall time may be past its
1634
+ # reset, but only a later retained observation may close and
1635
+ # clock it (#399). The ordinary legacy backfill keeps its
1636
+ # historical wall-time classification.
1637
+ is_closed = (
1638
+ 0 if only_missing else (1 if resets_dt < now_dt else 0)
1639
+ )
1640
+ projection_created_at = first_obs if only_missing else now_iso
1641
+ projection_updated_at = last_obs if only_missing else now_iso
1634
1642
 
1635
1643
  # Token + cost totals — recomputed via the shared helper,
1636
1644
  # which routes through get_entries() and falls back to
@@ -1681,8 +1689,8 @@ def _backfill_five_hour_blocks(
1681
1689
  totals["cache_read_tokens"],
1682
1690
  totals["cost_usd"],
1683
1691
  is_closed,
1684
- now_iso,
1685
- now_iso,
1692
+ projection_created_at,
1693
+ projection_updated_at,
1686
1694
  acct,
1687
1695
  ),
1688
1696
  )
@@ -1777,4 +1785,3 @@ def _backfill_five_hour_blocks(
1777
1785
  return 0
1778
1786
  return inserted
1779
1787
 
1780
-