cctally 1.82.1 → 1.83.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -0
- package/README.md +12 -5
- package/bin/_cctally_alerts.py +8 -1
- package/bin/_cctally_cache.py +912 -149
- package/bin/_cctally_config.py +43 -4
- package/bin/_cctally_core.py +933 -759
- package/bin/_cctally_dashboard.py +157 -47
- package/bin/_cctally_dashboard_cache_report.py +13 -6
- package/bin/_cctally_dashboard_conversation.py +1 -0
- package/bin/_cctally_dashboard_envelope.py +116 -8
- package/bin/_cctally_dashboard_share.py +50 -19
- package/bin/_cctally_dashboard_sources.py +223 -48
- package/bin/_cctally_db.py +605 -128
- package/bin/_cctally_doctor.py +417 -28
- package/bin/_cctally_five_hour.py +12 -5
- package/bin/_cctally_journal.py +2050 -156
- package/bin/_cctally_journal_repair.py +519 -0
- package/bin/_cctally_milestone_history.py +142 -56
- package/bin/_cctally_milestones.py +179 -111
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_project.py +24 -18
- package/bin/_cctally_quota.py +139 -25
- package/bin/_cctally_record.py +279 -108
- package/bin/_cctally_rederive.py +1052 -0
- package/bin/_cctally_reporting.py +58 -53
- package/bin/_cctally_setup.py +1 -0
- package/bin/_cctally_source_analytics.py +4 -1
- package/bin/_cctally_statusline.py +11 -11
- package/bin/_cctally_store.py +1039 -31
- package/bin/_cctally_sync_week.py +17 -8
- package/bin/_cctally_tui.py +350 -44
- package/bin/_cctally_update.py +133 -8
- package/bin/_cctally_weekrefs.py +14 -0
- package/bin/_lib_aggregators.py +10 -6
- package/bin/_lib_cache_report.py +101 -9
- package/bin/_lib_codex_pools.py +82 -0
- package/bin/_lib_conversation_query.py +81 -33
- package/bin/_lib_dashboard_sources.py +75 -0
- package/bin/_lib_diff_kernel.py +28 -15
- package/bin/_lib_doctor.py +342 -4
- package/bin/_lib_journal.py +924 -2
- package/bin/_lib_jsonl.py +43 -14
- package/bin/_lib_pricing.py +140 -21
- package/bin/_lib_rederive.py +395 -0
- package/bin/_lib_share.py +58 -2
- package/bin/cctally +56 -8
- package/dashboard/static/assets/{index-BKM43pxK.js → index-3bgCMVHb.js} +52 -52
- package/dashboard/static/assets/index-D27EIHEI.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +5 -1
- package/dashboard/static/assets/index-Dk1nplOz.css +0 -1
package/bin/_cctally_doctor.py
CHANGED
|
@@ -32,6 +32,7 @@ import os
|
|
|
32
32
|
import pathlib
|
|
33
33
|
import shutil
|
|
34
34
|
import sqlite3
|
|
35
|
+
import subprocess
|
|
35
36
|
import sys
|
|
36
37
|
|
|
37
38
|
import _cctally_core
|
|
@@ -40,28 +41,210 @@ from _cctally_core import _now_utc, eprint, now_utc_iso, parse_iso_datetime
|
|
|
40
41
|
from _lib_dashboard_json import encode_dashboard_json
|
|
41
42
|
|
|
42
43
|
|
|
44
|
+
#: Record types the deep journal conflict scan retains (#374). The selector
|
|
45
|
+
#: (`_lib_journal.resolve_effective_events`) reads only `evt`, `correction` and
|
|
46
|
+
#: `correction_batch`; `op` is kept because the rebuild-equivalent account
|
|
47
|
+
#: normalization is defined over evt/op records. Everything else — above all the
|
|
48
|
+
#: `obs` lines, ~97% of a real journal — is dropped as it is decoded, so the deep
|
|
49
|
+
#: gather's peak RSS tracks the decision history rather than the whole journal.
|
|
50
|
+
_CONFLICT_SCAN_RECORD_TYPES = frozenset(
|
|
51
|
+
{"evt", "correction", "correction_batch", "op"})
|
|
52
|
+
|
|
53
|
+
#: Doctor needs only recent evidence for this diagnostic. Bound both line count
|
|
54
|
+
#: and bytes so a corrupt single-line file cannot defeat the tail limit.
|
|
55
|
+
_GUARD_LOG_TAIL_LINES = 256
|
|
56
|
+
_GUARD_LOG_TAIL_BYTES = 64 * 1024
|
|
57
|
+
|
|
58
|
+
|
|
43
59
|
def _cctally():
|
|
44
60
|
"""Resolve the current `cctally` module at call-time (spec §3.1)."""
|
|
45
61
|
return sys.modules["cctally"]
|
|
46
62
|
|
|
47
63
|
|
|
64
|
+
def _stats_ro_guarded():
|
|
65
|
+
"""A `mode=ro` stats connection that participates in the #386 opener protocol.
|
|
66
|
+
|
|
67
|
+
Read-only does not mean side-effect-free: measured on this platform, a
|
|
68
|
+
`mode=ro` connection to a WAL database with absent sidecars CREATES
|
|
69
|
+
`stats.db-shm` and `stats.db-wal`. Spec §3.1's third clause therefore
|
|
70
|
+
covers these diagnostics, and both callers already degrade on
|
|
71
|
+
`sqlite3.Error` (which `StatsDbMaintenanceError` is).
|
|
72
|
+
"""
|
|
73
|
+
import _cctally_store
|
|
74
|
+
|
|
75
|
+
return _cctally_store.stats_open_guarded(
|
|
76
|
+
_cctally_core.DB_PATH,
|
|
77
|
+
connect=lambda p: sqlite3.connect(f"file:{p}?mode=ro", uri=True),
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
48
81
|
def _journal_heal_incident(kind: str, name: str, now_utc: dt.datetime) -> dict:
|
|
49
82
|
"""One auto-heal artifact record for the doctor journal leg (§9). Parses the
|
|
50
|
-
``%Y%m%dT%H%M%SZ``
|
|
51
|
-
|
|
52
|
-
|
|
83
|
+
legacy ``%Y%m%dT%H%M%SZ`` or collision-safe rebuild
|
|
84
|
+
``%Y%m%dT%H%M%S_%f`` timestamp trailing the name (quarantine dir
|
|
85
|
+
``<db>.db-<ts>`` or forensics file
|
|
86
|
+
``<db>.db-corruption-forensics-<ts>.json``) into an age; a name that
|
|
87
|
+
doesn't parse degrades to ``age_s=None``."""
|
|
53
88
|
base = name[:-5] if name.endswith(".json") else name
|
|
54
89
|
ts = base.rsplit("-", 1)[-1]
|
|
55
90
|
age_s = None
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
91
|
+
for timestamp_format in ("%Y%m%dT%H%M%SZ", "%Y%m%dT%H%M%S_%f"):
|
|
92
|
+
try:
|
|
93
|
+
parsed = dt.datetime.strptime(ts, timestamp_format).replace(
|
|
94
|
+
tzinfo=dt.timezone.utc)
|
|
95
|
+
age_s = int((now_utc - parsed).total_seconds())
|
|
96
|
+
break
|
|
97
|
+
except ValueError:
|
|
98
|
+
continue
|
|
62
99
|
return {"kind": kind, "name": name, "age_s": age_s}
|
|
63
100
|
|
|
64
101
|
|
|
102
|
+
def _read_guard_log_tail(path) -> list[str]:
|
|
103
|
+
"""Read at most the configured byte and line tail from one guard log."""
|
|
104
|
+
with path.open("rb") as fh:
|
|
105
|
+
fh.seek(0, os.SEEK_END)
|
|
106
|
+
size = fh.tell()
|
|
107
|
+
read_size = min(size, _GUARD_LOG_TAIL_BYTES)
|
|
108
|
+
fh.seek(size - read_size)
|
|
109
|
+
raw = fh.read(read_size)
|
|
110
|
+
if size > read_size:
|
|
111
|
+
# The byte window may start mid-line. Drop that fragment so every
|
|
112
|
+
# reported entry is one complete writer-guard record.
|
|
113
|
+
_, separator, raw = raw.partition(b"\n")
|
|
114
|
+
if not separator:
|
|
115
|
+
raw = b""
|
|
116
|
+
lines = [
|
|
117
|
+
line for line in raw.decode("utf-8", errors="replace").splitlines()
|
|
118
|
+
if line.strip()
|
|
119
|
+
]
|
|
120
|
+
return lines[-_GUARD_LOG_TAIL_LINES:]
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _gather_backup_sync_state(
|
|
124
|
+
app_dir,
|
|
125
|
+
*,
|
|
126
|
+
platform_name: str | None = None,
|
|
127
|
+
home_dir=None,
|
|
128
|
+
probe_time_machine: bool = True,
|
|
129
|
+
which=shutil.which,
|
|
130
|
+
run=subprocess.run,
|
|
131
|
+
) -> dict:
|
|
132
|
+
"""Classify known macOS file-level backup/sync coverage without mutation."""
|
|
133
|
+
platform_name = (
|
|
134
|
+
platform_name
|
|
135
|
+
or os.environ.get("CCTALLY_DOCTOR_FIXTURE_PLATFORM")
|
|
136
|
+
or sys.platform
|
|
137
|
+
)
|
|
138
|
+
if platform_name != "darwin":
|
|
139
|
+
return {"status": "unsupported", "provider": None}
|
|
140
|
+
|
|
141
|
+
try:
|
|
142
|
+
app_path = pathlib.Path(app_dir).resolve(strict=False)
|
|
143
|
+
home = pathlib.Path(
|
|
144
|
+
home_dir if home_dir is not None else pathlib.Path.home()
|
|
145
|
+
).resolve(strict=False)
|
|
146
|
+
except (OSError, RuntimeError):
|
|
147
|
+
return {"status": "unavailable", "provider": None}
|
|
148
|
+
|
|
149
|
+
def inside(root) -> bool:
|
|
150
|
+
try:
|
|
151
|
+
app_path.relative_to(pathlib.Path(root).resolve(strict=False))
|
|
152
|
+
return True
|
|
153
|
+
except (ValueError, OSError, RuntimeError):
|
|
154
|
+
return False
|
|
155
|
+
|
|
156
|
+
if inside(home / "Library/Mobile Documents/com~apple~CloudDocs"):
|
|
157
|
+
return {"status": "included", "provider": "iCloud Drive"}
|
|
158
|
+
if inside(home / "Dropbox"):
|
|
159
|
+
return {"status": "included", "provider": "Dropbox"}
|
|
160
|
+
cloud_storage = home / "Library/CloudStorage"
|
|
161
|
+
if inside(cloud_storage):
|
|
162
|
+
try:
|
|
163
|
+
relative = app_path.relative_to(cloud_storage.resolve(strict=False))
|
|
164
|
+
except (ValueError, OSError, RuntimeError):
|
|
165
|
+
relative = pathlib.Path()
|
|
166
|
+
if relative.parts and relative.parts[0].lower().startswith("dropbox"):
|
|
167
|
+
return {"status": "included", "provider": "Dropbox"}
|
|
168
|
+
|
|
169
|
+
# The dashboard gathers Doctor state on every envelope rebuild. Keep that
|
|
170
|
+
# hot path subprocess-free; the CLI's deep gather performs the bounded
|
|
171
|
+
# Time Machine probes below. Static cloud-root classification above is safe
|
|
172
|
+
# and cheap in either mode.
|
|
173
|
+
if not probe_time_machine:
|
|
174
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
175
|
+
|
|
176
|
+
try:
|
|
177
|
+
tmutil = which("tmutil")
|
|
178
|
+
except Exception:
|
|
179
|
+
tmutil = None
|
|
180
|
+
if not tmutil:
|
|
181
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
182
|
+
try:
|
|
183
|
+
destinations = run(
|
|
184
|
+
[tmutil, "destinationinfo"],
|
|
185
|
+
stdout=subprocess.PIPE,
|
|
186
|
+
stderr=subprocess.PIPE,
|
|
187
|
+
text=True,
|
|
188
|
+
timeout=2,
|
|
189
|
+
check=False,
|
|
190
|
+
)
|
|
191
|
+
except (OSError, subprocess.SubprocessError):
|
|
192
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
193
|
+
destination_text = f"{destinations.stdout}\n{destinations.stderr}".lower()
|
|
194
|
+
if destinations.returncode != 0:
|
|
195
|
+
status = (
|
|
196
|
+
"absent"
|
|
197
|
+
if "no destinations configured" in destination_text
|
|
198
|
+
else "unavailable"
|
|
199
|
+
)
|
|
200
|
+
return {"status": status, "provider": "Time Machine"}
|
|
201
|
+
if not destinations.stdout.strip():
|
|
202
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
203
|
+
try:
|
|
204
|
+
exclusion = run(
|
|
205
|
+
[tmutil, "isexcluded", str(app_path)],
|
|
206
|
+
stdout=subprocess.PIPE,
|
|
207
|
+
stderr=subprocess.PIPE,
|
|
208
|
+
text=True,
|
|
209
|
+
timeout=2,
|
|
210
|
+
check=False,
|
|
211
|
+
)
|
|
212
|
+
except (OSError, subprocess.SubprocessError):
|
|
213
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
214
|
+
if exclusion.returncode != 0:
|
|
215
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
216
|
+
output = exclusion.stdout.strip().lower()
|
|
217
|
+
if output.startswith("[excluded]"):
|
|
218
|
+
return {"status": "excluded", "provider": "Time Machine"}
|
|
219
|
+
if output.startswith("[included]"):
|
|
220
|
+
return {"status": "included", "provider": "Time Machine"}
|
|
221
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _gather_writer_guard_log(now_utc: dt.datetime):
|
|
225
|
+
"""Gather the bounded writer-guard tail; absent/unreadable is normal."""
|
|
226
|
+
import _cctally_store as _store_mod_guard
|
|
227
|
+
|
|
228
|
+
guard_path = _store_mod_guard._guard_log_path()
|
|
229
|
+
if not guard_path.exists():
|
|
230
|
+
return None
|
|
231
|
+
lines = _read_guard_log_tail(guard_path)
|
|
232
|
+
newest_age = None
|
|
233
|
+
if lines:
|
|
234
|
+
stamp = lines[-1].split("\t", 1)[0]
|
|
235
|
+
try:
|
|
236
|
+
when = _cctally_core.parse_iso_datetime(stamp, "guard log")
|
|
237
|
+
newest_age = max(0, int((now_utc - when).total_seconds()))
|
|
238
|
+
except Exception:
|
|
239
|
+
newest_age = None
|
|
240
|
+
return {
|
|
241
|
+
"entries": len(lines),
|
|
242
|
+
"newest_age_s": newest_age,
|
|
243
|
+
"path": str(guard_path),
|
|
244
|
+
"sample": lines[-1] if lines else None,
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
|
|
65
248
|
def _gather_statusline_pipeline(c, *, now_utc: dt.datetime) -> dict:
|
|
66
249
|
"""Read the #318 statusline pipeline without creating or pruning files."""
|
|
67
250
|
now_epoch = int(now_utc.timestamp())
|
|
@@ -203,7 +386,10 @@ def _gather_accounts_state(now_utc: "dt.datetime") -> dict:
|
|
|
203
386
|
if not _cctally_core.DB_PATH.exists():
|
|
204
387
|
return state
|
|
205
388
|
try:
|
|
206
|
-
|
|
389
|
+
# #386: a `mode=ro` reader participates in the opener protocol too —
|
|
390
|
+
# measured, it CREATES `-shm`/`-wal` when they are absent. See
|
|
391
|
+
# `_cctally_store.stats_open_guarded`.
|
|
392
|
+
conn = _stats_ro_guarded()
|
|
207
393
|
except sqlite3.Error:
|
|
208
394
|
return state
|
|
209
395
|
try:
|
|
@@ -314,13 +500,16 @@ def doctor_gather_state(
|
|
|
314
500
|
cache_lock.close()
|
|
315
501
|
cache_lock = None
|
|
316
502
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
503
|
+
import _cctally_store
|
|
504
|
+
|
|
505
|
+
with _cctally_store.suppress_interrupted_stats_recovery():
|
|
506
|
+
return _doctor_gather_state_impl(
|
|
507
|
+
now_utc=now_utc,
|
|
508
|
+
runtime_bind=runtime_bind,
|
|
509
|
+
deep=deep,
|
|
510
|
+
_cache_probe_allowed=cache_probe_allowed,
|
|
511
|
+
_cache_repair_marker=cache_repair_marker,
|
|
512
|
+
)
|
|
324
513
|
finally:
|
|
325
514
|
if cache_lock is not None:
|
|
326
515
|
try:
|
|
@@ -354,6 +543,11 @@ def _doctor_gather_state_impl(
|
|
|
354
543
|
if now_utc is None:
|
|
355
544
|
now_utc = _now_utc()
|
|
356
545
|
|
|
546
|
+
backup_sync_state = _gather_backup_sync_state(
|
|
547
|
+
_cctally_core.APP_DIR,
|
|
548
|
+
probe_time_machine=deep,
|
|
549
|
+
)
|
|
550
|
+
|
|
357
551
|
# ── Install ──────────────────────────────────────────────────────
|
|
358
552
|
# #279 S2 F5d: guard the only two unguarded statements in the
|
|
359
553
|
# otherwise fail-soft gather — an exception here would kill the whole
|
|
@@ -449,9 +643,29 @@ def _doctor_gather_state_impl(
|
|
|
449
643
|
|
|
450
644
|
# ── DB ───────────────────────────────────────────────────────────
|
|
451
645
|
try:
|
|
452
|
-
|
|
646
|
+
import _cctally_store
|
|
647
|
+
|
|
648
|
+
interrupted = _cctally_store.stats_interrupted_rebuild_evidence(
|
|
649
|
+
_cctally_core.DB_PATH
|
|
650
|
+
)
|
|
651
|
+
if interrupted is not None and interrupted.get("live") is True:
|
|
652
|
+
stats_db_status = {
|
|
653
|
+
"path": str(_cctally_core.DB_PATH),
|
|
654
|
+
"user_version": 0,
|
|
655
|
+
"registry_size": len(c._STATS_MIGRATIONS),
|
|
656
|
+
"migrations": [],
|
|
657
|
+
}
|
|
658
|
+
else:
|
|
659
|
+
stats_db_status = c._db_status_for(
|
|
660
|
+
_cctally_core.DB_PATH,
|
|
661
|
+
c._STATS_MIGRATIONS,
|
|
662
|
+
"stats.db",
|
|
663
|
+
recover_interrupted_stats=False,
|
|
664
|
+
)
|
|
453
665
|
if not _cctally_core.DB_PATH.exists():
|
|
454
666
|
stats_db_status["_file_exists"] = False
|
|
667
|
+
if interrupted is not None:
|
|
668
|
+
stats_db_status["_interrupted_rebuild"] = interrupted
|
|
455
669
|
except sqlite3.Error as exc:
|
|
456
670
|
stats_db_status = {"path": str(_cctally_core.DB_PATH), "user_version": 0,
|
|
457
671
|
"registry_size": len(c._STATS_MIGRATIONS),
|
|
@@ -494,7 +708,15 @@ def _doctor_gather_state_impl(
|
|
|
494
708
|
credited_weeks: list[dict] | None = None
|
|
495
709
|
try:
|
|
496
710
|
if _cctally_core.DB_PATH.exists():
|
|
497
|
-
|
|
711
|
+
# #386 spec section 3.1, third clause: EVERY opener of the live stats
|
|
712
|
+
# family participates in the replacement protocol, read-only probes
|
|
713
|
+
# included. The open mode stays read-WRITE deliberately — switching a
|
|
714
|
+
# WAL DB whose `-shm` may be absent to `mode=ro` fails
|
|
715
|
+
# SQLITE_CANTOPEN, which is not corruption and has been misread as
|
|
716
|
+
# such on this project twice. Participation, not read-only-ness, is
|
|
717
|
+
# what the clause requires.
|
|
718
|
+
import _cctally_store as _store_mod
|
|
719
|
+
conn = _store_mod.stats_open_guarded(_cctally_core.DB_PATH)
|
|
498
720
|
try:
|
|
499
721
|
try:
|
|
500
722
|
row = conn.execute(
|
|
@@ -876,7 +1098,15 @@ def _doctor_gather_state_impl(
|
|
|
876
1098
|
_path.exists()
|
|
877
1099
|
and (_label != "cache" or _cache_probe_allowed)
|
|
878
1100
|
):
|
|
879
|
-
|
|
1101
|
+
# #386: the stats leg holds a read-write handle for the whole
|
|
1102
|
+
# of a full quick_check — the longest-lived stats handle any
|
|
1103
|
+
# diagnostic takes — so it participates in the replacement
|
|
1104
|
+
# protocol. The cache leg keeps its own opener.
|
|
1105
|
+
if _label == "stats":
|
|
1106
|
+
import _cctally_store as _store_mod
|
|
1107
|
+
_conn = _store_mod.stats_open_guarded(_path)
|
|
1108
|
+
else:
|
|
1109
|
+
_conn = sqlite3.connect(str(_path))
|
|
880
1110
|
try:
|
|
881
1111
|
_row = _conn.execute(
|
|
882
1112
|
"PRAGMA quick_check(1)").fetchone()
|
|
@@ -1068,11 +1298,16 @@ def _doctor_gather_state_impl(
|
|
|
1068
1298
|
journal_present = False
|
|
1069
1299
|
journal_appendable = None
|
|
1070
1300
|
journal_segment_count = 0
|
|
1301
|
+
journal_has_bytes = False
|
|
1071
1302
|
journal_malformed_count = None
|
|
1072
1303
|
journal_torn_tail_count = None
|
|
1073
1304
|
journal_cursor_lag_bytes = None
|
|
1074
1305
|
journal_hw_segment = None
|
|
1075
1306
|
journal_cursor_segment = None
|
|
1307
|
+
journal_conflicts = None
|
|
1308
|
+
journal_protocol_violations = None
|
|
1309
|
+
journal_protocol_acknowledged = None
|
|
1310
|
+
journal_protocol_error = None
|
|
1076
1311
|
try:
|
|
1077
1312
|
jdir = _cctally_core.JOURNAL_DIR
|
|
1078
1313
|
journal_present = jdir.exists()
|
|
@@ -1087,6 +1322,35 @@ def _doctor_gather_state_impl(
|
|
|
1087
1322
|
except Exception:
|
|
1088
1323
|
segs = []
|
|
1089
1324
|
journal_segment_count = len(segs)
|
|
1325
|
+
# #402: the disposable stats index persists the most recent complete
|
|
1326
|
+
# selector result. Shallow Dashboard/TUI gathers read that bounded
|
|
1327
|
+
# summary instead of rescanning a production-sized journal and
|
|
1328
|
+
# therefore cannot turn known taint into a false OK.
|
|
1329
|
+
try:
|
|
1330
|
+
if _cctally_core.DB_PATH.exists():
|
|
1331
|
+
pc = _stats_ro_guarded()
|
|
1332
|
+
try:
|
|
1333
|
+
protocol_rows = [
|
|
1334
|
+
json.loads(str(row[0]))
|
|
1335
|
+
for row in pc.execute(
|
|
1336
|
+
"SELECT violation_json "
|
|
1337
|
+
"FROM journal_protocol_violations "
|
|
1338
|
+
"ORDER BY batch_id, kind, fingerprint"
|
|
1339
|
+
)
|
|
1340
|
+
]
|
|
1341
|
+
journal_protocol_violations = [
|
|
1342
|
+
item for item in protocol_rows
|
|
1343
|
+
if not item.get("auditId")
|
|
1344
|
+
]
|
|
1345
|
+
journal_protocol_acknowledged = [
|
|
1346
|
+
item for item in protocol_rows
|
|
1347
|
+
if item.get("auditId")
|
|
1348
|
+
]
|
|
1349
|
+
finally:
|
|
1350
|
+
pc.close()
|
|
1351
|
+
except (sqlite3.Error, ValueError, TypeError):
|
|
1352
|
+
journal_protocol_violations = None
|
|
1353
|
+
journal_protocol_acknowledged = None
|
|
1090
1354
|
sizes: dict = {}
|
|
1091
1355
|
for seg in segs:
|
|
1092
1356
|
try:
|
|
@@ -1095,12 +1359,19 @@ def _doctor_gather_state_impl(
|
|
|
1095
1359
|
sizes[seg] = 0
|
|
1096
1360
|
if segs:
|
|
1097
1361
|
journal_hw_segment = segs[-1]
|
|
1362
|
+
journal_has_bytes = _jr._has_retained_journal_bytes(
|
|
1363
|
+
sizes.values()
|
|
1364
|
+
)
|
|
1098
1365
|
# deep-gated malformed / torn-tail scan (reads the whole journal;
|
|
1099
1366
|
# the dashboard's per-rebuild gather stays deep=False so it never
|
|
1100
1367
|
# pays this at the 10× envelope — mirrors the quick_check legs).
|
|
1101
1368
|
if deep and segs:
|
|
1102
1369
|
malformed = 0
|
|
1103
1370
|
torn = 0
|
|
1371
|
+
decoded_records: list = []
|
|
1372
|
+
protocol_evidence = []
|
|
1373
|
+
prior_high_water = None
|
|
1374
|
+
cutover_value = None
|
|
1104
1375
|
for seg in segs:
|
|
1105
1376
|
try:
|
|
1106
1377
|
data = (jdir / seg).read_bytes()
|
|
@@ -1113,24 +1384,124 @@ def _doctor_gather_state_impl(
|
|
|
1113
1384
|
# every element except the last is a complete line; the last
|
|
1114
1385
|
# is either "" (ended in \n) or the torn partial — not a
|
|
1115
1386
|
# mid-file line, so it is never counted as malformed.
|
|
1387
|
+
offset = 0
|
|
1116
1388
|
for raw in data.split(b"\n")[:-1]:
|
|
1117
|
-
if
|
|
1389
|
+
if not raw:
|
|
1390
|
+
prior_high_water = (seg, offset + 1)
|
|
1391
|
+
offset += 1
|
|
1392
|
+
continue
|
|
1393
|
+
record = _jl.decode_line(raw)
|
|
1394
|
+
if record is None:
|
|
1118
1395
|
malformed += 1
|
|
1396
|
+
prior_high_water = (
|
|
1397
|
+
seg,
|
|
1398
|
+
offset + len(raw) + 1,
|
|
1399
|
+
)
|
|
1400
|
+
offset += len(raw) + 1
|
|
1401
|
+
continue
|
|
1402
|
+
_jr._capture_protocol_prefix_evidence(
|
|
1403
|
+
record,
|
|
1404
|
+
prior_high_water,
|
|
1405
|
+
protocol_evidence,
|
|
1406
|
+
)
|
|
1407
|
+
# first cutover op wins, exactly as
|
|
1408
|
+
# `find_accounts_cutover_op` scans — captured here so the
|
|
1409
|
+
# conflict scan does not decode the whole journal twice.
|
|
1410
|
+
if (cutover_value is None
|
|
1411
|
+
and record.get("id") == _jr.CUTOVER_OP_ID):
|
|
1412
|
+
payload = record.get("payload")
|
|
1413
|
+
if isinstance(payload, dict):
|
|
1414
|
+
cutover_value = payload.get(
|
|
1415
|
+
"claude_legacy_account")
|
|
1416
|
+
# RETAIN ONLY what the selector consumes. `obs` lines are
|
|
1417
|
+
# ~97% of a real journal (984k of 1.02M) and
|
|
1418
|
+
# `resolve_effective_events` ignores them entirely —
|
|
1419
|
+
# keeping them cost 4.3 GB of peak RSS for an identical
|
|
1420
|
+
# result (#374 review).
|
|
1421
|
+
if record.get("t") in _CONFLICT_SCAN_RECORD_TYPES:
|
|
1422
|
+
decoded_records.append(record)
|
|
1423
|
+
prior_high_water = (
|
|
1424
|
+
seg,
|
|
1425
|
+
offset + len(raw) + 1,
|
|
1426
|
+
)
|
|
1427
|
+
offset += len(raw) + 1
|
|
1119
1428
|
journal_malformed_count = malformed
|
|
1120
1429
|
journal_torn_tail_count = torn
|
|
1430
|
+
# #374: same-revision quarantine, via the SHARED selector over
|
|
1431
|
+
# rebuild-equivalent input. Raw `(id, rev)` grouping would report
|
|
1432
|
+
# lower-revision groups a completed rev-1 batch legitimately
|
|
1433
|
+
# superseded, and false account conflicts that the rebuild's
|
|
1434
|
+
# `_normalize_legacy_account_stamp` resolves — so normalize
|
|
1435
|
+
# exactly as `rebuild_stats_index` does, then select.
|
|
1436
|
+
try:
|
|
1437
|
+
cutover_claude = (
|
|
1438
|
+
cutover_value if cutover_value is not None
|
|
1439
|
+
else _jr.resolve_cutover_claude_account()
|
|
1440
|
+
)
|
|
1441
|
+
for record in decoded_records:
|
|
1442
|
+
_jr._normalize_legacy_account_stamp(
|
|
1443
|
+
record, cutover_claude)
|
|
1444
|
+
selection = _jl.resolve_effective_events(
|
|
1445
|
+
decoded_records,
|
|
1446
|
+
protocol_prefix_evidence=protocol_evidence,
|
|
1447
|
+
)
|
|
1448
|
+
except _jl.JournalProtocolError as exc:
|
|
1449
|
+
# Out-of-scope malformed known record: selection did not
|
|
1450
|
+
# finish, so conflicts/tainted-batch results are unavailable.
|
|
1451
|
+
journal_protocol_error = str(exc)
|
|
1452
|
+
journal_conflicts = None
|
|
1453
|
+
journal_protocol_violations = None
|
|
1454
|
+
journal_protocol_acknowledged = None
|
|
1455
|
+
except Exception:
|
|
1456
|
+
journal_conflicts = None
|
|
1457
|
+
journal_protocol_violations = None
|
|
1458
|
+
journal_protocol_acknowledged = None
|
|
1459
|
+
else:
|
|
1460
|
+
journal_conflicts = [
|
|
1461
|
+
conflict.to_dict() for conflict in selection.conflicts
|
|
1462
|
+
]
|
|
1463
|
+
journal_protocol_violations = [
|
|
1464
|
+
violation.to_dict()
|
|
1465
|
+
for violation in selection.protocol_violations
|
|
1466
|
+
]
|
|
1467
|
+
journal_protocol_acknowledged = [
|
|
1468
|
+
violation.to_dict()
|
|
1469
|
+
for violation in (
|
|
1470
|
+
selection.acknowledged_protocol_violations
|
|
1471
|
+
)
|
|
1472
|
+
]
|
|
1121
1473
|
# ingest cursor lag: unconsumed bytes between the stats index cursor
|
|
1122
1474
|
# and the journal high-water, in canonical (segment, offset) order.
|
|
1123
1475
|
cursor = None
|
|
1124
1476
|
try:
|
|
1125
1477
|
if _cctally_core.DB_PATH.exists():
|
|
1126
|
-
jc =
|
|
1127
|
-
f"file:{_cctally_core.DB_PATH}?mode=ro", uri=True)
|
|
1478
|
+
jc = _stats_ro_guarded() # #386 opener protocol
|
|
1128
1479
|
try:
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1480
|
+
cursor_columns = {
|
|
1481
|
+
str(row[1])
|
|
1482
|
+
for row in jc.execute(
|
|
1483
|
+
"PRAGMA table_info(journal_cursor)"
|
|
1484
|
+
)
|
|
1485
|
+
}
|
|
1486
|
+
if {
|
|
1487
|
+
"applied_segment", "applied_offset"
|
|
1488
|
+
} <= cursor_columns:
|
|
1489
|
+
crow = jc.execute(
|
|
1490
|
+
"SELECT segment, offset, applied_segment, "
|
|
1491
|
+
"applied_offset FROM journal_cursor "
|
|
1492
|
+
"WHERE id = 1").fetchone()
|
|
1493
|
+
if (
|
|
1494
|
+
crow is not None
|
|
1495
|
+
and crow[2] is not None
|
|
1496
|
+
and crow[3] is not None
|
|
1497
|
+
):
|
|
1498
|
+
cursor = (crow[2], int(crow[3]))
|
|
1499
|
+
else:
|
|
1500
|
+
legacy = jc.execute(
|
|
1501
|
+
"SELECT segment, offset FROM journal_cursor "
|
|
1502
|
+
"WHERE id = 1").fetchone()
|
|
1503
|
+
if legacy is not None:
|
|
1504
|
+
cursor = (legacy[0], int(legacy[1]))
|
|
1134
1505
|
except sqlite3.OperationalError:
|
|
1135
1506
|
pass # pre-cutover DB has no journal_cursor table
|
|
1136
1507
|
finally:
|
|
@@ -1181,6 +1552,16 @@ def _doctor_gather_state_impl(
|
|
|
1181
1552
|
d["age_s"] if d["age_s"] is not None else 0))
|
|
1182
1553
|
journal_heal_incidents = _incidents
|
|
1183
1554
|
|
|
1555
|
+
# #386/#389 stats sole-writer guard log (spec §6.4). Read-only, fail-soft: an
|
|
1556
|
+
# absent log is the NORMAL state and must read as INFO, never as a gather
|
|
1557
|
+
# failure. Read only the bounded tail; rotation and cross-process throttling
|
|
1558
|
+
# bound the writer side independently.
|
|
1559
|
+
journal_writer_guard = None
|
|
1560
|
+
try:
|
|
1561
|
+
journal_writer_guard = _gather_writer_guard_log(now_utc)
|
|
1562
|
+
except (OSError, Exception):
|
|
1563
|
+
journal_writer_guard = None
|
|
1564
|
+
|
|
1184
1565
|
cctally_version_tuple = _lib_changelog._read_latest_changelog_version()
|
|
1185
1566
|
cctally_version = (
|
|
1186
1567
|
cctally_version_tuple[0] if cctally_version_tuple else "unknown"
|
|
@@ -1251,6 +1632,11 @@ def _doctor_gather_state_impl(
|
|
|
1251
1632
|
locks_held=locks_held,
|
|
1252
1633
|
# #297: cache.db WAL size backstop (gathered outside the deep branch).
|
|
1253
1634
|
cache_db_wal_bytes=cache_db_wal_bytes,
|
|
1635
|
+
# #374: quarantined same-revision groups + structural protocol violation.
|
|
1636
|
+
journal_conflicts=journal_conflicts,
|
|
1637
|
+
journal_protocol_violations=journal_protocol_violations,
|
|
1638
|
+
journal_protocol_acknowledged=journal_protocol_acknowledged,
|
|
1639
|
+
journal_protocol_error=journal_protocol_error,
|
|
1254
1640
|
# #315: read-only cache free-page evidence for the reclaim hint.
|
|
1255
1641
|
cache_db_page_count=cache_db_page_count,
|
|
1256
1642
|
cache_db_freelist_count=cache_db_freelist_count,
|
|
@@ -1266,15 +1652,18 @@ def _doctor_gather_state_impl(
|
|
|
1266
1652
|
journal_present=journal_present,
|
|
1267
1653
|
journal_appendable=journal_appendable,
|
|
1268
1654
|
journal_segment_count=journal_segment_count,
|
|
1655
|
+
journal_has_bytes=journal_has_bytes,
|
|
1269
1656
|
journal_malformed_count=journal_malformed_count,
|
|
1270
1657
|
journal_torn_tail_count=journal_torn_tail_count,
|
|
1271
1658
|
journal_cursor_lag_bytes=journal_cursor_lag_bytes,
|
|
1272
1659
|
journal_hw_segment=journal_hw_segment,
|
|
1273
1660
|
journal_cursor_segment=journal_cursor_segment,
|
|
1274
1661
|
journal_heal_incidents=journal_heal_incidents,
|
|
1662
|
+
journal_writer_guard=journal_writer_guard,
|
|
1275
1663
|
# Multi-account attribution legs (#341).
|
|
1276
1664
|
accounts_state=_gather_accounts_state(now_utc),
|
|
1277
1665
|
cache_repair_marker=cache_repair_marker,
|
|
1666
|
+
backup_sync_state=backup_sync_state,
|
|
1278
1667
|
)
|
|
1279
1668
|
|
|
1280
1669
|
|
|
@@ -1629,8 +1629,16 @@ def _backfill_five_hour_blocks(
|
|
|
1629
1629
|
).fetchone()
|
|
1630
1630
|
crossed = 1 if cross_row is not None else 0
|
|
1631
1631
|
|
|
1632
|
-
#
|
|
1633
|
-
|
|
1632
|
+
# A rebuild's only-missing row is the trailing unjournaled
|
|
1633
|
+
# projection, never a close decision. Wall time may be past its
|
|
1634
|
+
# reset, but only a later retained observation may close and
|
|
1635
|
+
# clock it (#399). The ordinary legacy backfill keeps its
|
|
1636
|
+
# historical wall-time classification.
|
|
1637
|
+
is_closed = (
|
|
1638
|
+
0 if only_missing else (1 if resets_dt < now_dt else 0)
|
|
1639
|
+
)
|
|
1640
|
+
projection_created_at = first_obs if only_missing else now_iso
|
|
1641
|
+
projection_updated_at = last_obs if only_missing else now_iso
|
|
1634
1642
|
|
|
1635
1643
|
# Token + cost totals — recomputed via the shared helper,
|
|
1636
1644
|
# which routes through get_entries() and falls back to
|
|
@@ -1681,8 +1689,8 @@ def _backfill_five_hour_blocks(
|
|
|
1681
1689
|
totals["cache_read_tokens"],
|
|
1682
1690
|
totals["cost_usd"],
|
|
1683
1691
|
is_closed,
|
|
1684
|
-
|
|
1685
|
-
|
|
1692
|
+
projection_created_at,
|
|
1693
|
+
projection_updated_at,
|
|
1686
1694
|
acct,
|
|
1687
1695
|
),
|
|
1688
1696
|
)
|
|
@@ -1777,4 +1785,3 @@ def _backfill_five_hour_blocks(
|
|
|
1777
1785
|
return 0
|
|
1778
1786
|
return inserted
|
|
1779
1787
|
|
|
1780
|
-
|