cctally 1.82.0 → 1.83.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/README.md +52 -74
- package/bin/_cctally_alerts.py +8 -1
- package/bin/_cctally_cache.py +963 -149
- package/bin/_cctally_config.py +43 -4
- package/bin/_cctally_core.py +933 -759
- package/bin/_cctally_dashboard.py +157 -47
- package/bin/_cctally_dashboard_cache_report.py +13 -6
- package/bin/_cctally_dashboard_conversation.py +1 -0
- package/bin/_cctally_dashboard_envelope.py +186 -8
- package/bin/_cctally_dashboard_share.py +60 -20
- package/bin/_cctally_dashboard_sources.py +427 -128
- package/bin/_cctally_db.py +605 -128
- package/bin/_cctally_doctor.py +413 -28
- package/bin/_cctally_five_hour.py +12 -5
- package/bin/_cctally_journal.py +2050 -156
- package/bin/_cctally_journal_repair.py +519 -0
- package/bin/_cctally_milestone_history.py +142 -56
- package/bin/_cctally_milestones.py +179 -111
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_project.py +24 -18
- package/bin/_cctally_quota.py +139 -25
- package/bin/_cctally_record.py +279 -108
- package/bin/_cctally_rederive.py +1052 -0
- package/bin/_cctally_reporting.py +58 -53
- package/bin/_cctally_setup.py +1 -0
- package/bin/_cctally_source_analytics.py +4 -1
- package/bin/_cctally_statusline.py +11 -11
- package/bin/_cctally_store.py +1039 -31
- package/bin/_cctally_sync_week.py +17 -8
- package/bin/_cctally_tui.py +421 -54
- package/bin/_cctally_update.py +133 -8
- package/bin/_cctally_weekrefs.py +14 -0
- package/bin/_lib_aggregators.py +10 -6
- package/bin/_lib_cache_report.py +101 -9
- package/bin/_lib_codex_pools.py +82 -0
- package/bin/_lib_conversation_query.py +126 -33
- package/bin/_lib_dashboard_sources.py +126 -1
- package/bin/_lib_diff_kernel.py +28 -15
- package/bin/_lib_doctor.py +342 -4
- package/bin/_lib_journal.py +924 -2
- package/bin/_lib_jsonl.py +43 -14
- package/bin/_lib_pricing.py +140 -21
- package/bin/_lib_readme_refresh.py +401 -0
- package/bin/_lib_rederive.py +395 -0
- package/bin/_lib_share.py +58 -2
- package/bin/cctally +56 -8
- package/dashboard/static/assets/{index-DJP4gEB7.js → index-3bgCMVHb.js} +52 -52
- package/dashboard/static/assets/index-D27EIHEI.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +6 -1
- package/dashboard/static/assets/index-Dk1nplOz.css +0 -1
package/bin/_cctally_doctor.py
CHANGED
|
@@ -32,6 +32,7 @@ import os
|
|
|
32
32
|
import pathlib
|
|
33
33
|
import shutil
|
|
34
34
|
import sqlite3
|
|
35
|
+
import subprocess
|
|
35
36
|
import sys
|
|
36
37
|
|
|
37
38
|
import _cctally_core
|
|
@@ -40,28 +41,206 @@ from _cctally_core import _now_utc, eprint, now_utc_iso, parse_iso_datetime
|
|
|
40
41
|
from _lib_dashboard_json import encode_dashboard_json
|
|
41
42
|
|
|
42
43
|
|
|
44
|
+
#: Record types the deep journal conflict scan retains (#374). The selector
|
|
45
|
+
#: (`_lib_journal.resolve_effective_events`) reads only `evt`, `correction` and
|
|
46
|
+
#: `correction_batch`; `op` is kept because the rebuild-equivalent account
|
|
47
|
+
#: normalization is defined over evt/op records. Everything else — above all the
|
|
48
|
+
#: `obs` lines, ~97% of a real journal — is dropped as it is decoded, so the deep
|
|
49
|
+
#: gather's peak RSS tracks the decision history rather than the whole journal.
|
|
50
|
+
_CONFLICT_SCAN_RECORD_TYPES = frozenset(
|
|
51
|
+
{"evt", "correction", "correction_batch", "op"})
|
|
52
|
+
|
|
53
|
+
#: Doctor needs only recent evidence for this diagnostic. Bound both line count
|
|
54
|
+
#: and bytes so a corrupt single-line file cannot defeat the tail limit.
|
|
55
|
+
_GUARD_LOG_TAIL_LINES = 256
|
|
56
|
+
_GUARD_LOG_TAIL_BYTES = 64 * 1024
|
|
57
|
+
|
|
58
|
+
|
|
43
59
|
def _cctally():
|
|
44
60
|
"""Resolve the current `cctally` module at call-time (spec §3.1)."""
|
|
45
61
|
return sys.modules["cctally"]
|
|
46
62
|
|
|
47
63
|
|
|
64
|
+
def _stats_ro_guarded():
|
|
65
|
+
"""A `mode=ro` stats connection that participates in the #386 opener protocol.
|
|
66
|
+
|
|
67
|
+
Read-only does not mean side-effect-free: measured on this platform, a
|
|
68
|
+
`mode=ro` connection to a WAL database with absent sidecars CREATES
|
|
69
|
+
`stats.db-shm` and `stats.db-wal`. Spec §3.1's third clause therefore
|
|
70
|
+
covers these diagnostics, and both callers already degrade on
|
|
71
|
+
`sqlite3.Error` (which `StatsDbMaintenanceError` is).
|
|
72
|
+
"""
|
|
73
|
+
import _cctally_store
|
|
74
|
+
|
|
75
|
+
return _cctally_store.stats_open_guarded(
|
|
76
|
+
_cctally_core.DB_PATH,
|
|
77
|
+
connect=lambda p: sqlite3.connect(f"file:{p}?mode=ro", uri=True),
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
48
81
|
def _journal_heal_incident(kind: str, name: str, now_utc: dt.datetime) -> dict:
|
|
49
82
|
"""One auto-heal artifact record for the doctor journal leg (§9). Parses the
|
|
50
|
-
``%Y%m%dT%H%M%SZ``
|
|
51
|
-
|
|
52
|
-
|
|
83
|
+
legacy ``%Y%m%dT%H%M%SZ`` or collision-safe rebuild
|
|
84
|
+
``%Y%m%dT%H%M%S_%f`` timestamp trailing the name (quarantine dir
|
|
85
|
+
``<db>.db-<ts>`` or forensics file
|
|
86
|
+
``<db>.db-corruption-forensics-<ts>.json``) into an age; a name that
|
|
87
|
+
doesn't parse degrades to ``age_s=None``."""
|
|
53
88
|
base = name[:-5] if name.endswith(".json") else name
|
|
54
89
|
ts = base.rsplit("-", 1)[-1]
|
|
55
90
|
age_s = None
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
91
|
+
for timestamp_format in ("%Y%m%dT%H%M%SZ", "%Y%m%dT%H%M%S_%f"):
|
|
92
|
+
try:
|
|
93
|
+
parsed = dt.datetime.strptime(ts, timestamp_format).replace(
|
|
94
|
+
tzinfo=dt.timezone.utc)
|
|
95
|
+
age_s = int((now_utc - parsed).total_seconds())
|
|
96
|
+
break
|
|
97
|
+
except ValueError:
|
|
98
|
+
continue
|
|
62
99
|
return {"kind": kind, "name": name, "age_s": age_s}
|
|
63
100
|
|
|
64
101
|
|
|
102
|
+
def _read_guard_log_tail(path) -> list[str]:
|
|
103
|
+
"""Read at most the configured byte and line tail from one guard log."""
|
|
104
|
+
with path.open("rb") as fh:
|
|
105
|
+
fh.seek(0, os.SEEK_END)
|
|
106
|
+
size = fh.tell()
|
|
107
|
+
read_size = min(size, _GUARD_LOG_TAIL_BYTES)
|
|
108
|
+
fh.seek(size - read_size)
|
|
109
|
+
raw = fh.read(read_size)
|
|
110
|
+
if size > read_size:
|
|
111
|
+
# The byte window may start mid-line. Drop that fragment so every
|
|
112
|
+
# reported entry is one complete writer-guard record.
|
|
113
|
+
_, separator, raw = raw.partition(b"\n")
|
|
114
|
+
if not separator:
|
|
115
|
+
raw = b""
|
|
116
|
+
lines = [
|
|
117
|
+
line for line in raw.decode("utf-8", errors="replace").splitlines()
|
|
118
|
+
if line.strip()
|
|
119
|
+
]
|
|
120
|
+
return lines[-_GUARD_LOG_TAIL_LINES:]
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _gather_backup_sync_state(
|
|
124
|
+
app_dir,
|
|
125
|
+
*,
|
|
126
|
+
platform_name: str | None = None,
|
|
127
|
+
home_dir=None,
|
|
128
|
+
probe_time_machine: bool = True,
|
|
129
|
+
which=shutil.which,
|
|
130
|
+
run=subprocess.run,
|
|
131
|
+
) -> dict:
|
|
132
|
+
"""Classify known macOS file-level backup/sync coverage without mutation."""
|
|
133
|
+
platform_name = platform_name or sys.platform
|
|
134
|
+
if platform_name != "darwin":
|
|
135
|
+
return {"status": "unsupported", "provider": None}
|
|
136
|
+
|
|
137
|
+
try:
|
|
138
|
+
app_path = pathlib.Path(app_dir).resolve(strict=False)
|
|
139
|
+
home = pathlib.Path(
|
|
140
|
+
home_dir if home_dir is not None else pathlib.Path.home()
|
|
141
|
+
).resolve(strict=False)
|
|
142
|
+
except (OSError, RuntimeError):
|
|
143
|
+
return {"status": "unavailable", "provider": None}
|
|
144
|
+
|
|
145
|
+
def inside(root) -> bool:
|
|
146
|
+
try:
|
|
147
|
+
app_path.relative_to(pathlib.Path(root).resolve(strict=False))
|
|
148
|
+
return True
|
|
149
|
+
except (ValueError, OSError, RuntimeError):
|
|
150
|
+
return False
|
|
151
|
+
|
|
152
|
+
if inside(home / "Library/Mobile Documents/com~apple~CloudDocs"):
|
|
153
|
+
return {"status": "included", "provider": "iCloud Drive"}
|
|
154
|
+
if inside(home / "Dropbox"):
|
|
155
|
+
return {"status": "included", "provider": "Dropbox"}
|
|
156
|
+
cloud_storage = home / "Library/CloudStorage"
|
|
157
|
+
if inside(cloud_storage):
|
|
158
|
+
try:
|
|
159
|
+
relative = app_path.relative_to(cloud_storage.resolve(strict=False))
|
|
160
|
+
except (ValueError, OSError, RuntimeError):
|
|
161
|
+
relative = pathlib.Path()
|
|
162
|
+
if relative.parts and relative.parts[0].lower().startswith("dropbox"):
|
|
163
|
+
return {"status": "included", "provider": "Dropbox"}
|
|
164
|
+
|
|
165
|
+
# The dashboard gathers Doctor state on every envelope rebuild. Keep that
|
|
166
|
+
# hot path subprocess-free; the CLI's deep gather performs the bounded
|
|
167
|
+
# Time Machine probes below. Static cloud-root classification above is safe
|
|
168
|
+
# and cheap in either mode.
|
|
169
|
+
if not probe_time_machine:
|
|
170
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
171
|
+
|
|
172
|
+
try:
|
|
173
|
+
tmutil = which("tmutil")
|
|
174
|
+
except Exception:
|
|
175
|
+
tmutil = None
|
|
176
|
+
if not tmutil:
|
|
177
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
178
|
+
try:
|
|
179
|
+
destinations = run(
|
|
180
|
+
[tmutil, "destinationinfo"],
|
|
181
|
+
stdout=subprocess.PIPE,
|
|
182
|
+
stderr=subprocess.PIPE,
|
|
183
|
+
text=True,
|
|
184
|
+
timeout=2,
|
|
185
|
+
check=False,
|
|
186
|
+
)
|
|
187
|
+
except (OSError, subprocess.SubprocessError):
|
|
188
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
189
|
+
destination_text = f"{destinations.stdout}\n{destinations.stderr}".lower()
|
|
190
|
+
if destinations.returncode != 0:
|
|
191
|
+
status = (
|
|
192
|
+
"absent"
|
|
193
|
+
if "no destinations configured" in destination_text
|
|
194
|
+
else "unavailable"
|
|
195
|
+
)
|
|
196
|
+
return {"status": status, "provider": "Time Machine"}
|
|
197
|
+
if not destinations.stdout.strip():
|
|
198
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
199
|
+
try:
|
|
200
|
+
exclusion = run(
|
|
201
|
+
[tmutil, "isexcluded", str(app_path)],
|
|
202
|
+
stdout=subprocess.PIPE,
|
|
203
|
+
stderr=subprocess.PIPE,
|
|
204
|
+
text=True,
|
|
205
|
+
timeout=2,
|
|
206
|
+
check=False,
|
|
207
|
+
)
|
|
208
|
+
except (OSError, subprocess.SubprocessError):
|
|
209
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
210
|
+
if exclusion.returncode != 0:
|
|
211
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
212
|
+
output = exclusion.stdout.strip().lower()
|
|
213
|
+
if output.startswith("[excluded]"):
|
|
214
|
+
return {"status": "excluded", "provider": "Time Machine"}
|
|
215
|
+
if output.startswith("[included]"):
|
|
216
|
+
return {"status": "included", "provider": "Time Machine"}
|
|
217
|
+
return {"status": "unavailable", "provider": "Time Machine"}
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _gather_writer_guard_log(now_utc: dt.datetime):
|
|
221
|
+
"""Gather the bounded writer-guard tail; absent/unreadable is normal."""
|
|
222
|
+
import _cctally_store as _store_mod_guard
|
|
223
|
+
|
|
224
|
+
guard_path = _store_mod_guard._guard_log_path()
|
|
225
|
+
if not guard_path.exists():
|
|
226
|
+
return None
|
|
227
|
+
lines = _read_guard_log_tail(guard_path)
|
|
228
|
+
newest_age = None
|
|
229
|
+
if lines:
|
|
230
|
+
stamp = lines[-1].split("\t", 1)[0]
|
|
231
|
+
try:
|
|
232
|
+
when = _cctally_core.parse_iso_datetime(stamp, "guard log")
|
|
233
|
+
newest_age = max(0, int((now_utc - when).total_seconds()))
|
|
234
|
+
except Exception:
|
|
235
|
+
newest_age = None
|
|
236
|
+
return {
|
|
237
|
+
"entries": len(lines),
|
|
238
|
+
"newest_age_s": newest_age,
|
|
239
|
+
"path": str(guard_path),
|
|
240
|
+
"sample": lines[-1] if lines else None,
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
|
|
65
244
|
def _gather_statusline_pipeline(c, *, now_utc: dt.datetime) -> dict:
|
|
66
245
|
"""Read the #318 statusline pipeline without creating or pruning files."""
|
|
67
246
|
now_epoch = int(now_utc.timestamp())
|
|
@@ -203,7 +382,10 @@ def _gather_accounts_state(now_utc: "dt.datetime") -> dict:
|
|
|
203
382
|
if not _cctally_core.DB_PATH.exists():
|
|
204
383
|
return state
|
|
205
384
|
try:
|
|
206
|
-
|
|
385
|
+
# #386: a `mode=ro` reader participates in the opener protocol too —
|
|
386
|
+
# measured, it CREATES `-shm`/`-wal` when they are absent. See
|
|
387
|
+
# `_cctally_store.stats_open_guarded`.
|
|
388
|
+
conn = _stats_ro_guarded()
|
|
207
389
|
except sqlite3.Error:
|
|
208
390
|
return state
|
|
209
391
|
try:
|
|
@@ -314,13 +496,16 @@ def doctor_gather_state(
|
|
|
314
496
|
cache_lock.close()
|
|
315
497
|
cache_lock = None
|
|
316
498
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
499
|
+
import _cctally_store
|
|
500
|
+
|
|
501
|
+
with _cctally_store.suppress_interrupted_stats_recovery():
|
|
502
|
+
return _doctor_gather_state_impl(
|
|
503
|
+
now_utc=now_utc,
|
|
504
|
+
runtime_bind=runtime_bind,
|
|
505
|
+
deep=deep,
|
|
506
|
+
_cache_probe_allowed=cache_probe_allowed,
|
|
507
|
+
_cache_repair_marker=cache_repair_marker,
|
|
508
|
+
)
|
|
324
509
|
finally:
|
|
325
510
|
if cache_lock is not None:
|
|
326
511
|
try:
|
|
@@ -354,6 +539,11 @@ def _doctor_gather_state_impl(
|
|
|
354
539
|
if now_utc is None:
|
|
355
540
|
now_utc = _now_utc()
|
|
356
541
|
|
|
542
|
+
backup_sync_state = _gather_backup_sync_state(
|
|
543
|
+
_cctally_core.APP_DIR,
|
|
544
|
+
probe_time_machine=deep,
|
|
545
|
+
)
|
|
546
|
+
|
|
357
547
|
# ── Install ──────────────────────────────────────────────────────
|
|
358
548
|
# #279 S2 F5d: guard the only two unguarded statements in the
|
|
359
549
|
# otherwise fail-soft gather — an exception here would kill the whole
|
|
@@ -449,9 +639,29 @@ def _doctor_gather_state_impl(
|
|
|
449
639
|
|
|
450
640
|
# ── DB ───────────────────────────────────────────────────────────
|
|
451
641
|
try:
|
|
452
|
-
|
|
642
|
+
import _cctally_store
|
|
643
|
+
|
|
644
|
+
interrupted = _cctally_store.stats_interrupted_rebuild_evidence(
|
|
645
|
+
_cctally_core.DB_PATH
|
|
646
|
+
)
|
|
647
|
+
if interrupted is not None and interrupted.get("live") is True:
|
|
648
|
+
stats_db_status = {
|
|
649
|
+
"path": str(_cctally_core.DB_PATH),
|
|
650
|
+
"user_version": 0,
|
|
651
|
+
"registry_size": len(c._STATS_MIGRATIONS),
|
|
652
|
+
"migrations": [],
|
|
653
|
+
}
|
|
654
|
+
else:
|
|
655
|
+
stats_db_status = c._db_status_for(
|
|
656
|
+
_cctally_core.DB_PATH,
|
|
657
|
+
c._STATS_MIGRATIONS,
|
|
658
|
+
"stats.db",
|
|
659
|
+
recover_interrupted_stats=False,
|
|
660
|
+
)
|
|
453
661
|
if not _cctally_core.DB_PATH.exists():
|
|
454
662
|
stats_db_status["_file_exists"] = False
|
|
663
|
+
if interrupted is not None:
|
|
664
|
+
stats_db_status["_interrupted_rebuild"] = interrupted
|
|
455
665
|
except sqlite3.Error as exc:
|
|
456
666
|
stats_db_status = {"path": str(_cctally_core.DB_PATH), "user_version": 0,
|
|
457
667
|
"registry_size": len(c._STATS_MIGRATIONS),
|
|
@@ -494,7 +704,15 @@ def _doctor_gather_state_impl(
|
|
|
494
704
|
credited_weeks: list[dict] | None = None
|
|
495
705
|
try:
|
|
496
706
|
if _cctally_core.DB_PATH.exists():
|
|
497
|
-
|
|
707
|
+
# #386 spec section 3.1, third clause: EVERY opener of the live stats
|
|
708
|
+
# family participates in the replacement protocol, read-only probes
|
|
709
|
+
# included. The open mode stays read-WRITE deliberately — switching a
|
|
710
|
+
# WAL DB whose `-shm` may be absent to `mode=ro` fails
|
|
711
|
+
# SQLITE_CANTOPEN, which is not corruption and has been misread as
|
|
712
|
+
# such on this project twice. Participation, not read-only-ness, is
|
|
713
|
+
# what the clause requires.
|
|
714
|
+
import _cctally_store as _store_mod
|
|
715
|
+
conn = _store_mod.stats_open_guarded(_cctally_core.DB_PATH)
|
|
498
716
|
try:
|
|
499
717
|
try:
|
|
500
718
|
row = conn.execute(
|
|
@@ -876,7 +1094,15 @@ def _doctor_gather_state_impl(
|
|
|
876
1094
|
_path.exists()
|
|
877
1095
|
and (_label != "cache" or _cache_probe_allowed)
|
|
878
1096
|
):
|
|
879
|
-
|
|
1097
|
+
# #386: the stats leg holds a read-write handle for the whole
|
|
1098
|
+
# of a full quick_check — the longest-lived stats handle any
|
|
1099
|
+
# diagnostic takes — so it participates in the replacement
|
|
1100
|
+
# protocol. The cache leg keeps its own opener.
|
|
1101
|
+
if _label == "stats":
|
|
1102
|
+
import _cctally_store as _store_mod
|
|
1103
|
+
_conn = _store_mod.stats_open_guarded(_path)
|
|
1104
|
+
else:
|
|
1105
|
+
_conn = sqlite3.connect(str(_path))
|
|
880
1106
|
try:
|
|
881
1107
|
_row = _conn.execute(
|
|
882
1108
|
"PRAGMA quick_check(1)").fetchone()
|
|
@@ -1068,11 +1294,16 @@ def _doctor_gather_state_impl(
|
|
|
1068
1294
|
journal_present = False
|
|
1069
1295
|
journal_appendable = None
|
|
1070
1296
|
journal_segment_count = 0
|
|
1297
|
+
journal_has_bytes = False
|
|
1071
1298
|
journal_malformed_count = None
|
|
1072
1299
|
journal_torn_tail_count = None
|
|
1073
1300
|
journal_cursor_lag_bytes = None
|
|
1074
1301
|
journal_hw_segment = None
|
|
1075
1302
|
journal_cursor_segment = None
|
|
1303
|
+
journal_conflicts = None
|
|
1304
|
+
journal_protocol_violations = None
|
|
1305
|
+
journal_protocol_acknowledged = None
|
|
1306
|
+
journal_protocol_error = None
|
|
1076
1307
|
try:
|
|
1077
1308
|
jdir = _cctally_core.JOURNAL_DIR
|
|
1078
1309
|
journal_present = jdir.exists()
|
|
@@ -1087,6 +1318,35 @@ def _doctor_gather_state_impl(
|
|
|
1087
1318
|
except Exception:
|
|
1088
1319
|
segs = []
|
|
1089
1320
|
journal_segment_count = len(segs)
|
|
1321
|
+
# #402: the disposable stats index persists the most recent complete
|
|
1322
|
+
# selector result. Shallow Dashboard/TUI gathers read that bounded
|
|
1323
|
+
# summary instead of rescanning a production-sized journal and
|
|
1324
|
+
# therefore cannot turn known taint into a false OK.
|
|
1325
|
+
try:
|
|
1326
|
+
if _cctally_core.DB_PATH.exists():
|
|
1327
|
+
pc = _stats_ro_guarded()
|
|
1328
|
+
try:
|
|
1329
|
+
protocol_rows = [
|
|
1330
|
+
json.loads(str(row[0]))
|
|
1331
|
+
for row in pc.execute(
|
|
1332
|
+
"SELECT violation_json "
|
|
1333
|
+
"FROM journal_protocol_violations "
|
|
1334
|
+
"ORDER BY batch_id, kind, fingerprint"
|
|
1335
|
+
)
|
|
1336
|
+
]
|
|
1337
|
+
journal_protocol_violations = [
|
|
1338
|
+
item for item in protocol_rows
|
|
1339
|
+
if not item.get("auditId")
|
|
1340
|
+
]
|
|
1341
|
+
journal_protocol_acknowledged = [
|
|
1342
|
+
item for item in protocol_rows
|
|
1343
|
+
if item.get("auditId")
|
|
1344
|
+
]
|
|
1345
|
+
finally:
|
|
1346
|
+
pc.close()
|
|
1347
|
+
except (sqlite3.Error, ValueError, TypeError):
|
|
1348
|
+
journal_protocol_violations = None
|
|
1349
|
+
journal_protocol_acknowledged = None
|
|
1090
1350
|
sizes: dict = {}
|
|
1091
1351
|
for seg in segs:
|
|
1092
1352
|
try:
|
|
@@ -1095,12 +1355,19 @@ def _doctor_gather_state_impl(
|
|
|
1095
1355
|
sizes[seg] = 0
|
|
1096
1356
|
if segs:
|
|
1097
1357
|
journal_hw_segment = segs[-1]
|
|
1358
|
+
journal_has_bytes = _jr._has_retained_journal_bytes(
|
|
1359
|
+
sizes.values()
|
|
1360
|
+
)
|
|
1098
1361
|
# deep-gated malformed / torn-tail scan (reads the whole journal;
|
|
1099
1362
|
# the dashboard's per-rebuild gather stays deep=False so it never
|
|
1100
1363
|
# pays this at the 10× envelope — mirrors the quick_check legs).
|
|
1101
1364
|
if deep and segs:
|
|
1102
1365
|
malformed = 0
|
|
1103
1366
|
torn = 0
|
|
1367
|
+
decoded_records: list = []
|
|
1368
|
+
protocol_evidence = []
|
|
1369
|
+
prior_high_water = None
|
|
1370
|
+
cutover_value = None
|
|
1104
1371
|
for seg in segs:
|
|
1105
1372
|
try:
|
|
1106
1373
|
data = (jdir / seg).read_bytes()
|
|
@@ -1113,24 +1380,124 @@ def _doctor_gather_state_impl(
|
|
|
1113
1380
|
# every element except the last is a complete line; the last
|
|
1114
1381
|
# is either "" (ended in \n) or the torn partial — not a
|
|
1115
1382
|
# mid-file line, so it is never counted as malformed.
|
|
1383
|
+
offset = 0
|
|
1116
1384
|
for raw in data.split(b"\n")[:-1]:
|
|
1117
|
-
if
|
|
1385
|
+
if not raw:
|
|
1386
|
+
prior_high_water = (seg, offset + 1)
|
|
1387
|
+
offset += 1
|
|
1388
|
+
continue
|
|
1389
|
+
record = _jl.decode_line(raw)
|
|
1390
|
+
if record is None:
|
|
1118
1391
|
malformed += 1
|
|
1392
|
+
prior_high_water = (
|
|
1393
|
+
seg,
|
|
1394
|
+
offset + len(raw) + 1,
|
|
1395
|
+
)
|
|
1396
|
+
offset += len(raw) + 1
|
|
1397
|
+
continue
|
|
1398
|
+
_jr._capture_protocol_prefix_evidence(
|
|
1399
|
+
record,
|
|
1400
|
+
prior_high_water,
|
|
1401
|
+
protocol_evidence,
|
|
1402
|
+
)
|
|
1403
|
+
# first cutover op wins, exactly as
|
|
1404
|
+
# `find_accounts_cutover_op` scans — captured here so the
|
|
1405
|
+
# conflict scan does not decode the whole journal twice.
|
|
1406
|
+
if (cutover_value is None
|
|
1407
|
+
and record.get("id") == _jr.CUTOVER_OP_ID):
|
|
1408
|
+
payload = record.get("payload")
|
|
1409
|
+
if isinstance(payload, dict):
|
|
1410
|
+
cutover_value = payload.get(
|
|
1411
|
+
"claude_legacy_account")
|
|
1412
|
+
# RETAIN ONLY what the selector consumes. `obs` lines are
|
|
1413
|
+
# ~97% of a real journal (984k of 1.02M) and
|
|
1414
|
+
# `resolve_effective_events` ignores them entirely —
|
|
1415
|
+
# keeping them cost 4.3 GB of peak RSS for an identical
|
|
1416
|
+
# result (#374 review).
|
|
1417
|
+
if record.get("t") in _CONFLICT_SCAN_RECORD_TYPES:
|
|
1418
|
+
decoded_records.append(record)
|
|
1419
|
+
prior_high_water = (
|
|
1420
|
+
seg,
|
|
1421
|
+
offset + len(raw) + 1,
|
|
1422
|
+
)
|
|
1423
|
+
offset += len(raw) + 1
|
|
1119
1424
|
journal_malformed_count = malformed
|
|
1120
1425
|
journal_torn_tail_count = torn
|
|
1426
|
+
# #374: same-revision quarantine, via the SHARED selector over
|
|
1427
|
+
# rebuild-equivalent input. Raw `(id, rev)` grouping would report
|
|
1428
|
+
# lower-revision groups a completed rev-1 batch legitimately
|
|
1429
|
+
# superseded, and false account conflicts that the rebuild's
|
|
1430
|
+
# `_normalize_legacy_account_stamp` resolves — so normalize
|
|
1431
|
+
# exactly as `rebuild_stats_index` does, then select.
|
|
1432
|
+
try:
|
|
1433
|
+
cutover_claude = (
|
|
1434
|
+
cutover_value if cutover_value is not None
|
|
1435
|
+
else _jr.resolve_cutover_claude_account()
|
|
1436
|
+
)
|
|
1437
|
+
for record in decoded_records:
|
|
1438
|
+
_jr._normalize_legacy_account_stamp(
|
|
1439
|
+
record, cutover_claude)
|
|
1440
|
+
selection = _jl.resolve_effective_events(
|
|
1441
|
+
decoded_records,
|
|
1442
|
+
protocol_prefix_evidence=protocol_evidence,
|
|
1443
|
+
)
|
|
1444
|
+
except _jl.JournalProtocolError as exc:
|
|
1445
|
+
# Out-of-scope malformed known record: selection did not
|
|
1446
|
+
# finish, so conflicts/tainted-batch results are unavailable.
|
|
1447
|
+
journal_protocol_error = str(exc)
|
|
1448
|
+
journal_conflicts = None
|
|
1449
|
+
journal_protocol_violations = None
|
|
1450
|
+
journal_protocol_acknowledged = None
|
|
1451
|
+
except Exception:
|
|
1452
|
+
journal_conflicts = None
|
|
1453
|
+
journal_protocol_violations = None
|
|
1454
|
+
journal_protocol_acknowledged = None
|
|
1455
|
+
else:
|
|
1456
|
+
journal_conflicts = [
|
|
1457
|
+
conflict.to_dict() for conflict in selection.conflicts
|
|
1458
|
+
]
|
|
1459
|
+
journal_protocol_violations = [
|
|
1460
|
+
violation.to_dict()
|
|
1461
|
+
for violation in selection.protocol_violations
|
|
1462
|
+
]
|
|
1463
|
+
journal_protocol_acknowledged = [
|
|
1464
|
+
violation.to_dict()
|
|
1465
|
+
for violation in (
|
|
1466
|
+
selection.acknowledged_protocol_violations
|
|
1467
|
+
)
|
|
1468
|
+
]
|
|
1121
1469
|
# ingest cursor lag: unconsumed bytes between the stats index cursor
|
|
1122
1470
|
# and the journal high-water, in canonical (segment, offset) order.
|
|
1123
1471
|
cursor = None
|
|
1124
1472
|
try:
|
|
1125
1473
|
if _cctally_core.DB_PATH.exists():
|
|
1126
|
-
jc =
|
|
1127
|
-
f"file:{_cctally_core.DB_PATH}?mode=ro", uri=True)
|
|
1474
|
+
jc = _stats_ro_guarded() # #386 opener protocol
|
|
1128
1475
|
try:
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1476
|
+
cursor_columns = {
|
|
1477
|
+
str(row[1])
|
|
1478
|
+
for row in jc.execute(
|
|
1479
|
+
"PRAGMA table_info(journal_cursor)"
|
|
1480
|
+
)
|
|
1481
|
+
}
|
|
1482
|
+
if {
|
|
1483
|
+
"applied_segment", "applied_offset"
|
|
1484
|
+
} <= cursor_columns:
|
|
1485
|
+
crow = jc.execute(
|
|
1486
|
+
"SELECT segment, offset, applied_segment, "
|
|
1487
|
+
"applied_offset FROM journal_cursor "
|
|
1488
|
+
"WHERE id = 1").fetchone()
|
|
1489
|
+
if (
|
|
1490
|
+
crow is not None
|
|
1491
|
+
and crow[2] is not None
|
|
1492
|
+
and crow[3] is not None
|
|
1493
|
+
):
|
|
1494
|
+
cursor = (crow[2], int(crow[3]))
|
|
1495
|
+
else:
|
|
1496
|
+
legacy = jc.execute(
|
|
1497
|
+
"SELECT segment, offset FROM journal_cursor "
|
|
1498
|
+
"WHERE id = 1").fetchone()
|
|
1499
|
+
if legacy is not None:
|
|
1500
|
+
cursor = (legacy[0], int(legacy[1]))
|
|
1134
1501
|
except sqlite3.OperationalError:
|
|
1135
1502
|
pass # pre-cutover DB has no journal_cursor table
|
|
1136
1503
|
finally:
|
|
@@ -1181,6 +1548,16 @@ def _doctor_gather_state_impl(
|
|
|
1181
1548
|
d["age_s"] if d["age_s"] is not None else 0))
|
|
1182
1549
|
journal_heal_incidents = _incidents
|
|
1183
1550
|
|
|
1551
|
+
# #386/#389 stats sole-writer guard log (spec §6.4). Read-only, fail-soft: an
|
|
1552
|
+
# absent log is the NORMAL state and must read as INFO, never as a gather
|
|
1553
|
+
# failure. Read only the bounded tail; rotation and cross-process throttling
|
|
1554
|
+
# bound the writer side independently.
|
|
1555
|
+
journal_writer_guard = None
|
|
1556
|
+
try:
|
|
1557
|
+
journal_writer_guard = _gather_writer_guard_log(now_utc)
|
|
1558
|
+
except (OSError, Exception):
|
|
1559
|
+
journal_writer_guard = None
|
|
1560
|
+
|
|
1184
1561
|
cctally_version_tuple = _lib_changelog._read_latest_changelog_version()
|
|
1185
1562
|
cctally_version = (
|
|
1186
1563
|
cctally_version_tuple[0] if cctally_version_tuple else "unknown"
|
|
@@ -1251,6 +1628,11 @@ def _doctor_gather_state_impl(
|
|
|
1251
1628
|
locks_held=locks_held,
|
|
1252
1629
|
# #297: cache.db WAL size backstop (gathered outside the deep branch).
|
|
1253
1630
|
cache_db_wal_bytes=cache_db_wal_bytes,
|
|
1631
|
+
# #374: quarantined same-revision groups + structural protocol violation.
|
|
1632
|
+
journal_conflicts=journal_conflicts,
|
|
1633
|
+
journal_protocol_violations=journal_protocol_violations,
|
|
1634
|
+
journal_protocol_acknowledged=journal_protocol_acknowledged,
|
|
1635
|
+
journal_protocol_error=journal_protocol_error,
|
|
1254
1636
|
# #315: read-only cache free-page evidence for the reclaim hint.
|
|
1255
1637
|
cache_db_page_count=cache_db_page_count,
|
|
1256
1638
|
cache_db_freelist_count=cache_db_freelist_count,
|
|
@@ -1266,15 +1648,18 @@ def _doctor_gather_state_impl(
|
|
|
1266
1648
|
journal_present=journal_present,
|
|
1267
1649
|
journal_appendable=journal_appendable,
|
|
1268
1650
|
journal_segment_count=journal_segment_count,
|
|
1651
|
+
journal_has_bytes=journal_has_bytes,
|
|
1269
1652
|
journal_malformed_count=journal_malformed_count,
|
|
1270
1653
|
journal_torn_tail_count=journal_torn_tail_count,
|
|
1271
1654
|
journal_cursor_lag_bytes=journal_cursor_lag_bytes,
|
|
1272
1655
|
journal_hw_segment=journal_hw_segment,
|
|
1273
1656
|
journal_cursor_segment=journal_cursor_segment,
|
|
1274
1657
|
journal_heal_incidents=journal_heal_incidents,
|
|
1658
|
+
journal_writer_guard=journal_writer_guard,
|
|
1275
1659
|
# Multi-account attribution legs (#341).
|
|
1276
1660
|
accounts_state=_gather_accounts_state(now_utc),
|
|
1277
1661
|
cache_repair_marker=cache_repair_marker,
|
|
1662
|
+
backup_sync_state=backup_sync_state,
|
|
1278
1663
|
)
|
|
1279
1664
|
|
|
1280
1665
|
|
|
@@ -1629,8 +1629,16 @@ def _backfill_five_hour_blocks(
|
|
|
1629
1629
|
).fetchone()
|
|
1630
1630
|
crossed = 1 if cross_row is not None else 0
|
|
1631
1631
|
|
|
1632
|
-
#
|
|
1633
|
-
|
|
1632
|
+
# A rebuild's only-missing row is the trailing unjournaled
|
|
1633
|
+
# projection, never a close decision. Wall time may be past its
|
|
1634
|
+
# reset, but only a later retained observation may close and
|
|
1635
|
+
# clock it (#399). The ordinary legacy backfill keeps its
|
|
1636
|
+
# historical wall-time classification.
|
|
1637
|
+
is_closed = (
|
|
1638
|
+
0 if only_missing else (1 if resets_dt < now_dt else 0)
|
|
1639
|
+
)
|
|
1640
|
+
projection_created_at = first_obs if only_missing else now_iso
|
|
1641
|
+
projection_updated_at = last_obs if only_missing else now_iso
|
|
1634
1642
|
|
|
1635
1643
|
# Token + cost totals — recomputed via the shared helper,
|
|
1636
1644
|
# which routes through get_entries() and falls back to
|
|
@@ -1681,8 +1689,8 @@ def _backfill_five_hour_blocks(
|
|
|
1681
1689
|
totals["cache_read_tokens"],
|
|
1682
1690
|
totals["cost_usd"],
|
|
1683
1691
|
is_closed,
|
|
1684
|
-
|
|
1685
|
-
|
|
1692
|
+
projection_created_at,
|
|
1693
|
+
projection_updated_at,
|
|
1686
1694
|
acct,
|
|
1687
1695
|
),
|
|
1688
1696
|
)
|
|
@@ -1777,4 +1785,3 @@ def _backfill_five_hour_blocks(
|
|
|
1777
1785
|
return 0
|
|
1778
1786
|
return inserted
|
|
1779
1787
|
|
|
1780
|
-
|